text:
import os, argparse, sys
import numpy as np
from tqdm import tqdm

np.random.seed(44)

def run(args):

    RAW_FOLDER = os.path.join(args.home, args.monoDir)
    DATASET_FOLDER = os.path.join(args.datasetHomeDir, args.monoLang)
    LANG = args.monoLang

    sumEn = open(os.path.join(RAW_FOLDER, '{}_summaries.txt'.format(LANG)), 'r')
    docsEn = open(os.path.join(RAW_FOLDER, '{}_documents.txt'.format(LANG)), 'r')
    titlesEn = open(os.path.join(RAW_FOLDER, '{}_titles.txt'.format(LANG)), 'r')
    empty_idx, titlesNb = [], []

    for e, (s, d) in enumerate(zip(sumEn.readlines(), docsEn.readlines())):
        if len(s.strip().split())<20 or len(d.strip().split())<200:
            empty_idx.append(e) #check for ill extracted, but should be OK now
        else:
            titlesNb.append(e)
    sumEn.seek(0)
    docsEn.seek(0)
    totalEx = len(titlesNb) + len(empty_idx)
    print(" * from {} titles/documents are {}=empty".format(len(titlesNb), len(empty_idx)))
    pbar = tqdm(total=totalEx)

    examples = np.random.permutation(titlesNb)
    examples = examples[:args.totalExamples] #en=300000
    training_idx, val_idx, test_idx  =  np.split(examples, [args.svalid, args.stest]) #en=280000, 290000

    print(set(training_idx).intersection(set(empty_idx)))
    print(set(val_idx).intersection(set(empty_idx)))
    print(set(test_idx).intersection(set(empty_idx)))

    title = ".{}_titles.txt".format(LANG)
    doc = ".{}_documents.txt".format(LANG)
    sum = ".{}_summaries.txt".format(LANG)

    generate_files = {}
    for sp in ['train', 'val', 'test']:
        generate_files[sp] = {}
        for f in [title, doc, sum]:
            print("creating file... " +  os.path.join(DATASET_FOLDER, sp + f))
            generate_files[sp][f] = open(os.path.join(DATASET_FOLDER, sp + f), 'w')

    print("splits", len(training_idx), len(val_idx), len(test_idx))

    for i in range(totalEx):
        t = titlesEn.readline()
        d = docsEn.readline()
        s = sumEn.readline()
        if i in training_idx:
            currentSplit = 'train'
        elif i in val_idx:
            currentSplit = 'val'
        elif i in test_idx:
            currentSplit = 'test'
        else:
            continue
        generate_files[currentSplit][title].write(t)
        generate_files[currentSplit][doc].write(d)
        generate_files[currentSplit][sum].write(s)

        if (i % 200) == 0:
            pbar.update(200)
            generate_files[currentSplit][title].flush()
            generate_files[currentSplit][doc].flush()
            generate_files[currentSplit][sum].flush()

    for sp in ['train', 'val', 'test']:
        for f in [title, doc, sum]:
            generate_files[sp][f].close()

if __name__ == "__main__":

    parser = argparse.ArgumentParser(
       description=__doc__,
       formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument("--home", type=str, default='', required=True,
                        help="Home where all XWikis related files are.")
    parser.add_argument('--datasetHomeDir', help="Directory where the generated splits will be located.")
    parser.add_argument('--monoDir', help="Directory with the monolingual files.")
    parser.add_argument('--monoLang', help="Language of monolingual files.")
    parser.add_argument('--totalExamples', help="Nb. total examples.")
    parser.add_argument('--stest', help="List index to split for test.")
    parser.add_argument('--svalid', help="List index to split for valid.")

    args = parser.parse_args(sys.argv[1:])

    run(args)

max_stars_repo_name: lauhaide/clads

max_issues_repo_name: lauhaide/clads

max_forks_repo_name: lauhaide/clads


text:
# -*- coding: utf-8 -*-
"""
Created on Sat Jan  2 20:39:58 2021

@author: ghiggi
"""
##----------------------------------------------------------------------------.
import numpy as np

from modules.utils_autoregressive import check_ar_settings
from modules.utils_autoregressive import plot_ar_settings
from modules.utils_autoregressive import get_first_valid_idx
from modules.utils_autoregressive import get_last_valid_idx
from modules.utils_autoregressive import get_dict_Y
from modules.utils_autoregressive import get_dict_X_dynamic
from modules.utils_autoregressive import get_dict_X_bc
from modules.utils_autoregressive import get_dict_stack_info

##----------------------------------------------------------------------------.
### Example - Classical Autoregressive
input_k = np.array([-9, -6,-3])
output_k = np.array([0])
ar_iterations = 6
forecast_cycle = 3

check_ar_settings(input_k=input_k, output_k=output_k, forecast_cycle=forecast_cycle,
                  ar_iterations=ar_iterations, stack_most_recent_prediction=True)
plot_ar_settings(input_k=input_k, output_k=output_k, forecast_cycle=forecast_cycle, ar_iterations = ar_iterations)

print(get_first_valid_idx(input_k))
print(get_last_valid_idx(output_k=output_k,
                         forecast_cycle=forecast_cycle,
                         ar_iterations=ar_iterations))

##----------------------------------------------------------------------------.
### Example - Fixed DELTA T
delta_t = 6
n_input_lags = 3
n_output_timesteps = 1
forecast_cycle = delta_t
ar_iterations = 4
input_k = np.cumsum(-1*np.repeat(delta_t, repeats=n_input_lags))[::-1]
output_k = np.cumsum(np.concatenate((np.array([0]), np.repeat(delta_t, repeats=n_output_timesteps-1))))

check_ar_settings(input_k=input_k, output_k=output_k, forecast_cycle=forecast_cycle,
                  ar_iterations=ar_iterations, stack_most_recent_prediction=True)
plot_ar_settings(input_k=input_k, output_k=output_k, forecast_cycle=forecast_cycle, ar_iterations = ar_iterations)

print(get_first_valid_idx(input_k))
print(get_last_valid_idx(output_k=output_k,
                         forecast_cycle=forecast_cycle,
                         ar_iterations=ar_iterations))

##----------------------------------------------------------------------------.
### Example - Multi-Temporal Output
input_k = np.array([-7,-5,-3,-1])
output_k = np.array([0, 1, 3, 7])
ar_iterations = 6
forecast_cycle = 2

check_ar_settings(input_k=input_k, output_k=output_k, forecast_cycle=forecast_cycle,
                  ar_iterations=ar_iterations, stack_most_recent_prediction=True)
plot_ar_settings(input_k=input_k, output_k=output_k, forecast_cycle=forecast_cycle,
                 ar_iterations = ar_iterations, stack_most_recent_prediction = True)
plot_ar_settings(input_k=input_k, output_k=output_k, forecast_cycle=forecast_cycle,
                 ar_iterations = ar_iterations, stack_most_recent_prediction = False)

##----------------------------------------------------------------------------.
### Example - Chess pattern
input_k = np.array([-8,-6,-4,-2])
output_k = np.array([0, 2, 4, 7])
ar_iterations = 6
forecast_cycle = 1

check_ar_settings(input_k=input_k, output_k=output_k, forecast_cycle=forecast_cycle,
                  ar_iterations=ar_iterations, stack_most_recent_prediction=True)
plot_ar_settings(input_k, output_k, forecast_cycle, ar_iterations, stack_most_recent_prediction=True)
plot_ar_settings(input_k, output_k, forecast_cycle, ar_iterations, stack_most_recent_prediction=False)

#-----------------------------------------------------------------------------.
### Example dictionary for data retrieval and stacking
ar_iterations=5
forecast_cycle=3
output_k = np.array([0,3])
input_k = np.array([-9, -6, -3])

plot_ar_settings(input_k=input_k,
                 output_k=output_k,
                 forecast_cycle=forecast_cycle,
                 ar_iterations = ar_iterations,
                 stack_most_recent_prediction = False)

plot_ar_settings(input_k=input_k,
                 output_k=output_k,
                 forecast_cycle=forecast_cycle,
                 ar_iterations = ar_iterations,
                 stack_most_recent_prediction = True)

dict_Y = get_dict_Y(ar_iterations=ar_iterations,
                    forecast_cycle=forecast_cycle,
                    output_k=output_k)
dict_X_dynamic = get_dict_X_dynamic(ar_iterations=ar_iterations,
                                    forecast_cycle=forecast_cycle,
                                    input_k=input_k)
dict_X_bc = get_dict_X_bc(ar_iterations = ar_iterations,
                          forecast_cycle = forecast_cycle,
                          input_k = input_k)
dict_Y_to_stack, dict_Y_to_remove = get_dict_stack_info(ar_iterations = ar_iterations,
                                                        forecast_cycle = forecast_cycle,
                                                        input_k = input_k,
                                                        output_k = output_k,
                                                        stack_most_recent_prediction = False)

#----------------------------------------------------------------------------.
### Check validity of something do not work
input_k = np.array([-7,-5,-3,-1])
output_k = np.array([0, 1, 3, 7])
ar_iterations = 2
forecast_cycle = 5

check_ar_settings(input_k=input_k, output_k=output_k, forecast_cycle=forecast_cycle,
                  ar_iterations=ar_iterations, stack_most_recent_prediction=True)
plot_ar_settings(input_k, output_k, forecast_cycle, ar_iterations, hatch=True)

# - Use Hatch=False allow to still visualize the AR setting
plot_ar_settings(input_k, output_k, forecast_cycle, ar_iterations, hatch=False)

#-----------------------------------------------------------------------------.

ext: py

max_stars_repo_name: deepsphere/deepsphere-weather

max_issues_repo_name: deepsphere/deepsphere-weather

max_forks_repo_name: deepsphere/deepsphere-weather

reason: import numpy


text:
"""Linear covariance function."""

from typing import Optional

import numpy as np

import probnum.utils as _utils
from probnum.typing import IntArgType, ScalarArgType

from ._kernel import Kernel

class Linear(Kernel):
    r"""Linear kernel.

    Linear covariance function defined by

    .. math ::
        k(x_0, x_1) = x_0^\top x_1 + c.

    Parameters
    ----------
    input_dim :
        Input dimension of the kernel.
    constant
        Constant offset :math:`c`.

    See Also
    --------
    Polynomial : Polynomial covariance function.

    Examples
    --------
    >>> import numpy as np
    >>> from probnum.kernels import Linear
    >>> K = Linear(input_dim=2)
    >>> xs = np.array([[1, 2], [2, 3]])
    >>> K.matrix(xs)
    array([[ 5.,  8.],
           [ 8., 13.]])
    """

    def __init__(self, input_dim: IntArgType, constant: ScalarArgType = 0.0):
        self.constant = _utils.as_numpy_scalar(constant)
        super().__init__(input_dim=input_dim)

    def _evaluate(self, x0: np.ndarray, x1: Optional[np.ndarray]) -> np.ndarray:
        return self._euclidean_inner_products(x0, x1) + self.constant

max_stars_repo_name: probabilistic-numerics/probnum

max_issues_repo_name: probabilistic-numerics/probnum

max_forks_repo_name: probabilistic-numerics/probnum


text:
# Copyright (c) OpenMMLab. All rights reserved.
import copy

import numpy as np
import pycocotools.mask as maskUtils
import pytest
from mmcv.utils import build_from_cfg

from mmdet.core.mask import BitmapMasks, PolygonMasks
from mmdet.datasets.builder import PIPELINES

def _check_keys(results, results_translated):
    assert len(set(results.keys()).difference(set(
        results_translated.keys()))) == 0
    assert len(set(results_translated.keys()).difference(set(
        results.keys()))) == 0

def _pad(h, w, c, pad_val, axis=-1, dtype=np.float32):
    assert isinstance(pad_val, (int, float, tuple))
    if isinstance(pad_val, (int, float)):
        pad_val = tuple([pad_val] * c)
    assert len(pad_val) == c
    pad_data = np.stack([np.ones((h, w)) * pad_val[i] for i in range(c)],
                        axis=axis).astype(dtype)
    return pad_data

def _construct_img(results):
    h, w = results['img_info']['height'], results['img_info']['width']
    img = np.random.uniform(0, 1, (h, w, 3)) * 255
    img = img.astype(np.uint8)
    results['img'] = img
    results['img_shape'] = img.shape
    results['ori_shape'] = img.shape
    results['img_fields'] = ['img']

def _construct_ann_info(h=427, w=640, c=3):
    bboxes = np.array(
        [[222.62, 217.82, 241.81, 238.93], [50.5, 329.7, 130.23, 384.96],
         [175.47, 331.97, 254.8, 389.26]],
        dtype=np.float32)
    labels = np.array([9, 2, 2], dtype=np.int64)
    bboxes_ignore = np.array([[59., 253., 311., 337.]], dtype=np.float32)
    masks = [
        [[222.62, 217.82, 222.62, 238.93, 241.81, 238.93, 240.85, 218.78]],
        [[
            69.19, 332.17, 82.39, 330.25, 97.24, 329.7, 114.01, 331.35, 116.76,
            337.39, 119.78, 343.17, 128.03, 344.54, 128.86, 347.84, 124.18,
            350.59, 129.96, 358.01, 130.23, 366.54, 129.13, 377.81, 125.28,
            382.48, 119.78, 381.93, 117.31, 377.54, 116.21, 379.46, 114.83,
            382.21, 107.14, 383.31, 105.49, 378.36, 77.99, 377.54, 75.79,
            381.11, 69.74, 381.93, 66.72, 378.91, 65.07, 377.81, 63.15, 379.19,
            62.32, 383.31, 52.7, 384.96, 50.5, 379.46, 51.32, 375.61, 51.6,
            370.11, 51.6, 364.06, 53.52, 354.99, 56.27, 344.54, 59.57, 336.29,
            66.45, 332.72
        ]],
        [[
            175.47, 386.86, 175.87, 376.44, 177.08, 351.2, 189.1, 332.77,
            194.31, 331.97, 236.37, 332.77, 244.79, 342.39, 246.79, 346.79,
            248.39, 345.99, 251.6, 345.59, 254.8, 348.0, 254.8, 351.6, 250.0,
            352.0, 250.0, 354.81, 251.6, 358.41, 251.6, 364.42, 251.6, 370.03,
            252.8, 378.04, 252.8, 384.05, 250.8, 387.26, 246.39, 387.66,
            245.19, 386.46, 242.38, 388.86, 233.97, 389.26, 232.77, 388.06,
            232.77, 383.65, 195.91, 381.25, 195.91, 384.86, 191.1, 384.86,
            187.49, 385.26, 186.69, 382.85, 184.29, 382.45, 183.09, 387.26,
            178.68, 388.46, 176.28, 387.66
        ]]
    ]
    return dict(
        bboxes=bboxes, labels=labels, bboxes_ignore=bboxes_ignore, masks=masks)

def _load_bboxes(results):
    ann_info = results['ann_info']
    results['gt_bboxes'] = ann_info['bboxes'].copy()
    results['bbox_fields'] = ['gt_bboxes']
    gt_bboxes_ignore = ann_info.get('bboxes_ignore', None)
    if gt_bboxes_ignore is not None:
        results['gt_bboxes_ignore'] = gt_bboxes_ignore.copy()
        results['bbox_fields'].append('gt_bboxes_ignore')

def _load_labels(results):
    results['gt_labels'] = results['ann_info']['labels'].copy()

def _poly2mask(mask_ann, img_h, img_w):
    if isinstance(mask_ann, list):
        # polygon -- a single object might consist of multiple parts
        # we merge all parts into one mask rle code
        rles = maskUtils.frPyObjects(mask_ann, img_h, img_w)
        rle = maskUtils.merge(rles)
    elif isinstance(mask_ann['counts'], list):
        # uncompressed RLE
        rle = maskUtils.frPyObjects(mask_ann, img_h, img_w)
    else:
        # rle
        rle = mask_ann
    mask = maskUtils.decode(rle)
    return mask

def _process_polygons(polygons):
    polygons = [np.array(p) for p in polygons]
    valid_polygons = []
    for polygon in polygons:
        if len(polygon) % 2 == 0 and len(polygon) >= 6:
            valid_polygons.append(polygon)
    return valid_polygons

def _load_masks(results, poly2mask=True):
    h, w = results['img_info']['height'], results['img_info']['width']
    gt_masks = results['ann_info']['masks']
    if poly2mask:
        gt_masks = BitmapMasks([_poly2mask(mask, h, w) for mask in gt_masks],
                               h, w)
    else:
        gt_masks = PolygonMasks(
            [_process_polygons(polygons) for polygons in gt_masks], h, w)
    results['gt_masks'] = gt_masks
    results['mask_fields'] = ['gt_masks']

def _construct_semantic_seg(results):
    h, w = results['img_info']['height'], results['img_info']['width']
    seg_toy = (np.random.uniform(0, 1, (h, w)) * 255).astype(np.uint8)
    results['gt_semantic_seg'] = seg_toy
    results['seg_fields'] = ['gt_semantic_seg']

def construct_toy_data(poly2mask=True):
    img_info = dict(height=427, width=640)
    ann_info = _construct_ann_info(h=img_info['height'], w=img_info['width'])
    results = dict(img_info=img_info, ann_info=ann_info)
    # construct image, similar to 'LoadImageFromFile'
    _construct_img(results)
    # 'LoadAnnotations' (bboxes, labels, masks, semantic_seg)
    _load_bboxes(results)
    _load_labels(results)
    _load_masks(results, poly2mask)
    _construct_semantic_seg(results)
    return results

def test_translate():
    # test assertion for invalid value of level
    with pytest.raises(AssertionError):
        transform = dict(type='Translate', level=-1)
        build_from_cfg(transform, PIPELINES)

    # test assertion for invalid type of level
    with pytest.raises(AssertionError):
        transform = dict(type='Translate', level=[1])
        build_from_cfg(transform, PIPELINES)

    # test assertion for invalid prob
    with pytest.raises(AssertionError):
        transform = dict(type='Translate', level=1, prob=-0.5)
        build_from_cfg(transform, PIPELINES)

    # test assertion for the num of elements in tuple img_fill_val
    with pytest.raises(AssertionError):
        transform = dict(
            type='Translate', level=1, img_fill_val=(128, 128, 128, 128))
        build_from_cfg(transform, PIPELINES)

    # test ValueError for invalid type of img_fill_val
    with pytest.raises(ValueError):
        transform = dict(
            type='Translate', level=1, img_fill_val=[128, 128, 128])
        build_from_cfg(transform, PIPELINES)

    # test assertion for invalid value of img_fill_val
    with pytest.raises(AssertionError):
        transform = dict(
            type='Translate', level=1, img_fill_val=(128, -1, 256))
        build_from_cfg(transform, PIPELINES)

    # test assertion for invalid value of direction
    with pytest.raises(AssertionError):
        transform = dict(
            type='Translate', level=1, img_fill_val=128, direction='diagonal')
        build_from_cfg(transform, PIPELINES)

    # test assertion for invalid type of max_translate_offset
    with pytest.raises(AssertionError):
        transform = dict(
            type='Translate',
            level=1,
            img_fill_val=128,
            max_translate_offset=(250., ))
        build_from_cfg(transform, PIPELINES)

    # construct toy data example for unit test
    results = construct_toy_data()

    def _check_bbox_mask(results,
                         results_translated,
                         offset,
                         direction,
                         min_size=0.):
        # The key correspondence from bboxes to labels and masks.
        bbox2label = {
            'gt_bboxes': 'gt_labels',
            'gt_bboxes_ignore': 'gt_labels_ignore'
        }
        bbox2mask = {
            'gt_bboxes': 'gt_masks',
            'gt_bboxes_ignore': 'gt_masks_ignore'
        }

        def _translate_bbox(bboxes, offset, direction, max_h, max_w):
            if direction == 'horizontal':
                bboxes[:, 0::2] = bboxes[:, 0::2] + offset
            elif direction == 'vertical':
                bboxes[:, 1::2] = bboxes[:, 1::2] + offset
            else:
                raise ValueError
            bboxes[:, 0::2] = np.clip(bboxes[:, 0::2], 0, max_w)
            bboxes[:, 1::2] = np.clip(bboxes[:, 1::2], 0, max_h)
            return bboxes

        h, w, c = results_translated['img'].shape
        for key in results_translated.get('bbox_fields', []):
            label_key, mask_key = bbox2label[key], bbox2mask[key]
            # check length of key
            if label_key in results:
                assert len(results_translated[key]) == len(
                    results_translated[label_key])
            if mask_key in results:
                assert len(results_translated[key]) == len(
                    results_translated[mask_key])
            # construct gt_bboxes
            gt_bboxes = _translate_bbox(
                copy.deepcopy(results[key]), offset, direction, h, w)
            valid_inds = (gt_bboxes[:, 2] - gt_bboxes[:, 0] > min_size) & (
                gt_bboxes[:, 3] - gt_bboxes[:, 1] > min_size)
            gt_bboxes = gt_bboxes[valid_inds]
            # check bbox
            assert np.equal(gt_bboxes, results_translated[key]).all()

            # construct gt_masks
            if mask_key not in results:
                # e.g. 'gt_masks_ignore'
                continue
            masks, masks_translated = results[mask_key].to_ndarray(
            ), results_translated[mask_key].to_ndarray()
            assert masks.dtype == masks_translated.dtype
            if direction == 'horizontal':
                masks_pad = _pad(
                    h,
                    abs(offset),
                    masks.shape[0],
                    0,
                    axis=0,
                    dtype=masks.dtype)
                if offset <= 0:
                    # left shift
                    gt_masks = np.concatenate(
                        (masks[:, :, -offset:], masks_pad), axis=-1)
                else:
                    # right shift
                    gt_masks = np.concatenate(
                        (masks_pad, masks[:, :, :-offset]), axis=-1)
            else:
                masks_pad = _pad(
                    abs(offset),
                    w,
                    masks.shape[0],
                    0,
                    axis=0,
                    dtype=masks.dtype)
                if offset <= 0:
                    # top shift
                    gt_masks = np.concatenate(
                        (masks[:, -offset:, :], masks_pad), axis=1)
                else:
                    # bottom shift
                    gt_masks = np.concatenate(
                        (masks_pad, masks[:, :-offset, :]), axis=1)
            gt_masks = gt_masks[valid_inds]
            # check masks
            assert np.equal(gt_masks, masks_translated).all()

    def _check_img_seg(results, results_translated, keys, offset, fill_val,
                       direction):
        for key in keys:
            assert isinstance(results_translated[key], type(results[key]))
            # assert type(results[key]) == type(results_translated[key])
            data, data_translated = results[key], results_translated[key]
            if 'mask' in key:
                data, data_translated = data.to_ndarray(
                ), data_translated.to_ndarray()
            assert data.dtype == data_translated.dtype
            if 'img' in key:
                data, data_translated = data.transpose(
                    (2, 0, 1)), data_translated.transpose((2, 0, 1))
            elif 'seg' in key:
                data, data_translated = data[None, :, :], data_translated[
                    None, :, :]
            c, h, w = data.shape
            if direction == 'horizontal':
                data_pad = _pad(
                    h, abs(offset), c, fill_val, axis=0, dtype=data.dtype)
                if offset <= 0:
                    # left shift
                    data_gt = np.concatenate((data[:, :, -offset:], data_pad),
                                             axis=-1)
                else:
                    # right shift
                    data_gt = np.concatenate((data_pad, data[:, :, :-offset]),
                                             axis=-1)
            else:
                data_pad = _pad(
                    abs(offset), w, c, fill_val, axis=0, dtype=data.dtype)
                if offset <= 0:
                    # top shift
                    data_gt = np.concatenate((data[:, -offset:, :], data_pad),
                                             axis=1)
                else:
                    # bottom shift
                    data_gt = np.concatenate((data_pad, data[:, :-offset, :]),
                                             axis=1)
            if 'mask' in key:
                # TODO assertion here. ``data_translated`` must be a subset
                # (or equal) of ``data_gt``
                pass
            else:
                assert np.equal(data_gt, data_translated).all()

    def check_translate(results,
                        results_translated,
                        offset,
                        img_fill_val,
                        seg_ignore_label,
                        direction,
                        min_size=0):
        # check keys
        _check_keys(results, results_translated)
        # check image
        _check_img_seg(results, results_translated,
                       results.get('img_fields', ['img']), offset,
                       img_fill_val, direction)
        # check segmentation map
        _check_img_seg(results, results_translated,
                       results.get('seg_fields', []), offset, seg_ignore_label,
                       direction)
        # check masks and bboxes
        _check_bbox_mask(results, results_translated, offset, direction,
                         min_size)

    # test case when level=0 (without translate aug)
    img_fill_val = (104, 116, 124)
    seg_ignore_label = 255
    transform = dict(
        type='Translate',
        level=0,
        prob=1.0,
        img_fill_val=img_fill_val,
        seg_ignore_label=seg_ignore_label)
    translate_module = build_from_cfg(transform, PIPELINES)
    results_wo_translate = translate_module(copy.deepcopy(results))
    check_translate(
        copy.deepcopy(results),
        results_wo_translate,
        0,
        img_fill_val,
        seg_ignore_label,
        'horizontal',
    )

    # test case when level>0 and translate horizontally (left shift).
    transform = dict(
        type='Translate',
        level=8,
        prob=1.0,
        img_fill_val=img_fill_val,
        random_negative_prob=1.0,
        seg_ignore_label=seg_ignore_label)
    translate_module = build_from_cfg(transform, PIPELINES)
    offset = translate_module.offset
    results_translated = translate_module(copy.deepcopy(results))
    check_translate(
        copy.deepcopy(results),
        results_translated,
        -offset,
        img_fill_val,
        seg_ignore_label,
        'horizontal',
    )

    # test case when level>0 and translate horizontally (right shift).
    translate_module.random_negative_prob = 0.0
    results_translated = translate_module(copy.deepcopy(results))
    check_translate(
        copy.deepcopy(results),
        results_translated,
        offset,
        img_fill_val,
        seg_ignore_label,
        'horizontal',
    )

    # test case when level>0 and translate vertically (top shift).
    transform = dict(
        type='Translate',
        level=10,
        prob=1.0,
        img_fill_val=img_fill_val,
        seg_ignore_label=seg_ignore_label,
        random_negative_prob=1.0,
        direction='vertical')
    translate_module = build_from_cfg(transform, PIPELINES)
    offset = translate_module.offset
    results_translated = translate_module(copy.deepcopy(results))
    check_translate(
        copy.deepcopy(results), results_translated, -offset, img_fill_val,
        seg_ignore_label, 'vertical')

    # test case when level>0 and translate vertically (bottom shift).
    translate_module.random_negative_prob = 0.0
    results_translated = translate_module(copy.deepcopy(results))
    check_translate(
        copy.deepcopy(results), results_translated, offset, img_fill_val,
        seg_ignore_label, 'vertical')

    # test case when no translation is called (prob<=0)
    transform = dict(
        type='Translate',
        level=8,
        prob=0.0,
        img_fill_val=img_fill_val,
        random_negative_prob=0.0,
        seg_ignore_label=seg_ignore_label)
    translate_module = build_from_cfg(transform, PIPELINES)
    results_translated = translate_module(copy.deepcopy(results))

    # test translate vertically with PolygonMasks (top shift)
    results = construct_toy_data(False)
    transform = dict(
        type='Translate',
        level=10,
        prob=1.0,
        img_fill_val=img_fill_val,
        seg_ignore_label=seg_ignore_label,
        direction='vertical')
    translate_module = build_from_cfg(transform, PIPELINES)
    offset = translate_module.offset
    translate_module.random_negative_prob = 1.0
    results_translated = translate_module(copy.deepcopy(results))

    def _translated_gt(masks, direction, offset, out_shape):
        translated_masks = []
        for poly_per_obj in masks:
            translated_poly_per_obj = []
            for p in poly_per_obj:
                p = p.copy()
                if direction == 'horizontal':
                    p[0::2] = np.clip(p[0::2] + offset, 0, out_shape[1])
                elif direction == 'vertical':
                    p[1::2] = np.clip(p[1::2] + offset, 0, out_shape[0])
                if PolygonMasks([[p]], *out_shape).areas[0] > 0:
                    # filter invalid (area=0)
                    translated_poly_per_obj.append(p)
            if len(translated_poly_per_obj):
                translated_masks.append(translated_poly_per_obj)
        translated_masks = PolygonMasks(translated_masks, *out_shape)
        return translated_masks

    h, w = results['img_shape'][:2]
    for key in results.get('mask_fields', []):
        masks = results[key]
        translated_gt = _translated_gt(masks, 'vertical', -offset, (h, w))
        assert np.equal(results_translated[key].to_ndarray(),
                        translated_gt.to_ndarray()).all()

    # test translate horizontally with PolygonMasks (right shift)
    results = construct_toy_data(False)
    transform = dict(
        type='Translate',
        level=8,
        prob=1.0,
        img_fill_val=img_fill_val,
        random_negative_prob=0.0,
        seg_ignore_label=seg_ignore_label)
    translate_module = build_from_cfg(transform, PIPELINES)
    offset = translate_module.offset
    results_translated = translate_module(copy.deepcopy(results))
    h, w = results['img_shape'][:2]
    for key in results.get('mask_fields', []):
        masks = results[key]
        translated_gt = _translated_gt(masks, 'horizontal', offset, (h, w))
        assert np.equal(results_translated[key].to_ndarray(),
                        translated_gt.to_ndarray()).all()

    # test AutoAugment equipped with Translate
    policies = [[dict(type='Translate', level=10, prob=1.)]]
    autoaug = dict(type='AutoAugment', policies=policies)
    autoaug_module = build_from_cfg(autoaug, PIPELINES)
    autoaug_module(copy.deepcopy(results))

    policies = [[
        dict(type='Translate', level=10, prob=1.),
        dict(
            type='Translate',
            level=8,
            img_fill_val=img_fill_val,
            direction='vertical')
    ]]
    autoaug = dict(type='AutoAugment', policies=policies)
    autoaug_module = build_from_cfg(autoaug, PIPELINES)
    autoaug_module(copy.deepcopy(results))

max_stars_repo_name: Brym-Gyimah/mmdetection


text:
#!/usr/bin/env python
# =============================================================================
"""
Code Information:
    Date: 04/14/2019
	Programmer: John A. Betancourt G.
	Mail: john.betancourt93@gmail.com
    Web: www.linkedin.com/in/jhon-alberto-betancourt-gonzalez-345557129

Description: Project 5 - Udacity - self driving cars Nanodegree
    Extended Kalman Filter

Tested on:
    python 3.5
    UBUNTU 16.04
"""

# =============================================================================
# LIBRARIES AND DEPENDENCIES - LIBRARIES AND DEPENDENCIES - LIBRARIES AND DEPEN
# =============================================================================
import os
from multiprocessing import Pool
import subprocess

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import csv

# =============================================================================
def run_process(process):
    os.system('{}'.format(process))

def getClipboardData():
    p = subprocess.Popen(['xclip','-selection', 'clipboard', '-o'],
        stdout=subprocess.PIPE)
    retcode = p.wait()
    data = p.stdout.read()
    return data

def setClipboardData(data):
    p = subprocess.Popen(['xclip','-selection','clipboard'],
        stdin=subprocess.PIPE)
    p.stdin.write(data)
    p.stdin.close()
    retcode = p.wait()

def plot_kalman_graph(kalman_file_path, save_file=None):

    # -------------------------------------------------------------------------
    # Variables inizialitation
    ground_truth_x = []
    ground_truth_y = []
    ground_truth_vx = []
    ground_truth_vy = []
    estimate_x = []
    estimate_y = []
    estimate_vx = []
    estimate_vy = []
    rmse_x = []
    rmse_y = []
    rmse_vx = []
    rmse_vy = []

    # Read csv file for steering pid controller
    with open(kalman_file_path) as csvfile:
        reader = csv.reader(csvfile)
        for idx, line in enumerate(reader):
            if idx:
                ground_truth_x.append(float(line[0])) # ground_truth_x
                ground_truth_y.append(float(line[1])) # ground_truth_y
                ground_truth_vx.append(float(line[2])) # ground_truth_vx
                ground_truth_vy.append(float(line[3])) # ground_truth_vy
                estimate_x.append(float(line[4])) # estimate_x
                estimate_y.append(float(line[5])) # estimate_y
                estimate_vx.append(float(line[6])) # estimate_x
                estimate_vy.append(float(line[7])) # estimate_y
                rmse_x.append(float(line[8])) # rmse_x
                rmse_y.append(float(line[9])) # rmse_y
                rmse_vx.append(float(line[10])) # rmse_vx
                rmse_vy.append(float(line[11])) # rmse_vy

    # -------------------------------------------------------------------------
    plt.figure(figsize=(30,20))

    plt.subplot(2, 2, 1)
    plt.plot(ground_truth_x/np.max(ground_truth_x), '#0000FF')
    plt.plot(estimate_x/np.max(estimate_x), '#FF8000')
    plt.plot(rmse_x/np.max(rmse_x), '#FF0000')
    ground_truth_x_patch = mpatches.Patch(color='#0000FF', label='ground_truth_x - max:{}'.format(round(np.max(estimate_x), 2)))
    estimate_x_patch = mpatches.Patch(color='#FF8000', label='estimate_x - max:{}'.format(round(np.max(estimate_x), 2)))
    rmse_x_patch = mpatches.Patch(color='#FF0000', label='rmse_x - max:{}'.format(round(np.max(rmse_x), 2)))
    plt.legend(handles=[ground_truth_x_patch, estimate_x_patch, rmse_x_patch])
    plt.title('Cars X position')
    plt.xlabel('Iteration (t/[s])')
    plt.ylabel('Position [m]')
    plt.grid(True)

    plt.subplot(2, 2, 2)
    plt.plot(ground_truth_y/np.max(ground_truth_y), '#0000FF')
    plt.plot(estimate_y/np.max(estimate_y), '#FF8000')
    plt.plot(rmse_y/np.max(rmse_y), '#FF0000')
    ground_truth_y_patch = mpatches.Patch(color='#0000FF', label='ground_truth_y - max:{}'.format(round(np.max(estimate_y), 2)))
    estimate_y_patch = mpatches.Patch(color='#FF8000', label='estimate_y - max:{}'.format(round(np.max(estimate_y), 2)))
    rmse_y_patch = mpatches.Patch(color='#FF0000', label='rmse_y - max:{}'.format(round(np.max(rmse_y), 2)))
    plt.legend(handles=[ground_truth_y_patch, estimate_y_patch, rmse_y_patch])
    plt.title('Cars Y position')
    plt.xlabel('Iteration (t/[s])')
    plt.ylabel('Position [m]')
    plt.grid(True)

    plt.subplot(2, 2, 3)
    plt.plot(ground_truth_vx/np.max(ground_truth_vx), '#0000FF')
    plt.plot(estimate_vx/np.max(estimate_vx), '#8000FF')
    plt.plot(rmse_vx/np.max(rmse_vx), '#FF0000')
    ground_truth_vx_patch = mpatches.Patch(color='#0000FF', label='ground_truth_vx - max:{}'.format(round(np.max(ground_truth_vx), 2)))
    estimate_vx_patch = mpatches.Patch(color='#8000FF', label='estimate_vx - max:{}'.format(round(np.max(estimate_vx), 2)))
    rmse_vx_patch = mpatches.Patch(color='#FF0000', label='rmse_vx - max:{}'.format(round(np.max(rmse_vx), 2)))
    plt.legend(handles=[ground_truth_vx_patch, estimate_vx_patch, rmse_vx_patch])
    plt.title('Cars Vx Speed')
    plt.xlabel('Iteration (t/[s])')
    plt.ylabel('Vy Speed [m/s]')
    plt.grid(True)

    plt.subplot(2, 2, 4)
    plt.plot(ground_truth_vy/np.max(ground_truth_vy), '#0000FF')
    plt.plot(estimate_vy/np.max(estimate_vy), '#8000FF')
    plt.plot(rmse_vy/np.max(rmse_vy), '#FF0000')
    ground_truth_vy_patch = mpatches.Patch(color='#0000FF', label='ground_truth_vy - max:{}'.format(round(np.max(ground_truth_vy), 2)))
    estimate_vy_patch = mpatches.Patch(color='#8000FF', label='estimate_vy - max:{}'.format(round(np.max(estimate_vy), 2)))
    rmse_vy_patch = mpatches.Patch(color='#FF0000', label='rmse_vy - max:{}'.format(round(np.max(rmse_vy), 2)))
    plt.legend(handles=[ground_truth_vy_patch, estimate_vy_patch, rmse_vy_patch])
    plt.title('Cars Vy Speed')
    plt.xlabel('Iteration (t/[s])')
    plt.ylabel('Vy Speed [m/s]')
    plt.grid(True)

    # -------------------------------------------------------------------------
    if save_file is not None:
        plt.savefig(save_file)

    # Plot graphs
    plt.show()

def main():
    # Copy command to clipboard to Create Video from desktop
    video_name = "kalman_filter_result_{}.mp4".format(1)
    command = "ffmpeg -video_size {}x{} -framerate {} -f x11grab -i :0.0+100,200 {}".format(
        1200, 750, 30, os.path.join(os.getcwd(), "video_results", video_name))
    setClipboardData(command.encode())

    # Run make in build fodler
    os.system('cd build && make')

    # Run subprocess
    try:
        processes = (
            "{}".format(os.path.join(os.getcwd(), "term2_sim_linux", "term2_sim.x86_64")),
            "{}".format(os.path.join(os.getcwd(), "build", "ExtendedKF"))
        )
        # Run simulator and socket
        pool = Pool(processes=len(processes))
        pool.map(run_process, processes)
    except Exception as e:
        print(str(e))

    print("Process has finished")

# =============================================================================
if __name__=="__main__":

    # Run all stuff
    main()

    # Plot steering controller graph
    plot_kalman_graph(
        kalman_file_path="Extender_Kalman_Filter-Dataset_1.csv",
        save_file="writeup_files/Extender_Kalman_Filter-Dataset_1_graph.png"
        )
    plot_kalman_graph(
        kalman_file_path="Extender_Kalman_Filter-Dataset_2.csv",
        save_file="writeup_files/Extender_Kalman_Filter-Dataset_2_graph.png"
        )

# =============================================================================


text:
# -*- coding: utf-8 -*-
# @Time    : 2021/3/30 14:20
# @Author  : Daniel Zhang
# @Email   : zhangdan_nuaa@163.com
# @File    : utils.py
# @Software: PyCharm

from dataloader import load_data_label
from sklearn.cross_decomposition import PLSRegression
from sklearn.model_selection import cross_val_score
import scipy.stats as stats
import numpy as np
import time

from sklearn.svm import SVR
from sklearn.metrics import mean_absolute_error

class IRIV:
    '''Iteratively retaining informative variables (IRIV)'''
    def __init__(self, max_components=10):
        self.max_components = max_components
        self.back_elimination = Elimination(max_components)

    def iteration(self, data, label, iter_num=100, min_dimension=10):
        self.data, self.label = data, label
        for j in range(iter_num):
            start_time = time.time()
            store_variables, remove_variables = self.__calculate_informative_variable(data, label)
            if np.sum(remove_variables) == 0 or np.sum(store_variables) <= min_dimension:
                data = data[:, store_variables]
                print('The iterative rounds of IRIV have been finished, now enter into the process of backward elimination!\n')
                break

            data = data[:, store_variables]
            print('The %d th round of IRIV has finished!  ' % (j + 1))
            print('Remain %d / %d  variable, using time: %g seconds!\n' % (data.shape[1],
                                                                           self.data.shape[1],
                                                                           time.time() - start_time))

        data, _ = self.back_elimination.iteration(data, label)
        self.remain_data = data
        return data

    def __calculate_informative_variable(self, data, label):
        '''返回强弱信息变量 和无 干扰信息的索引'''
        rmsecv5, A = self.__calculate_rmsecv(data, label)
        rmsecv_origin = np.tile(rmsecv5[:, 0].reshape(-1, 1), (A.shape[1], ))
        rmsecv_replace = rmsecv5[:, 1:]

        rmsecv_exclude = rmsecv_replace.copy()
        rmsecv_include = rmsecv_replace.copy()
        rmsecv_exclude[A == 0] = rmsecv_origin[A == 0]
        rmsecv_include[A == 1] = rmsecv_origin[A == 1]
        exclude_mean = np.mean(rmsecv_exclude, axis=0)
        include_mean = np.mean(rmsecv_include, axis=0)

        p_val, DMEAN, H = [], [], []
        for i in range(A.shape[1]):
            _, pVal = stats.mannwhitneyu(rmsecv_exclude[:, i], rmsecv_include[:, i], alternative='two-sided')
            H.append(int(pVal <= 0.05))

            # # Just a trick, indicating uninformative and interfering variable if Pvalue>1
            temp_DMEAN = exclude_mean[i] - include_mean[i]
            if temp_DMEAN < 0:
                pVal = pVal + 1

            p_val.append(pVal)
            DMEAN.append(temp_DMEAN)
        p_val = np.stack(p_val)
        DMEAN = np.stack(DMEAN)
        H = np.stack(H)

        strong_inform = (H == 1) * (p_val < 1)
        weak_inform = (H == 0) * (p_val < 1)
        un_inform = (H == 0) * (p_val >= 1)
        interfering = (H == 1) * (p_val >= 1)
        remove_variables = un_inform | interfering
        store_variables = strong_inform | weak_inform
        return store_variables, remove_variables

    def __calculate_rmsecv(self, data, label):
        A, row = self.__generate_binary_matrix(data)

        rmsecv5 = np.zeros((row, data.shape[1]+1))  # PLS的五折交叉验证计算rmse
        for k, sub_a in enumerate(A):
            sub_data = data[:, sub_a == 1]

            n_component = min(np.sum(sub_a == 1), self.max_components)
            model = PLSRegression(n_components=n_component, max_iter=1000)
            score = cross_val_score(model, sub_data, label, cv=5, n_jobs=-1, scoring='neg_mean_squared_error').mean()
            rmsecv5[k, 0] = np.sqrt(-score)

        # 分别置换每一列特征的0 1， 得到B，同样方法交叉验证得到rmsecv
        for i in range(data.shape[1]):
            B = np.copy(A)
            B[:, i] = 1 - B[:, i]
            for k, sub_b in enumerate(B):
                sub_data = data[:, sub_b == 1]

                n_component = min(np.sum(sub_b == 1), self.max_components)
                model = PLSRegression(n_components=n_component, max_iter=1000)
                score = cross_val_score(model, sub_data, label, cv=5, n_jobs=-1, scoring='neg_mean_squared_error').mean()
                rmsecv5[k, i+1] = np.sqrt(-score)
        return rmsecv5, A

    def __generate_binary_matrix(self, data):
        # 新建二值矩阵A
        if data.shape[1] >= 500:
            row = 500
        elif data.shape[1] >= 300:
            row = 300
        elif data.shape[1] >= 100:
            row = 200
        elif data.shape[1] >= 50:
            row = 100
        else:
            row = 50

        A = np.ones((row, data.shape[1]))
        A[row // 2:] = 0

        while True:
            A = np.stack([np.random.permutation(sub_a) for sub_a in A.transpose()]).transpose()
            if not np.sum(np.sum(A, axis=1) == 0):
                break
        return A, row

    def remain_index(self):
        index = []
        for i in range(self.remain_data.shape[1]):
            index.append(np.where(np.sum(self.data - self.remain_data[:, i].reshape(-1, 1), axis=0) == 0)[0][0])
        return index

class Elimination:
    def __init__(self, max_components=10):
        self.max_components = max_components

    def iteration(self, data, label):
        delete_index = []
        base_score = self.__get_score(data, label)
        while True:
            scores = self.__get_partial_score(data, label, delete_index)
            if base_score < scores.min():
                break
            else:
                base_score = scores.min()
                index = np.argmin(scores)
                delete_index.append(index)
        return np.delete(data, delete_index, axis=1), delete_index

    def __get_partial_score(self, data, label, delete_index):
        scores = []
        for i in range(data.shape[1]):
            if i in delete_index:
                scores.append(np.inf)
                continue
            index = delete_index.copy()
            index.append(i)
            sub_data = np.delete(data, index, axis=1)

            n_component = min(sub_data.shape[1], self.max_components)
            model = PLSRegression(n_components=n_component, max_iter=1000)
            sub_score = cross_val_score(model, sub_data, label, cv=5, n_jobs=-1, scoring='neg_mean_squared_error').mean()
            scores.append(np.sqrt(-sub_score))
        return np.stack(scores)

    def __get_score(self, data, label):
        n_component = min(data.shape[1], self.max_components)
        model = PLSRegression(n_components=n_component, max_iter=1000)
        score = cross_val_score(model, data, label, cv=5, n_jobs=-1, scoring='neg_mean_squared_error').mean()
        return np.sqrt(-score)

def calculate_retained_index(train_data, train_label):
    start_time = time.time()
    iriv_model = IRIV()
    iriv_model.iteration(train_data, train_label)
    retained_index = iriv_model.remain_index()
    end_time = time.time()
    print('计算强弱信息变量完成，耗时 %.3fs' % (end_time - start_time))
    print('保留索引[从0开始]：', retained_index)

    return retained_index

if __name__ == '__main__':
    calculate_retained_index(train_data, train_label)

max_stars_repo_name: zd-daniel/IRIV

max_issues_repo_name: zd-daniel/IRIV

max_forks_repo_name: zd-daniel/IRIV

reason: import numpy,import scipy


text:
"""
Boolean Operations
~~~~~~~~~~~~~~~~~~

Perform boolean operations with closed surfaces (intersect, cut, etc.).

Boolean/topological operations (intersect, cut, etc.) methods are implemented
for :class:`pyvista.PolyData` mesh types only and are accessible directly from
any :class:`pyvista.PolyData` mesh. Check out :class:`pyvista.PolyDataFilters`
and take a look at the following filters:

* :func:`pyvista.PolyDataFilters.boolean_add`
* :func:`pyvista.PolyDataFilters.boolean_cut`
* :func:`pyvista.PolyDataFilters.boolean_difference`
* :func:`pyvista.PolyDataFilters.boolean_union`

For merging, the ``+`` operator can be used between any two meshes in PyVista
which simply calls the ``.merge()`` filter to combine any two meshes.
Similarly, the ``-`` operator can be used between any two :class:`pyvista.PolyData`
meshes in PyVista to cut the first mesh by the second.
"""

# sphinx_gallery_thumbnail_number = 6
import pyvista as pv
import numpy as np

def make_cube():
    x = np.linspace(-0.5, 0.5, 25)
    grid = pv.StructuredGrid(*np.meshgrid(x, x, x))
    return grid.extract_surface().triangulate()

# Create to examplee PolyData meshes for boolean operations
sphere = pv.Sphere(radius=0.65, center=(0, 0, 0))
cube = make_cube()

p = pv.Plotter()
p.add_mesh(sphere, color="yellow", opacity=0.5, show_edges=True)
p.add_mesh(cube, color="royalblue", opacity=0.5, show_edges=True)
p.show()

###############################################################################
# Boolean Add
# +++++++++++
#
# Add all of the two meshes together using the
# :func:`pyvista.PolyDataFilters.boolean_add` filter or the ``+`` operator.
#
# Order of operations does not matter for boolean add as the entirety of both
# meshes are appended together.

add = sphere + cube
add.plot(opacity=0.5, color=True, show_edges=True)

###############################################################################
# Boolean Cut
# +++++++++++
#
# Perform a boolean cut of ``a`` using ``b`` with the
# :func:`pyvista.PolyDataFilters.boolean_cut` filter or the ``-`` operator
# since both meshes are :class:`pyvista.PolyData`.
#
# Order of operations does not matter for boolean cut.

cut = cube - sphere

p = pv.Plotter()
p.add_mesh(cut, opacity=0.5, show_edges=True, color=True)
p.show()

###############################################################################
# Boolean Difference
# ++++++++++++++++++
#
# Combine two meshes and retains only the volume in common between the meshes
# using the :func:`pyvista.PolyDataFilters.boolean_difference` method.
#
# Note that the order of operations for a boolean difference will affect the
# results.

diff = sphere.boolean_difference(cube)

p = pv.Plotter()
p.add_mesh(diff, opacity=0.5, show_edges=True, color=True)
p.show()

###############################################################################

diff = cube.boolean_difference(sphere)

p = pv.Plotter()
p.add_mesh(diff, opacity=0.5, show_edges=True, color=True)
p.show()

###############################################################################
# Boolean Union
# +++++++++++++
#
# Combine two meshes and attempts to create a manifold mesh using the
# :func:`pyvista.PolyDataFilters.boolean_union` method.
#
# Order of operations does not matter for boolean union.

union = sphere.boolean_union(cube)

p = pv.Plotter()
p.add_mesh(union,  opacity=0.5, show_edges=True, color=True)
p.show()

max_stars_repo_name: whophil/pyvista

max_issues_repo_name: whophil/pyvista

max_forks_repo_name: whophil/pyvista


text:
import numpy as np
import tensorflow as tf
import cv2

class Waifu2x:
    __MODEL_PATH__ = "./GUI//src/saved_model/Waifu2x"

    def __init__(self, sess):

        if sess is None:
            config = tf.ConfigProto()
            config.gpu_options.per_process_gpu_memory_fraction = 0.2
            self.sess = tf.Session(config=config)
        else:
            self.sess = sess

        self.__build_model()
        self.__zero_init()

    def __build_model(self):
        self.__model = tf.contrib.saved_model.load_keras_model(self.__MODEL_PATH__)

    def __zero_init(self):
        feed = np.zeros(shape=(1, 512, 512, 3))
        self.__model.predict(feed)

    def __image_preprocessing(self, image):
        x, y, _ = image.shape
        image = cv2.resize(image, dsize=(2 * y, 2 * x))
        return np.array([image]) / 255.0

    def __image_postprocessing(self, image):
        image = np.clip(image[0], 0, 1) * 255
        return image.astype(np.uint8)

    def upscale(self, image):
        image = self.__image_preprocessing(image=image)
        image = self.__model.predict(image)
        image = self.__model.predict(image)
        return self.__image_postprocessing(image=image)


text:
#!/usr/bin/env python3

import pytest
import numpy as np

from scipy.spatial.distance import cdist
from sklearn.utils._testing import assert_array_almost_equal

from ..constants import *
from ..datasets import load_cell_cycle
from ..datasets import load_spirals
from ..tdata import TData
from ..transmorph import Transmorph

# For now, tests will not challenge the return value
# They will rather test for code inconsistencies
# TODO: Think of testing metrics

def test_transmorph_validate_parameters():

    # methods
    t = Transmorph(method='ot')
    assert t.method == TR_METHOD_OT
    t = Transmorph(method='gromov')
    assert t.method == TR_METHOD_GROMOV

    with pytest.raises(AssertionError):
        Transmorph(method='foo')

    # strategies
    for strategy in [
            'woti',
            'labels',
            'uniform'
    ]:
        t = Transmorph(weighting_strategy=strategy)

    with pytest.raises(AssertionError):
        Transmorph(weighting_strategy='foo')

    # label dependency
    Transmorph(label_dependency=0)
    Transmorph(label_dependency=0.5)
    Transmorph(label_dependency=1)

    with pytest.raises(AssertionError):
        Transmorph(label_dependency=-.1)
    with pytest.raises(AssertionError):
        Transmorph(label_dependency=1.1)

    # max_iter
    Transmorph(max_iter=100)
    with pytest.raises(AssertionError):
        Transmorph(max_iter=-1)

    # combinations
    with pytest.raises(AssertionError):
        Transmorph(method='gromov', label_dependency=1)

    with pytest.raises(AssertionError):
        Transmorph(method='gromov', unbalanced=True)

    with pytest.raises(AssertionError):
        Transmorph(method='ot', geodesic=True)

    Transmorph(method='gromov', weighting_strategy='labels')

def test_tdata_validate_parameters():
    xs, _ = load_spirals()
    n = len(xs)
    weights = np.ones(n) / n
    weights_n = np.ones(n) # should be corrected automatically to sum up to 1
    labels = np.ones(n)
    weights_w = np.ones(n+1) / (n+1)
    labels_w = np.ones(n+1)
    t = TData(xs)
    assert n == len(t)
    assert t.layers['raw'] is t.X
    t = TData(xs, weights)
    t = TData(xs, weights_n)
    assert abs(t.weights().sum() - 1) < 1e-6
    t = TData(xs, labels)

    with pytest.raises(AssertionError):
        TData(np.ndarray([]))
    with pytest.raises(AssertionError):
        TData(xs, weights_w)
    with pytest.raises(AssertionError):
        TData(xs, labels_w)

def test_tdata_pca_3d_2d():
    xs, xt = load_spirals()
    ys, _ = load_cell_cycle()
    txs = TData(xs)
    txt = TData(xt)
    txs.pca(n_components=2)
    txs.pca(n_components=2, other=txt)
    with pytest.raises(AssertionError):
        txs.pca(n_components=2, other=TData(ys))

def test_tdata_neighbors():
    xs, _ = load_spirals()
    t = TData(xs)
    t.neighbors()
    with pytest.raises(AssertionError):
        t.neighbors(layer='pca')
    t.pca(n_components=2)
    t.neighbors(layer='pca')

def test_tdata_representers():
    xs, _ = load_spirals()
    t = TData(xs)
    with pytest.raises(AssertionError):
        t.select_representers()
    t.neighbors()
    t.select_representers()

def test_tdata_distance():
    xs, yt = load_spirals()
    tx, ty = TData(xs, normalize=False), TData(yt, normalize=False)
    D1t = cdist(xs, yt, metric='sqeuclidean')
    D1 = tx.distance(ty, metric='sqeuclidean')
    D2t = cdist(xs, xs, metric='sqeuclidean')
    D2 = tx.distance(metric='sqeuclidean')
    with pytest.raises(AssertionError):
        tx.distance(layer='pca')
    tx.pca(n_components=2)
    tx.distance(layer='pca')
    with pytest.raises(AssertionError):
        tx.distance(ty, layer='pca')

def test_tdata_compute_weights_uniform():
    xs, _ = load_spirals()
    tx = TData(xs)
    tx.compute_weights(method=TR_WS_UNIFORM)
    assert_array_almost_equal(tx.weights(), np.ones(len(xs)) / len(xs))

def test_tdata_compute_weights_woti():
    xs, _ = load_spirals()
    tx = TData(xs)
    tx.compute_weights(method=TR_WS_AUTO)
    assert tx.weights() is not None

def test_tdata_compute_weights_label():
    xs, yt = load_spirals()
    xl, yl = np.ones(len(xs)), np.ones(len(yt))
    xl[:50] = 0
    yl[:50] = 0
    tx, ty = TData(xs, labels=xl), TData(yt, labels=yl)
    tx.compute_weights(method=TR_WS_LABELS, other=ty)
    assert abs(tx.weights()[xl == 0].sum() - ty.weights()[yl == 0].sum()) < 1e-6
    assert abs(tx.weights()[xl == 1].sum() - ty.weights()[yl == 1].sum()) < 1e-6

def test_tdata_barycenter():
    xs, yt = load_spirals()
    t = TData(xs)
    b1 = t.get_barycenter()
    t.compute_weights(method=TR_WS_UNIFORM)
    b2 = t.get_barycenter()
    assert_array_almost_equal(b1, b2)

def test_transmorph_str_log_no_crash():
    t = Transmorph()
    str(t)
    t._log("foo")
    t._log("foo", end='bar')
    t._log("foo", header=False)

def test_tdata_str_log_no_crash():
    xs, _ = load_spirals()
    t = TData(xs)
    str(t)
    t._log("foo")
    t._log("foo", end='bar')
    t._log("foo", header=False)

def test_fit_empty_xs():
    # Transmorph raises an exception when fitted and xs is empty
    xs, yt = load_cell_cycle()
    tm = Transmorph()
    with pytest.raises(AssertionError):
        tm.fit(np.array([]), yt)

def test_fit_empty_yt():
    # Transmorph raises an exception when fitted and yt is empty
    xs, yt = load_cell_cycle()
    tm = Transmorph()
    with pytest.raises(AssertionError):
        tm.fit(xs, np.array([]))

def test_ot():
    # Optimal transport based dataset integration
    # Minimal setup
    # Spirals dataset (d=3)
    xs, yt = load_spirals()
    tm = Transmorph()
    xt = tm.fit_transform(xs, yt, jitter=False)
    assert xt.shape == xs.shape, \
        "Shape inconsistency: (%i, %i) != (%i, %i)" \
        % (*xt.shape, *xs.shape)

def test_ot_custom_xy_weights():
    # Optimal transport based dataset integration
    # Minimal setup
    # Spirals dataset (d=3)
    xs, yt = load_spirals()
    tm = Transmorph(weighting_strategy='uniform')
    xt1 = tm.fit_transform(xs, yt, jitter=False)
    xt2 = tm.fit_transform(xs, yt, jitter=False, xs_weights=np.ones(len(xs))/len(xs))
    xt3 = tm.fit_transform(xs, yt, jitter=False, yt_weights=np.ones(len(yt))/len(yt))
    assert_array_almost_equal(xt1, xt2)
    assert_array_almost_equal(xt1, xt3)

def test_ot_weighted():
    # Optimal transport based dataset integration
    # Weihgted setup
    # Spirals dataset (d=3)
    xs, yt = load_spirals()
    tm = Transmorph(
        weighting_strategy='woti',
    )
    xt = tm.fit_transform(xs, yt, jitter=False)
    assert xt.shape == xs.shape, \
        "Shape inconsistency: (%i, %i) != (%i, %i)" \
        % (*xt.shape, *xs.shape)

def test_ot_sinkhorn():
    # Optimal transport based dataset integration
    # Sinkhorn setup
    # Spirals dataset (d=3)
    xs, yt = load_spirals()
    tm = Transmorph(
        entropy=True
    )
    xt = tm.fit_transform(xs, yt, jitter=False)
    assert xt.shape == xs.shape, \
        "Shape inconsistency: (%i, %i) != (%i, %i)" \
        % (*xt.shape, *xs.shape)

def test_ot_unbalanced():
    # Optimal transport based dataset integration
    # Unbalanced setup
    # Spirals dataset (d=3)
    xs, yt = load_spirals()
    tm = Transmorph(
        unbalanced=True,
    )
    xt = tm.fit_transform(xs, yt, jitter=False)
    assert xt.shape == xs.shape, \
        "Shape inconsistency: (%i, %i) != (%i, %i)" \
        % (*xt.shape, *xs.shape)

def test_ot_custom_M():
    # Optimal transport based dataset integration
    # Minimal setup, custom cost matrix
    # Spirals dataset (d=3)
    xs, yt = load_spirals()
    tm = Transmorph(
        metric='sqeuclidean',
    )
    xt = tm.fit_transform(xs, yt, jitter=False)
    M = cdist(xs, yt, metric="sqeuclidean")
    xtM = tm.fit_transform(xs, yt, Mxy=M, jitter=False)
    assert_array_almost_equal(xt, xtM)

def test_ot_custom_M_wrong():
    # Optimal transport based dataset integration
    # Minimal setup, custom cost matrix
    # Should return an error
    xs, yt = load_spirals()
    tm = Transmorph(
        metric='sqeuclidean',
    )
    M = cdist(yt, xs, metric="sqeuclidean") # Wrong size
    with pytest.raises(AssertionError):
        xtM = tm.fit_transform(xs, yt, Mxy=M, jitter=False)

def test_ot_normalized():
    # Optimal transport based dataset integration
    # Normalization
    xs, yt = load_spirals()
    tm = Transmorph(
        method='ot',
    )
    tm.fit_transform(xs, yt)

def test_ot_pca_3d_2pc():
    # Optimal transport based dataset integration
    # Integration in PC space
    xs, yt = load_spirals()
    tm = Transmorph(
        method='ot',
        n_comps=2
    )
    tm.fit_transform(xs, yt)

def test_ot_pca_3d_2pc_normalized():
    # Optimal transport based dataset integration
    # Integration in PC space + normalization
    xs, yt = load_spirals()
    tm = Transmorph(
        method='ot',
        normalize=True,
        n_comps=2
    )
    tm.fit_transform(xs, yt)

def test_ot_pca_3d_3pc():
    # Optimal transport based dataset integration
    # Integration in PC space
    xs, yt = load_spirals()
    tm = Transmorph(
        method='ot',
        n_comps=3
    )
    tm.fit_transform(xs, yt)

def test_ot_pca_3d_4pc():
    # Optimal transport based dataset integration
    # Integration in PC space
    # Should return an error
    xs, yt = load_spirals()
    tm = Transmorph(
        method='ot',
        n_comps=4
    )
    with pytest.raises(AssertionError):
        tm.fit_transform(xs, yt)

def test_ot_subsample():
    xs, yt = load_spirals()
    tm = Transmorph(
        method='ot',
        n_hops=1
    )
    xt = tm.fit_transform(xs, yt)
    assert xt.shape == xs.shape, \
        "Shape inconsistency: (%i, %i) != (%i, %i)" \
        % (*xt.shape, *xs.shape)

def test_gromov():
    # Gromov-Wasserstein based dataset integration
    # Minimal setup
    # Spirals dataset (d=3)
    xs, yt = load_spirals()
    tm = Transmorph(
        method='gromov',
    )
    xt = tm.fit_transform(xs, yt, jitter=False)
    assert xt.shape == xs.shape, \
        "Shape inconsistency: (%i, %i) != (%i, %i)" \
        % (*xt.shape, *xs.shape)

def test_gromov_custom_Mx():
    # Gromov-Wasserstein based dataset integration
    # Minimal setup, custom cost matrix (source)
    # Spirals dataset (d=3)
    xs, yt = load_spirals()
    tm = Transmorph(
        method='gromov',
        metric='sqeuclidean'
    )
    xt = tm.fit_transform(xs, yt, jitter=False)
    M = cdist(xs, xs, metric='sqeuclidean')
    xtM = tm.fit_transform(xs, yt, Mx=M, jitter=False)
    assert_array_almost_equal(xt, xtM)

def test_gromov_custom_Mx_wrong():
    # Gromov-Wasserstein based dataset integration
    # Minimal setup, custom cost matrix (source)
    # Should return an error
    xs, yt = load_spirals()
    tm = Transmorph(
        method='gromov',
        metric='sqeuclidean'
    )
    M = cdist(yt, yt, metric='sqeuclidean')
    with pytest.raises(AssertionError):
        xtM = tm.fit_transform(xs, yt, Mx=M, jitter=False)

def test_gromov_custom_My():
    # Gromov-Wasserstein based dataset integration
    # Minimal setup, custom cost matrix (reference)
    # Spiral datasets (d=3)
    xs, yt = load_spirals()
    tm = Transmorph(
        method='gromov',
        metric='sqeuclidean'
    )
    xt = tm.fit_transform(xs, yt, jitter=False)
    M = cdist(yt, yt, metric='sqeuclidean')
    xtM = tm.fit_transform(xs, yt, My=M, jitter=False)
    assert_array_almost_equal(xt, xtM)

def test_gromov_custom_My_wrong():
    # Gromov-Wasserstein based dataset integration
    # Minimal setup, custom cost matrix (reference)
    # Should return an error
    xs, yt = load_spirals()
    tm = Transmorph(
        method='gromov',
        metric='sqeuclidean'
    )
    M = cdist(xs, xs, metric='sqeuclidean') # Wrong shape
    with pytest.raises(AssertionError):
        xtM = tm.fit_transform(xs, yt, My=M, jitter=False)

def test_gromov_geodesic():
    xs, yt = load_spirals()
    tm = Transmorph(
        method='gromov',
        geodesic=True
    )
    xt = tm.fit_transform(xs, yt)
    assert xt.shape == xs.shape, \
        "Shape inconsistency: (%i, %i) != (%i, %i)" \
        % (*xt.shape, *xs.shape)

def test_gromov_subsample():
    xs, yt = load_spirals()
    tm = Transmorph(
        method='gromov',
        n_hops=1
    )
    xt = tm.fit_transform(xs, yt)
    assert xt.shape == xs.shape, \
        "Shape inconsistency: (%i, %i) != (%i, %i)" \
        % (*xt.shape, *xs.shape)

def test_label_transfer():
    # Optimal transport based dataset integration
    # Minimal setup
    # Spirals dataset (d=3)
    xs, yt = load_spirals()
    tm = Transmorph()
    tm.fit(xs, yt)
    y_labels = np.ones(yt.shape[0])
    x_labels = tm.label_transfer(y_labels)
    assert x_labels.shape[0] == xs.shape[0], \
        "Shape inconsistency: %i != %i" \
        % (x_labels.shape[0], xs.shape[0])

max_stars_repo_name: Risitop/transmorph

max_issues_repo_name: Risitop/WOTi

max_forks_repo_name: Risitop/WOTi

reason: import numpy,from scipy


text:
def write_run_file(model_file, feature_file, system_name, out_file):
    with open(out_file, 'w') as f:
        f.write(run_file_content(model_file, feature_file, system_name))

def run_file_content(model_file, feature_file, system_name):
    from lightgbm import Booster
    model = Booster(model_file=model_file)
    ret = ''

    to_rerank = load_ranking_data(feature_file)
    for topic_to_rerank in to_rerank:
        doc_to_score = []
        y_predictions = model.predict(topic_to_rerank['documentFeatures'])
        assert len(y_predictions) == len(topic_to_rerank['documents'])

        for i in range(len(topic_to_rerank['documents'])):
            doc_to_score += [{'documentId': topic_to_rerank['documents'][i], 'score': y_predictions[i]}]

        doc_to_score = sorted(doc_to_score, key=lambda i: i['score'], reverse=True)
        ret += topic_as_run_file(topic_to_rerank['topic'], doc_to_score, system_name)

    return ret.strip()

def topic_as_run_file(topic_nr, doc_to_score, system_name):
    ret = ''

    for i in range(len(doc_to_score)):
        ret += topic_nr + ' Q0 ' + doc_to_score[i]['documentId'] + ' ' + str(i+1) + ' ' + str(doc_to_score[i]['score']) + ' ' + system_name + '\n'

    return ret

def load_ranking_data(file_name):
    import json
    ret = []
    with open(file_name, 'r') as f:
        for l in f:
            l = json.loads(l)

            ret += [{
                'topic': l['topic'],
                'documents': [i['documentId'] for i in l['documents']],
                'documentFeatures': as_np_array(l['documents']),
            }]
    return ret

def as_np_array(docs):
    import numpy as np
    ret = []
    for doc in docs:
        last_feature=1
        features = doc['features']
        # relevance label and query id
        ret_features = [-100, -100]
        for feature_num in features:
            if (int(feature_num) -1) != last_feature:
                raise ValueError('Could not handle...')

            ret_features += [features[feature_num]]
            last_feature = int(feature_num)
        ret += [ret_features]

    return np.array(ret)


text:
#!usr/env/bin/python

import rospy
import time
from simulation.msg import *
import numpy as np
from random import randint

rospy.init_node("Backend")
command = rospy.Publisher("simulation_backend", commands, queue_size=19)
msg = commands()
time_departure_off = 120

cars_arrival = np.array([3*abs(8 - round(elem, 1)) for elem in np.random.rayleigh(2, 108)])
cars_stay = np.array([time_departure_off + 3*round(elem, 1) for elem in np.random.chisquare(3, 108)])
cars_size = [randint(1, 3) for _ in range(0, 108)]
ids = np.array(xrange(0, 107))

time_init = time.time()

while True:
    time.sleep(0.1)
    time_off = time.time() - time_init
    remove_index = []
    for i in range(0, len(cars_arrival)):
        if time_off > cars_arrival[i]:
            msg.state = "arrive"
            msg.id = ids[i]
            print "Publish a car"
            command.publish(msg)
        elif time_off > cars_stay[i]:
            msg.state = "return"
            msg.id = ids[i]
            print "Remove a car"
            command.publish(msg)
            remove_index.append(i)
    ids = np.delete(ids, remove_index, None)
    cars_arrival = np.delete(cars_arrival, remove_index, None)
    cars_stay = np.delete(cars_stay, remove_index, None)

max_stars_repo_name: mohakbhardwaj/auto-park

max_issues_repo_name: mohakbhardwaj/auto-park

max_forks_repo_name: mohakbhardwaj/auto-park


text:
import cv2
from numpy import *
import numpy as np
import math
##
##define sampling function
def sampling(image_original,ratio):
    size_original=image_original.shape
    length=size_original[0]
    width=size_original[1]
    length_adj=math.floor(length/ratio)
    width_adj=math.floor(width/ratio)
    rgb_or_gray=len(size_original)
    if rgb_or_gray==3:
        sampled_img=zeros((length_adj,width_adj,size_original[2]))
        for colorchannel in range(0,size_original[2]):
            for x_adj in range(0,length_adj):
                for y_adj in range(0,width_adj):
                    sampled_img[x_adj,y_adj,colorchannel]=np.mean(image_original[math.floor(ratio*x_adj):math.floor(ratio*(x_adj+1)),math.floor(ratio*y_adj):math.floor(ratio*(y_adj+1)),colorchannel])
    else:
        sampled_img=zeros((length_adj,width_adj))
        for x_adj in range(0,length_adj):
                for y_adj in range(0,width_adj):
                    sampled_img[x_adj,y_adj]=np.mean(image_original[math.floor(ratio*x_adj):math.floor(ratio*(x_adj+1)),math.floor(ratio*y_adj):math.floor(ratio*(y_adj+1))])

    sampled_img_output=np.rint(sampled_img).astype('uint8')
    return sampled_img_output
for i_c in range(1,16):
    dir_str="/Users/yangyunchen/Dropbox/Python/proj1_cse573/task3/images/pos/pos_"+str(i_c)+".jpg"
    img_p_o=cv2.imread(dir_str)
    img_p=cv2.imread(dir_str)

    temo=cv2.imread("/Users/yangyunchen/Dropbox/Python/proj1_cse573/task3/template.png")
    tem=temo[6:30,5:19]

    length_tem=tem.shape[0]
    width_tem=tem.shape[1]

    size_list=[]
    n_size=0
    score_list=[]
    loc_list=[]
    for ind in range(10,22):
        length_tema=math.floor(length_tem/ind*10)
        width_tema=math.floor(width_tem/ind*10)
        size_list.append([length_tema,width_tema])
        tem_re=sampling(tem,ind/10)
        match=cv2.matchTemplate(img_p,tem_re,cv2.TM_CCOEFF_NORMED)
        score_m=[cv2.minMaxLoc(match)[1]]
        loc_m=[cv2.minMaxLoc(match)[3]]
        score_list.append(score_m)
        loc_list.append(loc_m)

    max_ind=cv2.minMaxLoc(np.array(score_list))[3]
    loc_max=loc_list[max_ind[1]]

    cv2.rectangle(img_p_o,(loc_max[0][0],loc_max[0][1]),(loc_max[0][0]+size_list[max_ind[1]][1],loc_max[0][1]+size_list[max_ind[1]][0]),(0,0,255),2)

    cv2.imshow("cursor"+str(i_c),img_p_o)
    dir_str_2="/Users/yangyunchen/Dropbox/Python/proj1_cse573/task3/images/pos/poswrite/cursor"+str(i_c)+".png"
    cv2.imwrite(dir_str_2,img_p_o)

cv2.waitKey(0)
cv2.destroyAllWindows()

reason: import numpy,from numpy


text:
# Scalable Graph Embedding for Asymmetric Proximity ,AAAI, 2017
from __future__ import print_function
import random
import math
import numpy as np
import tensorflow as tf

class _LINE(object):

    def __init__(self, graph, rep_size = 128, n_sample = 200, max_step = 10, negative_ratio = 5):
        self.cur_epoch = 0
        self.g = graph
        self.node_size = graph.G.number_of_nodes()
        self.rep_size = rep_size
        self.n_sample = n_sample
        self.max_step = max_step
        self.negative_ratio = negative_ratio
        self.batch_size = 1000
        self.sess = tf.Session()
        cur_seed = random.getrandbits(32)
        initializer = tf.contrib.layers.xavier_initializer( # init weight
            uniform=False, seed=cur_seed)
        with tf.variable_scope("model", reuse=None, initializer=initializer):
            self.build_graph()
        self.sess.run(tf.global_variables_initializer())

    def build_graph(self):
        self.h = tf.placeholder(tf.int32, [None])
        self.t = tf.placeholder(tf.int32, [None])
        self.sign = tf.placeholder(tf.float32, [None])

        cur_seed = random.getrandbits(32)
        self.embeddings = tf.get_variable(name="embeddings", shape=[
                                          self.node_size, self.rep_size], initializer=tf.contrib.layers.xavier_initializer(uniform=False, seed=cur_seed))
        self.context_embeddings = tf.get_variable(name="context_embeddings", shape=[
                                                  self.node_size, self.rep_size], initializer=tf.contrib.layers.xavier_initializer(uniform=False, seed=cur_seed))
        self.h_e = tf.nn.embedding_lookup(self.embeddings, self.h)
        self.t_e_context = tf.nn.embedding_lookup(self.context_embeddings, self.t)
        self.loss = -tf.reduce_mean(tf.log_sigmoid(self.sign*tf.reduce_sum(tf.multiply(self.h_e, self.t_e_context), axis=1)))
        optimizer = tf.train.AdamOptimizer(0.001)
        self.train_op = optimizer.minimize(self.loss)

    def walker(self, start_node, max_step = 10, alpha = 0.85):

        G = self.g.G
        walk = [start_node]
        while len(walk) < max_step:
            cur = walk[-1]
            cur_nbrs = list(G.neighbors(cur))
            if np.random.rand() < alpha:
                if len(cur_nbrs) > 0:
                    walk.append(random.choice(cur_nbrs))
                else:
                    break
            else:
                break
        return walk[-1]

    def train_one_epoch(self):
        # 原coding是 epoch*sample=200*200=40000
        sum_loss = 0.0
        batches = self.batch_iter()
        batch_id = 0
        for batch in batches:
            h, t, sign = batch
            feed_dict = {
                self.h: h,
                self.t: t,
                self.sign: sign,
            }
            _, cur_loss = self.sess.run([self.train_op, self.loss], feed_dict)
            sum_loss += cur_loss
            batch_id += 1
        print('epoch:{} sum of loss:{!s}'.format(self.cur_epoch, sum_loss))
        self.cur_epoch += 1

    def batch_iter(self):
        look_up = self.g.look_up_dict
        look_back = self.g.look_back_list
        numNodes = self.node_size

        # positive or negative mod
        mod = 0
        mod_size = 1 + self.negative_ratio
        h = []
        t = []
        sign = 0

        start_index = 0
        end_index = min(start_index+self.batch_size, numNodes)
        while start_index < numNodes:
            if mod == 0:
                sign = 1.
                h = []
                t = []
                for i in range(start_index, end_index):
                    ppr_node = self.walker(look_back[i])
                    cur_h = i
                    cur_t = look_up[ppr_node]
                    h.append(cur_h)
                    t.append(cur_t)
            else:
                sign = -1.
                t = []
                for i in range(len(h)):
                    t.append(random.randint(0, numNodes-1))

            yield h, t, [sign]
            mod += 1
            mod %= mod_size
            if mod == 0:
                start_index = end_index
                end_index = min(start_index+self.batch_size, numNodes)

    def get_embeddings(self):
        vectors_s = {}
        vectors_t = {}
        embeddings = self.embeddings.eval(session=self.sess)
        context_embeddings = self.context_embeddings.eval(session=self.sess)
        # embeddings = self.sess.run(tf.nn.l2_normalize(self.embeddings.eval(session=self.sess), 1))
        look_back = self.g.look_back_list
        for i, embedding in enumerate(embeddings):
            vectors_s[look_back[i]] = embedding
        for i, embedding in enumerate(context_embeddings):
            vectors_t[look_back[i]] = embedding
        return vectors_s, vectors_t

class App(object):

    def __init__(self, graph, rep_size=128, n_sample=3, max_step=10, epoch = 50, negative_ratio=3):
        self.rep_size = rep_size
        self.model = _LINE(graph, rep_size, n_sample, max_step, negative_ratio)
        for i in range(epoch):
            self.model.train_one_epoch()

        self.get_embeddings()

    def get_embeddings(self):
        ## self.last_vectors = self.vectors
        self.vectors_s = {}
        self.vectors_t = {}
        self.vectors_s, self.vectors_t = self.model.get_embeddings()

    def save_embeddings(self, filename):
        #需要重写双向量保存的代码
        fout = open(filename, 'w')
        node_num = len(self.vectors.keys())
        fout.write("{} {}\n".format(node_num, self.rep_size))
        for node, vec in self.vectors.items():
            fout.write("{} {}\n".format(node,
                                        ' '.join([str(x) for x in vec])))
        fout.close()

max_stars_repo_name: barcawy/OpenNE

max_issues_repo_name: barcawy/OpenNE

max_forks_repo_name: barcawy/OpenNE


text:
#!/usr/bin/python

import json
import numpy as np
import gensim, gensim.models.lsimodel
from gensim import models
import sys

def iter_documents(corpus,lexicon,threshold_artistPop,threshold_artistCount):
    """Iterate over all playlists, yielding a document (=list of artistId with duplicates) at a
    time.
    """
    for playlist_id in corpus:
        acc = {}
        token_list = []
        for song_id in corpus[playlist_id]:
            artist_id = lexicon["songs"][song_id]["artistId"]
            if lexicon["artists"][artist_id]["count"] < threshold_artistPop:
                continue
            if artist_id in acc:
                acc[artist_id] += 1
            else:
                acc[artist_id] = 1
            token_list.append(artist_id)
        if len(acc) >= threshold_artistCount:
            yield token_list

def save_Idx2ArtistName_LUT(corpus, lexicon, threshold_artistPop, threshold_artistCount):
    """save the dictionary in LSI_LUT.txt"""
    LUT = {}
    for tokens in iter_documents(corpus, lexicon, threshold_artistPop, threshold_artistCount):
        for artist_id in tokens:
            if artist_id not in LUT:
                LUT[artist_id] = lexicon["artists"][artist_id]["name"]
    fstream = open("LSI_LUT.txt", 'w')
    for artist_id, name in LUT.items():
        try:
            fstream.write(name+": "+artist_id+"\n")
        except:
            #print "WARNING: Encoding failure."
            pass
    fstream.close()

class MyCorpus(object):
    """corpus formated for gensim"""
    def __init__(self, corpus, lexicon, threshold_artistCount, threshold_artistPop):
        self.dictionary = gensim.corpora.Dictionary(iter_documents(corpus, lexicon,\
        threshold_artistPop, threshold_artistCount))
        self.corpus = corpus
        self.lexicon = lexicon
        self.threshold_artistPop = threshold_artistPop
        self.threshold_artistCount = threshold_artistCount

    def __iter__(self):
        for tokens in iter_documents(self.corpus, self.lexicon, self.threshold_artistPop,\
            self.threshold_artistCount):
            yield self.dictionary.doc2bow(tokens)

if __name__ == "__main__":

    CORPUS_PATH = "/home/mat/Documents/Git/backup/pygrooveshark/src/playlists/corpus.json"
    LEXICON_PATH = "/home/mat/Documents/Git/backup/pygrooveshark/src/playlists/lexicon.json"

    CORPUS = None   #contain all the playlists as lists of songId
    LEXICON = None  #contain all the data for each artist/song

    T_ARTIST_POP = 200
    T_ARTIST_COUNT = 3
    NUM_TOPICS = 25

    ### LOADING THE DATA
    print "\nLoading corpus from "+CORPUS_PATH+" ...",
    sys.stdout.flush()
    with open(CORPUS_PATH, 'r') as jsonStream:
        CORPUS = json.load(jsonStream)
    print "ok"
    print "\nLoading lexicon from "+LEXICON_PATH+" ...",
    sys.stdout.flush()
    with open(LEXICON_PATH, 'r') as jsonStream:
        LEXICON = json.load(jsonStream)
    print "ok"

    save_Idx2ArtistName_LUT(CORPUS,LEXICON, threshold_artistPop=T_ARTIST_POP, threshold_artistCount=T_ARTIST_COUNT)

    ### FORMAT THE DATASET
    gensimCorpus = MyCorpus(CORPUS,LEXICON, threshold_artistPop=T_ARTIST_POP, threshold_artistCount=T_ARTIST_COUNT)
    tfidf = models.TfidfModel(gensimCorpus)

    ### PERFORM LSI
    print "LSI ...",
    sys.stdout.flush()
    lsi = gensim.models.lsimodel.LsiModel(tfidf[gensimCorpus], num_topics=NUM_TOPICS)
    print "ok"

    QUERY_ = [80911]*3+[676]*5+[3531]*3+[50]*5+[673]*1+[836]*5
    QUERY = gensimCorpus.dictionary.doc2bow([str(id) for id in QUERY_])

    print "###############################"
    print "##           QUERY           ##"
    print "###############################"
    for artistId in QUERY_:
        print LEXICON["artists"][str(artistId)]["name"]#,":\t",count
    print ""

    proj = [x[1] for x in lsi[QUERY]]
    print proj
    artistRating = np.dot(lsi.projection.u, np.array(proj))
    print "artistRating ", artistRating.shape
    for artistId, count in QUERY:
        artistRating[artistId] = -1e10
    prediction = np.argsort(artistRating)[::-1]
    print "prediction  ", prediction.shape

    print "###############################"
    print "##           answer          ##"
    print "###############################"
    for k in xrange(40):
        artistId = prediction[k]
        print LEXICON["artists"][gensimCorpus.dictionary.get(artistId)]["name"], ":\t",artistRating[artistId]

max_stars_repo_name: mpagli/GroovesharkRecommender

max_issues_repo_name: mpagli/GroovesharkRecommender

max_forks_repo_name: mpagli/GroovesharkRecommender


text:
# Copyright 2020 DeepMind Technologies Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Scenario factory."""

import concurrent
import random
from typing import Callable, Collection, Iterable, List, Mapping, Sequence, Tuple, TypeVar

from absl import logging
import dm_env
from ml_collections import config_dict
import numpy as np

from meltingpot.python import bot as bot_factory
from meltingpot.python import substrate as substrate_factory
from meltingpot.python.configs import scenarios as scenario_config
from meltingpot.python.utils.scenarios.wrappers import agent_slot_wrapper
from meltingpot.python.utils.scenarios.wrappers import all_observations_wrapper
from meltingpot.python.utils.scenarios.wrappers import base
from meltingpot.python.utils.scenarios.wrappers import default_observation_wrapper

AVAILABLE_SCENARIOS = frozenset(scenario_config.SCENARIOS)

SCENARIOS_BY_SUBSTRATE: Mapping[
    str, Collection[str]] = scenario_config.scenarios_by_substrate(
        scenario_config.SCENARIOS)

PERMITTED_OBSERVATIONS = frozenset({
    'INVENTORY',
    'READY_TO_SHOOT',
    'RGB',
})

T = TypeVar('T')

def _step_fn(policy: bot_factory.Policy) -> Callable[[dm_env.TimeStep], int]:
  """Returns a stateful step function where the state is encapsulated.

  Args:
    policy: the underlying policy to use.

  Returns:
    A step function that returns an action in response to a timestep.
  """
  state = policy.initial_state()

  def step(timestep: dm_env.TimeStep) -> int:
    nonlocal state
    action, state = policy.step(timestep=timestep, prev_state=state)
    return action

  return step

def _restrict_observation(
    observation: Mapping[str, T],
    permitted_observations: Collection[str],
) -> Mapping[str, T]:
  """Restricts an observation to only the permitted keys."""
  return {
      key: observation[key]
      for key in observation if key in permitted_observations
  }

def _restrict_observations(
    observations: Iterable[Mapping[str, T]],
    permitted_observations: Collection[str],
) -> Sequence[Mapping[str, T]]:
  """Restricts multiple observations to only the permitted keys."""
  return [
      _restrict_observation(observation, permitted_observations)
      for observation in observations
  ]

def _partition(
    values: Sequence[T],
    is_focal: Sequence[bool],
) -> Tuple[Sequence[T], Sequence[T]]:
  """Partitions a sequence into focal and background sequences."""
  focal_values = []
  background_values = []
  for focal, value in zip(is_focal, values):
    if focal:
      focal_values.append(value)
    else:
      background_values.append(value)
  return focal_values, background_values

def _merge(
    focal_values: Sequence[T],
    background_values: Sequence[T],
    is_focal: Sequence[bool],
) -> Sequence[T]:
  """Merges focal and background sequences into one."""
  focal_values = iter(focal_values)
  background_values = iter(background_values)
  return [
      next(focal_values if focal else background_values) for focal in is_focal
  ]

class Scenario(base.Wrapper):
  """An substrate where a number of player slots are filled by bots."""

  def __init__(
      self,
      substrate,
      bots: Mapping[str, bot_factory.Policy],
      is_focal: Sequence[bool],
      permitted_observations: Collection[str] = PERMITTED_OBSERVATIONS,
  ) -> None:
    """Initializes the scenario.

    Args:
      substrate: the substrate to add bots to.
      bots: the bots to sample from (with replacement) each episode.
      is_focal: which player slots are allocated to focal players.
      permitted_observations: the observations exposed by the scenario to focal
        agents.
    """
    super().__init__(substrate)
    self._bots = dict(bots)
    num_players = len(substrate.action_spec())
    if len(is_focal) != num_players:
      raise ValueError(f'is_focal is length {len(is_focal)} but substrate is '
                       f'{num_players}-player.')
    self._is_focal = is_focal
    self._num_focal = sum(is_focal)
    self._num_bots = num_players - self._num_focal
    self._permitted_observations = frozenset(permitted_observations)
    self._executor = concurrent.futures.ThreadPoolExecutor(
        max_workers=self._num_bots)
    self._bot_step_fns: List[Callable[[dm_env.TimeStep], int]] = []
    self._action_futures: List[concurrent.futures.Future] = []

  def close(self):
    """See base class."""
    for bot in self._bots.values():
      bot.close()
    self._executor.shutdown(wait=False)
    super().close()

  def _sample_bots(self) -> Sequence[str]:
    """Returns a sample of bot names to fill the background slots."""
    return random.choices(tuple(self._bots), k=self._num_bots)

  def _set_bots(self, bot_names: Sequence[str]) -> None:
    """Sets the currently active bots.

    Args:
      bot_names: names of bots to set active.
    """
    logging.info('Resampled bots: %s', bot_names)
    self._bot_step_fns = [_step_fn(self._bots[name]) for name in bot_names]
    for future in self._action_futures:
      future.cancel()
    self._action_futures.clear()

  def _send_timesteps(self, timesteps: Sequence[dm_env.TimeStep]) -> None:
    """Sends timesteps to bots for asynchronous processing."""
    assert not self._action_futures
    for bot_step, timestep in zip(self._bot_step_fns, timesteps):
      future = self._executor.submit(bot_step, timestep=timestep)
      self._action_futures.append(future)

  def _await_actions(self) -> Sequence[int]:
    """Waits for the bots actions form the last timestep sent."""
    assert self._action_futures
    actions = [future.result() for future in self._action_futures]
    self._action_futures.clear()
    return actions

  def _split_timestep(
      self, timestep: dm_env.TimeStep
  ) -> Tuple[dm_env.TimeStep, Sequence[dm_env.TimeStep]]:
    """Splits multiplayer timestep as needed by agents and bots."""
    agent_rewards, bot_rewards = _partition(timestep.reward, self._is_focal)
    agent_observations, bot_observations = _partition(timestep.observation,
                                                      self._is_focal)
    agent_timestep = timestep._replace(
        reward=agent_rewards,
        observation=_restrict_observations(agent_observations,
                                           self._permitted_observations),
    )
    bot_timesteps = [
        timestep._replace(observation=observation, reward=reward)
        for observation, reward in zip(bot_observations, bot_rewards)
    ]
    return agent_timestep, bot_timesteps

  def reset(self) -> dm_env.TimeStep:
    """See base class."""
    bot_names = self._sample_bots()
    self._set_bots(bot_names)
    timestep = super().reset()
    agent_timestep, bot_timesteps = self._split_timestep(timestep)
    self._send_timesteps(bot_timesteps)
    return agent_timestep

  def step(self, action: Sequence[int]) -> dm_env.TimeStep:
    """See base class."""
    agent_actions = action
    bot_actions = self._await_actions()
    actions = _merge(agent_actions, bot_actions, self._is_focal)
    timestep = super().step(actions)
    agent_timestep, bot_timesteps = self._split_timestep(timestep)
    self._send_timesteps(bot_timesteps)
    return agent_timestep

  def action_spec(self) -> Sequence[dm_env.specs.DiscreteArray]:
    """See base class."""
    agent_action_spec, _ = _partition(super().action_spec(), self._is_focal)
    return agent_action_spec

  def observation_spec(self) -> Sequence[Mapping[str, dm_env.specs.Array]]:
    """See base class."""
    agent_observation_spec, _ = _partition(super().observation_spec(),
                                           self._is_focal)
    return _restrict_observations(agent_observation_spec,
                                  self._permitted_observations)

  def reward_spec(self) -> Sequence[dm_env.specs.Array]:
    """See base class."""
    # TODO(b/192925212): better typing to avoid pytype disables.
    reward_spec: Sequence[dm_env.specs.Array] = super().reward_spec()  # pytype: disable=annotation-type-mismatch
    agent_reward_spec, _ = _partition(reward_spec, self._is_focal)
    return agent_reward_spec

def get_config(scenario_name: str) -> config_dict.ConfigDict:
  """Returns a config for the specified scenario.

  Args:
    scenario_name: Name of the scenario. Must be in AVAILABLE_SCENARIOS.
  """
  if scenario_name not in AVAILABLE_SCENARIOS:
    raise ValueError(f'Unknown scenario {scenario_name!r}')
  scenario = scenario_config.SCENARIOS[scenario_name]
  substrate = substrate_factory.get_config(scenario.substrate)
  bots = {name: bot_factory.get_config(name) for name in scenario.bots}
  config = config_dict.create(
      substrate=substrate,
      num_players=scenario.num_focal_agents,
      bots=bots,
      num_bots=scenario.num_background_bots,
      is_focal=tuple([True] * scenario.num_focal_agents +
                     [False] * scenario.num_background_bots),
  )
  return config.lock()

def build(config: config_dict.ConfigDict) -> Scenario:
  """Builds a scenario for the given config.

  Args:
    config: config resulting from `get_config`.

  Returns:
    The test scenario.
  """
  substrate = substrate_factory.build(config.substrate)
  bots = {
      bot_name: bot_factory.build(bot_config)
      for bot_name, bot_config in config.bots.items()
  }

  # Add observations needed by some bots. These are removed for focal players.
  substrate_observations = set(substrate.observation_spec()[0])
  substrate = all_observations_wrapper.Wrapper(
      substrate, observations_to_share=['POSITION'], share_actions=True)
  substrate = agent_slot_wrapper.Wrapper(substrate)
  if 'INVENTORY' not in substrate_observations:
    substrate = default_observation_wrapper.Wrapper(
        substrate, key='INVENTORY', default_value=np.zeros([1]))

  return Scenario(
      substrate=substrate,
      bots=bots,
      is_focal=config.is_focal,
      permitted_observations=PERMITTED_OBSERVATIONS & substrate_observations)

max_stars_repo_name: vishalbelsare/meltingpot

max_issues_repo_name: vishalbelsare/meltingpot

max_forks_repo_name: vishalbelsare/meltingpot


text:
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import math
import pathlib

import numpy as np

HERE = pathlib.Path(__file__).resolve().parent
ASTEROID = 1
VACANT = 0

def to_int(value):
    if value == "#":  # Asteroid
        return ASTEROID
    if value == ".":  # Empty
        return VACANT
    raise ValueError("Unexpected element", value)

def parse_asteroids(content):
    lines = content.strip().split("\n")
    rows = len(lines)
    columns = len(lines[0])

    values = np.zeros((rows, columns), dtype=np.uint8)
    for row, line in enumerate(lines):
        assert len(line) == columns
        values[row, :] = [to_int(char) for char in line]

    return values

def normalize_direction(a, b):
    if a == 0 and b == 0:
        return 1, (0, 0)

    common_factor = abs(math.gcd(a, b))
    a_without, remainder = divmod(a, common_factor)
    assert remainder == 0
    b_without, remainder = divmod(b, common_factor)
    assert remainder == 0
    return common_factor, (a_without, b_without)

def line_of_sight(values, target_row, target_column):
    rows, columns = values.shape
    asteroids = collections.defaultdict(list)

    for row in range(rows):
        delta_row = target_row - row  # points down
        for column in range(columns):
            if row == target_row and column == target_column:
                continue

            delta_column = column - target_column  # points right
            if values[row, column] != ASTEROID:
                continue

            factor, direction = normalize_direction(delta_row, delta_column)
            asteroids[direction].append(factor)

    return asteroids

def all_line_of_sight(values):
    rows, columns = values.shape
    counts = np.zeros((rows, columns), dtype=np.uint64)

    for row in range(rows):
        for column in range(columns):
            if values[row, column] != ASTEROID:
                counts[row, column] = 0
                continue

            asteroids = line_of_sight(values, row, column)
            counts[row, column] = len(asteroids)

    return counts

def all_vaporized(values, target_row, target_column):
    rows, columns = values.shape
    asteroids = line_of_sight(values, target_row, target_column)
    directions_theta = {}

    for direction, multipliers in asteroids.items():
        # Row indices **increase** in the downward y direction.
        d_row, d_column = direction
        theta = np.arctan2(d_row, d_column)  # Row indices are y, columns are x
        directions_theta[theta] = (direction, sorted(multipliers))

    removed = []
    sorted_theta = np.array(sorted(directions_theta.keys(), reverse=True))
    # This is in the range [-pi, pi] (but reverse, since we go counter clockwise)
    in_first_quadrant, = np.where(
        np.logical_and(0 < sorted_theta, sorted_theta <= np.pi / 2)
    )
    start_index = min(in_first_quadrant)
    laser_theta = np.array(
        list(sorted_theta[start_index:]) + list(sorted_theta[:start_index])
    )

    updated = True
    while updated:
        updated = False

        for theta in laser_theta:
            direction, multipliers = directions_theta[theta]
            if not multipliers:
                continue

            multiplier = multipliers[0]
            delta_row, delta_column = direction
            delta_row *= multiplier  # Un-normalize
            delta_column *= multiplier  # Un-normalize
            # Undo delta_row = target_row - row
            row = target_row - delta_row
            assert 0 <= row < rows
            # Undo delta_column = column - target_column
            column = target_column + delta_column
            assert 0 <= column < columns, (column, columns)
            removed.append((row, column))
            updated = True

            # Update for next iteration.
            multipliers[:] = multipliers[1:]

    return removed

def main():
    filename = HERE / "input.txt"
    with open(filename, "r") as file_obj:
        content = file_obj.read().strip()

    values = parse_asteroids(content)
    all_counts = all_line_of_sight(values)
    best_count = max(all_counts.flatten())
    print(best_count)

    row_matches, col_matches = np.where(all_counts == best_count)
    row_match, = row_matches
    col_match, = col_matches
    removed = all_vaporized(values, row_match, col_match)
    print(removed[200 - 1][::-1])

if __name__ == "__main__":
    main()


text:
from __future__ import absolute_import, division, print_function

from argparse import ArgumentParser
from collections import Counter
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

def print_nearest_words(args):
    word = args.word.lower().strip()

    # Load the word vectors
    embeddings_index = {}
    f = open(args.vectors)
    for line in f:
        values = line.split(' ')
        w = values[0]
        coefs = np.asarray(values[1:], dtype='float32')
        embeddings_index[w] = coefs
    f.close()

    w_v = np.zeros_like(embeddings_index[w])
    for w in word.split():
        if w not in embeddings_index.keys():
            continue

        w_v += embeddings_index[w]

    # Get the similarity scores
    score_dict = {}
    for w in embeddings_index.keys():
        if word == w:
            continue

        score = cosine_similarity(w_v.reshape(1, -1), embeddings_index[w].reshape(1, -1))[0][0]
        score_dict[w] = score

    closest = Counter(score_dict).most_common(args.num_words)

    close_words = []
    for word, score in closest:
        if args.verbose:
            print(score, word)
        else:
            close_words.append(word)

    if not args.verbose:
        print(', '.join(close_words))

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument('--vectors', default='vectors.txt', help='Word vector file')
    parser.add_argument('--vocab', default='vocab.txt', help='Vocab file')
    parser.add_argument('--word', default='dollar', help='Input word')
    parser.add_argument('--verbose', type=bool, default=False, help='Print score')
    parser.add_argument('--num_words', type=int, default=5, help='Number of closest words to print')
    args = parser.parse_args()

    print_nearest_words(args)

max_stars_repo_name: hardikp/fnlp

max_issues_repo_name: hardikp/fnlp

max_forks_repo_name: hardikp/fnlp