text:
# -*- coding: utf-8 -*-
"""
Oxford Experiment:
    ibeis TestResult --db Oxford -p smk:nWords=[64000],nAssign=[1],SV=[False],can_match_sameimg=True -a oxford

Zebra Experiment:
    python -m ibeis draw_rank_cmc --db GZ_Master1 --show \
        -p :proot=smk,num_words=[64000],fg_on=False,nAssign=[1],SV=[False] \
           :proot=vsmany,fg_on=False,SV=[False] \
        -a ctrl:qmingt=2

    python -m ibeis draw_rank_cmc --db PZ_Master1 --show \
        -p :proot=smk,num_words=[64000],fg_on=False,nAssign=[1],SV=[False] \
           :proot=vsmany,fg_on=False,SV=[False] \
        -a ctrl:qmingt=2
"""
from __future__ import absolute_import, division, print_function, unicode_literals
import dtool
import six
import utool as ut
import numpy as np
from ibeis.algo.smk import match_chips5 as mc5
from ibeis.algo.smk import vocab_indexer
from ibeis.algo.smk import inverted_index
from ibeis.algo.smk import smk_funcs
from ibeis import core_annots
from ibeis.algo import Config as old_config
(print, rrr, profile) = ut.inject2(__name__)

class MatchHeuristicsConfig(dtool.Config):
    _param_info_list = [
        ut.ParamInfo('can_match_self', False),
        ut.ParamInfo('can_match_samename', True),
        ut.ParamInfo('can_match_sameimg', False),
    ]

class SMKRequestConfig(dtool.Config):
    """ Figure out how to do this """
    _param_info_list = [
        ut.ParamInfo('proot', 'smk'),
        ut.ParamInfo('smk_alpha', 3.0),
        ut.ParamInfo('smk_thresh', 0.0),
        #ut.ParamInfo('smk_thresh', -1.0),
        ut.ParamInfo('agg', True),
        ut.ParamInfo('data_ma', False),  # hack for query only multiple assignment
        ut.ParamInfo('word_weight_method', 'idf', shortprefix='wwm'),  # hack for query only multiple assignment
        ut.ParamInfo('smk_version', 3),
    ]
    _sub_config_list = [
        core_annots.ChipConfig,
        core_annots.FeatConfig,
        old_config.SpatialVerifyConfig,
        vocab_indexer.VocabConfig,
        inverted_index.InvertedIndexConfig,
        MatchHeuristicsConfig,
    ]

@ut.reloadable_class
class SMKRequest(mc5.EstimatorRequest):
    r"""
    qreq_-like object. Trying to work on becoming more scikit-ish

    CommandLine:
        python -m ibeis.algo.smk.smk_pipeline SMKRequest --profile
        python -m ibeis.algo.smk.smk_pipeline SMKRequest --show

        python -m ibeis draw_rank_cmc --db GZ_ALL --show \
            -p :proot=smk,num_words=[64000,4000],nAssign=[1,5],sv_on=[False,True] \
            -a ctrl:qmingt=2

        python -m ibeis draw_rank_cmc --db PZ_MTEST --show \
            -p :proot=smk,num_words=[64000,8000,4000],nAssign=[1,2,4],sv_on=[True,False] \
                default:proot=vsmany,sv_on=[True,False] \
            -a default:qmingt=2

        python -m ibeis draw_rank_cmc --db PZ_MTEST --show \
            -p :proot=smk,num_words=[64000],nAssign=[1],sv_on=[True] \
                default:proot=vsmany,sv_on=[True] \
            -a default:qmingt=2

        python -m ibeis draw_rank_cmc --db PZ_Master1 --show \
            -p :proot=smk,num_words=[64000],nAssign=[1],sv_on=[False] \
            -a ctrl:qmingt=2

        python -m ibeis draw_rank_cmc --db PZ_Master1 \
            -p :proot=smk,num_words=[64000],nAssign=[1],sv_on=[True] \
            -a ctrl:qmingt=2,qindex=60:80 --profile

        python -m ibeis draw_rank_cmc --db GZ_ALL \
            -p :proot=smk,num_words=[64000],nAssign=[1],sv_on=[True] \
            -a ctrl:qmingt=2,qindex=40:60 --profile

    Example:
        >>> from ibeis.algo.smk.smk_pipeline import *  # NOQA
        >>> import ibeis
        >>> ibs, aid_list = ibeis.testdata_aids(defaultdb='PZ_MTEST')
        >>> qaids = aid_list[0:2]
        >>> daids = aid_list[:]
        >>> config = {'nAssign': 2, 'num_words': 64000, 'sv_on': True}
        >>> qreq_ = SMKRequest(ibs, qaids, daids, config)
        >>> qreq_.ensure_data()
        >>> cm_list = qreq_.execute()
        >>> ut.quit_if_noshow()
        >>> ut.qtensure()
        >>> cm_list[0].ishow_analysis(qreq_, fnum=1, viz_name_score=False)
        >>> cm_list[1].ishow_analysis(qreq_, fnum=2, viz_name_score=False)
        >>> ut.show_if_requested()

    """
    def __init__(qreq_, ibs=None, qaids=None, daids=None, config=None):
        super(SMKRequest, qreq_).__init__()
        if config is None:
            config = {}

        qreq_.ibs = ibs
        qreq_.qaids = qaids
        qreq_.daids = daids

        qreq_.config = config

        #qreq_.vocab = None
        #qreq_.dinva = None

        qreq_.qinva = None
        qreq_.dinva = None
        qreq_.smk = SMK()

        # Hack to work with existing hs code
        qreq_.stack_config = SMKRequestConfig(**config)
        # Flat config
        qreq_.qparams = dtool.base.StackedConfig([
            dict(qreq_.stack_config.parse_items())
        ])
        #    # TODO: add vocab, inva, features
        qreq_.cachedir = ut.ensuredir((ibs.cachedir, 'smk'))

    def dump_vectors(qreq_):
        """
        Example:
            >>> from ibeis.algo.smk.smk_pipeline import *  # NOQA
            >>> import ibeis
            >>> ibs, aid_list = ibeis.testdata_aids(defaultdb='PZ_MTEST', a='default:mingt=2,pername=2')
            >>> qaids = aid_list[0:2]
            >>> daids = aid_list[:]
            >>> config = {'nAssign': 1, 'num_words': 8000,
            >>>           'sv_on': True}
            >>> qreq_ = SMKRequest(ibs, qaids, daids, config)
            >>> qreq_.ensure_data()
        """
        inva = qreq_.dinva
        X = qreq_.dinva.get_annot(qreq_.daids[0])
        n_words = inva.wx_list[-1] + 1
        n_dims = X.agg_rvecs.shape[1]
        n_annots = len(qreq_.daids)
        X.agg_rvecs.dtype
        vlads = np.zeros((n_annots, n_words, n_dims), dtype=np.float32)
        ids_ = list(zip(qreq_.dnids, qreq_.daids))
        for count, (nid, aid) in enumerate(ut.ProgIter(ids_, label='vlad')):
            # X.rrr()
            X = qreq_.dinva.get_annot(aid)
            out = vlads[count]
            out[X.wx_list] = X.agg_rvecs
            # X.to_dense(out=out)
        # Flatten out
        vlads.shape = (n_annots, n_words * n_dims)
        ut.print_object_size(vlads)
        fname = 'vlad_%d_d%d_%s' % (n_annots, n_words * n_dims, qreq_.ibs.get_dbname())
        fpath = ut.truepath('~/' + fname + '.mat')
        import scipy.io
        mdict = {
            'vlads': vlads,
            'nids': qreq_.dnids,
        }
        scipy.io.savemat(fpath, mdict)

    def ensure_data(qreq_):
        """
            >>> import ibeis
            qreq_ = ibeis.testdata_qreq_(
                defaultdb='Oxford', a='oxford',
                p='default:proot=smk,nAssign=1,num_words=64000,SV=False,can_match_sameimg=True,dim_size=None')
        """
        print('Ensure data for %s' % (qreq_,))

        #qreq_.cachedir = ut.ensuredir((ibs.cachedir, 'smk'))
        qreq_.ensure_nids()

        def make_cacher(name, cfgstr=None):
            if cfgstr is None:
                cfgstr = ut.hashstr27(qreq_.get_cfgstr())
            if False and ut.is_developer():
                return ut.Cacher(
                    fname=name + '_' + qreq_.ibs.get_dbname(),
                    cfgstr=cfgstr,
                    cache_dir=ut.ensuredir(ut.truepath('~/Desktop/smkcache'))
                )
            else:
                wrp = ut.DynStruct()
                def ensure(func):
                    return func()
                wrp.ensure = ensure
                return wrp

        import copy
        dconfig = copy.deepcopy(qreq_.qparams)
        qconfig = qreq_.qparams
        if qreq_.qparams['data_ma']:
            # Disable database-dise multi-assignment
            dconfig['nAssign'] = 1
        wwm = qreq_.qparams['word_weight_method']

        depc = qreq_.ibs.depc
        vocab_aids = qreq_.daids

        cheat = False
        if cheat:
            import ibeis
            ut.cprint('CHEATING', 'red')
            vocab_aids = ibeis.init.filter_annots.sample_annots_wrt_ref(
                qreq_.ibs, qreq_.daids, {'exclude_ref_contact': True},
                qreq_.qaids, verbose=1)
            vocab_rowid = depc.get_rowids('vocab', (vocab_aids,), config=dconfig, ensure=False)[0]
            assert vocab_rowid is not None

        depc = qreq_.ibs.depc
        dinva_pcfgstr = depc.stacked_config(
            None, 'inverted_agg_assign', config=dconfig).get_cfgstr()
        qinva_pcfgstr = depc.stacked_config(
            None, 'inverted_agg_assign', config=qconfig).get_cfgstr()
        dannot_vuuid = qreq_.ibs.get_annot_hashid_visual_uuid(qreq_.daids).strip('_')
        qannot_vuuid = qreq_.ibs.get_annot_hashid_visual_uuid(qreq_.qaids).strip('_')
        tannot_vuuid = dannot_vuuid
        dannot_suuid = qreq_.ibs.get_annot_hashid_semantic_uuid(qreq_.daids).strip('_')
        qannot_suuid = qreq_.ibs.get_annot_hashid_semantic_uuid(qreq_.qaids).strip('_')

        dinva_phashid = ut.hashstr27(dinva_pcfgstr + tannot_vuuid)
        qinva_phashid = ut.hashstr27(qinva_pcfgstr + tannot_vuuid)
        dinva_cfgstr = '_'.join([dannot_vuuid, dinva_phashid])
        qinva_cfgstr = '_'.join([qannot_vuuid, qinva_phashid])

        #vocab = inverted_index.new_load_vocab(ibs, qreq_.daids, config)
        dinva_cacher = make_cacher('inva', dinva_cfgstr)
        qinva_cacher = make_cacher('inva', qinva_cfgstr)
        dwwm_cacher  = make_cacher('word_weight', wwm + dinva_cfgstr)

        gamma_phashid = ut.hashstr27(qreq_.get_pipe_cfgstr() + tannot_vuuid)
        dgamma_cfgstr = '_'.join([dannot_suuid, gamma_phashid])
        qgamma_cfgstr = '_'.join([qannot_suuid, gamma_phashid])
        dgamma_cacher = make_cacher('dgamma', cfgstr=dgamma_cfgstr)
        qgamma_cacher = make_cacher('qgamma', cfgstr=qgamma_cfgstr)

        dinva = dinva_cacher.ensure(
            lambda: inverted_index.InvertedAnnots.from_depc(
                depc, qreq_.daids, vocab_aids, dconfig))

        qinva = qinva_cacher.ensure(
            lambda: inverted_index.InvertedAnnots.from_depc(
                depc, qreq_.qaids, vocab_aids, qconfig))

        dinva.wx_to_aids = dinva.compute_inverted_list()

        wx_to_weight = dwwm_cacher.ensure(
            lambda: dinva.compute_word_weights(wwm))
        dinva.wx_to_weight = wx_to_weight
        qinva.wx_to_weight = wx_to_weight

        thresh = qreq_.qparams['smk_thresh']
        alpha = qreq_.qparams['smk_alpha']

        dinva.gamma_list = dgamma_cacher.ensure(
            lambda: dinva.compute_gammas(alpha, thresh))

        qinva.gamma_list = qgamma_cacher.ensure(
            lambda: qinva.compute_gammas(alpha, thresh))

        qreq_.qinva = qinva
        qreq_.dinva = dinva

        print('loading keypoints')
        if qreq_.qparams.sv_on:
            qreq_.data_kpts = qreq_.ibs.get_annot_kpts(
                qreq_.daids, config2_=qreq_.extern_data_config2)

        print('building aid index')
        qreq_.daid_to_didx = ut.make_index_lookup(qreq_.daids)

    def execute_pipeline(qreq_):
        """
        >>> from ibeis.algo.smk.smk_pipeline import *  # NOQA
        >>> ibs, smk, qreq_ = testdata_smk()
        >>> cm_list = qreq_.execute()
        """
        smk = qreq_.smk
        cm_list = smk.predict_matches(qreq_)
        return cm_list

    def get_qreq_qannot_kpts(qreq_, qaids):
        return qreq_.ibs.get_annot_kpts(
            qaids, config2_=qreq_.extern_query_config2)

    def get_qreq_dannot_kpts(qreq_, daids):
        didx_list = ut.take(qreq_.daid_to_didx, daids)
        return ut.take(qreq_.data_kpts, didx_list)
        #return qreq_.ibs.get_annot_kpts(
        #    daids, config2_=qreq_.extern_data_config2)

@ut.reloadable_class
class SMK(ut.NiceRepr):
    """
    Harness class that controls the execution of the SMK algorithm

    K(X, Y) = gamma(X) * gamma(Y) * sum([Mc(Xc, Yc) for c in words])
    """

    def predict_matches(smk, qreq_, verbose=True):
        """
        >>> from ibeis.algo.smk.smk_pipeline import *  # NOQA
        >>> ibs, smk, qreq_ = testdata_smk()
        >>> verbose = True
        """
        print('Predicting matches')
        #assert qreq_.qinva.vocab is qreq_.dinva.vocab
        #X_list = qreq_.qinva.inverted_annots(qreq_.qaids)
        #Y_list = qreq_.dinva.inverted_annots(qreq_.daids)
        #verbose = 2
        _prog = ut.ProgPartial(lbl='smk query', bs=verbose <= 1, enabled=verbose)
        daids = np.array(qreq_.daids)
        cm_list = [smk.match_single(qaid, daids, qreq_, verbose=verbose > 1)
                   for qaid in _prog(qreq_.qaids)]
        return cm_list

    @profile
    def match_single(smk, qaid, daids, qreq_, verbose=True):
        """
        CommandLine:
            python -m ibeis.algo.smk.smk_pipeline SMK.match_single --profile
            python -m ibeis.algo.smk.smk_pipeline SMK.match_single --show

            python -m ibeis SMK.match_single -a ctrl:qmingt=2 --profile --db PZ_Master1
            python -m ibeis SMK.match_single -a ctrl --profile --db GZ_ALL

        Example:
            >>> # FUTURE_ENABLE
            >>> from ibeis.algo.smk.smk_pipeline import *  # NOQA
            >>> import ibeis
            >>> qreq_ = ibeis.testdata_qreq_(defaultdb='PZ_MTEST')
            >>> ibs = qreq_.ibs
            >>> daids = qreq_.daids
            >>> #ibs, daids = ibeis.testdata_aids(defaultdb='PZ_MTEST', default_set='dcfg')
            >>> qreq_ = SMKRequest(ibs, daids[0:1], daids, {'agg': True,
            >>>                                             'num_words': 1000,
            >>>                                             'sv_on': True})
            >>> qreq_.ensure_data()
            >>> qaid = qreq_.qaids[0]
            >>> daids = qreq_.daids
            >>> daid = daids[1]
            >>> verbose = True
            >>> cm = qreq_.smk.match_single(qaid, daids, qreq_)
            >>> ut.quit_if_noshow()
            >>> ut.qtensure()
            >>> cm.ishow_analysis(qreq_)
            >>> ut.show_if_requested()
        """
        from ibeis.algo.hots import chip_match
        from ibeis.algo.hots import pipeline

        alpha  = qreq_.qparams['smk_alpha']
        thresh = qreq_.qparams['smk_thresh']
        agg    = qreq_.qparams['agg']
        #nAnnotPerName   = qreq_.qparams.nAnnotPerNameSVER

        sv_on   = qreq_.qparams.sv_on
        if sv_on:
            nNameShortList  = qreq_.qparams.nNameShortlistSVER
            shortsize = nNameShortList
        else:
            shortsize = None

        X = qreq_.qinva.get_annot(qaid)

        # Determine which database annotations need to be checked
        #with ut.Timer('searching qaid=%r' % (qaid,), verbose=verbose):
        hit_inva_wxs = ut.take(qreq_.dinva.wx_to_aids, X.wx_list)
        hit_daids = np.array(list(set(ut.iflatten(hit_inva_wxs))))

        # Mark impossible daids
        #with ut.Timer('checking impossible daids=%r' % (qaid,), verbose=verbose):
        valid_flags = check_can_match(qaid, hit_daids, qreq_)
        valid_daids = hit_daids.compress(valid_flags)

        shortlist = ut.Shortlist(shortsize)
        #gammaX = smk.gamma(X, wx_to_weight, agg, alpha, thresh)
        _prog = ut.ProgPartial(lbl='smk scoring qaid=%r' % (qaid,),
                               enabled=verbose, bs=True, adjust=True)

        wx_to_weight = qreq_.dinva.wx_to_weight

        debug = False
        if debug:
            qnid = qreq_.get_qreq_annot_nids([qaid])[0]
            daids = np.array(qreq_.daids)
            dnids = qreq_.get_qreq_annot_nids(daids)
            correct_aids = daids[np.where(dnids == qnid)[0]]
            daid = correct_aids[0]

        if agg:
            for daid in _prog(valid_daids):
                Y = qreq_.dinva.get_annot(daid)
                item = match_kernel_agg(X, Y, wx_to_weight, alpha, thresh)
                shortlist.insert(item)
        else:
            for daid in _prog(valid_daids):
                Y = qreq_.dinva.get_annot(daid)
                item = match_kernel_sep(X, Y, wx_to_weight, alpha, thresh)
                shortlist.insert(item)

        # Build chipmatches for the shortlist results

        #with ut.Timer('build cms', verbose=verbose):
        cm = chip_match.ChipMatch(qaid=qaid, fsv_col_lbls=['smk'])
        cm.daid_list = []
        cm.fm_list = []
        cm.fsv_list = []
        _prog = ut.ProgPartial(lbl='smk build cm qaid=%r' % (qaid,),
                               enabled=verbose, bs=True, adjust=True)
        for item in _prog(shortlist):
            (score, score_list, Y, X_idx, Y_idx) = item
            X_fxs = ut.take(X.fxs_list, X_idx)
            Y_fxs = ut.take(Y.fxs_list, Y_idx)
            # Only build matches for those that sver will use
            if agg:
                X_maws = ut.take(X.maws_list, X_idx)
                Y_maws = ut.take(Y.maws_list, Y_idx)
                fm, fs = smk_funcs.build_matches_agg(X_fxs, Y_fxs, X_maws,
                                                     Y_maws, score_list)
            else:
                fm, fs = smk_funcs.build_matches_sep(X_fxs, Y_fxs, score_list)
            if len(fm) > 0:
                #assert not np.any(np.isnan(fs))
                daid = Y.aid
                fsv = fs[:, None]
                cm.daid_list.append(daid)
                cm.fm_list.append(fm)
                cm.fsv_list.append(fsv)
        cm._update_daid_index()
        cm.arraycast_self()
        cm.score_name_maxcsum(qreq_)

        #if False:
        #    cm.assert_self(qreq_=qreq_, verbose=True)

        if sv_on:
            cm = pipeline.sver_single_chipmatch(qreq_, cm, verbose=verbose)
            cm.score_name_maxcsum(qreq_)

        return cm

def word_isect(X, Y, wx_to_weight):
    isect_words = sorted(X.words.intersection(Y.words))
    X_idx = ut.take(X.wx_to_idx, isect_words)
    Y_idx = ut.take(Y.wx_to_idx, isect_words)
    weights = np.array(ut.take(wx_to_weight, isect_words))
    return X_idx, Y_idx, weights

def match_kernel_agg(X, Y, wx_to_weight, alpha, thresh):
    import utool
    with utool.embed_on_exception_context:
        gammaXY = X.gamma * Y.gamma
        # Words in common define matches
        X_idx, Y_idx, weights = word_isect(X, Y, wx_to_weight)

        PhisX, flagsX = X.Phis_flags(X_idx)
        PhisY, flagsY = Y.Phis_flags(Y_idx)
        score_list = smk_funcs.match_scores_agg(
            PhisX, PhisY, flagsX, flagsY, alpha, thresh)

        norm_weights = (weights * gammaXY)
        score_list *= norm_weights
        score = score_list.sum()
        item = (score, score_list, Y, X_idx, Y_idx)
    return item

def match_kernel_sep(X, Y, wx_to_weight, alpha, thresh):
    gammaXY = X.gamma * Y.gamma
    # Words in common define matches
    X_idx, Y_idx, weights = word_isect(X, Y, wx_to_weight)

    phisX_list, flagsY_list = X.phis_flags_list(X_idx)
    phisY_list, flagsX_list = Y.phis_flags_list(Y_idx)
    scores_list = smk_funcs.match_scores_sep(
        phisX_list, phisY_list, flagsX_list, flagsY_list, alpha,
        thresh)

    norm_weights = (weights * gammaXY)
    for scores, w in zip(scores_list, norm_weights):
        scores *= w

    score = [s.sum() for s in scores_list].sum()
    item = (score, scores_list, Y, X_idx, Y_idx)
    return item

def check_can_match(qaid, hit_daids, qreq_):
    can_match_samename = qreq_.qparams.can_match_samename
    can_match_sameimg = qreq_.qparams.can_match_sameimg
    can_match_self = False
    valid_flags = np.ones(len(hit_daids), dtype=np.bool)
    # Check that the two annots meet the conditions
    if not can_match_self:
        valid_flags[hit_daids == qaid] = False
    if not can_match_samename:
        qnid = qreq_.get_qreq_annot_nids([qaid])[0]
        hit_dnids = qreq_.get_qreq_annot_nids(hit_daids)
        valid_flags[hit_dnids == qnid] = False
    if not can_match_sameimg:
        qgid = qreq_.get_qreq_annot_gids([qaid])[0]
        hit_dgids = qreq_.get_qreq_annot_gids(hit_daids)
        valid_flags[hit_dgids == qgid] = False
    return valid_flags

def testdata_smk(*args, **kwargs):
    """
    >>> from ibeis.algo.smk.smk_pipeline import *  # NOQA
    >>> kwargs = {}
    """
    import ibeis
    import sklearn
    import sklearn.cross_validation
    # import sklearn.model_selection
    ibs, aid_list = ibeis.testdata_aids(defaultdb='PZ_MTEST')
    nid_list = np.array(ibs.annots(aid_list).nids)
    rng = ut.ensure_rng(0)
    xvalkw = dict(n_folds=4, shuffle=False, random_state=rng)

    skf = sklearn.cross_validation.StratifiedKFold(nid_list, **xvalkw)
    train_idx, test_idx = six.next(iter(skf))
    daids = ut.take(aid_list, train_idx)
    qaids = ut.take(aid_list, test_idx)

    config = {
        'num_words': 1000,
    }
    config.update(**kwargs)
    qreq_ = SMKRequest(ibs, qaids, daids, config)
    smk = qreq_.smk
    #qreq_ = ibs.new_query_request(qaids, daids, cfgdict={'pipeline_root': 'smk', 'proot': 'smk'})
    #qreq_ = ibs.new_query_request(qaids, daids, cfgdict={})
    return ibs, smk, qreq_

if __name__ == '__main__':
    r"""
    CommandLine:
        export PYTHONPATH=$PYTHONPATH:/home/joncrall/code/ibeis/ibeis/algo/smk
        python ~/code/ibeis/ibeis/algo/smk/smk_pipeline.py
        python ~/code/ibeis/ibeis/algo/smk/smk_pipeline.py --allexamples
    """
    import multiprocessing
    multiprocessing.freeze_support()  # for win32
    import utool as ut  # NOQA
    ut.doctest_funcs()

max_stars_repo_name: brmscheiner/ibeis

max_issues_repo_name: brmscheiner/ibeis

max_forks_repo_name: brmscheiner/ibeis

reason: import numpy,import scipy


text:
# -*- coding: utf-8 -*-
#!/usr/bin/env python

# def warn(*args, **kwargs):
#     pass
# import warnings
# warnings.warn = warn

import getopt
import argparse
import time

import sys, getopt

import h5py, numpy as np, os

import itertools
# import matplotlib.pyplot as plt
import eventHelper as eH
# import eventHelper as ed
from eventHelper import f_events
from MLpredictor import MLpred
from quotes import quotes
from progress import progress
from keras.preprocessing.sequence import pad_sequences
from VCNN1 import predictLabel
from DPredict import DP
from numpy import zeros, newaxis
from Fast5_parsing import fast5_parse
from file_indexing import file_index
from keras import backend as K
# import psutil

starttt = time.time()

print("For CNN based prediction Tensorflow will be used.."+'\n')
# ########################################### file control ########################################

parser = argparse.ArgumentParser(description='PELICAN--> Fast5 Analysis for modified A Nucleotides')

parser.add_argument('-i', action='store',
                    dest='path_input',
                    help='Provide Fast5 folder path --> ex: /path/')

parser.add_argument('-n', action='store',
                    dest='nucleotide_Base',
                    help='Provide nucleotide base type to predict modifications(ex:A)--> Default: A (Do Not use this option in current version)')

parser.add_argument('-b', action='store',
                    default=3,
                    dest='file_batch',
                    help='Process files in batches depending on your available resources..smaller batch size is recommended for less computational resources')

parser.add_argument('-T', action='store_false',
                    default=False,
                    dest='WriteToFast5',
                    help='Default: Do not writes modified sequences to fast5 files under mFastq --> Set -f to turn it off')

parser.add_argument('--version', action='version',
                    version='%(prog)s 1.0')

results = parser.parse_args()
inp = results.path_input
nb= results.nucleotide_Base
overw=results.WriteToFast5

fBatch_size=results.file_batch
fBatch_size=int(fBatch_size)

print('\n'+"### "+quotes()+" ###"+'\n')
##############################################################

l_filename=file_index(inp)

# for filenames in os.listdir(str(inp)):
#     if filenames.endswith(".fast5"):
#         l_filename.append(inp+filenames.replace('\n',''))
print("File Processing Started..."+'\n')
print('Total Number of files to be processed: '+str(len(l_filename)))

############ Rock n Roll --> show begins Fast5 file processing ####################################

cnt=0
ef=str(len(l_filename))

# def get_mem_usage():
#     process = psutil.Process(os.getpid())
#     return process.memory_info()

fCounter=0
while fCounter < len(l_filename):
    start = fCounter
    end = fCounter+fBatch_size
    fBatch = l_filename[start:end]

    holder=[]
    sigholder=[]
    for r in fBatch:
        try:
            events, fastq_decoded, raw_signal = fast5_parse(r)
        except AttributeError:
            continue
        final_eves=eH.event_scrapper(events)    ### event scrapping
        # print(final_eves)

        seq_no=3    ### inititalize sequence number

        for e, i in enumerate(final_eves):

            seq_no=seq_no+int(i[1]) ### update sequence number with steps
            # print(len(''.join(fastq_decoded[1])))
            len_seq=len(''.join(fastq_decoded[1]))  ### length of sequence
        ##### Tail pass
            if e > 2 and e < len(final_eves)-2 and i[0][2:][:1] == 'A':
                f_e=f_events(final_eves,e)
                if len(f_e) != 0:
                    start=[]
                    end=[]
                    for s in f_e:
                        for t in range(5):
                            start.append(s[t][2])
                            end.append(s[t][4])
                    #         print(s[t][2])
                    # print(min(start))
                    # print((max(end)))
                    # print(raw_signal[min(start):][:(max(end)-min(start))+1])
                    min_st=min(start)
                    max_stl=(max(end)-min(start))+1

                    sig='_'.join(map(str, raw_signal[min_st:][:max_stl]))
                    a_seq_no=(len_seq-seq_no)+1   ### transverse seqno

                    if len(sig) <= 1000:
                        # print(fastq_decoded[0])
                        id=fastq_decoded[0]

                        # print(str(len_seq)+' '+str(a_seq_no)+' '+str(seq_no)+' '+str(min_st)+'_'+str(max_stl)+' '+str(len(sig)))
                        # print()
                        holder.append(id+' '+str(seq_no)+' '+str(a_seq_no))
                        sigholder.append(sig.split('_'))
        cnt=cnt+1
    fsignals=[list(map(int, i)) for i in sigholder]
    psignals=pad_sequences(fsignals, padding="post", maxlen=1000)
    # print(X.shape)
    X = psignals[:, :, newaxis]
    # print(X.shape)

    plabels, proba = DP(X,len(X))
    # print(proba)
    filename = 'm1A_Prediction_output3.txt'

    if os.path.exists(filename):
        a_write = 'a' # append if already exists
    else:
        a_write = 'w' # make a new file if not
    file3=open(filename,a_write)
    for s, v, m in zip(holder,plabels, proba):
        if m[1] > 0.5:
            # print(s,v)
            sv=s+' '+str(v)+' '+str(m[1])
            # print(sv)
            file3.write(sv+'\n')
    file3.close()
    progress(cnt, ef, status='Processing..Please wait..')

    fCounter += fBatch_size

    # time.sleep(0.5)
print("\nTotal files processed: ", cnt)
# print((time.time() - start))
stopttt = time.time()
print(stopttt-starttt)

# mem = get_mem_usage()
# print('mem: {}'.format(t, mem))

ext: py

max_stars_repo_name: Janga-Lab/Pelican

max_issues_repo_name: Janga-Lab/Pelican

max_forks_repo_name: Janga-Lab/Pelican

reason: from numpy


text:
#!/usr/bin/env python3
#
# 3. I/O handling (reproducible experiments, snapshotting)
#
import numpy as np
import torch

# set logger and enforce reproducibility
from GPErks.log.logger import get_logger
from GPErks.utils.random import set_seed
log = get_logger()
seed = 8
set_seed(seed)  # reproducible sampling

# function to learn (normally a high-dimensional, expensive deterministic model)
from GPErks.utils.test_functions import currin_exp
f = lambda X: np.array([currin_exp(x) for x in X])
D = 2

# build dataset
from GPErks.gp.data.dataset import Dataset
dataset = Dataset.build_from_function(
    f,
    D,
    n_train_samples=20,
    n_test_samples=25,
    design="lhs",
    seed=seed,
)

# choose likelihood
from gpytorch.likelihoods import GaussianLikelihood
likelihood = GaussianLikelihood()

# choose mean function
from gpytorch.means import LinearMean
mean_function = LinearMean(input_size=dataset.input_size)

# choose kernel
from gpytorch.kernels import RBFKernel, ScaleKernel
kernel = ScaleKernel(RBFKernel(ard_num_dims=dataset.input_size))

# choose metrics
from torchmetrics import MeanSquaredError, R2Score
metrics = [MeanSquaredError(), R2Score()]

# define experiment
from GPErks.gp.experiment import GPExperiment
experiment = GPExperiment(
    dataset,
    likelihood,
    mean_function,
    kernel,
    n_restarts=3,
    metrics=metrics,
    seed=seed  # reproducible training
)

# dump experiment in config file
config_file = "./example_3.ini"
experiment.save_to_config_file(config_file)

# load experiment from config file
del experiment

from GPErks.gp.experiment import load_experiment_from_config_file
experiment = load_experiment_from_config_file(
    config_file,
    dataset  # notice that we still need to provide the dataset used!
)

# choose training options: device + optimizer
device = "cpu"
optimizer = torch.optim.Adam(experiment.model.parameters(), lr=0.1)

# train model
from GPErks.train.emulator import GPEmulator
emulator = GPEmulator(experiment, device)

# snapshotting
from GPErks.serialization.path import posix_path
from GPErks.train.snapshot import (
    EveryEpochSnapshottingCriterion,
    EveryNEpochsSnapshottingCriterion,
    NeverSaveSnapshottingCriterion
)
import os

snapshot_dir = posix_path(os.getcwd(), "snapshot", "example_3")
train_restart_template = "restart_{restart}"
train_epoch_template = "epoch_{epoch}.pth"

snapshot_file = train_epoch_template
snpc = EveryEpochSnapshottingCriterion(
    posix_path(snapshot_dir, train_restart_template),
    snapshot_file
)

# training
emulator.train(optimizer, snapshotting_criterion=snpc)

# inference on stored test set
from GPErks.perks.inference import Inference
inference = Inference(emulator)
inference.summary()

# loading emulator
best_model_file = posix_path(
    snapshot_dir,
    "best_model.pth"
)
best_model_state = torch.load(best_model_file, map_location=torch.device(device))

emulator1 = GPEmulator(experiment, device)
emulator1.model.load_state_dict(best_model_state)

inference = Inference(emulator1)
inference.summary()

max_stars_repo_name: stelong/GPErks

max_issues_repo_name: stelong/GPErks

max_forks_repo_name: stelong/GPErks

reason: import numpy


text:
import numpy as np

def estimateTDOA(
        photonMeasurements,
        pulsarObject,
        maxMLEResolution=None,
        MLEBins=20
):
    pulsarPeriod = pulsarObject.getPeriod(photonMeasurements[0]['t']['value'])

    if not maxMLEResolution:
        maxMLEResolution = pulsarPeriod/100
    else:
        maxMLEResolution = pulsarPeriod * maxMLEResolution

    currentTimeResolution = pulsarPeriod/MLEBins
    tSearchLowerBound = 0
    tSearchUpperBound = currentTimeResolution * (MLEBins-1)
    while currentTimeResolution > maxMLEResolution:
        tSearchVector = np.linspace(tSearchLowerBound, tSearchUpperBound, MLEBins)
        likelihoodVector = np.zeros(MLEBins)

        for tSearchIndex in range(len(tSearchVector)):
            currentTimeOffset = tSearchVector[tSearchIndex]
            fluxValues = np.array([
                pulsarObject.signalIntegral(
                    photon['t']['value'] + currentTimeOffset,
                    photon['t']['value'] + currentTimeResolution + currentTimeOffset
                )
                for photon in photonMeasurements
            ])
            likelihoodVector[tSearchIndex] = (
                np.sum(
                    np.log(fluxValues)
                )
            )
        myArgMax = np.argmax(likelihoodVector)
        maxLikelihoodTDOA = tSearchVector[myArgMax]

        currentTimeResolution = currentTimeResolution/MLEBins

        tSearchLowerBound = maxLikelihoodTDOA
        tSearchUpperBound = maxLikelihoodTDOA + (currentTimeResolution * (MLEBins-1))

    return maxLikelihoodTDOA


text:
import matplotlib.pyplot as plt
import numpy as np
import scipy.special

import GooseMPL as gplt

plt.style.use(["goose", "goose-latex"])

# generate CDF

mu = 0.0
sigma = 1.0
x = np.sort(np.random.normal(mu, sigma, size=10000))
Px = np.linspace(0, 1, x.size)

# draw random number from CDF

y = gplt.random_from_cdf(shape=(10000), P=Px, x=x)

# plot result

prob_x, bin_x = gplt.histogram(x, bins=100, return_edges=False, density=True)
prob_y, bin_y = gplt.histogram(y, bins=100, return_edges=False, density=True)

xi = np.linspace(-5, 5, 10000)
Pi = 1.0 / (sigma * np.sqrt(2.0 * np.pi)) * np.exp(-0.5 * ((xi - mu) / sigma) ** 2.0)
Ci = 0.5 * (1.0 + scipy.special.erf((xi - mu) / (sigma * np.sqrt(2.0))))

fig, axes = gplt.subplots(ncols=2)

axes[0].plot(xi, Pi, color="k")
axes[0].plot(bin_x, prob_x, color="r")
axes[0].plot(bin_y, prob_y, color="g")
axes[0].set_xlabel(r"$x$")
axes[0].set_ylabel(r"$p$")

axes[1].plot(xi, Ci, color="k", label="analytical")
axes[1].plot(x, Px, color="r", label="discrete CDF")
axes[1].plot(np.sort(y), np.linspace(0, 1, y.size), color="g", label="drawn random numbers")
axes[1].set_xlabel(r"$x$")
axes[1].set_ylabel(r"$P$")
axes[1].legend()

plt.savefig("random_from_cdf.svg")
plt.close()


text:
#!/usr/bin/env python3

import sys
sys.path.append('..')

import numpy as np
import tensorflow as tf

from wrappers.gym_atari import Sandbox
from agent import PolicyAgent
from networks import policy_gradient
from utils import load_config, log_feedback

# -----------------------------

physical_devices = tf.config.experimental.list_physical_devices("GPU")
tf.config.experimental.set_memory_growth(physical_devices[0], True)
print("GPU is", "available" if physical_devices else "NOT AVAILABLE")
print("Eager mode:", tf.executing_eagerly())

# -----------------------------

config = load_config('config.yml')['pong-policy']
log_dir = "metrics/"

# -----------------------------

sandbox = Sandbox(config)
env, action_space = sandbox.build_env(config['env_name'])

model = policy_gradient(config['input_shape'], config['window_length'], action_space, config['learning_rate'])
model.summary()

agent = PolicyAgent(config, sandbox, env, action_space)

# -----------------------------

timestamp, summary_writer = log_feedback(log_dir)
print("Job ID:", timestamp)

frame_count = 0
episode_count = 0

loss = 0

episode_reward_history = []
episode_reward = 0
eval_reward = config['min_max'][0]
min_reward = config['min_max'][0]

life = 0
max_life = 0

# -----------------------------

print("Training...")
terminal, state, info = sandbox.reset(env)
prev_info = info

while True:
    action = agent.act(state, model)
    state_next, reward, terminal, info = sandbox.step(env, action, prev_info)
    agent.push(state, action, reward)

    if terminal:
        loss = agent.learn_policy(model)

        episode_reward = 0
        episode_count += 1

        max_life = max(life, max_life)
        life = 0
    else:
        episode_reward += reward
        life += 1

    prev_info = info
    state = state_next

    episode_reward_history.append(episode_reward)
    if len(episode_reward_history) > 100:
        del episode_reward_history[:1]
    running_reward = np.mean(episode_reward_history)

    if terminal:
        print("Frame: {}, Episode: {}, Reward: {}, Loss: {}, Max Life: {}".format(frame_count, episode_count, running_reward, loss, max_life))

    with summary_writer.as_default():
        tf.summary.scalar('loss', loss, step=episode_count)
        tf.summary.scalar('running_reward', running_reward, step=episode_count)
        tf.summary.scalar('eval_reward', eval_reward, step=episode_count)
        tf.summary.scalar('max_life', max_life, step=episode_count)

    if terminal and running_reward > (min_reward + 1):
        agent.save(model, log_dir + timestamp)
        eval_reward = agent.evaluate(model, (log_dir + timestamp), episode_count)
        min_reward = running_reward

    if running_reward == config['min_max'][1]:
        agent.save(model, log_dir + timestamp)
        print("Solved at episode {}!".format(episode_count))
        break

    frame_count += 1

env.close()

max_stars_repo_name: epochlab/rl

max_issues_repo_name: epochlab/rl

max_forks_repo_name: epochlab/rl


text:
import cv2
import sys
import numpy as np

try:
    imagePath = sys.argv[1]
    image = cv2.imread(imagePath)
except:
    image = cv2.imread("hough_demo.jpg")

gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)

edges = cv2.GaussianBlur(gray, (5, 5), 0)
edges = cv2.Canny(gray, 50, 150, 3)

try:
    plines = cv2.HoughLinesP(edges, 1, np.pi/180, 50, None, 100, 5)[0]
    print plines

    for pl in plines:
        cv2.line(image, (pl[0], pl[1]), (pl[2], pl[3]), (255, 0, 0), 3)

    cv2.imshow("HoughLinesP", image)
    cv2.waitKey(0)

except TypeError:
    print "The HoughlinesP function returns None, try decrease the threshold!"

cv2.destroyAllWindows()


text:
# Copyright (c) Gorilla-Lab. All rights reserved.
import argparse
import glob
import os
import os.path as osp
from random import sample

import numpy as np
import torch

def random_sample(coords: np.ndarray, colors: np.ndarray, semantic_labels: np.ndarray,
                  instance_labels: np.ndarray, ratio: float):
    num_points = coords.shape[0]
    num_sample = int(num_points * ratio)
    sample_ids = sample(range(num_points), num_sample)

    # downsample
    coords = coords[sample_ids]
    colors = colors[sample_ids]
    semantic_labels = semantic_labels[sample_ids]
    instance_labels = instance_labels[sample_ids]

    return coords, colors, semantic_labels, instance_labels

def get_parser():
    parser = argparse.ArgumentParser(description='downsample s3dis by voxelization')
    parser.add_argument(
        '--data-dir', type=str, default='./preprocess', help='directory save processed data')
    parser.add_argument('--ratio', type=float, default=0.25, help='random downsample ratio')
    parser.add_argument(
        '--voxel-size',
        type=float,
        default=None,
        help='voxelization size (priority is higher than voxel-size)')
    parser.add_argument('--verbose', action='store_true', help='show partition information or not')

    args_cfg = parser.parse_args()

    return args_cfg

if __name__ == '__main__':
    args = get_parser()

    data_dir = args.data_dir
    # voxelize or not
    voxelize_flag = args.voxel_size is not None
    if voxelize_flag:
        print('processing: voxelize')
        save_dir = f'{data_dir}_voxelize'
    else:
        print('processing: random sample')
        save_dir = f'{data_dir}_sample'
    os.makedirs(save_dir, exist_ok=True)

    # for data_file in [osp.join(data_dir, "Area_6_office_17.pth")]:
    for data_file in glob.glob(osp.join(data_dir, '*.pth')):
        # for data_file in glob.glob(osp.join(data_dir, "*.pth")):
        (coords, colors, semantic_labels, instance_labels, room_label,
         scene) = torch.load(data_file)

        if args.verbose:
            print(f'processing: {scene}')

        save_path = osp.join(save_dir, f'{scene}_inst_nostuff.pth')
        if os.path.exists(save_path):
            continue

        if voxelize_flag:
            raise NotImplementedError
        else:
            coords, colors, semantic_labels, instance_labels = \
                random_sample(coords, colors, semantic_labels, instance_labels, args.ratio)

        torch.save((coords, colors, semantic_labels, instance_labels, room_label, scene), save_path)

max_stars_repo_name: thangvubk/SoftGroup

max_issues_repo_name: thangvubk/SoftGroup

max_forks_repo_name: thangvubk/SoftGroup


text:
#Skisser grafen av polynomfunskjonen 2x^3 - x^2 - 7x + 6

import numpy as np # Import numpy og gi den aliaset np
import matplotlib.pyplot as plt # Import matplotlib.pyplot og gi den aliaset plt

# Definer funskjonen med python syntax
def f(x):
    return 2 * x ** 3 - x ** 2 - 7 * x + 6

x = np.linspace(-4, 4, 50) # Hent 50 verdier mellom -4 og 4

y = f(x) # evaluer f1(x) på disse 50 ulike verdiene
plt.plot(x, y) # plot den inn i grafen

# Horisontal og vertikal linje der hvor x og y er lik 0
plt.axhline(y=0, color='black', linestyle='--')
plt.axvline(x=0, color='black', linestyle='--')

plt.show()


text:
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Mon Dec  4 15:25:54 2017

@author: coelhorp
"""

from scipy.linalg import eigh
import autograd.numpy as np
from pymanopt.manifolds import Rotations
from pymanopt import Problem
from pymanopt.solvers import SteepestDescent
from pyriemann.utils.distance import distance_riemann
from pyriemann.utils.base import invsqrtm, sqrtm, logm

from functools import partial

def gen_symm(n):
    A = np.random.randn(n,n)
    return A + A.T

def gen_spd(n):
    A = gen_symm(n)
    w,v = eigh(A)
    w = np.diag(np.random.rand(len(w)))
    return np.dot(v, np.dot(w, v.T))

def gen_anti(n):
    A = np.random.randn(n,n)
    return A - A.T

def gen_orth(n):
    A = gen_symm(n)
    _,Q = eigh(A)
    return Q

def cost_function_pair_euc(M, Mtilde, Q):
    t1 = M
    t2 = np.dot(Q, np.dot(Mtilde, Q.T))
    return np.linalg.norm(t1 - t2)**2

def cost_function_pair_rie(M, Mtilde, Q):
    t1 = M
    t2 = np.dot(Q, np.dot(Mtilde, Q.T))
    return distance_riemann(t1, t2)**2

def cost_function_full(Q, M, Mtilde, weights=None, dist=None):
    if weights is None:
        weights = np.ones(len(M))
    else:
        weights = np.array(weights)

    if dist is None:
        dist = 'euc'

    cost_function_pair = {}
    cost_function_pair['euc'] = cost_function_pair_euc
    cost_function_pair['rie'] = cost_function_pair_rie

    c = []
    for Mi, Mitilde in zip(M, Mtilde):
        ci = cost_function_pair[dist](Mi, Mitilde, Q)
        c.append(ci)
    c = np.array(c)

    return np.dot(c, weights)

def egrad_function_pair_rie(M, Mtilde, Q):
    Mtilde_invsqrt = invsqrtm(Mtilde)
    M_sqrt = sqrtm(M)
    term_aux = np.dot(Q, np.dot(M, Q.T))
    term_aux = np.dot(Mtilde_invsqrt, np.dot(term_aux, Mtilde_invsqrt))
    return 4 * np.dot(np.dot(Mtilde_invsqrt, logm(term_aux)), np.dot(M_sqrt, Q))

def egrad_function_full_rie(Q, M, Mtilde, weights=None):

    if weights is None:
        weights = np.ones(len(M))
    else:
        weights = np.array(weights)

    g = []
    for Mi, Mitilde, wi in zip(M, Mtilde, weights):
        gi = egrad_function_pair_rie(Mi, Mitilde, Q)
        g.append(gi * wi)
    g = np.sum(g, axis=0)

    return g

def get_rotation_matrix(M, Mtilde, weights=None, dist=None):

    if dist is None:
        dist = 'euc'

    n = M[0].shape[0]

    # (1) Instantiate a manifold
    manifold = Rotations(n)

    # (2) Define cost function and a problem
    if dist == 'euc':
        cost = partial(cost_function_full, M=M, Mtilde=Mtilde, weights=weights, dist=dist)
        problem = Problem(manifold=manifold, cost=cost, verbosity=0)
    elif dist == 'rie':
        cost = partial(cost_function_full, M=M, Mtilde=Mtilde, weights=weights, dist=dist)
        egrad = partial(egrad_function_full_rie, M=M, Mtilde=Mtilde, weights=weights)
        problem = Problem(manifold=manifold, cost=cost, egrad=egrad, verbosity=0)

    # (3) Instantiate a Pymanopt solver
    solver = SteepestDescent(mingradnorm=1e-3)

    # let Pymanopt do the rest
    Q_opt = solver.solve(problem)

    return Q_opt

reason: from scipy


text:
import numpy as np
import pandas as pd

from bokeh.plotting import *

# Here is some code to read in some stock data from the Yahoo Finance API
AAPL = pd.read_csv(
    "http://ichart.yahoo.com/table.csv?s=AAPL&a=0&b=1&c=2000",
    parse_dates=['Date'])
GOOG = pd.read_csv(
    "http://ichart.yahoo.com/table.csv?s=GOOG&a=0&b=1&c=2000",
    parse_dates=['Date'])
MSFT = pd.read_csv(
    "http://ichart.yahoo.com/table.csv?s=MSFT&a=0&b=1&c=2000",
    parse_dates=['Date'])
IBM = pd.read_csv(
    "http://ichart.yahoo.com/table.csv?s=IBM&a=0&b=1&c=2000",
    parse_dates=['Date'])

output_file("stocks.html", title="stocks.py example")

# EXERCISE: turn on plot hold
hold()

# EXERCISE: finish this line plot, and add more for the other stocks. Each one should
# have a legend, and its own color.
line(
    AAPL['Date'],                                       # x coordinates
    AAPL['Adj Close'],                                  # y coordinates
    color='#A6CEE3',                                    # set a color for the line
    legend='AAPL',                                      # attach a legend label
    x_axis_type = "datetime",                           # NOTE: only needed on first
    tools="pan,wheel_zoom,box_zoom,reset,previewsave"   # NOTE: only needed on first
)
line(GOOG['Date'], GOOG['Adj Close'], color='#B2DF8A', legend='GOOG')
line(IBM['Date'], IBM['Adj Close'], color='#33A02C', legend='IBM')
line(MSFT['Date'], MSFT['Adj Close'], color='#FB9A99', legend='MSFT')

# EXERCISE: style the plot, set a title, lighten the gridlines, etc.
curplot().title = "Stock Closing Prices"
grid().grid_line_alpha=0.3

# EXERCISE: start a new figure
figure()

# Here is some code to compute the 30-day moving average for AAPL
aapl = AAPL['Adj Close']
aapl_dates = AAPL['Date']

window_size = 30
window = np.ones(window_size)/float(window_size)
aapl_avg = np.convolve(aapl, window, 'same')

# EXERCISE: plot a scatter of circles for the individual AAPL prices with legend
# 'close'. Remember to set the x axis type and tools on the first renderer
scatter(aapl_dates, aapl, size=4, color='#A6CEE3', legend='close',
        x_axis_type="datetime", tools="pan,wheel_zoom,box_zoom,reset,previewsave")

# EXERCISE: plot a line of the AAPL moving average data with the legeng 'avg'
line(aapl_dates, aapl_avg, color='red', legend='avg')

# EXERCISE: style the plot, set a title, lighten the gridlines, etc.
curplot().title = "AAPL One-Month Average"
grid().grid_line_alpha=0.3

show()  # open a browser

max_stars_repo_name: tswicegood/bokeh

max_issues_repo_name: csaid/bokeh

max_forks_repo_name: csaid/bokeh


text:
from typing import Dict, Tuple, Union, Any
import logging
import re
from enum import Enum

from attr.validators import instance_of
import numpy as np
import attr

from pysight.ascii_list_file_parser import timepatch_switch
from pysight.ascii_list_file_parser.tabulation import Tabulate
from pysight.ascii_list_file_parser.distribute_data import DistributeData

NANOSECONDS_PER_FSTCHAN = 6.4e-9  # Multiscaler const parameter
MULTISCALER_RESOLUTION = 100e-12  # 100 picoseconds resolution
MULTISCALER_EOS_DEADTIME = 96  # 96 nanoseconds end-of-sweep deadtime

class LstFormat(Enum):
    ASCII = "ascii"
    BINARY = "binary"

@attr.s(slots=True)
class ReadMeta:
    """
    Manage pipeline of file IO process. Parses metadata, doesn't read the actual
    data in it.

    :param str filename: List file name
    :param str input_start: Data type in analog channel 'START' (6)
    :param str input_stop1: Data type in analog channel 'STOP1' (1)
    :param str input_stop2: Data type in analog channel 'STOP2' (2)
    :param float binwidth: Multiscaler resolution in seconds (100-800 picoseconds)
    :param bool use_sweeps: Use the sweeps counter as a new line indicator
    :param float mirror_phase: Phase delay for the line signal (in us)
    """

    filename = attr.ib(validator=instance_of(str))
    debug = attr.ib(default=False, validator=instance_of(bool))
    input_start = attr.ib(default="Frames", validator=instance_of(str))
    input_stop1 = attr.ib(default="PMT1", validator=instance_of(str))
    input_stop2 = attr.ib(default="Lines", validator=instance_of(str))
    input_stop3 = attr.ib(default="Empty", validator=instance_of(str))
    input_stop4 = attr.ib(default="Empty", validator=instance_of(str))
    input_stop5 = attr.ib(default="Empty", validator=instance_of(str))
    binwidth = attr.ib(default=800e-12, validator=instance_of(float))
    use_sweeps = attr.ib(default=False, validator=instance_of(bool))
    mirror_phase = attr.ib(default=-2.78, validator=instance_of(float))
    is_binary = attr.ib(init=False)
    timepatch = attr.ib(init=False)
    data_range = attr.ib(init=False)
    time_after = attr.ib(init=False)
    acq_delay = attr.ib(init=False)
    bitshift = attr.ib(init=False)
    start_of_data_pos = attr.ib(init=False)
    dict_of_input_channels = attr.ib(init=False)
    lst_metadata = attr.ib(init=False)
    fill_fraction = attr.ib(init=False)
    num_of_channels = attr.ib(init=False)
    help_dict = attr.ib(init=False)

    def run(self):
        """ Open file and find the needed parameters """
        self.determine_binary()
        metadata: str = self.__get_metadata()
        self.timepatch: str = self.get_timepatch(metadata)
        if self.timepatch == "3" and not self.is_binary:
            raise NotImplementedError(
                'Timepatch value "3" is currently not supported for hex files. Please message the package owner.'
            )
        self.data_range: int = self.get_range(metadata)

        self.start_of_data_pos: int = self.get_start_pos()
        self.time_after: int = self.__get_hold_after(cur_str=metadata)
        self.acq_delay: int = self.__get_fstchan(cur_str=metadata)
        self.bitshift: int = self.__get_bitshift(cur_str=metadata)
        self.fill_fraction: float = self.__calc_actual_fill_fraction()
        self.dict_of_input_channels, self.num_of_channels = self.find_active_channels(
            metadata
        )
        # Grab some additional metadata from the file, to be saved later
        self.__parse_extra_metadata(metadata)
        self.lst_metadata["data_range"] = str(self.data_range)

    def __attrs_post_init__(self):
        self.help_dict = {
            "1": (self.input_stop1, "001"),
            "2": (self.input_stop2, "010"),
            "3": (self.input_stop3, "011"),
            "4": (self.input_stop4, "100"),
            "5": (self.input_stop5, "101"),
            "6": (self.input_start, "110"),
        }

    def __get_metadata(self) -> str:
        """
        Read the file's metadata to be parsed by other functions.

        :return str: String with the .lst file's metadata
        """
        file_mode = "rb" if self.is_binary else "r"
        with open(self.filename, file_mode) as f:
            metadata = f.read(5000)

        return metadata

    @staticmethod
    def hex_to_bin_dict() -> Dict:
        """ Create a simple dictionary that maps a hex input into a 4 letter binary output. """
        diction = {
            "0": "0000",
            "1": "0001",
            "2": "0010",
            "3": "0011",
            "4": "0100",
            "5": "0101",
            "6": "0110",
            "7": "0111",
            "8": "1000",
            "9": "1001",
            "a": "1010",
            "b": "1011",
            "c": "1100",
            "d": "1101",
            "e": "1110",
            "f": "1111",
        }
        return diction

    def determine_binary(self):
        """
        Determine whether we're dealing with a binary or a hex file.
        """
        if self.filename == "":
            raise ValueError("No filename given.")

        try:
            with open(self.filename, "r") as f:
                txt = f.read(400)
        except FileNotFoundError:
            raise FileNotFoundError(f"File {self.filename} doesn't exist.")
        except UnicodeDecodeError:
            with open(self.filename, "rb") as f:
                txt = f.read(400).decode()
        except Exception:
            raise Exception(f"File {self.filename} read unsuccessfully.")

        reg = re.compile(r"\nmpafmt=(\w{3})")
        match = reg.findall(txt)
        if len(match) == 0:
            raise RuntimeError("Could not resolve file type.")
        if match[0] == "dat":
            self.is_binary = True
        elif match[0] == "asc":
            self.is_binary = False

    def get_range(self, cur_str) -> int:
        """
        Finds the "range" of the current file in the proper units
        :return int: range as defined by MCS6A
        """
        if self.filename == "":
            raise ValueError("No filename given.")

        if self.is_binary:
            format_str: Union[str, bytes] = rb"range=(\d+)"
        else:
            format_str = r"range=(\d+)"

        format_range = re.compile(format_str)
        range_lst = int(re.search(format_range, cur_str).group(1))

        return range_lst

    def get_timepatch(self, cur_str: str) -> str:
        """
        Get the time patch value out of of a list file.

        :param cur_str: Start of file to be analyzed.
        :return str: Time patch value as string.
        """
        if self.filename == "":
            raise ValueError("No filename given.")

        if self.is_binary:
            format_str: bytes = rb"time_patch=(\w+)"
        else:
            format_str: str = r"time_patch=(\w+)"

        format_timepatch = re.compile(format_str)
        timepatch = re.search(format_timepatch, cur_str).group(1)
        try:
            timepatch = timepatch.decode("utf-8")
        except AttributeError:
            assert isinstance(timepatch, str)
        finally:
            if self.is_binary and timepatch in ("1a", "2a", "22", "32", "2"):
                raise NotImplementedError(
                    f"The timepatch used ({timepatch}) isn't supported "
                    "for binary files since it uses a 6-byte word representation. "
                    "Please disallow this option in the MPANT software."
                )
            return timepatch

    def __get_bitshift(self, cur_str: str) -> int:
        """
        Finds and parses the bitshift in the current file's metadata.
        The bitshift is the power of two which represents the value,
        in time bins, of the "range" value of the list file.
        The bitshift is given in a hexadecimal format, so the function
        has to convert it to decimal aswell.
        :param cur_str: Text metadata of the current file.
        :return int: Bitshift value (2 ** bitshift)
        """
        if self.is_binary:
            format_str: bytes = rb"bitshift=(\w+)"
        else:
            format_str: str = r"bitshift=(\w+)"

        format_timepatch = re.compile(format_str)
        bitshift = re.search(format_timepatch, cur_str).group(1)
        try:
            bitshift = bitshift.decode("utf-8")
        except AttributeError:
            assert isinstance(bitshift, str)

        bitshift = 2 ** int(bitshift, 16)
        return bitshift

    def find_active_channels(self, cur_str) -> Tuple[Dict[str, str], int]:
        """
        Create a dictionary containing the active channels.

        :param cur_str: String to be analyzed.
        """
        if self.filename == "":
            raise ValueError("No filename given.")

        if self.is_binary:
            format_str = rb"\[CHN(\d)\].+?active=(\d)"
        else:
            format_str = r"\[CHN(\d)\].+?active=(\d)"

        format_active = re.compile(format_str, re.DOTALL)
        matches = format_active.findall(cur_str)
        dict_of_inputs = {}  # DataType: BinaryNumber (e.g. "Lines": "010")
        if self.is_binary:
            matches = [(match[0].decode(), match[1].decode()) for match in matches]
        for cur_match in matches[:-1]:
            if cur_match[1] == "1":  # channel is active, populate dict
                dict_of_inputs[self.help_dict[str(cur_match[0])][0]] = self.help_dict[
                    cur_match[0]
                ][1]
            elif (cur_match[1] == "0") and (
                self.help_dict[cur_match[0]][0] != "Empty"
            ):  # Inactive channel accroding to the multiscaler, but was marked as active by the user
                raise UserWarning(
                    f"Channel {cur_match[0]} didn't record data but was marked as active by the user."
                )

        if matches[-1][1] == "1":
            dict_of_inputs[self.help_dict["6"][0]] = self.help_dict["6"][1]
        elif (matches[-1][0] == "0") and (self.help_dict["6"][0] != "Empty"):
            raise UserWarning(
                "Channel 6 (START) didn't record data but was marked as active by the user."
            )

        num_of_channels = sum([1 for key in dict_of_inputs if "PMT" in key])

        assert len(dict_of_inputs) >= 1
        if "Empty" in dict_of_inputs.keys():
            logging.warning(
                f"At least one channel ({dict_of_inputs['Empty']})"
                " contained recorded data but was marked as 'Empty'."
            )
            dict_of_inputs.pop("Empty")

        return dict_of_inputs, num_of_channels

    def get_start_pos(self) -> int:
        """
        Returns the start position of the data

        :return int: Integer of file position for f.seek() method
        """
        if self.filename == "":
            raise ValueError("No filename given.")

        if self.is_binary:
            format_str = rb"DATA]\r\n"
            file_mode = "rb"
        else:
            format_str = r"DATA]\n"
            file_mode = "r"

        format_data = re.compile(format_str)
        pos_in_file: int = 0
        line_num: int = 0
        with open(self.filename, file_mode) as f:
            while pos_in_file == 0 and line_num < 1000:
                line = f.readline()
                line_num += 1
                match = re.search(format_data, line)
                if match is not None:
                    pos_in_file = f.tell()
                    return pos_in_file  # to have the [DATA] as header
        return -1

    def __get_hold_after(self, cur_str) -> int:
        """
        Read the time (in ns, and convert to timebins) that is considered a
        "hold after", or "hold-off", after a single sweep. Add that time, along with a 96 ns
        inherit delay, to all future times of the sweep.

        :param cur_str: String to parse
        :return int: Final time that has to be added to all sweeps, in timebins
        """
        if self.is_binary:
            format_str: bytes = rb"holdafter=([\w\+]+)"
        else:
            format_str: str = r"holdafter=([\w\+]+)"

        format_holdafter = re.compile(format_str)
        holdafter = int(re.search(format_holdafter, cur_str).group(1), 16)

        time_after_sweep = int(
            ((MULTISCALER_EOS_DEADTIME + (holdafter * 6.4)) * 10 ** (-9))
            / MULTISCALER_RESOLUTION
        )
        return time_after_sweep

    def __get_fstchan(self, cur_str: str) -> int:
        """
        Read the acquisition delay of each sweep, called "fstchan" in the list files

        :param str cur_str: Metadata to be parsed
        :return int: Acq delay in timebins
        """
        if self.is_binary:
            format_str: Union[bytes, str] = rb"fstchan=(\w+)"
        else:
            format_str = r"fstchan=(\w+)"

        format_fstchan = re.compile(format_str)
        fstchan = (
            int(re.search(format_fstchan, cur_str).group(1), 16)
            * NANOSECONDS_PER_FSTCHAN
        )  # in nanoseconds
        acq_delay = int(fstchan / MULTISCALER_RESOLUTION)
        return acq_delay

    def __parse_extra_metadata(self, metadata):
        """
        Update self.lst_metadata with some additional information

        :param metadata: Data from the start of the lst file.
        """
        list_to_parse = [
            "fstchan",
            "holdafter",
            "periods",
            "rtpreset",
            "cycles",
            "sequences",
            "range",
            "sweepmode",
            "fdac",
            "swpreset",
        ]
        self.lst_metadata = {}
        for cur_str in list_to_parse:
            self.__parse_str(metadata, cur_str)

        self.lst_metadata["mirror_phase"] = str(self.mirror_phase)

    def __parse_str(self, metadata, str_to_parse):
        """
        Find str_to_parse in metadata and place the corresponding value in
        the dictionary self.lst_metadata
        """
        if self.is_binary:
            self.__parse_str_bin(metadata, str_to_parse)
        else:
            self.__parse_str_ascii(metadata, str_to_parse)

    def __parse_str_bin(self, metadata, str_to_parse):
        """Parse a binary file for metadata."""
        format_str: bytes = str_to_parse.encode("utf-8") + rb"=(\w+)"
        format_regex = re.compile(format_str)
        try:
            self.lst_metadata[str_to_parse] = (
                re.search(format_regex, metadata).group(1).decode()
            )
        except AttributeError:  # field is non-existent
            pass

    def __parse_str_ascii(self, metadata, str_to_parse):
        """Parse an ASCII file for metadata."""
        format_str: str = str_to_parse + r"=(\w+)"
        format_regex = re.compile(format_str)
        try:
            self.lst_metadata[str_to_parse] = re.search(format_regex, metadata).group(1)
        except AttributeError:  # field is non-existent
            pass

    def __calc_actual_fill_fraction(self) -> float:
        """
        If we're using sweeps as lines then the true fill fraction is determined
        by the multiscaler's parameters, like hold_after and acquisiton delay.
        :return float: True fill fraction
        """
        if self.use_sweeps:
            fill_frac = self.data_range / (
                self.acq_delay + self.data_range + self.time_after
            )
            return fill_frac * 100  # in percent
        else:
            return -1.0

def ascii_parsing(cur_file: ReadMeta, raw_data: np.ndarray, config: Dict[str, Any]):
    """Preliminary readout of data from ASCII file."""
    dict_of_slices_hex = timepatch_switch.ChoiceManagerHex().process(cur_file.timepatch)
    tabulated_data = Tabulate(
        data_range=cur_file.data_range,
        data=raw_data,
        dict_of_inputs=cur_file.dict_of_input_channels,
        use_tag_bits=config["tagbits"]["tag_bits"],
        dict_of_slices_hex=dict_of_slices_hex,
        bitshift=cur_file.bitshift,
        time_after_sweep=cur_file.time_after,
        acq_delay=cur_file.acq_delay,
        num_of_channels=cur_file.num_of_channels,
    )
    tabulated_data.run()

    del raw_data
    separated_data = DistributeData(
        df=tabulated_data.df_after_timepatch,
        dict_of_inputs=tabulated_data.dict_of_inputs,
        use_tag_bits=config["tagbits"]["tag_bits"],
    )
    separated_data.run()
    return separated_data.data_to_grab, separated_data.dict_of_data

max_stars_repo_name: liorgolgher/python-pysight

max_issues_repo_name: liorgolgher/python-pysight

max_forks_repo_name: HagaiHargil/python-pysight


text:
# Libraries
from bs4 import BeautifulSoup
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pandas_datareader.data as data
import seaborn as sns
import datetime
import json
from yahoofinancials import YahooFinancials
from selenium import webdriver
from selenium.webdriver.chrome.webdriver import WebDriver
from webdriver_manager.chrome import ChromeDriverManager
import re
import requests
import warnings

driver = webdriver.Chrome()

asset_dict = {}

warnings.filterwarnings('ignore')

def get_table(soup):
    for t in soup.select('table'):
        header = t.select('thead tr th')
        if len(header) > 2:
            if (header[0].get_text().strip() == 'Symbol'
                and header[2].get_text().strip().startswith('% Holding')):
                return t
    raise Exception('could not find symbol list table')

# Scrapes ETF holdings from barchart.com
def get_etf_holdings():
    '''
    etf_symbol: str

    return: pd.DataFrame
    '''
    url = 'https://www.barchart.com/stocks/quotes/VONE/constituents?page=all'

    # Loads the ETF constituents page and reads the holdings table
    browser = WebDriver() # webdriver.PhantomJS()
    browser.get(url)
    html = browser.page_source
    soup = BeautifulSoup(html, 'html')
    table = get_table(soup)

    # Reads the holdings table line by line and appends each asset to a
    # dictionary along with the holdings percentage
    for row in table.select('tr')[1:26]:
        try:
            cells = row.select('td')
            # print(row)
            symbol = cells[0].get_text().strip()
            # print(symbol)
            # name = cells[1].text.strip()
            celltext = cells[2].get_text().strip()
            percent = float(celltext.rstrip('%'))
            # shares = int(cells[3].text.strip().replace(',', ''))
            if symbol != "" and percent != 0.0:
                asset_dict[symbol] = {
                    'percent': percent,
                }
        except BaseException as ex:
            print(ex)
    browser.quit()
    return pd.DataFrame(asset_dict)

constituent = get_etf_holdings()

constituent.T

# Assets
russell_index = ["^RUI"]
tickers = []

# Set timeframe
start = "2019-03-31"
end = "2019-04-29"

def pullTickers():

    list(asset_dict.keys())

    for key in asset_dict.keys():
        tickers.append(key.replace('.','-'))
    return tickers

pullTickers()

marketCaps = {}

def pullFundamentals():

    for i in tickers:
        yahoo_financials = YahooFinancials(i)
        sharesOutstanding = yahoo_financials.get_key_statistics_data()[i]['sharesOutstanding']

        prices = yahoo_financials.get_historical_price_data('2019-04-01', '2019-04-02', 'daily')[i]['prices'][0]['adjclose']

        mcap = sharesOutstanding * prices

        marketCaps[i] = mcap

        print(marketCaps)

pullFundamentals()

replicatedWeights = {}

aggregateValue = sum(marketCaps.values())

for i in tickers:
    weights = marketCaps[i] / aggregateValue

    replicatedWeights[i] = weights

    print(replicatedWeights)

rweights = list(replicatedWeights.values())

print(rweights)

def replicateETF():
    prices = data.DataReader(tickers, data_source="yahoo", start=start, end=end)['Adj Close']
    x = 0
    for i in tickers:
        prices[i] *= rweights[x]
        x += 1

    prices['ETF'] = prices.sum(axis = 1)

    return prices['ETF']

replicateETF()

etfReturn = replicateETF().pct_change()[1:]

print(etfReturn)

def russell():
    # Pull data from Yahoo! Finance for Russell 1000 Index
    df = data.DataReader(russell_index, data_source="yahoo", start=start, end=end)

    #percentage = df["Adj Close"].pct_change()
    #russellReturn = percentage[1:]

    return df["Adj Close"]

russell()

russellReturn = russell().pct_change()[1:]

russellTotalReturn = russellReturn.values.flatten()

print(russellTotalReturn)

def trackingError():

    returnDifference = etfTotalReturn - russellTotalReturn

    squaredValues = returnDifference ** 2

    sumation = squaredValues.sum()

    Error = (sumation / (20 - 1)) ** (1/2)

    print("Tracking Error is " + "{:.2%}".format(Error))

trackingError()

# create figure and axis objects with subplots()
fig,ax = plt.subplots()
# make a plot
fig.suptitle('Daily Adj Closing Price', fontsize=16)
plt.rcParams['figure.figsize'] = (12,8)
ax.plot(figsize = (12,8))
ax2.plot(figsize = (12,8))
ax.plot(replicateETF(), color="red", marker="o")
# set x-axis label
ax.set_xlabel("Date")
# set y-axis label
ax.set_ylabel("Replicated PF",color="red",fontsize=14)
ax2=ax.twinx()
ax2.plot(russell(), color="blue", marker="o")
ax2.set_ylabel("^RUI",color="blue",fontsize=14)
plt.show()

fig=plt.figure(figsize = (12,8))
fig.suptitle('Daily Returns', fontsize=16)

fig.show()
ax=fig.add_subplot(111)

plt.plot(russellTotalReturn, color="blue")
plt.plot(etfTotalReturn, color="red")

ax.set_ylabel("Return %",fontsize=14)
ax.set_xlabel("Days",fontsize=14)


text:
# MIT License
#
# Copyright (c) 2016 David Sandberg
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import unittest
import tensorflow as tf
import numpy as np
from src import facenet

class DemuxEmbeddingsTest(unittest.TestCase):

    def testDemuxEmbeddings(self):
        batch_size = 3*12
        embedding_size = 16
        alpha = 0.2

        with tf.Graph().as_default():

            embeddings = tf.placeholder(tf.float64, shape=(batch_size, embedding_size), name='embeddings')
            anchor, positive, negative = tf.unstack(tf.reshape(embeddings, [-1,3,embedding_size]), 3, 1)
            triplet_loss = facenet.triplet_loss(anchor, positive, negative, alpha)

            sess = tf.Session()
            with sess.as_default():
                np.random.seed(seed=666)
                emb = np.random.uniform(size=(batch_size, embedding_size))
                tf_triplet_loss = sess.run(triplet_loss, feed_dict={embeddings:emb})

                pos_dist_sqr = np.sum(np.square(emb[0::3,:]-emb[1::3,:]),1)
                neg_dist_sqr = np.sum(np.square(emb[0::3,:]-emb[2::3,:]),1)
                np_triplet_loss = np.mean(np.maximum(0.0, pos_dist_sqr - neg_dist_sqr + alpha))

                np.testing.assert_almost_equal(tf_triplet_loss, np_triplet_loss, decimal=5, err_msg='Triplet loss is incorrect')

if __name__ == "__main__":
    unittest.main()

max_issues_repo_name: shuyuandeqipa/mycaptcha

max_forks_repo_name: shuyuandeqipa/mycaptcha


text:
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
from torch import nn
from torch.autograd import Function
from torch.autograd.function import once_differentiable
from torch.nn.modules.utils import _pair

from maskrcnn_benchmark import _C

class _ROIAlignRotatedKeep(Function):
    @staticmethod
    def forward(ctx, input, roi, output_size, spatial_scale, sampling_ratio):
        ctx.save_for_backward(roi)
        ctx.output_size = _pair(output_size)
        ctx.spatial_scale = spatial_scale
        ctx.sampling_ratio = sampling_ratio
        ctx.input_shape = input.size()
        output = _C.roi_align_rotated_keep_forward(
            input, roi, spatial_scale, output_size[0], output_size[1], sampling_ratio
        )
        return output

    @staticmethod
    @once_differentiable
    def backward(ctx, grad_output):
        rois, = ctx.saved_tensors
        output_size = ctx.output_size
        spatial_scale = ctx.spatial_scale
        sampling_ratio = ctx.sampling_ratio
        bs, ch, h, w = ctx.input_shape
        grad_input = _C.roi_align_rotated_keep_backward(
            grad_output,
            rois,
            spatial_scale,
            output_size[0],
            output_size[1],
            bs,
            ch,
            h,
            w,
            sampling_ratio,
        )
        return grad_input, None, None, None, None, None

roi_align_rotated_keep = _ROIAlignRotatedKeep.apply

class ROIAlignRotatedKeep(nn.Module):
    def __init__(self, output_size, spatial_scale, sampling_ratio):
        """
        Args:
            output_size (tuple): h, w
            spatial_scale (float): scale the input boxes by this number
            sampling_ratio (int): number of inputs samples to take for each output
                sample. 0 to take samples densely.

        Note:
            ROIAlignRotated supports continuous coordinate by default:
            Given a continuous coordinate c, its two neighboring pixel indices (in our
            pixel model) are computed by floor(c - 0.5) and ceil(c - 0.5). For example,
            c=1.3 has pixel neighbors with discrete indices [0] and [1] (which are sampled
            from the underlying signal at continuous coordinates 0.5 and 1.5).
        """
        super(ROIAlignRotatedKeep, self).__init__()
        self.output_size = output_size
        self.spatial_scale = spatial_scale
        self.sampling_ratio = sampling_ratio

    def forward(self, input, rois):
        """
        Args:
            input: NCHW images
            rois: Bx6 boxes. First column is the index into N.
                The other 5 columns are (x_ctr, y_ctr, width, height, angle_degrees).
        """
        assert rois.dim() == 2 and rois.size(1) == 6
        return roi_align_rotated_keep(
            input, rois, self.output_size, self.spatial_scale, self.sampling_ratio
        )

    def __repr__(self):
        tmpstr = self.__class__.__name__ + "("
        tmpstr += "output_size=" + str(self.output_size)
        tmpstr += ", spatial_scale=" + str(self.spatial_scale)
        tmpstr += ", sampling_ratio=" + str(self.sampling_ratio)
        tmpstr += ")"
        return tmpstr

if __name__ == "__main__":
    import os

    # os.environ['CUDA_VISIBLE_DEVICES'] = '4'
    import torch
    import torch.nn as nn
    import torch.nn.functional as F
    import cv2
    import numpy as np
    from PIL import Image

    # from torch.autograd import Variable
    # from rroi_pooling.modules.rroi_pool import RRoIPool

    # import network
    imname = 'gt_show.jpg'

    im = cv2.imread(imname)

    cv2.line(im, (1353, 500), (1000, 853), (255, 0, 255), 3)
    cv2.line(im, (1000, 853), (1000 - 353, 500), (255, 0, 255), 3)
    cv2.line(im, (1000 - 353, 500), (1000, 500 - 353), (255, 0, 255), 3)
    cv2.line(im, (1000, 500 - 353), (1353, 500), (255, 0, 255), 3)
    cv2.line(im, (1250, 500), (1000, 750), (255, 0, 255), 3)
    cv2.line(im, (1000, 750), (750, 500), (255, 0, 255), 3)
    cv2.line(im, (750, 500), (1000, 250), (255, 0, 255), 3)
    cv2.line(im, (1000, 250), (1250, 500), (255, 0, 255), 3)

    # cv2.imshow('win1', im.copy())

    ma = np.expand_dims(im, 0).transpose(0, 3, 1, 2)
    iminfo = np.array([im.shape[0], im.shape[1], 1])
    rois = np.array([[0, 1000, 500, 500, 200, 45], [0, 1000, 500, 500, 200, 0], [0, 100, 100, 200, 200, 45]],
                    dtype=np.float32)
    print('ma:', ma.shape, iminfo)

    ma = torch.tensor(ma).float().cuda()
    iminfo = torch.tensor(iminfo).cuda()  # network.np_to_variable(iminfo, is_cuda=True)
    rois = torch.tensor(rois).cuda()  # network.np_to_variable(rois, is_cuda=True)
    print('ma.requires_grad:', ma.requires_grad)
    ma.requires_grad = True
    rroi_pool = ROIAlignRotatedKeep((100, 200), 1.0 / 1, 2)

    pooled = rroi_pool(ma, rois)

    print('pooled:', pooled.size())

    crop = pooled.data.cpu().numpy()

    print(crop.shape, np.unique(crop))

    crop_trans = crop.transpose(0, 2, 3, 1)
    crop1 = Image.fromarray(crop_trans[0].astype(np.uint8))
    crop2 = Image.fromarray(crop_trans[1].astype(np.uint8))
    crop3 = Image.fromarray(cv2.cvtColor(crop_trans[2].astype(np.uint8), cv2.COLOR_BGR2RGB))
    crop1.save('crop1.jpg', 'jpeg')
    crop2.save('crop2.jpg', 'jpeg')
    crop3.save('crop3.jpg', 'jpeg')

    print(crop.shape)

    mean = torch.mean(pooled)

    # mean.backward()

    grad = torch.autograd.grad(mean, ma)

    print(
        'grad:',
        type(grad),
        len(grad),
        np.unique(grad[0].data.cpu().numpy()),
        np.where(grad[0].data.cpu().numpy() > 0)
    )
    grad_np = grad[0].data.cpu().numpy()
    print(grad_np.shape)
    grad_np = np.transpose(grad_np[0], (1, 2, 0))
    max_grad = np.max(grad_np)
    grad_pic = Image.fromarray(((grad_np / max_grad) * 255).astype(np.uint8))
    grad_pic.save('crop_grad.jpg', 'jpeg')

max_stars_repo_name: Shualite/UBNet

max_issues_repo_name: Shualite/UBNet

max_forks_repo_name: Shualite/UBNet


text:
#!/usr/bin/env python
import logging
from argparse import ArgumentParser
from pathlib import Path
from numpy import arange
from matplotlib.pyplot import show
from gridaurora.filterload import getSystemT
from gridaurora.plots import comparejgr2013, plotAllTrans
import seaborn as sns

sns.color_palette("cubehelix")
sns.set(context="paper", style="whitegrid", font_scale=2, rc={"image.cmap": "cubehelix_r"})

R = Path(__file__).parent

def main():
    p = ArgumentParser(description="computes optical transmission and compares (debug)")
    p.add_argument("--path", help="path to HDF5 data")
    p.add_argument("-a", "--altkm", help="observer altitude (km)", type=float, default=0.0)
    p.add_argument("--zenang", help="zenith angle (deg)", type=float, default=0.0)
    p.add_argument("-m", "--makeplot", help="[eps png]", nargs="+")
    p = p.parse_args()

    dpath = Path(p.path).expanduser() if p.path else R / "gridaurora/precompute"
    bg3fn = dpath / "BG3transmittance.h5"
    windfn = dpath / "ixonWindowT.h5"
    qefn = dpath / "emccdQE.h5"

    reqLambda = arange(200, 1200, 1)
    # reqLambda = linspace(200,1000,500) #so coarse it misses features

    optT = getSystemT(reqLambda, bg3fn, windfn, qefn, p.altkm, p.zenang)
    # %%
    try:
        comparejgr2013(p.altkm, p.zenang, bg3fn, windfn, qefn)
        # %% considering atmosphere
        plotAllTrans(optT, False)
        plotAllTrans(optT, True)

        # plotOptMod(ver,VERgray,tTC,Ek,Eki) #called in readTranscar.py
        show()

    except Exception as e:
        logging.warning(f"problem plotting    {e}")
        print(optT)

if __name__ == "__main__":
    main()

max_stars_repo_name: scivision/gridaurora

max_issues_repo_name: scivision/gridaurora

max_forks_repo_name: scivision/gridaurora


text:
from autode import atoms
from autode.atoms import Atom
import numpy as np
import pytest

def test_atoms():

    assert atoms.get_maximal_valance(atom_label='C') == 4
    assert atoms.get_maximal_valance(atom_label='Aa') == 6
    assert atoms.get_atomic_weight(atom_label='C') == 12.01
    assert atoms.get_atomic_weight(atom_label='Aa') == 70
    assert 0.9 < atoms.get_vdw_radius(atom_label='H') < 1.2
    assert 2 < atoms.get_vdw_radius(atom_label='Aa') < 3

    assert atoms.is_pi_atom(atom_label='C', valency=3) is True
    assert atoms.is_pi_atom(atom_label='C', valency=4) is False
    assert atoms.is_pi_atom(atom_label='Aa', valency=9) is False

def test_atom_collection():

    h2 = atoms.AtomCollection()
    assert h2.n_atoms == 0
    assert h2.coordinates is None

    # Cannot set coordinates without atoms
    with pytest.raises(ValueError):
        h2.coordinates = np.array([1.0, 1.0, 1.0])

    h2.atoms = [Atom('H', 0.0, 0.0, 0.0), Atom('H')]
    assert h2.n_atoms == 2

    assert type(h2.coordinates) == np.ndarray

    # Should be able to set coordinate from a flat array (row major)
    h2.coordinates = np.zeros(shape=(6,))
    assert h2.coordinates[0] is not None
    assert h2.n_atoms == 2

    assert np.isclose(h2.distance(0, 1), 0.0, atol=1E-5)

    coord = h2.coordinates[0]
    coord += 1.0

    # Shift of coordinates should not be in place
    assert not np.allclose(h2.coordinates[0], coord)

    # Cannot set coordinates with anything but a 3xn_atoms flat array, or
    # 2-dimensional array (matrix)
    with pytest.raises(AssertionError):
        h2.coordinates = np.array([])

    with pytest.raises(AssertionError):
        h2.coordinates = np.array([1.0, 0.1])

    with pytest.raises(AssertionError):
        h2.coordinates = np.array([[[1.0], [1.0]]])

    with pytest.raises(ValueError):
        h2.distance(-1, 0)

    with pytest.raises(ValueError):
        h2.distance(0, 2)

def test_atom():

    h = Atom(atomic_symbol='H', x=0.0, y=0.0, z=0.0)
    assert h.label == 'H'
    assert h.atomic_number == 1
    assert h.atomic_symbol == 'H'
    assert not h.is_metal
    assert h.tm_row is None
    assert h.group == 1
    assert h.period == 1

    assert type(h.coord) == np.ndarray
    assert len(h.coord) == 3
    assert h.coord[0] == 0
    assert h.coord[1] == 0
    assert h.coord[2] == 0

    # Translate the H atom by 1 A in the z direction
    h.translate(vec=np.array([0.0, 0.0, 1.0]))
    assert np.linalg.norm(h.coord - np.array([0.0, 0.0, 1.0])) < 1E-6

    # Rotate the atom 180° (pi radians) in the x axis
    h.rotate(axis=np.array([1.0, 0.0, 0.0]), theta=np.pi)
    assert np.linalg.norm(h.coord - np.array([0.0, 0.0, -1.0])) < 1E-6

    # Perform a rotation about a different origin e.g. (1, 0, -1)
    h.rotate(axis=np.array([0.0, 0.0, 1.0]),
             theta=np.pi,
             origin=np.array([1.0, 0.0, -1.0]))
    assert np.linalg.norm(h.coord - np.array([2.0, 0.0, -1.0])) < 1E-6

    # Ensure that the atoms has a string representation
    assert len(str(h)) > 0

    assert Atom('C').atomic_number == 6
    assert Atom('C').period == 2
    assert Atom('C').group == 14

    dummy = atoms.DummyAtom(0.0, 0.0, 0.0)
    assert dummy.atomic_number == 0
    assert dummy.period == 0
    assert dummy.group == 0

    fe = Atom(atomic_symbol='Fe')
    assert fe.tm_row == 1

def test_periodic_table():

    with pytest.raises(ValueError):
        _ = atoms.PeriodicTable.period(n=0)   # Periods are indexed from 1
        _ = atoms.PeriodicTable.period(n=8)   # and don't exceed 8

        _ = atoms.PeriodicTable.period(n=19)   # Groups don't exceed 18

    period2 = atoms.PeriodicTable.period(n=2)
    assert len(period2) == 8
    assert period2[0] == 'Li'

    assert len(atoms.PeriodicTable.period(n=1)) == 2
    assert len(atoms.PeriodicTable.period(n=3)) == 8
    assert len(atoms.PeriodicTable.period(n=4)) == 18

    group13 = atoms.PeriodicTable.group(n=13)
    assert 'B' in group13

    with pytest.raises(Exception):
        _ = atoms.PeriodicTable.group(0)   # No group 0
        _ = atoms.PeriodicTable.group(19)  # or 19

    with pytest.raises(IndexError):
        _ = atoms.PeriodicTable.element(0, 0)
        _ = atoms.PeriodicTable.element(0, 3)

    with pytest.raises(Exception):
        _ = atoms.PeriodicTable.transition_metals(row=0)
        _ = atoms.PeriodicTable.transition_metals(row=10)

    assert 'Fe' in atoms.PeriodicTable.transition_metals(row=1)

    assert atoms.PeriodicTable.element(2, 13) == 'B'

max_stars_repo_name: avanteijlingen/autodE

max_issues_repo_name: avanteijlingen/autodE

max_forks_repo_name: avanteijlingen/autodE


text:
import argparse
import time
import ml_dataset_loader.datasets as data_loader
import numpy as np
import pandas as pd
import xgboost as xgb
from sklearn.metrics import mean_squared_error, accuracy_score

def main():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--rows', type=int, default=None,
                        help='Max rows to benchmark for each dataset.')
    parser.add_argument('--num_rounds', type=int, default=500, help='Boosting rounds.')
    args = parser.parse_args()

    X,y = data_loader.get_airline(num_rows = args.rows)

    params = {'objective':'binary:logistic', 'tree_method':'gpu_hist'}
    dtrain = xgb.DMatrix(X, y)

    times = []
    for n_gpus in range(1,9):
        print("Running XGBoost with {} GPUs ...".format(n_gpus))
        params['n_gpus'] = n_gpus
        start = time.time()
        bst = xgb.train(params, dtrain, args.num_rounds)
        times.append( time.time() - start)
        del bst

    print(times)

if __name__ == "__main__":
    main()

max_stars_repo_name: cwpearson/GBM-Benchmarks

max_issues_repo_name: cwpearson/GBM-Benchmarks

max_forks_repo_name: cwpearson/GBM-Benchmarks


text:
'''
Non-parametric computation of entropy and mutual-information

Adapted by G Varoquaux for code created by R Brette, itself
from several papers (see in the code).

These computations rely on nearest-neighbor statistics
'''
import numpy as np

from scipy.special import gamma,psi
from scipy import ndimage
from scipy.linalg import det
from numpy import pi

from sklearn.neighbors import NearestNeighbors

__all__=['entropy', 'mutual_information', 'entropy_gaussian']

EPS = np.finfo(float).eps

def nearest_distances(X, k=1):
    '''
    X = array(N,M)
    N = number of points
    M = number of dimensions

    returns the distance to the kth nearest neighbor for every point in X
    '''
    knn = NearestNeighbors(n_neighbors=k)
    knn.fit(X)
    d, _ = knn.kneighbors(X) # the first nearest neighbor is itself
    return d[:, -1] # returns the distance to the kth nearest neighbor

def entropy_gaussian(C):
    '''
    Entropy of a gaussian variable with covariance matrix C
    '''
    if np.isscalar(C): # C is the variance
        return .5*(1 + np.log(2*pi)) + .5*np.log(C)
    else:
        n = C.shape[0] # dimension
        return .5*n*(1 + np.log(2*pi)) + .5*np.log(abs(det(C)))

def entropy(X, k=1):
    ''' Returns the entropy of the X.

    Parameters
    ===========

    X : array-like, shape (n_samples, n_features)
        The data the entropy of which is computed

    k : int, optional
        number of nearest neighbors for density estimation

    Notes
    ======

    Kozachenko, L. F. & Leonenko, N. N. 1987 Sample estimate of entropy
    of a random vector. Probl. Inf. Transm. 23, 95-101.
    See also: Evans, D. 2008 A computationally efficient estimator for
    mutual information, Proc. R. Soc. A 464 (2093), 1203-1215.
    and:
    Kraskov A, Stogbauer H, Grassberger P. (2004). Estimating mutual
    information. Phys Rev E 69(6 Pt 2):066138.
    '''

    # Distance to kth nearest neighbor
    r = nearest_distances(X, k) # squared distances
    n, d = X.shape
    volume_unit_ball = (pi**(.5*d)) / gamma(.5*d + 1)
    '''
    F. Perez-Cruz, (2008). Estimation of Information Theoretic Measures
    for Continuous Random Variables. Advances in Neural Information
    Processing Systems 21 (NIPS). Vancouver (Canada), December.

    return d*mean(log(r))+log(volume_unit_ball)+log(n-1)-log(k)
    '''
    return (d*np.mean(np.log(r + np.finfo(X.dtype).eps))
            + np.log(volume_unit_ball) + psi(n) - psi(k))

def mutual_information(variables, k=1):
    '''
    Returns the mutual information between any number of variables.
    Each variable is a matrix X = array(n_samples, n_features)
    where
      n = number of samples
      dx,dy = number of dimensions

    Optionally, the following keyword argument can be specified:
      k = number of nearest neighbors for density estimation

    Example: mutual_information((X, Y)), mutual_information((X, Y, Z), k=5)
    '''
    if len(variables) < 2:
        raise AttributeError(
                "Mutual information must involve at least 2 variables")
    all_vars = np.hstack(variables)
    return (sum([entropy(X, k=k) for X in variables])
            - entropy(all_vars, k=k))

def mutual_information_2d(x, y, sigma=1, normalized=False):
    """
    Computes (normalized) mutual information between two 1D variate from a
    joint histogram.

    Parameters
    ----------
    x : 1D array
        first variable

    y : 1D array
        second variable

    sigma: float
        sigma for Gaussian smoothing of the joint histogram

    Returns
    -------
    nmi: float
        the computed similariy measure

    """
    bins = (256, 256)

    jh = np.histogram2d(x, y, bins=bins)[0]

    # smooth the jh with a gaussian filter of given sigma
    ndimage.gaussian_filter(jh, sigma=sigma, mode='constant',
                                 output=jh)

    # compute marginal histograms
    jh = jh + EPS
    sh = np.sum(jh)
    jh = jh / sh
    s1 = np.sum(jh, axis=0).reshape((-1, jh.shape[0]))
    s2 = np.sum(jh, axis=1).reshape((jh.shape[1], -1))

    # Normalised Mutual Information of:
    # Studholme,  jhill & jhawkes (1998).
    # "A normalized entropy measure of 3-D medical image alignment".
    # in Proc. Medical Imaging 1998, vol. 3338, San Diego, CA, pp. 132-143.
    if normalized:
        mi = ((np.sum(s1 * np.log(s1)) + np.sum(s2 * np.log(s2)))
                / np.sum(jh * np.log(jh))) - 1
    else:
        mi = ( np.sum(jh * np.log(jh)) - np.sum(s1 * np.log(s1))
               - np.sum(s2 * np.log(s2)))

    return mi

###############################################################################
# Tests

def test_entropy():
    # Testing against correlated Gaussian variables
    # (analytical results are known)
    # Entropy of a 3-dimensional gaussian variable
    rng = np.random.RandomState(0)
    n = 50000
    d = 3
    P = np.array([[1, 0, 0], [0, 1, .5], [0, 0, 1]])
    C = np.dot(P, P.T)
    Y = rng.randn(d, n)
    X = np.dot(P, Y)
    H_th = entropy_gaussian(C)
    H_est = entropy(X.T, k=5)
    # Our estimated entropy should always be less that the actual one
    # (entropy estimation undershoots) but not too much
    np.testing.assert_array_less(H_est, H_th)
    np.testing.assert_array_less(.9*H_th, H_est)

def test_mutual_information():
    # Mutual information between two correlated gaussian variables
    # Entropy of a 2-dimensional gaussian variable
    n = 50000
    rng = np.random.RandomState(0)
    #P = np.random.randn(2, 2)
    P = np.array([[1, 0], [0.5, 1]])
    C = np.dot(P, P.T)
    U = rng.randn(2, n)
    Z = np.dot(P, U).T
    X = Z[:, 0]
    X = X.reshape(len(X), 1)
    Y = Z[:, 1]
    Y = Y.reshape(len(Y), 1)
    # in bits
    MI_est = mutual_information((X, Y), k=5)
    MI_th = (entropy_gaussian(C[0, 0])
             + entropy_gaussian(C[1, 1])
             - entropy_gaussian(C)
            )
    # Our estimator should undershoot once again: it will undershoot more
    # for the 2D estimation that for the 1D estimation
    print  MI_est, MI_th
    np.testing.assert_array_less(MI_est, MI_th)
    np.testing.assert_array_less(MI_th, MI_est  + .3)

def test_degenerate():
    # Test that our estimators are well-behaved with regards to
    # degenerate solutions
    rng = np.random.RandomState(0)
    x = rng.randn(50000)
    X = np.c_[x, x]
    assert np.isfinite(entropy(X))
    assert np.isfinite(mutual_information((x[:, np.newaxis],
                                           x[:,  np.newaxis])))
    assert 2.9 < mutual_information_2d(x, x) < 3.1

def test_mutual_information_2d():
    # Mutual information between two correlated gaussian variables
    # Entropy of a 2-dimensional gaussian variable
    n = 50000
    rng = np.random.RandomState(0)
    #P = np.random.randn(2, 2)
    P = np.array([[1, 0], [.9, .1]])
    C = np.dot(P, P.T)
    U = rng.randn(2, n)
    Z = np.dot(P, U).T
    X = Z[:, 0]
    X = X.reshape(len(X), 1)
    Y = Z[:, 1]
    Y = Y.reshape(len(Y), 1)
    # in bits
    MI_est = mutual_information_2d(X.ravel(), Y.ravel())
    MI_th = (entropy_gaussian(C[0, 0])
             + entropy_gaussian(C[1, 1])
             - entropy_gaussian(C)
            )
    print  MI_est, MI_th
    # Our estimator should undershoot once again: it will undershoot more
    # for the 2D estimation that for the 1D estimation
    np.testing.assert_array_less(MI_est, MI_th)
    np.testing.assert_array_less(MI_th, MI_est  + .2)

if __name__ == '__main__':
    # Run our tests
    test_entropy()
    test_mutual_information()
    test_degenerate()
    test_mutual_information_2d()

max_stars_repo_name: jjhenkel/dockerizeme

max_issues_repo_name: jjhenkel/dockerizeme

max_forks_repo_name: jjhenkel/dockerizeme

reason: import numpy,from numpy,from scipy


text:
"""
 masker functions for each mode to fill img subclass of the main Vivus object
"""

import logging
import numpy as np

from PIL import Image

from .imgproc import *
from .loader  import *

mlg = logging.getLogger(__name__)

def getGraySmask():
    """
    helper function to load grayscale image and
    scalemask image based on input.png
    """

    # grayscale image
    gray = np.array(Image.open('input.png').convert('L'))

    # scalemask from gray image
    smask = getScalemask(gray)

    return gray, smask

def getGrayclnPolar(gray, smask):
    """
    helper function that cleans input grayscale image
    and returns versions in cartesian and polar coordinates
    """

    # cleaned grayscale image with masks for the probe and scales
    graycln = getCleanImage(gray, smask)

    # polar transformation
    polar   = getPolar(graycln)

    return graycln, polar

def fullMasker():
    """
    masker() for mode 0, uses hollow type weights to use for both contours
    note: graycln version of the input is not computed here
    """

    # input image processing
    gray, smask = getGraySmask()

    # grayscale of the mask image
    img  = getGrayNNmask('hollow')
    graymask = np.array(img)

    # polar transformation
    polar = getPolar(graymask)

    # same enhanced polar image mask for both boundaries
    ienh  = getEnhancement(polar, p.threshMaxInn, p.threshMinInn)
    oenh  = np.array(ienh)

    mlg.info(' fullMasker: done')
    return ProcImages(gray, smask, None, polar, oenh, ienh)

def hybridMasker():
    """
    masker() for mode 1, uses filled type weights to use only for inner contour
    outer contour is treated in the same way as mode=2 (deterministically)
    """

    # input image processing
    gray, smask = getGraySmask()

    # grayscale mask image
    img  = getGrayNNmask('filled')
    graymask = np.array(img)

    # polar transformation
    polar = getPolar(graymask)

    # enhanced polar image mask for inner boundary
    # note that outer thresholds are used here!
    ienh    = getEnhancement(polar, p.threshMaxOut, p.threshMinOut)

    # cleaned grayscale image with masks for the probe and scales
    graycln, polar = getGrayclnPolar(gray, smask)

    # enhanced polar image mask for outer boundary
    oenh    = getEnhancement(polar, p.threshMaxOut, p.threshMinOut)

    mlg.info(' hybridMasker: done')
    return ProcImages(gray, smask, graycln, polar, oenh, ienh)

def simpleMasker():
    """
    masker() for mode 2, no weights used, purely deterministic approach
    """

    # input image processing
    gray, smask = getGraySmask()

    # cleaned grayscale image with masks for the probe and scales
    graycln, polar = getGrayclnPolar(gray, smask)

    # enhanced polar image mask for outer boundary
    oenh    = getEnhancement(polar, p.threshMaxOut, p.threshMinOut)

    # enhanced polar image mask for iner boundary
    ienh    = getEnhancement(polar, p.threshMaxInn, p.threshMinInn)

    mlg.info(' simpleMasker: done')
    return ProcImages(gray, smask, graycln, polar, oenh, ienh)

max_stars_repo_name: tuguldurs/vivus

max_issues_repo_name: tuguldurs/vivus

max_forks_repo_name: UltraSound-AI/vivus-sdk