text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import numpy as np
import tensorflow as tf
from data_reader import load_data, DataReader, FasttextModel, DataReaderFastText
from gensim.test.utils import get_tmpfile
from gensim.models.callbacks import CallbackAny2Vec
import model

flags = tf.flags

# data
flags.DEFINE_string('load_model',
                    r"C:\Users\Ohad.Volk\Desktop\Oshri\tf-lstm-char-cnn-master\tf-lstm-char-cnn-master\trained_models\AMI_2020-03-28--12-43-24\epoch011_3.5723.model",
                    'filename of the model to load')
# we need data only to compute vocabulary
flags.DEFINE_string('data_dir',
                    r"C:\Users\Ohad.Volk\Desktop\Oshri\tf-lstm-char-cnn-master\tf-lstm-char-cnn-master\data_sets\AMI",
                    'data directory')
flags.DEFINE_string('fasttext_model_path',
                    "E:\\RNNLM Project\\Models\\Wikipedia\\Lower Case\\15 Epoch\\Wikipedia_epoch15.model",
                    'fasttext trained model path')
flags.DEFINE_integer('num_samples', 2000, 'how many words to generate')
flags.DEFINE_float('temperature', 1.0, 'sampling temperature')
flags.DEFINE_string('embedding', "kim fasttext", 'embedding method')
# model params
flags.DEFINE_integer('rnn_size', 650, 'size of LSTM internal state')
flags.DEFINE_integer('highway_layers', 2, 'number of highway layers')
flags.DEFINE_integer('char_embed_size', 15, 'dimensionality of character embeddings')
flags.DEFINE_string('kernels', '[1,2,3,4,5,6,7]', 'CNN kernel widths')
flags.DEFINE_string('kernel_features', '[50,100,150,200,200,200,200]', 'number of features in the CNN kernel')
flags.DEFINE_integer('rnn_layers', 2, 'number of layers in the LSTM')
flags.DEFINE_float('dropout', 0.5, 'dropout. 0 = no dropout')
flags.DEFINE_integer('batch_size', 1, 'number of sequences to train on in parallel')
flags.DEFINE_integer('num_unroll_steps', 1, 'number of timesteps to unroll for')

# optimization
flags.DEFINE_integer('max_word_length', 65, 'maximum word length')

# bookkeeping
flags.DEFINE_integer('seed', 3435, 'random number generator seed')
flags.DEFINE_string('EOS', '+',
                    '<EOS> symbol. should be a single unused character (like +) for PTB and blank for others')

FLAGS = flags.FLAGS

class EpochSaver(CallbackAny2Vec):

    def __init__(self, path_prefix):
        self.path_prefix = path_prefix
        self.epoch = 0

    def on_epoch_begin(self, model):
        print("Epoch #{} start".format(self.epoch))

    def on_epoch_end(self, model):
        if self.epoch in [1, 5, 10, 15, 20]:
            output_path = get_tmpfile('{}_epoch{}.model'.format(self.path_prefix, self.epoch))
            model.save(output_path)
            print('model save')
        print("Epoch #{} end".format(self.epoch))
        self.epoch += 1

def main(_):
    ''' Loads trained model and evaluates it on test split '''

    if FLAGS.load_model is None:
        print('Please specify checkpoint file to load model from')
        return -1

    if not os.path.exists(FLAGS.load_model + '.meta'):
        print('Checkpoint file not found', FLAGS.load_model)
        return -1

    word_vocab, char_vocab, word_tensors, char_tensors, max_word_length, words_list = \
        load_data(FLAGS.data_dir, FLAGS.max_word_length, eos=FLAGS.EOS)

    fasttext_model = FasttextModel(fasttext_path=FLAGS.fasttext_model_path).get_fasttext_model()

    print('initialized test dataset reader')

    with tf.Graph().as_default(), tf.Session() as session:

        # tensorflow seed must be inside graph
        tf.set_random_seed(FLAGS.seed)
        np.random.seed(seed=FLAGS.seed)

        ''' build inference graph '''
        with tf.variable_scope("Model"):
            m = model.inference_graph(
                char_vocab_size=char_vocab.size,
                word_vocab_size=word_vocab.size,
                char_embed_size=FLAGS.char_embed_size,
                batch_size=FLAGS.batch_size,
                num_highway_layers=FLAGS.highway_layers,
                num_rnn_layers=FLAGS.rnn_layers,
                rnn_size=FLAGS.rnn_size,
                max_word_length=max_word_length,
                kernels=eval(FLAGS.kernels),
                kernel_features=eval(FLAGS.kernel_features),
                num_unroll_steps=FLAGS.num_unroll_steps,
                dropout=0,
                embedding=FLAGS.embedding,
                fasttext_word_dim=300,
                acoustic_features_dim=4)
            # we need global step only because we want to read it from the model
            global_step = tf.Variable(0, dtype=tf.int32, name='global_step')

        saver = tf.train.Saver()
        saver.restore(session, FLAGS.load_model)
        print('Loaded model from', FLAGS.load_model, 'saved at global step', global_step.eval())

        ''' training starts here '''
        rnn_state = session.run(m.initial_rnn_state)
        logits = np.ones((word_vocab.size,))
        rnn_state = session.run(m.initial_rnn_state)
        for i in range(FLAGS.num_samples):
            logits = logits / FLAGS.temperature
            prob = np.exp(logits)
            prob /= np.sum(prob)
            prob = prob.ravel()
            ix = np.random.choice(range(len(prob)), p=prob)

            word = word_vocab.token(ix)
            words_tf = fasttext_model.wv[word]
            input_2 = np.reshape((np.concatenate((words_tf, np.zeros(4))).T), (-1, 1))
            if word == '|':  # EOS
                print('<unk>', end=' ')
            elif word == '+':
                print('\n')
            else:
                print(word, end=' ')

            char_input = np.zeros((1, 1, max_word_length))
            for i, c in enumerate('{' + word + '}'):
                char_input[0, 0, i] = char_vocab[c]

            logits, rnn_state = session.run([m.logits, m.final_rnn_state],
                                            {m.input: char_input,
                                             m.input2: input_2,
                                             m.initial_rnn_state: rnn_state})
            logits = np.array(logits)

if __name__ == "__main__":
    tf.app.run()

max_stars_repo_name: oshToy/Character-Aware-LM

max_issues_repo_name: oshToy/Character-Aware-LM

max_forks_repo_name: oshToy/Character-Aware-LM


text:
from network import Network
from layers import Relu, Softmax, Linear, Convolution, Pooling
from loss import CrossEntropyLoss
from optimizer import SGDOptimizer
from optimizer import AdagradOptimizer
from solve_net import solve_net
from mnist import load_mnist_for_cnn

import theano.tensor as T
import theano

theano.config.floatX = 'float32'

train_data, test_data, train_label, test_label = load_mnist_for_cnn('data')
model = Network()
model.add(Convolution('conv1', 5, 1, 4, 0.1))   # output size: N x 4 x 24 x 24
model.add(Relu('relu1'))
model.add(Pooling('pool1', 2))                  # output size: N x 4 x 12 x 12
model.add(Convolution('conv2', 3, 4, 8, 0.1))   # output size: N x 8 x 10 x 10
model.add(Relu('relu2'))
model.add(Pooling('pool2', 2))                  # output size: N x 8 x 5 x 5
model.add(Linear('fc3', 8 * 5 * 5, 10, 0.1))          # input reshaped to N x 200 in Linear layer
model.add(Softmax('softmax'))

loss = CrossEntropyLoss(name='xent')

# optim = SGDOptimizer(learning_rate=0.005, weight_decay=0.005, momentum=0.9)
optim = AdagradOptimizer(learning_rate=0.001, eps=5e-4)

input_placeholder = T.ftensor4('input')
label_placeholder = T.fmatrix('label')
model.compile(input_placeholder, label_placeholder, loss, optim)

solve_net(model, train_data, train_label, test_data, test_label,
          batch_size=32, max_epoch=100, disp_freq=1000, test_freq=10000)

ext: py

reason: import theano


text:
import json
import tqdm
from gensim.models.doc2vec import Doc2Vec
from gensim.models import KeyedVectors
import numpy as np

#学習用jsonファイル読み込み
with open('data/div_corpus_data.json', mode='r') as f:
    data = json.load(f)

# #データ参照用jsonファイル読み込み
# with open('data/ref_data.json', mode='r') as f:
#     ref_data = json.load(f)

en_data = data['en']
ja_data = data['ja']
en_data_keys = en_data.keys()
ja_data_keys = ja_data.keys()

print('english data len :',len(en_data))
print('japanese data len :',len(ja_data))

#Doc2vecモデル読み込み
en_doc2vec_model = Doc2Vec.load('model/en_doc2vec.model')
ja_doc2vec_model = Doc2Vec.load('model/ja_doc2vec.model')

#Word2vecモデル読み込み
en_word2vec_model = KeyedVectors.load_word2vec_format('model/en_word2vec.bin', binary=True)
ja_word2vec_model = KeyedVectors.load_word2vec_format('model/ja_word2vec.bin', binary=True)

#テスト
test_class_ids = ['11610212', '30562111', '51100303']
check_list = ['major', 'name', 'summary', 'url']

#出力ファイル
output_data = []

#日本語講義推薦データ作成
print("日本語講義計算中...")
for ja_class_id in tqdm.tqdm(ja_data_keys):
    #一時保存用
    tmp_class_data = dict()
    tmp_similarClasses_list = []
    tmp_similarlevel_list = []

    #注目講義保存
    tmp_class_data['code'] = ja_class_id

    #類似講義
    wmd_list = []
    similar_class_id_list = []

    tmp_similar_list = ja_doc2vec_model.docvecs.most_similar(ja_class_id, topn=30) #類似講義上位500件
    for similar_class in tmp_similar_list:
        similar_class_id = similar_class[0]
        tmp_wmd = ja_word2vec_model.wmdistance(ja_data[ja_class_id].split(' '), ja_data[similar_class_id].split(' '))
        wmd_list.append(tmp_wmd)
        similar_class_id_list.append(similar_class_id)

    #wmd上位10件
    wmd_list = np.array(wmd_list)
    argsort_wmd = np.argsort(wmd_list)[:10]

    #類似講義検索
    for index in list(argsort_wmd):
        #一時保存用
        tmp_similar_class_data = dict()
        similar_class_id = similar_class_id_list[index]
        similar_class_level = wmd_list[index]
        tmp_similar_class_data['code'] = similar_class_id
        tmp_similar_class_data['year'] = '2021'
        tmp_similarClasses_list.append(tmp_similar_class_data)
        tmp_similarlevel_list.append(similar_class_level)

    tmp_class_data['similarClasses'] = tmp_similarClasses_list
    tmp_class_data['similarLevel'] = tmp_similarlevel_list

    output_data.append(tmp_class_data)
print("日本語講義計算終了")

#英語講義推薦データ作成
print("英語講義計算中...")
for en_class_id in tqdm.tqdm(en_data_keys):
    #一時保存用
    tmp_class_data = dict()
    tmp_similarClasses_list = []
    tmp_similarlevel_list = []

    #注目講義保存
    tmp_class_data['code'] = en_class_id

    #類似講義
    wmd_list = []
    similar_class_id_list = []

    tmp_similar_list = en_doc2vec_model.docvecs.most_similar(en_class_id, topn=100) #類似講義上位500件
    for similar_class in tmp_similar_list:
        similar_class_id = similar_class[0]
        tmp_wmd = en_word2vec_model.wmdistance(en_data[en_class_id].split(' '), en_data[similar_class_id].split(' '))
        wmd_list.append(tmp_wmd)
        similar_class_id_list.append(similar_class_id)

    #wmd上位10件
    wmd_list = np.array(wmd_list)
    argsort_wmd = np.argsort(wmd_list)[:10]

    #類似講義検索
    for index in list(argsort_wmd):
        #一時保存用
        tmp_similar_class_data = dict()
        similar_class_id = similar_class_id_list[index]
        similar_class_level = wmd_list[index]
        tmp_similar_class_data['code'] = similar_class_id
        tmp_similar_class_data['year'] = '2021'
        tmp_similarClasses_list.append(tmp_similar_class_data)
        tmp_similarlevel_list.append(similar_class_level)

    tmp_class_data['similarClasses'] = tmp_similarClasses_list
    tmp_class_data['similarLevel'] = tmp_similarlevel_list

    output_data.append(tmp_class_data)
print("英語講義計算終了")

with open('data/recommend_wmd.json', 'w') as f:
    json.dump(output_data, f, ensure_ascii=False, indent=4)
print("データ出力")

reason: import numpy


text:
"""Utility functions of Objects.

Utility functions for handling Objects.

"""
import builtins
from win32com.client import CDispatch, DispatchBaseClass, CoClassBaseClass
from pywintypes import com_error
from contextlib import contextmanager
from collections.abc import Sequence
from collections import UserDict, UserString
from collections.abc import Mapping

import numpy as np

from fairypptx import registry_utils

def get_type(instance):
    """Return the Capitalized Object Type Name."""
    if instance is None:
        return None
    return instance.__class__.__name__.strip("_").capitalize()
    #except AttributeError:
    #    return None
    # return getattr(type(instance), "__com_interface__").__name__.strip("_").capitalize()

def is_object(instance, name=None):
    """Return whether ``instance`` is regarded as Powerpoint Object.

    Args:
        instance: the checked instance.
        name(str): the Object Type name.

    Note
    --------------
    Return ``True``, if ``instance`` has ``__com_interface__``.
    Hence, this check is very weak so ``instance`` may be not related to
    PowerPoint, even if ``True`` is returned.

    """
    flag = isinstance(instance, (DispatchBaseClass, CoClassBaseClass))
    c_name = get_type(instance)
    if name:
        return flag and (name.capitalize() == c_name)
    else:
        return flag

def upstream(instance, name):
    """Return `name` Object as long as `instance.parent..parent` is achievable.

    Args:
        instance: instance.
        name(str): the Target ObjectType Name.

    Raises: ValueError:
    """
    target = instance
    while True:
        type_name = get_type(target)
        # print("type_name", type_name, name.capitalize())
        if type_name.capitalize() == name.capitalize():
            return target
        try:
            target = target.Parent
        except AttributeError as e:
            raise ValueError(f"`{name}` is not an ancestor of `{instance.__class__}`.")

def setattr(instance, attr, value):
    """Extension of `setattr` so that `.` specifier is valid.
    Args:
        instance: Object.
        attr (str or Sequence of str): specifier.
        value: value

    Raises:
        ValueError: When `value` is not invalid.
        AttributeError: When attribute is not existent.

    """
    elems = _listify(attr)
    target = instance
    for elem in elems[:-1]:
        target = builtins.getattr(target, elem)
    try:
        builtins.setattr(target, elems[-1], value)
    except AttributeError as e:
        raise AttributeError(f"`{value}` cannot be set to `{attr}`") from e
    except com_error as e:
        raise ValueError(f"`{value}` cannot be set to `{attr}`") from e

_NOT_SPECIFIED = object()

def getattr(instance, attr, default=_NOT_SPECIFIED):
    """Extension of `setattr` so that `.` specifier is valid.
    Args:
        instance: Object.
        attr (str or Sequence of str): specifier.
        default(Any): the default value
                      when attribute is not exist.
    Raises:
        AttributeError: Attribute is not existent and default is not set.
    """
    if default is not _NOT_SPECIFIED:
        try:
            return getattr(instance, attr)
        except AttributeError:
            return default
    else:
        elems = _listify(attr)
        target = instance
        for elem in elems:
            target = builtins.getattr(target, elem)
        return target

def hasattr(instance, attr):
    """Extension of `builtins.hasattr` so that `.` specifier is valid."""
    try:
        getattr(instance, attr)
    except AttributeError:
        return False
    return True

def to_api2(api_object):
    """Convert to the second version Object.

    In Powerpoint Object model,
    some classes have 2 apis. (e.g. `TextRange` and `TextRange2`.)

    This function returns 2 version of `api`.

    TODO: This list is not complete.
    """
    if is_object(api_object) and get_type(api_object).endswith("2"):
        return api_object

    def _to_textrange2_api(textrange_api):
        shape_obj = upstream(api_object, "Shape")
        start = textrange_api.Start
        length = textrange_api.Length
        return shape_obj.TextFrame2.TextRange.GetCharacters(start, length)

    if is_object(api_object, "TextFrame"):
        shape_api = upstream(api_object, "Shape")
        return shape_api.TextFrame2
    if is_object(api_object, "TextRange"):
        return _to_textrange2_api(api_object)
    if is_object(api_object, "Font"):
        tr_api = upstream(api_object, "TextRange")
        tr_api2 = _to_textrange2_api(tr_api)
        return tr_api2.Font
    if is_object(api_object, "ParagraphFormat"):
        tr_api = upstream(api_object, "TextRange")
        tr_api2 = _to_textrange2_api(tr_api)
        return tr_api2.ParagraphFormat
    if hasattr(api_object, "api"):
        return to_api2(api_object)  # Maybe high-level class is given?
    raise ValueError("Cannot interpret the given `api_object`. ", api_object.__class__)

@contextmanager
def stored(instance, attrs):
    """Store the values of `instance`'s `attrs`.
    Args:
        instance: Object
        attrs (str, Sequence of str or Sequence of Sequence):
    """

    stock = dict()
    if isinstance(attrs, str):
        attrs = [attrs]
    # Stocked Phase.
    for attr in attrs:
        elems = _listify(attr)
        stock[tuple(elems)] = getattr(instance, elems)

    # Rollback Phase
    def _rollback():
        for key, value in stock.items():
            setattr(instance, key, value)

    try:
        yield
    except Exception as e:
        _rollback()
        raise e
    else:
        _rollback()

def _listify(attr):
    if isinstance(attr, str):
        attr = attr.split(".")
    if not isinstance(attr, Sequence):
        raise ValueError(f"`{attr}` is not valid specifier.")
    return attr

class ObjectDictMixin(UserDict):
    """Mixin for Object Dict.

    Args:
        arg: the target of Object or Dict.
        name: the Name of Object.

    Protocol of Derivative classes.
    --------------------------------

    * `data`: class attribute is required as a template.
    * `name`: Object Name.
              If `None`, then the classname is the same as the Object name.
    * `readonly`: `data` is stored,
                   however, when `setattr` is called at `apply`,
                   the keys of `readonly` is skipped.
                   Typically, read-only properties of Object
                   is set here.

    Provided Service
    ------------------------------
    * conversion between `Object` and UserDict.
    * fetch / register of Object / UserDict.
    * key access to Object API.
        - [TODO] `key` of `dict` must be case incensitive.
            - It leads to th duplicated (key, value).

    Note
    ----------------------------------
    `fetch` <-> `register`.

    Usage
    ----------------------------------

    * `apply`: When you want to apply the content of `this class`
               to `Object`, using the stored dict.
    * `to_dict`: You you want to store the data from `Object` as `dict`.

    - When you implement `to_dict`, typically you also implement`apply`.
    - Some properties are read-only, so you may have to call the function other than `setattr`.
      For those cases, please implement your customized `apply`.

    The expected protocol of these classes as follows:
    what this class intend to realize this protocol.

    - __init__(self, arg):
        - If `arg` is Object, then `self.api` points to `arg` and `self.data` represents
        - If `arg` is `Mapping`, then `self.api` is None  and `self.data` is equivalent to `arg`.

    - def apply(self, api_object):
        - `self.data`'s contents are reflected onto `api_object`.

    - `fetch` / `register` / `api`:  (You can easily guess the contents.)

    I feel if these interfaces are realized, the feel of usage does not change from perspective of users.
    """

    data = dict()
    name = None
    readonly = []

    def __init__(self, arg=None, **kwargs):
        self.cls = type(self)
        if self.cls.name:
            self.name = self.cls.name
        else:
            self.name = self.cls.__name__
        self.data, self._api = self._construct(arg, **kwargs)

    @property
    def api(self):
        return self._api

    def detached(self):
        """Return this class without `_api`."""
        return self.cls(self)

    def __getstate__(self):
        # This is for serialization of `pickle`.
        return {"data": self.data, "name": self.name, "readonly": self.readonly}

    def __setitem__(self, key, item):
        super().__setitem__(key, item)
        if self._api:
            if not hasattr(self._api, key):
                api_type = get_type(self._api)
                raise KeyError(f"`{key}` does not exist in `{api_type}`.")
            setattr(self._api, key, item)
        else:
            # When `api` is not given, this behaves as normal `UserDict`.
            pass

    def __setattr__(self, name, value):

        # Without `_api`, the behavior is the same as `UserDict`.
        if "_api" not in self.__dict__:
            object.__setattr__(self, name, value)
            return
        if self._api is None:
            object.__setattr__(self, name, value)
            return

        if name in self.__dict__ or name in type(self).__dict__:
            object.__setattr__(self, name, value)
        elif hasattr(self.api, name):
            setattr(self.api, name, value)
            self[name] = value
        else:
            # TODO: Maybe require modification.
            object.__setattr__(self, name, value)

    def to_dict(self, api_object):
        """Convert `Object` to `dict`.

        As a default,  values of `keys` of `class.data`
        is copied.
        """
        keys = self.cls.data.keys()
        data = {key: getattr(api_object, key) for key in keys}
        return data

    def apply(self, api_object):
        """Apply `self` to `api_object`."""
        readonly_props = set(self.readonly)
        for key, value in self.data.items():
            if key not in readonly_props:
                setattr(api_object, key, value)
        return api_object

    def register(self, key, disk=False):
        """Register to the storage."""
        name = self.name
        registry_utils.register(name, key, self.data, extension=".json", disk=disk)

    @classmethod
    def fetch(cls, key, disk=True):
        """Construct the instance with `key` object."""
        name = self.name
        data = registry_utils.fetch(name, key, disk=True)
        return cls(data)

    def _construct(self, arg, **kwargs):
        name = self.name
        api = None
        if arg is None:
            data = dict(self.cls.data)
        elif is_object(arg, name):
            data = self.to_dict(arg)
            api = arg
        elif isinstance(arg, Mapping):
            data = dict(arg)
        else:
            raise ValueError(f"Cannot interpret `{arg}`.")
        # If specified, update is performed.
        data.update(kwargs)
        return data, api

class ObjectDictMixin2(Mapping):
    """Represents `api` / `api2`'s extension for

    For some `Objects`, `api` and `api2` exist,
    (c.f. ParagraphFormat, TextRange)

    data2 is added to `ObjectDictMixin`
    """

    data = dict()
    data2 = dict()

    readonly = []
    name = None

    def __init__(self, arg=None):
        cls = type(self)
        if cls.name is None:
            cls.name = cls.__name__
        self._construct(arg)
        assert bool(self.data) or bool(self.data2)

    def to_dict(self, api):
        cls = type(self)
        return {key: getattr(api, key) for key in self.data}

    def to_dict2(self, api2):
        cls = type(self)
        return {key: getattr(api2, key) for key in self.data2}

    def apply(self, api):
        api2 = to_api2(api)
        readonly_props = set(self.readonly)

        for key, value in self.data.items():
            if key not in readonly_props:
                if value is not None:
                    setattr(api, key, value)

        for key, value in self.data2.items():
            if key not in readonly_props:
                if value is not None:
                    setattr(api2, key, value)
        return api

    @property
    def api(self):
        return self._api

    @property
    def api2(self):
        if self.api:
            return to_api2(self.api)
        else:
            return None

    def to_api2(self, api):
        return to_api2(api)

    def _construct(self, arg):
        cls = type(self)
        self._api = None
        if arg is None:
            self.data = cls.data.copy()
            self.data2 = cls.data2.copy()
        elif is_object(arg, cls.name):
            self._api = arg
            self.data = self.to_dict(self.api)
            self.data2 = self.to_dict2(self.api2)
        elif isinstance(arg, Mapping):
            for key, value in arg.items():
                self[key] = value
        else:
            raise ValueError("Given `arg` is not appropriate.", arg)

    def __repr__(self):
        return repr(self.data) + "\n" + repr(self.data2)

    def items(self):
        import itertools
        return itertools.chain(self.data.items(), self.data2.items())

    def __len__(self):
        return len(self.data) + len(self.data2)

    def __getitem__(self, key):
        if key in self.data:
            return self.data[key]
        if key in self.data2:
            return self.data2[key]
        raise KeyError(key)

    def __setitem__(self, key, item):
        is_api_set =  self.api and (key not in self.readonly)

        if key in self.data:
            self.data[key] = item
            if is_api_set:
                if item is not None:
                    setattr(self.api, key, item)
            return
        elif key in self.data2:
            self.data2[key] = item
            if is_api_set:
                if item is not None:
                    setattr(self.api2, key, item)
            return
        elif is_api_set:
            # This is a fallback,
            try:
                setattr(self.api, key, item)
            except AttributeError as e:
                pass
            else:
                self.data[key] = item
                return
            try:
                setattr(self.api2, key, item)
            except AttributeError as e:
                pass
            else:
                self.data2[key] = item
                return
        raise KeyError("Cannot set key", key)

    def __delitem__(self, key):
        del self.data[key]
        del self.data2[key]

    def __iter__(self):
        import itertools
        return itertools.chain(iter(self.data), iter(self.data2))

    def __contains__(self, key):
        return key in self.data or key in self.data2

    def __getstate__(self):
        """For `pickle` serialization
        """
        return {"name": self.name,
                "data": self.data,
                "data2": self.data2,
                "readonly": self.readonly}

    def register(self, key, disk=False):
        """Register to the storage."""
        name = type(self).name
        registry_utils.register(name, key, self, extension=".pkl", disk=disk)

    @classmethod
    def fetch(cls, key, disk=True):
        """Construct the instance with `key` object."""
        name = cls.name
        return registry_utils.fetch(name, key, disk=True)

class ObjectClassMixin:
    """Provide functions useful for classes which corresponds to Object Class.

    Provided Service
    -----------------
    * Fetch / Setter of `api` attribute.
    * __getattr__ / __setattr__ are revised so that when attribute is not found for the class,

    Extension Cases
    ----------------
    * Implement `fetch_api` for the case when `arg` is None or other types.
    * Set `name` attribute if the name of the class is not equal to Object Type Name.
    """

    name = None

    def __init__(self, arg=None):
        self.cls = type(self)
        if self.cls.name:
            self.name = name
        else:
            self.name = self.cls.__name__

        self._api = self._fetch_api(arg)

    def fetch_api(self, arg):
        """Fetch `api` from `arg`."""
        raise ValueError("Cannot interpret `arg`; `{arg}`.")

    @property
    def api(self):
        return self._api

    @property
    def app(self):
        from fairypptx import Application

        return Application(self._api.Application)

    def _fetch_api(self, arg):
        if isinstance(arg, self.cls):
            return arg.api
        elif is_object(arg, self.name):
            return arg
        return self.fetch_api(arg)

    def __getattr__(self, name):
        if "_api" not in self.__dict__:
            raise AttributeError
        return getattr(self.__dict__["_api"], name)

    def __setattr__(self, name, value):
        if "_api" not in self.__dict__:
            object.__setattr__(self, name, value)

        if name in self.__dict__ or name in type(self).__dict__:
            object.__setattr__(self, name, value)
        elif hasattr(self.api, name):
            setattr(self.api, name, value)
        else:
            # TODO: Maybe require modification.
            object.__setattr__(self, name, value)

class ObjectItems:
    """Utility class for handing `api.Item(index)` function.
    Args
        api_object: `.api` object for the parent.
                    This must have the attribute `Item`.
        child_cls: The wrapper class used for each element of Items.
    """

    def __init__(self, api_object, child_cls):
        self.api = api_object
        self.cls = child_cls

    def __len__(self):
        return self.api.Count

    def __getitem__(self, key):
        if isinstance(key, (int, np.integer)):
            if key < 0:
                key = key + len(self) - 1
            if not (0 <= key < len(self)):
                raise IndexError(
                    f"Size is {len(self)}, index is {key} is out of range."
                )
            return self.cls(self.api.Item(key + 1))
        elif isinstance(key, slice):
            indices = range(*key.indices(len(self)))
            return [self[index] for index in indices]
        elif isinstance(key, Sequence):
            return [self[index] for index in key]
        raise TypeError(
            f"Key's type is invalid; key = `{key}`, type(key) = `{type(key)}`"
        )

    def normalize(self, key):
        """Return int or Sequence of int, which represents indices.

        If key is int, the return is in [0, len(sel) - 1].
        If key specifies, slice or Sequence,  each element of the return is in [0, len(self) - 1].

        Raise:
            IndexError.
            TypeError.
        """
        if isinstance(key, int):
            if key < 0:
                key = key + len(self)
            if not (0 <= key < len(self)):
                raise IndexError(
                    f"Size is {len(self)}, index is {key} is out of range."
                )
            return key

        elif isinstance(key, slice):
            indices = list(range(*key.indices(len(self))))
            return indices

        elif isinstance(key, Sequence):
            for elem in key:
                if not (0 <= elem < len(self)):
                    raise IndexError(
                        f"index out of range; len=`{len(self)}`, but index is `{elem}`"
                    )
            return key

        raise TypeError(f"Invalid Argument; `{key}`")

if __name__ == "__main__":
    pass

max_stars_repo_name: Sillte/fairypptx

max_issues_repo_name: Sillte/fairypptx

max_forks_repo_name: Sillte/fairypptx


text:
# general imports
import numpy as np
from abc import ABC, abstractmethod

# local modul imports
from . import channel

class FlowField:
    def __init__(self, name, flowfield_dict, channel):
        self.name = name
        self.channel = channel
        # number of channels of each half cell
        self.n_channels = flowfield_dict['channel_number']
        length = flowfield_dict['length']
        width = flowfield_dict['width']
        area = length * width

        # rib width dictates and resets channel length if provided
        if 'rib_width' in flowfield_dict:
            self.rib_width = flowfield_dict['rib_width']
            self.width_straight_channels = \
                (channel.width + self.rib_width) * self.n_channels
            self.channel.length = \
                area / (channel.width + self.rib_width) * self.n_channels
            area_factor = area / (channel.base_area * self.n_channels)
        else:
            area_factor = area / (channel.base_area * self.n_channels)
            if area_factor < 1.0:
                raise ValueError('width and length of cell result in a cell '
                                 'surface area  smaller than the area '
                                 'covered by channels')
            self.rib_width = channel.width * (area_factor - 1.0)

        self.width_straight_channels = \
            (channel.width + self.rib_width) * self.n_channels
        self.length_straight_channels = area / self.width_straight_channels
        self.active_area = area_factor * channel.base_area
        # self.active_area = area_factor * self.channel.base_area
        # factor active area with ribs / active channel area
        self.active_area_dx = area_factor * channel.base_area_dx
        # self.active_area_dx = area_factor * self.channel.base_area_dx

max_stars_repo_name: lufire/pemfc-core

max_issues_repo_name: lufire/pemfc-core

max_forks_repo_name: lufire/pemfc-core


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Maximum Lyapunov exponent of a closed noisy curve.

A trajectory in the form of a closed curve should have a Lyapunov
exponent equal to zero (or the average divergence should not vary with
time).  But our curves for the average divergence appear to be
oscillatory and don't look very flat.  What's wrong?
"""

import numpy as np
import matplotlib.pyplot as plt
from nolitsa import lyapunov, utils

t = np.linspace(0, 100 * np.pi, 5000)
x = np.sin(t) + np.sin(2 * t) + np.sin(3 * t) + np.sin(5 * t)
x = utils.corrupt(x, np.random.normal(size=5000), snr=1000)

# Time delay.
tau = 25

window = 100

# Embedding dimension.
dim = [10]

d = lyapunov.mle_embed(x, dim=dim, tau=tau, maxt=300, window=window)[0]

plt.title('Maximum Lyapunov exponent for a closed curve')
plt.xlabel(r'Time $t$')
plt.ylabel(r'Average divergence $\langle d_i(t) \rangle$')
plt.plot(t[:300], d)

plt.show()

max_stars_repo_name: manu-mannattil/nolitsa

max_issues_repo_name: tanmaymaloo/nolitsa

max_forks_repo_name: tanmaymaloo/nolitsa


text:
#! /usr/bin/env python3
# -*- coding: utf-8 -*-

import numpy as np
from ._networktools import distances
from math import lgamma, exp, log

lfact = lambda n: lgamma(n+1)

def lptilde(k, a, c, norm=0):
	return (
		  lfact(a)
		- lfact(k)
		- lfact(a-k)
		- lfact(2*a)
		+ lfact(c+k)
		+ lfact(2*a-c-k)
		+ norm
		)

ptilde = lambda k, a, c, norm: exp(lptilde(k, a, c, norm))

class strengthsurrogate_tester(object):
	"""
	Creates a tester for the goodness of strength-preserving surrogates as per Ansmann and Lehnertz, Phys. Rev. E 84, 026103 (2011), chapter C2. The names of parameters are analogous to that section. See the file goodness_example.py for an example of usage.

	Parameters
	----------
	QR: list of weighted networks
		These are surrogates generated with a reference method (such as the same method with a very high step size).

	a: integer
		The size of the lists of surrogates to be tested.

	c: integer
		The number of reference surrogates in each ε-ball used for testing. This should be much smaller than a. A reasonable choice for this parameter is 10.
	"""

	def __init__(self, QR, a, c):
		assert c<a
		self.a = a
		self.c = c
		np.random.shuffle(QR)
		Q = QR[:a]
		self.R = R = QR[a:]
		self.b = b = len(R)
		dists = distances(Q,R)
		dists.sort(axis=0)

		n = QR.shape[1]
		d = (n*(n-1))/2

		# safer and more robust than:
		# epsilons = ((dists[c+1,:]**d + dists[c,:]**d)/2)**(1./d)
		self.epsilons = dists[c,:]*(0.5+0.5*(dists[c+1,:]/dists[c,:])**d)**(1./d)
		if not np.all(np.isfinite(self.epsilons)):
			self.epsilons = dists[c+1,:]

	def chi(self, P):
		"""
		Calculates the quantifier of surrogate goodness χ. If this value is consistently around 1, the tested surrogates are good.

		Parameters
		----------
		P: list of a weighted networks
			The surrogates to be tested.
		"""

		assert self.a==len(P), "The number of networks in P must be a."
		dists = distances(P,self.R)
		ks = (dists<self.epsilons).sum(axis=0)
		norm = -lptilde(self.c,self.a,self.c)
		return (
			sum(ptilde(k, self.a, self.c, norm) for k in ks)
			* sum(ptilde(j, self.a, self.c, norm) for j in range(self.a+1))
			/ sum(ptilde(j, self.a, self.c, norm)**2 for j in range(self.a+1))
			/ self.b
			)

max_stars_repo_name: neurophysik/networktools

max_issues_repo_name: neurophysik/networktools

max_forks_repo_name: neurophysik/networktools


text:
import torch
import torchvision
import torchvision.transforms as transforms
import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from torch.utils.data.sampler import SubsetRandomSampler
import matplotlib.pyplot as plt

# Loading datasets and defining loaders.
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)
testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=False, transform=transform)
classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
my_dim = 128 * 5 * 5

indices = list(range(len(trainset)))
split = int(20 / 100 * len(indices))
val_i = np.random.choice(indices, size=split, replace=False)
train_i = list(set(indices) - set(val_i))

train_batch_size = 64
trainloader = torch.utils.data.DataLoader(trainset, batch_size=train_batch_size,
                                          sampler=SubsetRandomSampler(train_i), num_workers=4)
validloader = torch.utils.data.DataLoader(trainset, batch_size=train_batch_size,
                                          sampler=SubsetRandomSampler(val_i), num_workers=4)
test_batch_size = 1
testloader = torch.utils.data.DataLoader(testset, batch_size=test_batch_size, shuffle=False, num_workers=4)

train_size = len(train_i)  # 40000
valid_size = split  # 10000
test_size = len(testloader.dataset)  # 10000
# Defining the cnn.
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        # input channel = 3 = RGB, output channel = 6 = kernels (conv depth), kernel size = 5 = filter size
        self.conv1 = nn.Conv2d(3, 48, 5)  # RGB, depth, filer size
        self.pool = nn.MaxPool2d(2, 2)  # window size, stride
        self.conv2 = nn.Conv2d(48, 128, 5)
        self.fc1 = nn.Linear(my_dim, 120)  # first dim was my dim
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)
        self.bn1 = nn.BatchNorm1d(120)
        self.bn2 = nn.BatchNorm1d(84)
        self.dropout1 = nn.Dropout(p=0.5)
        self.dropout2 = nn.Dropout(p=0.5)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        # output -> 48*28*28
        # maxpooling -> 48 * 14 * 14
        x = self.pool(F.relu(self.conv2(x)))
        # output -> 128 * 10 * 10
        # maxpooling -> 128 * 5 * 5
        x = x.view(-1, my_dim)  # my_dim is a quarter
        x = F.relu(self.dropout1(self.bn1(self.fc1(x))))
        x = F.relu(self.dropout2(self.bn2(self.fc2(x))))
        x = self.fc3(x)
        return x

def train(net, optimizer, criterion):
    print("Training..")
    running_loss = 0.0
    losses1 = []
    correct = 0
    for idx, (data, labels) in enumerate(trainloader):
        inputs, labels = data.cuda(async=True), labels.cuda(async=True)
        optimizer.zero_grad()
        outputs = net(inputs)
        _, predicted = torch.max(outputs.data, 1)
        correct += (predicted == labels).sum()
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        losses1.append(loss.item())
        # print statistics
        running_loss += loss.item()
        if idx % 625 == 624:    # print every 625 mini-batches
            print('[%d, %5d] loss: %.3f' % (epoch + 1, idx + 1, running_loss / 625))
            running_loss = 0.0
    print("hits: {}".format(correct))
    losses1_avg.append(sum(losses1) / len(losses1))

def validation(net, criterion):
    print("Validating..")
    running_loss = 0.0
    losses2 = []
    correct = 0
    for idx, (data, labels) in enumerate(validloader):
        inputs, labels = data.cuda(async=True), labels.cuda(async=True)
        outputs = net(inputs)
        _, predicted = torch.max(outputs.data, 1)
        correct += (predicted == labels).sum()
        loss = criterion(outputs, labels)
        losses2.append(loss.item())
        # print statistics
        running_loss += loss.item()
        if idx % 157 == 156:    # print every 10000 mini-batches
            print('[%d, %5d] loss: %.3f' % (epoch + 1, idx + 1, running_loss / 157))
            running_loss = 0.0
    losses2_avg.append(sum(losses2) / len(losses2))
    print("hits: {}".format(correct))
    if correct > 7500:
        print("a good accuracy!")
        #test(net, criterion)

def test(net, criterion):
    correct = 0
    total = 0
    class_correct = list(0. for i in range(10))
    class_total = list(0. for i in range(10))
    pred_str = ""
    losses = []
    y_pred = []
    y_test = []
    with torch.no_grad():
        for idx, (images, labels) in enumerate(testloader):
            y_test.append(labels.item())
            images, labels = images.cuda(async=True), labels.cuda(async=True)
            outputs = net(images)
            loss = criterion(outputs, labels)
            losses.append(loss.item())
            _, predicted = torch.max(outputs.data, 1)  # there was also 'outputs' alone (no .data)
            y_pred.append(predicted)
            c = (predicted == labels).squeeze()
            for i in range(test_batch_size):  # labels[i], c[i] - not good for 0-dim tensor.
                label = labels.item()
                class_correct[label] += c.item()
                class_total[label] += 1
            total += labels.size(0)
            correct += (predicted == labels).sum().item()
            if pred_str == "":
                pred_str = str(predicted.item())
            else:
                pred_str = pred_str + "\n" + str(predicted.item())
        print("Test Hits: {}".format(correct))
        if correct > 7557:
            print("Saving a better test.pred file! ({} hits)".format(correct))
            my_file = "test{}.pred".format(correct)
            o_file = open(my_file, 'w')
            o_file.write(pred_str)  # writing the test.pred file
            o_file.close()
    print('Test Accuracy: {}%'.format(100 * correct / total))
    for i in range(10):
        print('Test Accuracy of %5s : %2d %%' % (classes[i], 100 * class_correct[i] / class_total[i]))
    print("Test Loss: {}".format(sum(losses) / len(losses)))
    return y_pred, y_test

# The following is all the commands to run the first model. change the epochs from 0 to x to apply.
losses1_avg = []
losses2_avg = []
epochs = 0

net1 = Net()
net1.cuda()

# Defining hyper parameters, optimizations, loss function.
criterion1 = nn.CrossEntropyLoss()
optimizer1 = optim.Adam(net1.parameters(), lr=0.0005)  # momentum=0.9

for epoch in range(epochs):
    net1.train()
    train(net1, optimizer1, criterion1)
    net1.eval()
    validation(net1, criterion1)
    # for testing, uncomment the following (I already have good results).
    #net.eval()
    #test(net1, criterion1)

#------------------------------------------- PART 2 -------------------------------------------#

# Loading datasets and defining loaders.
transform = transforms.Compose([transforms.Resize(224),
                                transforms.ToTensor(),
                                transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)
testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=False, transform=transform)

indices = list(range(len(trainset)))
split = int(20 / 100 * len(indices))
val_i = np.random.choice(indices, size=split, replace=False)
train_i = list(set(indices) - set(val_i))

train_batch_size = 64
trainloader = torch.utils.data.DataLoader(trainset, batch_size=train_batch_size,
                                          sampler=SubsetRandomSampler(train_i), num_workers=4)
validloader = torch.utils.data.DataLoader(trainset, batch_size=train_batch_size,
                                          sampler=SubsetRandomSampler(val_i), num_workers=4)
test_batch_size = 1
testloader = torch.utils.data.DataLoader(testset, batch_size=test_batch_size, shuffle=False)

train_size = len(train_i)  # 40000
valid_size = split  # 10000
test_size = len(testloader.dataset)  # 10000

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model_conv = torchvision.models.resnet18(pretrained=True)
for param in model_conv.parameters():
    param.requires_grad = False

# Parameters of newly constructed modules have requires_grad=True by default

num_ftrs = model_conv.fc.in_features
model_conv.fc = nn.Linear(num_ftrs, 10)
model_conv.cuda()
model_conv = model_conv.to(device)
criterion2 = nn.CrossEntropyLoss()

# Observe that only parameters of final layer are being optimized
optimizer_conv = optim.Adam(model_conv.fc.parameters(), lr=0.001)

losses1_avg = []
losses2_avg = []
epochs = 1

for epoch in range(epochs):

    model_conv.train()
    train(model_conv, optimizer_conv, criterion2)

    model_conv.eval()
    validation(model_conv, criterion2)

model_conv.eval()
y_pred, y_test = test(model_conv, criterion2)

from sklearn.metrics import classification_report, confusion_matrix
import seaborn as sn
import pandas as pd

for ix in range(10):
    print(ix, confusion_matrix(y_test,y_pred)[ix].sum())
cm = confusion_matrix(y_test,y_pred)
print(cm)

# Visualizing of confusion matrix of final model
df_cm = pd.DataFrame(cm, range(10), range(10))
plt.figure(figsize = (10,7))
sn.set(font_scale=1.4)
sn.heatmap(df_cm, annot=True,annot_kws={"size": 12})# font size
plt.show()

# Plotting first model results!
plt.plot(np.arange(1, epochs+1), losses1_avg, 'ro-', label='Train', linewidth=2)
plt.plot(np.arange(1, epochs+1), losses2_avg, 'go-', label='Validation', linewidth=2)
plt.legend()
plt.title("Loss on the Training Set and Validation Set")
plt.show()


text:
import sys
import mosek.array
from mosek import array
#import numpy
import operator
import random
import new

class AbstractMethodError(Exception): pass

def is_basestring(s):
  return isinstance(s,basestring)

class PrintTrace:
  def __enter__(self):
    return self
  def __exit__(self,exc_type,exc_value,tb):
    if (exc_type and exc_value and tb):
      import traceback
      traceback.print_tb(tb)
      print exc_value
def printtrace(f):
  def _(*args):
    with PrintTrace() as _:
      return f(*args)
  return _

if sys.maxint > 0x7fffffff:
  int64 = int
  int32 = int
else:
  int64 = long
  int32 = int

class Tools:

  @staticmethod
  def __arraycopy(src,srcoffset,tgt,tgtoffset,size):
    if len(tgt) < tgtoffset+size or len(src) < srcoffset+size:
      raise IndexError("Copy slice out of range")
    tgt[tgtoffset:tgtoffset+size] = src[srcoffset:srcoffset+size]

  _arraycopy__3II_3III =  __arraycopy
  _arraycopy__3LI_3LII = __arraycopy
  _arraycopy__3_3II_3_3III = __arraycopy
  _arraycopy__3_3FI_3_3FII = __arraycopy
  _arraycopy__3FI_3FII = __arraycopy
  _arraycopy__3IL_3ILL = __arraycopy
  _arraycopy__3LL_3LLL = __arraycopy
  _arraycopy__3FL_3FLL = __arraycopy

  @staticmethod
  #def __arrayclone(a): return array.array(a)
  def __arrayclone(a): return a[:]

  _arraycopy__3F = __arrayclone
  _arraycopy__3I = __arrayclone
  _arraycopy__3L = __arrayclone
  _arraycopy__3S = __arrayclone

  @staticmethod
  def __arraylength(a): return len(a)

  _arraylength__3Omosek_fusion_Variable_2   = __arraylength
  _arraylength__3Omosek_fusion_Constranit_2 = __arraylength
  _arraylength__3Omosek_fusion_Matrix_2     = __arraylength
  _arraylength__3S   = __arraylength
  _arraylength__3I   = __arraylength
  _arraylength__3L   = __arraylength
  _arraylength__3F   = __arraylength
  _arraylength__3_3F = __arraylength
  _arraylength__3_3I = __arraylength
  _uarraylength__3I  = __arraylength
  _uarraylength__3L  = __arraylength
  _uarraylength__3F  = __arraylength
  _uarraylength__3Omosek_fusion_Variable_2   = __arraylength
  _uarraylength__3Omosek_fusion_Constraint_2 = __arraylength
  _uarraylength__3Omosek_fusion_Matrix_2     = __arraylength
  _uarraylength__3_3S = __arraylength
  _uarraylength__3_3F = __arraylength
  _uarraylength__3_3I = __arraylength

  @staticmethod
  def __matrixheight(m): return len(m)
  @staticmethod
  def __matrixwidth(m): return 0 if len(m) == 0 else len(m[0])

  _matrixheight__3_3F = __matrixheight
  _matrixwidth__3_3F  = __matrixwidth
  _matrixheight__3_3Omosek_fusion_Matrix_2 = __matrixheight
  _matrixwidth__3_3Omosek_fusion_Matrix_2  = __matrixwidth

  @staticmethod
  def _range_L (last): return array.arange(last)
  @staticmethod
  def _range_LL(first,last): return array.arange(first,last)
  @staticmethod
  def _range_LLL(first,last,step): return array.arange(first,last,step)

  @staticmethod
  def _range_I (last): return array.arange(last)
  @staticmethod
  def _range_II(first,last): return array.arange(first,last)
  @staticmethod
  def _range_III(first,last,step): return array.arange(first,last,step)

  @staticmethod
  def _range_L (last): return range(last)
  @staticmethod
  def _range_LL(first,last): return range(first,last)
  @staticmethod
  def _range_LLL(first,last,step): return range(first,last,step)

  @staticmethod
  def range(*args): return array.arange(*args)

  @staticmethod
  def _zeros_I (num): return  array.zeros(num,float)
  @staticmethod
  def _zeros_II (dimi,dimj): return  array.zeros((dimi,dimj),float)
  #def zeros (dimi,dimj=None):
  #  if dimj is None: return _zeros_I(dimi)
  #  else: _zeros_II(dimi,dimj)
  @staticmethod
  def _ones_I(num):  return  array.ones(num,float)

  @staticmethod
  def _asint_U(i):   return i
  @staticmethod
  def _asuint_I(i):  return i
  @staticmethod
  def _asint_J(i):   return i
  @staticmethod
  def _asuint_L(i):  return i
  @staticmethod
  def _as64bit_U(i): return long(i)
  @staticmethod
  def _as64bit_I(i): return long(i)
  @staticmethod
  def _as32bit_J(i): return int(i)
  @staticmethod
  def _as32bit_L(i): return int(i)

  @staticmethod
  def _sort__3III(val,first,last):
    tmp = list(val[first:last])
    tmp.sort()
    val[first:last] = tmp

  _sort__3LII = _sort__3III

  __rand = random.Random()
  @staticmethod
  def _randInt_I(max):
    #return random.randint(0,max-1)
    return Tools.__rand.randint(0,max-1)

  @staticmethod
  def _argsort__3L_3I_3LII(perm,val1,val2,first,last):
    p = list(perm[first:last])
    p.sort(lambda lhs,rhs: cmp(val1[lhs],val1[rhs]) or cmp(val2[lhs],val2[rhs]))
    perm[first:last] = p

  @staticmethod
  def _argsort__3L_3I_3I(perm,val1,val2):
    Tools._argsort__3I_3I_3III(perm,val1,val2,0,len(perm))

  @staticmethod
  def _argsort__3L_3I(perm,vals):
    Tools._argsort__3I_3III(perm,vals,0,len(perm))

  @staticmethod
  def _argsort__3L_3III(perm,val,first,last):
    p = list(perm[first:last])
    p.sort(lambda lhs,rhs: cmp(val[lhs],val[rhs]))
    perm[first:last] = p

  @staticmethod
  def _sort__3I_II (vals,first,last):
    tmp = vals[first:last]
    tmp.sort()
    vals[first:last] = tmp

  @staticmethod
  def _makevector_FI(val,num): return array.doublearray([val] * num)
  @staticmethod
  def _makevector_II(val,num): return array.int32array([val] * num)
  @staticmethod
  def _makevector_LI(val,num): return array.int64array([long(val)] * num)
  @staticmethod
  def _repeatrange_III(first,last,num):
    res = array.zeros((last-first)*num,array.int32)
    ra = array.arange(first,last)
    l = last-first
    for i in range(num):
      res[i*l:(i+1)*l] = ra
    return res
  @staticmethod
  def _repeatrange_LLL(first,last,num):
    res = array.zeros((last-first)*num,array.int64)
    ra = array.arange(first,last)
    l = last-first
    for i in range(num):
      res[i*l:(i+1)*l] = ra
    return res

#    public static method vectoradd   :: [double] (var lhs :: [double],var rhs :: [double]);
  @staticmethod
  def _vectoradd__3F_3F (v1,v2):
    return array.array([ v1[i]+v2[i] for i in xrange(len(v1)) ])
  @staticmethod
  def _vectorsub__3F_3F (v1,v2):
    return array.array([ v1[i]-v2[i] for i in xrange(len(v1)) ])
    #return v1-v2
#    public static method vectorsub   :: [double] (var lhs :: [double],var rhs :: [double]);
  @staticmethod
  def _vectorneg__3F(v):
    return array.array([ -i for i in v ])
    #return -v
#    public static method dot         :: double (var lhs :: [double],var rhs :: [double]);
#    public static method sum         :: double (var rhs :: [double]);
  @staticmethod
  def  _stringvalue_I(v): return str(v)
  @staticmethod
  def  _stringvalue_L(v): return str(v)

  @staticmethod
  def _toDouble_S(v):
    return float(v)

  @staticmethod
  def _toInt_S(v):
    return int(v)

#
#
#
#    public static method joinvals    :: string (var vals :: [int32]);
#

#class BaseException(Exception):
#  def _toString_ (self): return str(self)
#
#class BaseError(Exception):
#  def _toString_ (self): return str(self)
#
#class UnexpectedError(Exception):
#  def _toString_ (self): return str(self)

class StringBuffer:
  def __init__(self):
    self.__buf = []
  def __a(self,v):
    self.__buf.append(str(v))
    return self
  def __a_array(self,v):
    if v is None: self.__buf.append("None")
    else: self.__buf.append(str(v))
    return self

  _a_I   = __a
  _a_L   = __a
  _a_U   = __a
  _a_F   = __a
  _a_S   = __a
  _a_B   = __a
  _a__3I = __a_array
  def _a__3L(self,v):
    if v is not None: self.__buf.extend([ '[',','.join(['%i' % i for i in v]),']'])
    else:             self.__buf.append('None')
    return self
  def _a__3F(self,v):
    if v is not None: self.__buf.extend([ '[',','.join(['%g' % i for i in v]),']'])
    else:             self.__buf.append('None')
    return self
  _a__3S = __a_array
  def _lf_(self):
    self.__buf.append('\n')
    return self
  def _clear_ (self):
    self.__buf = []
    return self
  def _toString_(self):
    return ''.join(self.__buf)
  def _toString_(self):
    return ''.join(self.__buf)
  def _consolePrint_(self):
    print(self._toString_())
    self._clear_()
    return(self)

class IntMap:
  def __init__ (self):
    self.__d = {}
  def _hasItem_L(self,key): return self.__d.has_key(key)
  def _getItem_L(self,key): return self.__d[key]
  def _setItem_LI(self,key,val): self.__d[key] = val

class OutputFileStream:
  def __init__(self,filename):
    try:
      self.__outfile = open(filename,'wt')
    except IOError,e:
      raise mosek.fusion.IOError(str(e))
  def _write_S(self,v):
    self.__outfile.write(str(v))
    return self
  def _write_I(self,v):
    self.__outfile.write(str(v))
    return self
  def _write_L(self,v):
    self.__outfile.write(str(v))
    return self
  def _write_F(self,v):
    self.__outfile.write(str(v))
    return self
  def _lf_(self):
    self.__outfile.write('\n')
    self.__outfile.flush()
    return self
  def _close_(self): self.__outfile.close()

class BinaryOutputStream:
  def __init__(self,filename):
    try:
      self.__outfile = open(filename,'bt')
    except IOError,e:
      raise mosek.fusion.IOError(str(e))
  def _write_S(self,v):
    s = v.encode('utf-8')
    self.__outfile.write(struct.pack('<i',len(s)))
    self.__outfile.write(s)
    return self
  def _write_I(self,v):
    self.__outfile.write(struct.pack('<i',v))
    return self
  def _write_L(self,v):
    self.__outfile.write(struct.pack('<q',v))
    return self
  def _write_F(self,v):
    self.__outfile.write(struct.pack('<d',v))
    return self
  def _write_B(self,v):
    self.__outfile.write(struct.pack('<b',v))
    return self
  def _write__3I(self,v):
    self.__outfile.write(struct.pack('<%di' % len(v),v))
    return self
  def _write__3L(self,v):
    self.__outfile.write(struct.pack('<%dq' % len(v),v))
    return self
  def _write__3F(self,v):
    self.__outfile.write(struct.pack('<%dd' % len(v),v))
    return self
  def _write__3B(self,v):
    self.__outfile.write(''.join([ chr(c) for c in v]))
    return self

  def _close_(self): self.__outfile.close()

def new_object_array (size): return [ None ] * size
def new_basestr_array (size): return [None] * size
def new_bool_array (size): return array.zeros(size,bool)
def new_int32_array (size): return array.zeros(size,array.int32)
def new_int64_array (size): return array.zeros(size,array.int64)
def new_double_array (size): return array.zeros(size,array.float64)
def isArray (v):
  return v is None or operator.isSequenceType(v)
  #return v is None or isinstance(v,numpy.ndarray) or isinstance(v,mosek.array.ndarray) or isinstance(v,list)
def is_int(v):
  return isinstance(v,int) or isinstance(v,long)
def is_long(v):
  return isinstance(v,int) or isinstance(v,long)
def is_float(v):
  return isinstance(v,float)

# Argument checking methods
def methodargs(*convs):
  def _transform(func):
    def _conv(*args):
      try:
        if len(args) != len(convs):
          raise ValueError("Expected %d arguments, got %d" % (len(conv),len(args)))
        return func(*[ c(a) for (a,c) in zip(args,convs) ])
      except Exception, e:
        raise
        #raise ValueError('%s:%d: %s' % (func.func_code.co_filename,func.func_code.co_firstlineno,e))
    return _conv
  return _transform

def arg_id(a): return a
def arg_array(ndim,t):
  def _conv(a):
    if a is None: return None
    # test depth:
    v = a
    d = 0
    try:
      while not isinstance(v,basestring):
        v = v[0]
        d += 1
    except:
      pass
    if d != ndim:
      raise ValueError("Expected array of %d dimensions, got %d" % (ndim,d))
    if t is arg_id:
      return array.array(a)
    else:
      return array.array(a,t)
  return _conv

def abstractmethod(f):
  def _transform(*args):
    raise AbstractMethodError("Cannot invoke abstract method %s" % (f.__name__))

arg_bool   = bool
arg_int32  = array.int32
arg_int64  = array.int64
arg_double = array.float64
arg_string = unicode

class FusionException(Exception):
  def _toString_ (self):
    return '%s: %s' % (self.__class__.__name__,str(self))
  def toString(self):
    return '%s: %s' % (self.__class__.__name__,str(self))
class Debug:
  @staticmethod
  def _o_(): return Debug()
  def p(self,v):
    sys.stdout.write(str(v))
    return self
  _p_S = p
  _p_I = p
  _p_L = p
  _p_F = p
  _p_B = p

  def p_array (self,v):
    if v is None:
        sys.stdout.write('None')
    else:
        sys.stdout.write('[ %s ]' % ','.join([str(i) for i in v]))
    return self

  _p__3I = p_array
  _p__3L = p_array
  _p__3F = p_array

  def _lf_(self):
    sys.stdout.write("\n")
    return self

def newobject(cls,init,*args):
    tmp = new.instance(cls)
    init(tmp,*args)
    return tmp


text:
from keras.preprocessing.image import img_to_array
from keras.models import load_model
import numpy as np
import argparse
import imutils
import cv2

ap = argparse.ArgumentParser()
ap.add_argument("-m1", "--model1", required=True, help="path to model1")
ap.add_argument("-m2", "--model2", required=True, help="path to model2")
ap.add_argument("-i", "--image", required=True, help="path to image")
args = vars(ap.parse_args())

image = cv2.imread(args["image"])
orig = image.copy()

image = cv2.resize(image, (64, 64))
image = image.astype("float") / 255.0
image = img_to_array(image)
image = np.expand_dims(image, axis=0)

model1 = load_model(args["model1"])
model2 = load_model(args["model2"])
print("models loaded")

(other, xray) = model1.predict(image)[0]
label2 = "Xray" if xray > other else "Other"
proba = "Xray" if xray > other else other
label = "{}: {:.2f}%".format(label2, proba * 100)

if label2 == "Xray":

    (infected, healthy) = model2.predict(image)[0]
    label2 = "Healthy" if healthy > infected else "Infected"
    proba = "Healthy" if healthy > infected else "Infected"
    label = "{}: {:.2f}%".format(label2, proba * 100)
    output = imutils.resize(orig, width=400)
    cv2.putText(output, label, (10, 25), cv2.FONT_HERSHEY_SIMPLEX,  0.7, (0, 255, 0), 2)
    cv2.imshow("Output", output)
    cv2.waitKey(0)
else:

    output = imutils.resize(orig, width=400)
    cv2.putText(output, label, (10, 25), cv2.FONT_HERSHEY_SIMPLEX,  0.7, (0, 255, 0), 2)
    cv2.imshow("Output", output)
    cv2.waitKey(0)

max_stars_repo_name: ChirilaLaura/covid-z

max_issues_repo_name: ChirilaLaura/covid-z

max_forks_repo_name: ChirilaLaura/covid-z


text:
# !/usr/bin/env python
#
#
# Author :
# Date: May 19 2018
# About:
from unittest.mock import inplace

'''

https://www.dataquest.io/blog/k-nearest-neighbors-in-python/
https://stackoverflow.com/questions/29530232/python-pandas-check-if-any-value-is-nan-in-dataframe
https://stackoverflow.com/questions/40393663/how-do-i-define-a-dataframe-in-python
https://stackoverflow.com/questions/18689823/pandas-dataframe-replace-nan-values-with-average-of-columns
https://stackoverflow.com/questions/31323499/sklearn-error-valueerror-input-contains-nan-infinity-or-a-value-too-large-for

Tact-id:836

'''

import pandas as pd
import math
import sys
import numpy as np
from scipy.spatial import distance

"""
    This method will find the euclidean distance
"""
'''
def euclidean_distance(row):
    """
    A simple euclidean distance function
    """

    inner_value = 0
    for k in columns:
        inner_value += (row[k] - selected_player[k]) ** 2
    return math.sqrt(inner_value)
'''

"""
    This method will find the top similar developers
    max: range

"""
def find_top_similar_entitties(max, nba, distance_frame, primary_column):

    for i in range(max):

        current_farthest = distance_frame.iloc[i]["idx"]
        #print('closest player index: '+str(int(current_farthest)))
        close_to_the_top_founder = nba.loc[int(current_farthest)][primary_column]

        current_distance = distance_frame.iloc[i]["dist"]
        percentile = 100 - (100 / 18.9714833602) * current_distance

        current_distance = round(current_distance, 2)

        if percentile <0:
            percentile = 0

        percentile = round(percentile, 2)

        print('similar '+str(i)+' : '+str(close_to_the_top_founder) + ' - distance : '+str(current_distance) + ", Percentile : "+ (str(percentile)))

"""
    This method will find the similar developers using KNN
"""
def find_similar_developers(filepath, columns, primary_column, given_entity_name):

    with open(filepath, 'r') as csvfile:
        dev_df = pd.read_csv(csvfile)

    #print the column name
    #print(dev_df.columns.values)

    # apply mean on NaN vlaues
    dev_df.fillna(round(dev_df.mean()), inplace=True)

    #print(dev_df)

    # remove duplicate index (https://stackoverflow.com/questions/27236275/what-does-valueerror-cannot-reindex-from-a-duplicate-axis-mean)
    #dev_df = dev_df[~dev_df.index.duplicated()]

    selected_player = dev_df[dev_df[primary_column] == given_entity_name].iloc[0]

    nba_numeric = dev_df[columns] #it select the only numeric column
    #print(nba_numeric)

    #print('mean : \n'+str(nba_numeric.mean()))

    #abc = nba_numeric - nba_numeric.mean()
    #return

    # the normalization calculation
    nba_normalized = (nba_numeric - nba_numeric.mean()) / nba_numeric.std()
    #print(nba_normalized) #print the value

    # Not sure whether it should be mean or zero (need to verify with ore sources)
    #nba_normalized.fillna(0, inplace=True)
    nba_normalized.fillna(round(nba_normalized.mean()), inplace=True)

    top_founder_normalized = nba_normalized[dev_df[primary_column] == given_entity_name]
    #print(top_founder_normalized)

    euclidean_distances = nba_normalized.apply(lambda row: distance.euclidean(row, top_founder_normalized), axis=1)
    #print(euclidean_distances)
    #return

    distance_frame = pd.DataFrame(data={"dist": euclidean_distances, "idx": euclidean_distances.index})

    distance_frame.sort_values(by=["dist"], inplace=True)

    second_smallest = distance_frame.iloc[1]["idx"]
    most_nearer_entity = dev_df.loc[int(second_smallest)][primary_column]

    print('Direct similarity : '+most_nearer_entity)

    print('Top 10 Similar developer to '+given_entity_name)
    find_top_similar_entitties(10, dev_df, distance_frame, primary_column)

    print('\nTop 5 developers Sorted')
    find_top_similar_entitties(5, dev_df, distance_frame, primary_column)

def test_dummy():
    filepath = "developer_score_5_criteria.csv"

    columns = [
        'LinkedIn content',
        'Public coding activities',
        'Github Analysis',
        'Stackoverflow Analysis',
        'Tech Keys involved'
    ]

    primary_column = "Name"
    top_developer_name = 'https://www.linkedin.com/in/geekopedia/'

    find_similar_developers(filepath, columns, primary_column, top_developer_name)

if __name__ == '__main__':
    test_dummy()

'''
    Variables:
        LinkedIn content
        Public coding activities
        Github Analysis
        Stackoverflow Analysis
        Tech Keys involved

'''

max_stars_repo_name: teamtact/tact-developer-similarity

max_issues_repo_name: teamtact/tact-developer-similarity

max_forks_repo_name: teamtact/tact-developer-similarity

reason: import numpy,from scipy


text:
from seaice import *
from firedrake import (
    PeriodicSquareMesh,
    SpatialCoordinate,
    as_vector,
    pi,
    SquareMesh,
    sin,
    as_matrix,
)
import numpy as np
import pytest

@pytest.mark.parametrize(
    "state, norm_type, theta, family",
    [
        (a, b, c, d)
        for a in [True, False]
        for b in ["L2", "H1"]
        for c in [0, 0.5, 1]
        for d in ["CR", "CG"]
    ],
)
def test_evp_convergence(state, norm_type, theta, family):
    timestep = 1
    dumpfreq = 10 ** 6
    timescale = 2
    number_of_triangles = [5, 10, 20, 40, 80]
    length = 5 * 10 ** 5
    pi_x = pi / length

    zero = Constant(0)
    zero_vector = Constant(as_vector([0, 0]))

    dirname = "./output/test-output/test.pvd"
    plot_dirname = "./output/test-output/srt-conv.png"

    error_values = []

    timestepping = TimesteppingParameters(timescale=timescale, timestep=timestep)
    output = OutputParameters(dirname=dirname, dumpfreq=dumpfreq)
    solver = SolverParameters()
    params = SeaIceParameters(rho_a=zero, C_a=zero, cor=zero)

    for values in number_of_triangles:
        mesh = SquareMesh(values, values, length)
        x, y = SpatialCoordinate(mesh)

        v_exp = as_vector(
            [-sin(pi_x * x) * sin(pi_x * y), -sin(pi_x * x) * sin(pi_x * y)]
        )
        sigma_exp = as_matrix(
            [
                [-sin(pi_x * x) * sin(pi_x * y), -sin(pi_x * x) * sin(pi_x * y)],
                [-sin(pi_x * x) * sin(pi_x * y), -sin(pi_x * x) * sin(pi_x * y)],
            ]
        )

        ocean_curr = as_vector(
            [0.1 * (2 * y - length) / length, -0.1 * (length - 2 * x) / length]
        )

        ic = {"u": v_exp, "a": 1, "h": 1, "s": sigma_exp}

        stabilised = {"state": state, "alpha": 1}

        conditions = Conditions(
            ic=ic, ocean_curr=ocean_curr, stabilised=stabilised, family=family
        )

        evp = ElasticViscousPlastic(
            mesh=mesh,
            conditions=conditions,
            timestepping=timestepping,
            output=output,
            params=params,
            solver_params=solver,
        )

        u1, s1 = split(evp.w1)
        u0, s0 = split(evp.w0)

        uh = (1 - theta) * u0 + theta * u1
        sh = (1 - theta) * s0 + theta * s1

        eqn = inner(params.rho * evp.h * (u1 - u0), evp.p) * dx
        eqn += timestep * inner(sh, grad(evp.p)) * dx
        eqn -= (
            timestep
            * inner(
                params.rho_w
                * params.C_w
                * sqrt(dot(ocean_curr - uh, ocean_curr - uh))
                * (ocean_curr - uh),
                evp.p,
            )
            * dx
        )

        # source terms in momentum equation=-==
        eqn += timestep * inner(div(sigma_exp), evp.p) * dx
        eqn += (
            timestep
            * inner(
                params.rho_w
                * params.C_w
                * sqrt(dot(ocean_curr - v_exp, ocean_curr - v_exp))
                * (ocean_curr - v_exp),
                evp.p,
            )
            * dx
        )

        zeta_exp = evp.zeta(evp.h, evp.a, evp.delta(v_exp))
        ep_dot_exp = evp.strain(grad(v_exp))
        rheology_exp = params.e ** 2 * sigma_exp + Identity(2) * 0.5 * (
            (1 - params.e ** 2) * tr(sigma_exp) + evp.Ice_Strength(evp.h, evp.a)
        )
        zeta = evp.zeta(evp.h, evp.a, evp.delta(uh))

        eqn += inner(s1 - s0 + 0.5 * timestep * evp.rheology / params.T, evp.q) * dx
        eqn -= inner(evp.q * zeta * timestep / params.T, evp.ep_dot) * dx

        # source terms in rheology
        eqn -= inner(0.5 * timestep * rheology_exp / params.T, evp.q) * dx
        eqn += inner(evp.q * zeta_exp * timestep / params.T, ep_dot_exp) * dx

        evp.assemble(eqn, evp.w1, evp.bcs, solver.srt_params)

        t = 0

        u1, s1 = evp.w1.split()

        evp.dump(u1, s1, t=0)

        while t < timescale - 0.5 * timestep:
            u0, s0 = evp.w0.split()
            evp.solve(evp.usolver)
            evp.update(evp.w0, evp.w1)
            t += timestep
            evp.dump(u1, s1, t=t)
            evp.progress(t)

        error_values.append(Error.compute(u1, v_exp, norm_type))

    h = [sqrt(2) * length / x for x in number_of_triangles]
    error_slope = float(
        format(np.polyfit(np.log(h), np.log(error_values), 1)[0], ".3f")
    )

    assert round(error_slope - 2, 1) == 0

if __name__ == "__main__":
    import sys

    pytest.main(sys.argv)


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import random
import numpy as np
import tensorflow as tf
import tensorflow_hub as hub
import h5py

import util
from nn_md import NNMD

class BiaffineMD(NNMD):

  def add_model_specific_valuables(self, config):
    if config["lm_path"]:
      self.lm_file = h5py.File(self.config["lm_path"], "r")
    else:
      self.lm_file = None
    self.lm_layers = self.config["lm_layers"]
    self.lm_size = self.config["lm_size"]
    self.eval_data = None # Load eval data lazily.

    input_props = []
    input_props.append((tf.string, [None, None])) # Tokens.
    input_props.append((tf.float32, [None, None, self.lm_size, self.lm_layers])) # LM embeddings.
    input_props.append((tf.int32, [None])) # Text lengths.
    input_props.append((tf.bool, [])) # Is training.
    input_props.append((tf.int32, [None])) # Gold starts.
    input_props.append((tf.int32, [None])) # Gold ends.
    return input_props

  def restore(self, session):
    # Don't try to restore unused variables from the TF-Hub ELMo module.
    vars_to_restore = [v for v in tf.global_variables() if "module/" not in v.name]
    saver = tf.train.Saver(vars_to_restore)
    checkpoint_path = os.path.join(self.config["log_dir"], "model.max.ckpt")
    print("Restoring from {}".format(checkpoint_path))
    session.run(tf.global_variables_initializer())
    saver.restore(session, checkpoint_path)

  def load_lm_embeddings(self, doc_key):
    if self.lm_file is None:
      return np.zeros([0, 0, self.lm_size, self.lm_layers])
    file_key = doc_key.replace("/", ":")
    if not file_key in self.lm_file and file_key[:-2] in self.lm_file:
      file_key = file_key[:-2]
    group = self.lm_file[file_key]
    num_sentences = len(list(group.keys()))
    sentences = [group[str(i)][...] for i in range(num_sentences)]
    lm_emb = np.zeros([num_sentences, max(s.shape[0] for s in sentences), self.lm_size, self.lm_layers])
    for i, s in enumerate(sentences):
      lm_emb[i, :s.shape[0], :, :] = s
    return lm_emb

  def tensorize_example(self, example, is_training):
    clusters = example["clusters"]

    gold_mentions = sorted(tuple(m) for m in util.flatten(clusters))

    sentences = example["sentences"]
    num_words = sum(len(s) for s in sentences)
    speakers = util.flatten(example["speakers"])

    assert num_words == len(speakers)

    max_sentence_length = max(len(s) for s in sentences)
    text_len = np.array([len(s) for s in sentences])
    tokens = [[""] * max_sentence_length for _ in sentences]

    for i, sentence in enumerate(sentences):
      for j, word in enumerate(sentence):
        tokens[i][j] = word

    tokens = np.array(tokens)

    doc_key = example["doc_key"]

    gold_starts, gold_ends = self.tensorize_mentions(gold_mentions)

    lm_emb = self.load_lm_embeddings(doc_key)

    example_tensors = (tokens, lm_emb, text_len, is_training, gold_starts, gold_ends)

    if is_training and len(sentences) > self.config["max_training_sentences"]:
      return self.truncate_example(*example_tensors)
    else:
      return example_tensors

  def truncate_example(self, tokens,  lm_emb, text_len, is_training, gold_starts, gold_ends):
    max_training_sentences = self.config["max_training_sentences"]
    num_sentences = tokens.shape[0]
    assert num_sentences > max_training_sentences

    sentence_offset = random.randint(0, num_sentences - max_training_sentences)
    word_offset = text_len[:sentence_offset].sum()
    num_words = text_len[sentence_offset:sentence_offset + max_training_sentences].sum()
    tokens = tokens[sentence_offset:sentence_offset + max_training_sentences, :]
    lm_emb = lm_emb[sentence_offset:sentence_offset + max_training_sentences, :, :, :]
    text_len = text_len[sentence_offset:sentence_offset + max_training_sentences]

    gold_spans = np.logical_and(gold_ends >= word_offset, gold_starts < word_offset + num_words)
    gold_starts = gold_starts[gold_spans] - word_offset
    gold_ends = gold_ends[gold_spans] - word_offset

    return tokens, lm_emb, text_len, is_training, gold_starts, gold_ends

  def get_predictions_and_loss(self, inputs):
    tokens, lm_emb, text_len, is_training, gold_starts, gold_ends = inputs
    self.dropout = self.get_dropout(self.config["dropout_rate"], is_training)
    self.lexical_dropout = self.get_dropout(self.config["lexical_dropout_rate"], is_training)
    self.lstm_dropout = self.get_dropout(self.config["lstm_dropout_rate"], is_training)

    num_sentences = tf.shape(tokens)[0]
    max_sentence_length = tf.shape(tokens)[1]

    if not self.lm_file:
      elmo_module = hub.Module("https://tfhub.dev/google/elmo/2")
      lm_embeddings = elmo_module(
          inputs={"tokens": tokens, "sequence_len": text_len},
          signature="tokens", as_dict=True)
      word_emb = lm_embeddings["word_emb"]  # [num_sentences, max_sentence_length, 512]
      lm_emb = tf.stack([tf.concat([word_emb, word_emb], -1),
                         lm_embeddings["lstm_outputs1"],
                         lm_embeddings["lstm_outputs2"]], -1)  # [num_sentences, max_sentence_length, 1024, 3]
    lm_emb_size = util.shape(lm_emb, 2)
    lm_num_layers = util.shape(lm_emb, 3)
    with tf.variable_scope("lm_aggregation"):
      self.lm_weights = tf.nn.softmax(tf.get_variable("lm_scores", [lm_num_layers], initializer=tf.constant_initializer(0.0)))
      self.lm_scaling = tf.get_variable("lm_scaling", [], initializer=tf.constant_initializer(1.0))
    flattened_lm_emb = tf.reshape(lm_emb, [num_sentences * max_sentence_length * lm_emb_size, lm_num_layers])
    flattened_aggregated_lm_emb = tf.matmul(flattened_lm_emb, tf.expand_dims(self.lm_weights, 1)) # [num_sentences * max_sentence_length * emb, 1]
    aggregated_lm_emb = tf.reshape(flattened_aggregated_lm_emb, [num_sentences, max_sentence_length, lm_emb_size])
    aggregated_lm_emb *= self.lm_scaling

    context_emb = aggregated_lm_emb

    context_emb = tf.nn.dropout(context_emb, self.lexical_dropout) # [num_sentences, max_sentence_length, emb]

    text_len_mask = tf.sequence_mask(text_len, maxlen=max_sentence_length) # [num_sentence, max_sentence_length]

    num_words = tf.reduce_sum(text_len)
    sentence_indices = tf.tile(tf.expand_dims(tf.range(num_sentences), 1),
                               [1, max_sentence_length])  # [num_sentences, max_sentence_length]
    flattened_sentence_indices = self.flatten_emb_by_sentence(sentence_indices, text_len_mask)  # [num_words]

    candidate_starts = tf.tile(tf.expand_dims(tf.range(num_words), 1),
                               [1, max_sentence_length])  # [num_words, max_sentence_length]
    candidate_ends = candidate_starts + tf.expand_dims(tf.range(max_sentence_length), 0)  # [num_words, max_sentence_length]
    candidate_start_sentence_indices = tf.gather(flattened_sentence_indices,
                                                 candidate_starts)  # [num_words, max_sentence_length]
    candidate_end_sentence_indices = tf.gather(flattened_sentence_indices,
                                               tf.minimum(candidate_ends, num_words - 1))  # [num_words, max_sentence_length]
    candidate_mask = tf.logical_and(candidate_ends < num_words, tf.equal(candidate_start_sentence_indices,
                                                                         candidate_end_sentence_indices))  # [num_words, max_sentence_length]
    flattened_candidate_mask = tf.reshape(candidate_mask, [-1])  # [num_words * max_sentence_length]

    candidate_starts = tf.boolean_mask(tf.reshape(candidate_starts, [-1]), flattened_candidate_mask)  # [num_candidates]
    candidate_ends = tf.boolean_mask(tf.reshape(candidate_ends, [-1]), flattened_candidate_mask)  # [num_candidates]

    candidate_labels = self.get_candidate_labels(candidate_starts, candidate_ends, gold_starts, gold_ends)  # [num_candidates]

    candidate_scores_mask = tf.logical_and(tf.expand_dims(text_len_mask,[1]),tf.expand_dims(text_len_mask,[2])) #[num_sentence, max_sentence_length,max_sentence_length]
    sentence_ends_leq_starts = tf.tile(tf.expand_dims(tf.logical_not(tf.sequence_mask(tf.range(max_sentence_length),max_sentence_length)), 0),[num_sentences,1,1]) #[num_sentence, max_sentence_length,max_sentence_length]
    candidate_scores_mask = tf.logical_and(candidate_scores_mask,sentence_ends_leq_starts)

    flattened_candidate_scores_mask = tf.reshape(candidate_scores_mask,[-1]) #[num_sentence * max_sentence_length * max_sentence_length]

    context_outputs = self.lstm_contextualize(context_emb, text_len, text_len_mask,self.lstm_dropout,False) # [num_sentence, max_sentence_length, emb]

    with tf.variable_scope("candidate_starts_ffnn"):
      candidate_starts_emb = util.projection(context_outputs,self.config["ffnn_size"]) #[num_sentences, max_sentences_length,emb]
    with tf.variable_scope("candidate_ends_ffnn"):
      candidate_ends_emb = util.projection(context_outputs,self.config["ffnn_size"]) #[num_sentences, max_sentences_length, emb]

    candidate_mention_scores = util.bilinear_classifier(candidate_starts_emb,candidate_ends_emb,self.dropout)#[num_sentence, max_sentence_length,max_sentence_length]
    candidate_mention_scores = tf.boolean_mask(tf.reshape(candidate_mention_scores,[-1]),flattened_candidate_scores_mask)

    loss = self.sigmoid_loss(candidate_mention_scores, candidate_labels)
    top_span_starts, top_span_ends = self.get_top_mentions(num_words,candidate_starts,candidate_ends,candidate_mention_scores)

    return [top_span_starts, top_span_ends], loss

max_stars_repo_name: cbiehl/dali-md

max_issues_repo_name: cbiehl/dali-md

max_forks_repo_name: cbiehl/dali-md


text:
import logging
import math

import numpy as np
from sklearn.base import TransformerMixin

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class FFTAligner(TransformerMixin):
    def __init__(self):
        self.best_offset_ = None
        self.best_score_ = None
        self.get_score_ = False

    def fit(self, substring, vidstring, get_score=False):
        substring, vidstring = [
            list(map(int, s))
            if isinstance(s, str) else s
            for s in [substring, vidstring]
        ]
        substring, vidstring = map(
            lambda s: 2 * np.array(s).astype(float) - 1, [substring, vidstring])
        total_bits = math.log(len(substring) + len(vidstring), 2)
        total_length = int(2 ** math.ceil(total_bits))
        extra_zeros = total_length - len(substring) - len(vidstring)
        subft = np.fft.fft(np.append(np.zeros(extra_zeros + len(vidstring)), substring))
        vidft = np.fft.fft(np.flip(np.append(vidstring, np.zeros(len(substring) + extra_zeros)), 0))
        convolve = np.real(np.fft.ifft(subft * vidft))
        best_idx = np.argmax(convolve)
        self.best_offset_ = len(convolve) - 1 - best_idx - len(substring)
        self.best_score_ = convolve[best_idx]
        self.get_score_ = get_score
        return self

    def transform(self, *_):
        if self.get_score_:
            return self.best_score_, self.best_offset_
        else:
            return self.best_offset_

class MaxScoreAligner(TransformerMixin):
    def __init__(self, base_aligner):
        if isinstance(base_aligner, type):
            self.base_aligner = base_aligner()
        else:
            self.base_aligner = base_aligner
        self._scores = []

    def fit(self, substrings, vidstrings):
        logger.info('computing alignments...')
        if not isinstance(substrings, list):
            substrings = [substrings]
        if not isinstance(vidstrings, list):
            vidstrings = [vidstrings]
        for substring in substrings:
            for vidstring in vidstrings:
                self._scores.append(self.base_aligner.fit_transform(
                    substring, vidstring, get_score=True))
        logger.info('...done')
        return self

    def transform(self, *_):
        return max(self._scores)[1]

max_issues_repo_name: xiaopinggai-webrtc/subsync

max_forks_repo_name: xiaopinggai-webrtc/subsync


text:
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 23 15:10:29 2016

@author: suraj
"""

# gensim modules
from gensim import utils
from gensim.models.doc2vec import LabeledSentence
from gensim.models import Doc2Vec

# numpy
import numpy

# random
from random import shuffle

# classifier
from sklearn.linear_model import LogisticRegression

class LabeledLineSentence(object):
    def __init__(self, sources):
        self.sources = sources

        flipped = {}

        # make sure that keys are unique
        for key, value in sources.items():
            if value not in flipped:
                flipped[value] = [key]
            else:
                raise Exception('Non-unique prefix encountered')

    def __iter__(self):
        for source, prefix in self.sources.items():
            with utils.smart_open(source) as fin:
                for item_no, line in enumerate(fin):
                    yield LabeledSentence(utils.to_unicode(line).split(), [prefix + '_%s' % item_no])

    def to_array(self):
        self.sentences = []
        for source, prefix in self.sources.items():
            with utils.smart_open(source) as fin:
                for item_no, line in enumerate(fin):
                    self.sentences.append(LabeledSentence(utils.to_unicode(line).split(), [prefix + '_%s' % item_no]))
        return self.sentences

    def sentences_perm(self):
        shuffle(self.sentences)
        return self.sentences

print 'start'
sources = {'test-neg.txt':'TEST_NEG', 'test-pos.txt':'TEST_POS', 'train-neg.txt':'TRAIN_NEG', 'train-pos.txt':'TRAIN_POS'}

sentences = LabeledLineSentence(sources)

model = Doc2Vec(min_count=1, window=10, size=100, sample=1e-4, negative=5, workers=8)

model.build_vocab(sentences.to_array())

for epoch in range(20):
    model.train(sentences.sentences_perm())

model.save('./imdb.d2v')

print 'DONE'

max_stars_repo_name: suraj-jayakumar/sentiment-analysis

max_issues_repo_name: sjayakum/sentiment-analysis

max_forks_repo_name: sjayakum/sentiment-analysis


text:
# --------------
#Importing header files
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

#Code starts here
#load the data set
data = pd.read_csv(path)

loan_status= data['Loan_Status'].value_counts()
# initialize the figure
plt.figure(figsize=[14,8])

# label the axes
plt.xlabel("loan_status")
plt.ylabel("Loans")

# title the plot
plt.title("Distribution of Loans status across loans")

# build and show the plot
plt.bar(loan_status.index,loan_status)

# --------------
#Code starts here

#Plotting an unstacked bar plot
property_and_loan=data.groupby(['Property_Area', 'Loan_Status'])
property_and_loan=property_and_loan.size().unstack()
property_and_loan.plot(kind='bar', stacked=False, figsize=(15,10))

#Changing the x-axis label
plt.xlabel('Property_Area')

#Changing the y-axis label
plt.ylabel('Loan_Status')

#Rotating the ticks of X-axis
plt.xticks(rotation=45)

#Code ends here

# --------------
#Code starts here

education_and_loan = data.groupby(['Education','Loan_Status']).size().unstack()
education_and_loan.plot(kind = 'bar',stacked = True, figsize=(10,15))

plt.xlabel('Education Status')
plt.ylabel('Loan Status')

plt.xticks(rotation = 45)

# --------------
#Code starts here
graduate= data[data['Education'] == 'Graduate']

not_graduate = data[data['Education'] == 'Not Graduate']

graduate['LoanAmount'].plot(kind = 'density',label = 'Graduate')

not_graduate['LoanAmount'].plot(kind = 'density',label = 'Not Graduate')

#Code ends here

#For automatic legend display
plt.legend()

# --------------
#Code starts here

# initialize figure and axis
fig,(ax_1,ax_2,ax_3) = plt.subplots(1,3, figsize=(20,20))

#create scatter plot for ApplicantIncome and LoanAmount
ax_1.scatter(data['ApplicantIncome'],data["LoanAmount"])
ax_1.set_title('Applicant Income')

#create and plot scatter plot for CoapplicantIncome and LoanAmount
ax_2.scatter(data['CoapplicantIncome'],data["LoanAmount"])
ax_2.set_title('Coapplicant Income')
#create new column 'Total Income'
data['TotalIncome']= data['ApplicantIncome']+data['CoapplicantIncome']
#scater plot between TotalIncome and LoanAmount
ax_3.scatter(data["TotalIncome"],data["LoanAmount"])
ax_3.set_title('Total Income')

max_stars_repo_name: Abasshiekh/ga-learner-dsb-repo

max_issues_repo_name: Abasshiekh/ga-learner-dsb-repo

max_forks_repo_name: Abasshiekh/ga-learner-dsb-repo


text:
# This is a sample Python program that trains a simple CatBoost model using SageMaker scikit-learn Docker image, and then performs inference.
# This implementation will work on your *local computer* or in the *AWS Cloud*.
#
# Prerequisites:
#   1. Install required Python packages:
#      `pip install -r requirements.txt`
#   2. Docker Desktop installed and running on your computer:
#      `docker ps`
#   3. You should have AWS credentials configured on your local machine
#      in order to be able to pull the docker image from ECR.
###############################################################################################

import os

import pandas as pd
from sagemaker.predictor import csv_serializer
from sagemaker.sklearn import SKLearn
from sklearn.datasets import load_boston
from sklearn.model_selection import train_test_split

DUMMY_IAM_ROLE = 'arn:aws:iam::111111111111:role/service-role/AmazonSageMaker-ExecutionRole-20200101T000001'

local_train = './data/train/boston_train.csv'
local_validation = './data/validation/boston_validation.csv'
local_test = './data/test/boston_test.csv'

def download_training_and_eval_data():
    if os.path.isfile('./data/train/boston_train.csv') and \
            os.path.isfile('./data/validation/boston_validation.csv') and \
            os.path.isfile('./data/test/boston_test.csv'):
        print('Training dataset exist. Skipping Download')
    else:
        print('Downloading training dataset')

        os.makedirs("./data", exist_ok=True)
        os.makedirs("./data/train", exist_ok=True)
        os.makedirs("./data/validation", exist_ok=True)
        os.makedirs("./data/test", exist_ok=True)

        data = load_boston()

        X_train, X_test, y_train, y_test = train_test_split(data.data, data.target, test_size=0.25, random_state=45)
        X_val, X_test, y_val, y_test = train_test_split(X_test, y_test, test_size=0.5, random_state=45)

        trainX = pd.DataFrame(X_train, columns=data.feature_names)
        trainX['target'] = y_train

        valX = pd.DataFrame(X_test, columns=data.feature_names)
        valX['target'] = y_test

        testX = pd.DataFrame(X_test, columns=data.feature_names)

        trainX.to_csv(local_train, header=None, index=False)
        valX.to_csv(local_validation, header=None, index=False)
        testX.to_csv(local_test, header=None, index=False)

        print('Downloading completed')

def main():
    download_training_and_eval_data()

    print('Starting model training.')
    print('Note: if launching for the first time in local mode, container image download might take a few minutes to complete.')

    sklearn = SKLearn(
        entry_point="catboost_train_deploy.py",
        source_dir='code',
        framework_version="0.23-1",
        instance_type="local",
        role=DUMMY_IAM_ROLE,
    )

    train_location = 'file://' + local_train
    validation_location = 'file://' + local_validation

    sklearn.fit({'train': train_location, 'validation': validation_location})
    print('Completed model training')

    print('Deploying endpoint in local mode')
    predictor = sklearn.deploy(1, 'local', serializer=csv_serializer)

    with open(local_test, 'r') as f:
        payload = f.read().strip()

    predictions = predictor.predict(payload)
    print('predictions: {}'.format(predictions))

    predictor.delete_endpoint(predictor.endpoint)

if __name__ == "__main__":
    main()

max_stars_repo_name: aws-samples/amazon-sagemaker-local-mode

max_issues_repo_name: aws-samples/amazon-sagemaker-local-mode

max_forks_repo_name: aws-samples/amazon-sagemaker-local-mode

reason: from sage


text:
#-*- coding:utf-8 -*-
#!\usr\bin\env py3

import os
import re
import sys
import numpy as np
import pdb
"""
Program: walk_files.py
Function: walk all files to check their modified date
Author: Junjie Pan (junjie.pan@nuance.com)
"""

class WalkThrough(object):
    '''
    '''
    def __init__(self, options, logger):
        self.logger = logger
        self.options = options
        self.count = 0
        if np.array(options.values()).all() == "":
                self.usage()
                exit(-1)
        self.fo = open(self.options['output'], 'w', encoding='utf-8')
        self.fo.write("===================================================\nCreated Time: %s -> %s\n===================================================n"%(self.options["start"], self.options["end"]))

    def process(self, root):
        format = "%Y-%m-%d %H:%M:%S"
        th_time = time.mktime(time.strptime(self.options["start"], format))
        ed_time = time.mktime(time.strptime(self.options["end"], format))

        for parent, dirnames, filenames in os.walk(root):
            for dirname in dirnames:
                if dirname == "..":
                    continue
                self.process(os.path.join(parent, dirname))
            for filename in filenames:
                mtime = os.path.getmtime(os.path.join(parent, filename))
                if mtime >= th_time and mtime <= ed_time:
                    self.count += 1
                    self.logger.info("[%d] %s"%(self.count, os.path.join(os.path.abspath(parent), filename)))
                    self.fo.write("[%d] %s\n"%(self.count, os.path.join(os.path.abspath(parent), filename)))

    def usage(self):
        self.logger.error('''Usage Example:\nwalk_files.py -s "2017-1-1 12:12:12" -e "2018-1-1 12:12:12" -r root_dir''')

if __name__ == '__main__':
    import time
    import logging
    from argparse import ArgumentParser

    parser = ArgumentParser(description='walk_files')
    parser.add_argument("--version", action="version", version="walk_files 1.0")
    parser.add_argument("-s", "--start", action="store", dest="start", default="", help='modified time.')
    parser.add_argument("-e", "--end", action="store", dest="end", default="", help='modified time.')
    parser.add_argument("-r", "--root", action="store", dest="root", default=r"", help='root path.')
    parser.add_argument("-o", "--output", action="store", dest="output", default=r"output.txt", help='root path.')

    args = parser.parse_args()
    options = vars(args)

    logger = logging.getLogger()
    formatter = logging.Formatter('[%(asctime)s][*%(levelname)s*] - %(message)s', '%Y%m%d-%H:%M:%S')
    file_handler = logging.FileHandler('LOG-walk_files.txt', 'w','utf-8')
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)

    stream_handler = logging.StreamHandler()
    stream_handler.setFormatter(formatter)
    logger.addHandler(stream_handler)
    logger.setLevel(logging.INFO)

    allStartTP = time.time()
    appInst = WalkThrough(options, logger)
    appInst.process(options["root"])
    allEndTP = time.time()
    logger.info("Operation Finished [Time Cost:%0.3f Seconds]" % float(allEndTP - allStartTP))

max_stars_repo_name: JeffpanUK/FileWalker

max_issues_repo_name: JeffpanUK/FileWalker

max_forks_repo_name: JeffpanUK/FileWalker


text:
import emc2
import numpy as np

def test_mie_file():
    KAZR = emc2.core.instruments.KAZR('nsa')
    assert "cl" in KAZR.mie_table.keys()
    assert "alpha_p" in KAZR.mie_table["cl"].variables.keys()
    assert KAZR.mie_table["cl"]["alpha_p"][0] == 1.463017e-16
    assert np.all(KAZR.mie_table["cl"]["wavelength"] == 8.6e3)

def test_to_netcdf():
    model = emc2.core.model.TestAllStratiform()
    KAZR = emc2.core.instruments.KAZR('nsa')
    model = emc2.simulator.main.make_simulated_data(model, KAZR, 8)
    model.subcolumns_to_netcdf('test.nc')
    model.ds.close()

    model = emc2.core.model.TestModel()
    model.load_subcolumns_from_netcdf('test.nc')
    assert "sub_col_Ze_cl_strat" in [x for x in model.ds.variables.keys()]
    assert "conv_frac_subcolumns_pi" in [x for x in model.ds.variables.keys()]
    model.ds.close()