text:
import coloredlogs
import logging
from tqdm import tqdm
import torch
import torchtext
import pdb
# from allennlp.modules import Elmo
# from allennlp.modules.elmo import batch_to_ids
from scipy.spatial.distance import cdist
from sklearn import metrics
import numpy as np
from transformers import BertTokenizer, BertModel
from models.base_models import BERTSequenceModel
from seqeval.metrics import f1_score, precision_score, recall_score, accuracy_score

logger = logging.getLogger('NearestNeighbor Log')
coloredlogs.install(logger=logger, level='DEBUG', fmt='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

class NearestNeighborClassifier():
    def __init__(self, config):
        self.vectors = config.get('vectors', 'elmo')
        self.device = torch.device(config.get('device', 'cpu'))

        if self.vectors == 'elmo':
            self.elmo = Elmo(options_file="https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_options.json",
                             weight_file="https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway_5.5B/elmo_2x4096_512_2048cnn_2xhighway_5.5B_weights.hdf5",
                             num_output_representations=1,
                             dropout=0,
                             requires_grad=False)
            self.elmo.to(self.device)
        elif self.vectors == 'glove':
            self.glove = torchtext.vocab.GloVe(name='840B', dim=300)
        elif self.vectors == 'bert':
            self.bert_tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
            self.bert = BERTSequenceModel(config['learner_params'])
            self.bert.to(self.device)

        logger.info('Nearest neighbor classifier instantiated')

    def vectorize(self, batch_x, batch_len, batch_y):
        with torch.no_grad():
            if self.vectors == 'elmo':
                char_ids = batch_to_ids(batch_x)
                char_ids = char_ids.to(self.device)
                batch_x = self.elmo(char_ids)['elmo_representations'][0]
            elif self.vectors == 'glove':
                max_batch_len = max(batch_len)
                vec_batch_x = torch.ones((len(batch_x), max_batch_len, 300))
                for i, sent in enumerate(batch_x):
                    sent_emb = self.glove.get_vecs_by_tokens(sent, lower_case_backup=True)
                    vec_batch_x[i, :len(sent_emb)] = sent_emb
                batch_x = vec_batch_x.to(self.device)
            elif self.vectors == 'bert':
                max_batch_len = max(batch_len) + 2
                input_ids = torch.zeros((len(batch_x), max_batch_len)).long()
                for i, sent in enumerate(batch_x):
                    sent_token_ids = self.bert_tokenizer.encode(sent, add_special_tokens=True)
                    input_ids[i, :len(sent_token_ids)] = torch.tensor(sent_token_ids)
                batch_x = input_ids.to(self.device)
                batch_x = self.bert(batch_x, max_batch_len)
        batch_len = torch.tensor(batch_len).to(self.device)
        batch_y = torch.tensor(batch_y).to(self.device)
        return batch_x, batch_len, batch_y

    def training(self, train_episodes, val_episodes):
        return 0

    def testing(self, test_episodes, label_map):
        self.bert.load_state_dict(torch.load('MetaLearningForNER/saved_models/SupervisedLearner-stable.h5'))

        map_to_label = {v:k for k,v in label_map.items()}

#         episode_accuracies, episode_precisions, episode_recalls, episode_f1s = [], [], [], []
        all_true_labels = []
        all_predictions = []
        for episode_id, episode in enumerate(tqdm(test_episodes)):
            batch_x, batch_len, batch_y = next(iter(episode.support_loader))
            support_repr, _, support_labels = self.vectorize(batch_x, batch_len, batch_y)
            support_repr = support_repr.reshape(support_repr.shape[0] * support_repr.shape[1], -1)
            support_labels = support_labels.view(-1)
            support_repr = support_repr[support_labels != -1].cpu().numpy()
            support_labels = support_labels[support_labels != -1].cpu().numpy()

            batch_x, batch_len, batch_y = next(iter(episode.query_loader))
            query_repr, _, true_labels = self.vectorize(batch_x, batch_len, batch_y)
            query_bs, query_seqlen = query_repr.shape[0], query_repr.shape[1]
            query_repr = query_repr.reshape(query_repr.shape[0] * query_repr.shape[1], -1)
            true_labels = true_labels.view(-1)
            # query_repr = query_repr[true_labels != -1].cpu().numpy()
            query_repr = query_repr.cpu().numpy()
            # true_labels = true_labels[true_labels != -1].cpu().numpy()
            true_labels = true_labels.cpu().numpy()

            dist = cdist(query_repr, support_repr, metric='cosine')
            nearest_neighbor = np.argmin(dist, axis=1)
            predictions = support_labels[nearest_neighbor]

            true_labels = true_labels.reshape(query_bs, query_seqlen)
            predictions = predictions.reshape(query_bs, query_seqlen)

            seq_true_labels, seq_predictions = [], []
            for i in range(len(true_labels)):
                true_i = true_labels[i]
                pred_i = predictions[i]
                seq_predictions.append([map_to_label[val] for val in pred_i[true_i != -1]])
                seq_true_labels.append([map_to_label[val] for val in true_i[true_i != -1]])

            all_predictions.extend(seq_predictions)
            all_true_labels.extend(seq_true_labels)

            accuracy = accuracy_score(seq_true_labels, seq_predictions)
            precision = precision_score(seq_true_labels, seq_predictions)
            recall = recall_score(seq_true_labels, seq_predictions)
            f1 = f1_score(seq_true_labels, seq_predictions)
            # logger.info('Episode {}/{}, task {} [query set]: Accuracy = {:.5f}, precision = {:.5f}, '
            #             'recall = {:.5f}, F1 score = {:.5f}'.format(episode_id + 1, len(test_episodes), episode.task_id,
            #                                                         accuracy, precision, recall, f1))

#             episode_accuracies.append(accuracy)
#             episode_precisions.append(precision)
#             episode_recalls.append(recall)
#             episode_f1s.append(f1_score)

        accuracy = accuracy_score(all_true_labels, all_predictions)
        precision = precision_score(all_true_labels, all_predictions)
        recall = recall_score(all_true_labels, all_predictions)
        f1 = f1_score(all_true_labels, all_predictions)
        logger.info('Avg meta-testing metrics: Accuracy = {:.5f}, precision = {:.5f}, recall = {:.5f}, '
                    'F1 score = {:.5f}'.format(accuracy,precision, recall, f1))
        return f1

max_stars_repo_name: saiajaym/MetaLearningForNER

max_issues_repo_name: saiajaym/MetaLearningForNER

max_forks_repo_name: saiajaym/MetaLearningForNER

reason: import numpy,from scipy


text:
# Copyright (c) OpenMMLab. All rights reserved.
import os.path as osp
from tempfile import NamedTemporaryFile

import mmcv
import numpy as np
import pytest
import torch

import mmdeploy.backend.onnxruntime as ort_apis
from mmdeploy.codebase import import_codebase
from mmdeploy.utils import Backend, Codebase
from mmdeploy.utils.test import SwitchBackendWrapper, backend_checker

IMAGE_H = 192
IMAGE_W = 256

try:
    import_codebase(Codebase.MMPOSE)
except ImportError:
    pytest.skip(
        f'{Codebase.MMPOSE} is not installed.', allow_module_level=True)

@backend_checker(Backend.ONNXRUNTIME)
class TestEnd2EndModel:

    @classmethod
    def setup_class(cls):
        # force add backend wrapper regardless of plugins
        from mmdeploy.backend.onnxruntime import ORTWrapper
        ort_apis.__dict__.update({'ORTWrapper': ORTWrapper})

        # simplify backend inference
        cls.wrapper = SwitchBackendWrapper(ORTWrapper)
        cls.outputs = {
            'outputs': torch.rand(1, 1, IMAGE_H, IMAGE_W),
        }
        cls.wrapper.set(outputs=cls.outputs)
        deploy_cfg = mmcv.Config(
            {'onnx_config': {
                'output_names': ['outputs']
            }})

        from mmdeploy.utils import load_config
        model_cfg_path = 'tests/test_codebase/test_mmpose/data/model.py'
        model_cfg = load_config(model_cfg_path)[0]
        from mmdeploy.codebase.mmpose.deploy.pose_detection_model import \
            End2EndModel
        cls.end2end_model = End2EndModel(
            Backend.ONNXRUNTIME, [''],
            device='cpu',
            deploy_cfg=deploy_cfg,
            model_cfg=model_cfg)

    @classmethod
    def teardown_class(cls):
        cls.wrapper.recover()

    def test_forward(self):
        img = torch.rand(1, 3, IMAGE_H, IMAGE_W)
        img_metas = [{
            'image_file':
            'tests/test_codebase/test_mmpose' + '/data/imgs/dataset/blank.jpg',
            'center': torch.tensor([0.5, 0.5]),
            'scale': 1.,
            'location': torch.tensor([0.5, 0.5]),
            'bbox_score': 0.5
        }]
        results = self.end2end_model.forward(img, img_metas)
        assert results is not None, 'failed to get output using '\
            'End2EndModel'

    def test_forward_test(self):
        imgs = torch.rand(2, 3, IMAGE_H, IMAGE_W)
        results = self.end2end_model.forward_test(imgs)
        assert isinstance(results[0], np.ndarray)

    def test_show_result(self):
        input_img = np.zeros([IMAGE_H, IMAGE_W, 3])
        img_path = NamedTemporaryFile(suffix='.jpg').name

        pred_bbox = torch.rand(1, 5)
        pred_keypoint = torch.rand((1, 10, 2))
        result = [{'bbox': pred_bbox, 'keypoints': pred_keypoint}]
        self.end2end_model.show_result(
            input_img, result, '', show=False, out_file=img_path)
        assert osp.exists(img_path), 'Fails to create drawn image.'

@backend_checker(Backend.ONNXRUNTIME)
def test_build_pose_detection_model():
    from mmdeploy.utils import load_config
    model_cfg_path = 'tests/test_codebase/test_mmpose/data/model.py'
    model_cfg = load_config(model_cfg_path)[0]
    deploy_cfg = mmcv.Config(
        dict(
            backend_config=dict(type=Backend.ONNXRUNTIME.value),
            onnx_config=dict(output_names=['outputs']),
            codebase_config=dict(type=Codebase.MMPOSE.value)))

    from mmdeploy.backend.onnxruntime import ORTWrapper
    ort_apis.__dict__.update({'ORTWrapper': ORTWrapper})

    # simplify backend inference
    with SwitchBackendWrapper(ORTWrapper) as wrapper:
        wrapper.set(model_cfg=model_cfg, deploy_cfg=deploy_cfg)
        from mmdeploy.codebase.mmpose.deploy.pose_detection_model import (
            End2EndModel, build_pose_detection_model)
        posedetector = build_pose_detection_model([''], model_cfg, deploy_cfg,
                                                  'cpu')
        assert isinstance(posedetector, End2EndModel)

ext: py

max_stars_repo_name: xizi/mmdeploy

max_issues_repo_name: xizi/mmdeploy

max_forks_repo_name: xizi/mmdeploy

reason: import numpy


text:
"""
This script evaluates a embedding model in a semantic similarity perspective.
It uses the dataset of ASSIN sentence similarity shared task and the method
of Hartmann which achieved the best results in the competition.
ASSIN shared-task website:
http://propor2016.di.fc.ul.pt/?page_id=381
Paper of Hartmann can be found at:
http://www.linguamatica.com/index.php/linguamatica/article/download/v8n2-6/365
"""

from gensim.models import FastText
from sklearn.linear_model import LinearRegression
from commons import read_xml
from assin_eval import eval_similarity
from assin_eval import eval_rte
from gensim.models import KeyedVectors
from xml.dom import minidom
from numpy import array
from os import path
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
import pickle
import argparse
import unicodedata

DATA_DIR = './data/'
TEST_DIR = path.join(DATA_DIR, 'assin-test-gold/')

def gensim_embedding_difference(data): #, field1, field2
    """Calculate the similarity between the sum of all embeddings."""
    distances = []
    for pair in data:
        #print("t: ", normalize_terms(pair.t.lower()))
        e1 = [i if i in embeddings else 'teste' for i in normalize_terms(pair.t.lower())]
        e2 = [i if i in embeddings else 'teste' for i in normalize_terms(pair.h.lower())]
        distances.append([embeddings.n_similarity(e1, e2)])
    return distances

def evaluate_testset(x, y, test):
    """Docstring."""
    l_reg = LinearRegression()
    l_reg.fit(x, y)
    test_predict = l_reg.predict(test)
    return test_predict

def write_xml(filename, pred):
    """Docstring."""
    with open(filename) as fp:
        xml = minidom.parse(fp)
    pairs = xml.getElementsByTagName('pair')
    for pair in pairs:
        #print('pred: ', pred)
        #print('pairs.index: ', pairs.index(pair))
        #print('pair: ', str(pred[pairs.index(pair)]))
        pair.setAttribute('similarity', str(pred[pairs.index(pair)]))
    with open(filename, 'w') as fp:
        fp.write(xml.toxml())

def normalize_terms(terms):
    # Remove Numerals
    #terms = remove_numerals(terms)

    # Remove Punctuation and tokenize
    terms = remove_punctuation(terms)

    # Remove StopWords
    filtered_words = [word for word in terms if word not in stopwords.words('portuguese')]

    # Remove Accents
    #filtered_words = [remove_accents(term).lower() for term in terms]

    # Stemming
    #st = nltk.stem.RSLPStemmer()
    #st = nltk.stem.SnowballStemmer('portuguese')
    #filtered_stem = [st.stem(term) for term in terms]
    #filtered_stem = [st.stem(filtered_word) for filtered_word in terms]

    return filtered_words

def remove_punctuation(term):
    """Remove Punctuation and tokenize"""
    tokenizer = RegexpTokenizer(r'\w+')
    return tokenizer.tokenize(term)

if __name__ == '__main__':
    # Parser descriptors
    parser = argparse.ArgumentParser(
        description='''Sentence similarity evaluation for word embeddings in
        brazilian and european variants of Portuguese language. It is expected
        a word embedding model in text format.''')

    parser.add_argument('embedding',
                        type=str,
                        help='embedding model')

    parser.add_argument('lang',
                        choices=['br', 'pt'],
                        help='{br, eu} choose PT-BR or PT-EU testset')

    args = parser.parse_args()
    lang = args.lang
    emb = args.embedding

    # Loading embedding model
    embeddings = KeyedVectors.load_word2vec_format(emb,
                                                   binary=False,
                                                   unicode_errors="ignore")

    pairs_train = read_xml('%sassin2-train.xml' % (DATA_DIR), True)

    pairs_test = read_xml('%sassin-ptbr-test.xml' % (TEST_DIR), True)

    # Loading evaluation data and parsing it
    #with open('%sassin-pt%s-train.pkl' % (DATA_DIR, lang), 'rb') as fp:
        #data = pickle.load(fp)

    #with open('%sassin-pt%s-test-gold.pkl' % (DATA_DIR, lang), 'rb') as fp:
        #test = pickle.load(fp)

    # Getting features
    #features = gensim_embedding_difference(data, 'tokens_t1', 'tokens_t2')
    features = gensim_embedding_difference(pairs_train)
    #features_test = gensim_embedding_difference(test, 'tokens_t1', 'tokens_t2')
    features_test = gensim_embedding_difference(pairs_test)

    # Predicting similarities
    #results = array([float(i['result']) for i in data])

    results = array([float(i.similarity) for i in pairs_train])
    results_test = evaluate_testset(features, results, features_test)

    write_xml('%soutput.xml' % DATA_DIR, results_test)

    # Evaluating
    pairs_gold = read_xml('%sassin-pt%s-test.xml' % (TEST_DIR, lang), True)
    pairs_sys = read_xml('%soutput.xml' % DATA_DIR, True)
    #eval_rte(pairs_gold, pairs_sys)
    eval_similarity(pairs_gold, pairs_sys)

#python evaluate_assin2.py ./models/skip_s300.txt br

reason: from numpy


text:
import numpy
from kernel_tuner import tune_kernel, run_kernel
from numba import cuda
import json
import argparse
from numpyencoder import NumpyEncoder

# Setup CLI parser
parser = argparse.ArgumentParser(description="Scan tuner")
parser.add_argument("--size", "-s", type=int, default=1, help="problem size to the benchmark (e.g.: 2)")
parser.add_argument("--technique", "-t", type=str, default="brute_force", help="tuning technique to use for the benchmark (e.g.: annealing)")
arguments = parser.parse_args()

size = arguments.size
problem_sizes = [1, 8, 32, 64]
problem_size = problem_sizes[size - 1]

# Use host code in combination with CUDA kernel
kernel_files = ['scan_host.cu', '../../../src/kernels/scan/scan_kernel_no_template.cu']

block_sizes = [2**i for i in range(4, 10)]
block_sizes.remove(32)
grid_sizes = [2**i for i in range(0, 10)]

tune_params = dict()
tune_params["BLOCK_SIZE"] = block_sizes
tune_params["GRID_SIZE"] = grid_sizes
tune_params["PRECISION"] = [32, 64]
tune_params["UNROLL_LOOP_1"] = [0, 1]
tune_params["UNROLL_LOOP_2"] = [0, 1]

restrict = ["GRID_SIZE <= BLOCK_SIZE"]

strategy_options = {}
if arguments.technique == "genetic_algorithm":
    strategy_options = {"maxiter": 50, "popsize": 10}

tuning_results = tune_kernel("RunBenchmark", kernel_files, problem_size, [], tune_params, strategy=arguments.technique, restrictions=restrict, lang="C", block_size_names=["BLOCK_SIZE"],
    compiler_options=["-I ../../../src/kernels/scan/", "-I ../../../src/programs/common/", "-I ../../../src/programs/cuda-common/", f"-DPROBLEM_SIZE={size}", "-DNOT_USING_HELPERS=1"],
    iterations=2, strategy_options=strategy_options)

# Save the results as a JSON file
with open("scan-results.json", 'w') as f:
    json.dump(tuning_results, f, indent=4, cls=NumpyEncoder)

# Get the best configuration
best_parameter_config = min(tuning_results[0], key=lambda x: x['time'])
best_parameters = dict()

# Filter out parameters from results
for k, v in best_parameter_config.items():
    if k not in tune_params:
        continue

    best_parameters[k] = v

# Add problem size and tuning technique to results
best_parameters["PROBLEM_SIZE"] = size
best_parameters["TUNING_TECHNIQUE"] = arguments.technique

# Save the best results as a JSON file
with open("best-scan-results.json", 'w') as f:
    json.dump(best_parameters, f, indent=4, cls=NumpyEncoder)

max_stars_repo_name: Robyroc/BAT

max_issues_repo_name: Robyroc/BAT

max_forks_repo_name: Robyroc/BAT

reason: import numpy,from numpy,from numba


text:
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function, division

import unittest
import numpy as np
import warnings
import paddle

class TestTensorTypePromotion(unittest.TestCase):

    def setUp(self):
        self.x = paddle.to_tensor([2, 3])
        self.y = paddle.to_tensor([1.0, 2.0])

    def test_operator(self):
        with warnings.catch_warnings(record=True) as context:
            warnings.simplefilter("always")
            self.x + self.y
            self.assertTrue(
                "The dtype of left and right variables are not the same" in str(
                    context[-1].message))

        with warnings.catch_warnings(record=True) as context:
            warnings.simplefilter("always")
            self.x - self.y
            self.assertTrue(
                "The dtype of left and right variables are not the same" in str(
                    context[-1].message))

        with warnings.catch_warnings(record=True) as context:
            warnings.simplefilter("always")
            self.x * self.y
            self.assertTrue(
                "The dtype of left and right variables are not the same" in str(
                    context[-1].message))

        with warnings.catch_warnings(record=True) as context:
            warnings.simplefilter("always")
            self.x / self.y
            self.assertTrue(
                "The dtype of left and right variables are not the same" in str(
                    context[-1].message))

if __name__ == '__main__':
    unittest.main()


text:
# MIT License
#
# Copyright (c) 2018-2019 Tskit Developers
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
Module responsible for various utility functions used in other modules.
"""

import numpy as np

def safe_np_int_cast(int_array, dtype, copy=False):
    """
    A few functions require arrays of certain dtypes (e.g. node indices are np.int32,
    genotypes are np.int8, etc. Standard numpy integer arrays are of (dtype=np.int64),
    so need casting. This function casts but checks bounds to avoid wrap-around
    conversion errors. Strangely, numpy seems not to have this functionality built-in.

    If copy=False, and the original array is a numpy array of exactly the same dtype
    required, simply return the original rather than making a copy (same as the numpy
    .astype(copy=...) function)
    """
    if not isinstance(int_array, np.ndarray):
        int_array = np.array(int_array)
        # Since this is a new numpy array anyway, it's always a copy, so economize by
        # setting copy=False
        copy = False
    if int_array.size == 0:
        return int_array.astype(dtype, copy=copy)  # Allow empty arrays of any type
    try:
        return int_array.astype(dtype, casting='safe', copy=copy)
    except TypeError:
        bounds = np.iinfo(dtype)
        if np.any(int_array < bounds.min) or np.any(int_array > bounds.max):
            raise OverflowError("Cannot convert safely to {} type".format(dtype))
        if int_array.dtype.kind == 'i' and np.dtype(dtype).kind == 'u':
            # Allow casting from int to unsigned int, since we have checked bounds
            casting = 'unsafe'
        else:
            # Raise a TypeError when we try to convert from, e.g., a float.
            casting = 'same_kind'
        return int_array.astype(dtype, casting=casting, copy=copy)

max_stars_repo_name: andrewkern/tskit

max_issues_repo_name: andrewkern/tskit

max_forks_repo_name: andrewkern/tskit


text:
import numpy as np
import torch
import librosa

class DataLoader(torch.utils.data.Dataset):
    def __init__(self, person_list, voice_list, batch_size, dataset_path, deform_type, phonemes_length=None, mel=False, seed=None):
        self.person_list = person_list
        self.voice_list  = voice_list
        self.batch_size  = batch_size

        self.dataset_path    = dataset_path
        self.deform_type     = deform_type
        self.phonemes_length = phonemes_length
        self.mel_basis       = librosa.filters.mel(sr=24000, n_fft=1024) if mel else None

        self.person_nbatches  = len(person_list) // batch_size[0]
        self.phoneme_nbatches = len(voice_list) * 32 // batch_size[1]

        if seed is not None:
            self.reset_shuffle(seed)
        else:
            self.shuffle = np.arange(len(self))

    def __len__(self):
        return self.person_nbatches * self.phoneme_nbatches

    def reset_shuffle(self, seed=0):
        np.random.seed(seed)
        self.shuffle = np.random.permutation(len(self))

    def __getitem__(self, batch_idx):
        '''
        Returns
        -------
        data : [n_speakers, n_utterances, phoenemes_length, nfft // 2]
        '''

        if batch_idx < 0 or batch_idx >= len(self):
            raise IndexError()

        batch_idx = self.shuffle[batch_idx]

        # インデックスをperson方向とphoneme方向に分解
        phoneme_batch_idx, person_batch_idx = divmod(batch_idx, self.person_nbatches)

        # 読み込むべきファイルのpersonの範囲を求める
        person_start_idx = person_batch_idx * self.batch_size[0]
        person_end_idx   = (person_batch_idx + 1) * self.batch_size[0]

        # 読み込むべきファイルのvoiceの範囲を求める
        voice_start_idx = phoneme_batch_idx * self.batch_size[1] // 32
        voice_end_idx   = ((phoneme_batch_idx + 1) * self.batch_size[1] - 1) // 32 + 1

        # 複数のvoiceを結合したデータから取り出すべき範囲を求める
        voice_start_phoneme = phoneme_batch_idx * self.batch_size[1] % 32
        voice_end_phoneme   = voice_start_phoneme + self.batch_size[1]

        data = list()
        for person_idx in range(person_start_idx, person_end_idx):

            person_data = list()
            for voice_idx in range(voice_start_idx, voice_end_idx):
                specific = {
                    'person'     : self.person_list[person_idx] + 1,
                    'voice'      : self.voice_list[voice_idx] + 1,
                    'deform_type': self.deform_type,
                }
                pack = np.load(self.dataset_path % specific, allow_pickle=True)

                if self.deform_type == 'stretch':
                    sp = pack['sp']
                elif self.deform_type == 'padding':
                    sp = np.array([self._zero_padding(x[:self.phonemes_length], self.phonemes_length) for x in pack['sp']])
                elif self.deform_type == 'variable':
                    sp = np.array([x for x in pack['sp']])

                person_data.extend(sp)
            data.append(person_data[voice_start_phoneme:voice_end_phoneme])
        data = np.array(data)

        if self.mel_basis is not None:
            data = np.dot(data, self.mel_basis.T)
        else:
            data = data[:, :, 1:]

        data  = torch.from_numpy(data).float().to('cuda')
        return data

    @staticmethod
    def _zero_padding(x, target_length):
        y_pad = target_length - len(x)
        return np.pad(x, ((0, y_pad), (0, 0)), mode='constant') if y_pad > 0 else x


text:
import os
import collections
import json
import attr
import copy

import numpy as np

import torch
import torch.utils.data

from text2qdmr.model.modules import abstract_preproc
from text2qdmr.utils import registry
from text2qdmr.utils.serialization import ComplexEncoder, ComplexDecoder

from text2qdmr.datasets.qdmr import load_tables, BreakItem
from text2qdmr.datasets.utils.extract_values import ValueUnit
from qdmr2sparql.structures import prepare_dict_for_json, load_grounding_list_from_file, RdfGraph

class ZippedDataset(torch.utils.data.Dataset):
    def __init__(self, *components, process_func=None):
        assert len(components) >= 1
        lengths = [len(c) for c in components]

        self.process_func = process_func if process_func is not None else lambda x: x

        assert all(lengths[0] == other for other in lengths[1:]), f"Lengths don't match: {lengths}"
        self.components = components
        self.part = 'spider'

    def __getitem__(self, idx):
        item = tuple(c[idx] for c in self.components)
        item_processed = self.process_func(item)
        return item_processed

    def __len__(self):
        return len(self.components[0])

class TwoZippedDataset(torch.utils.data.Dataset):
    def __init__(self, *components, process_func=None):
        assert len(components) >= 1
        lengths = [len(c) for c in components]

        self.process_func = process_func if process_func is not None else lambda x: x

        assert all(lengths[0] == other for other in lengths[1:]), f"Lengths don't match: {lengths}"
        # components
        self.components = {'spider': [], 'full_break': []}
        self.names = list(self.components.keys())

        for items in zip(*components):
            if hasattr(items[0], "subset_name"):
                is_spider = items[0].subset_name == "SPIDER"
            else:
                is_spider = items[0]['subset_name'] == 'SPIDER'

            self.components['spider' if is_spider else 'full_break'].append(items)

        print('spider len', len(self.components['spider']))
        print('full_break len', len(self.components['full_break']))

        self.part = 'all'

    def __getitem__(self, idx):
        if self.part == 'all':
            # choice = np.random.choice(self.names)
            # use randomness from torch because it is taken care of autoatically when having multiple dataloading workers
            choice = torch.randint(0, len(self.names), (1, )).item()
            # choice = int(torch.bernoulli(torch.tensor([0.75])).item())
            choice = self.names[choice]
            item = self.components[choice][idx % len(self.components[choice])]
        elif self.part == 'spider':
            item = self.components['spider'][idx]
        elif self.part == 'full_break':
            item = self.components['full_break'][idx]

        item_processed = self.process_func(item)
        return item_processed

    def __len__(self):
        if self.part == 'all':
            return 2 * max(len(self.components['spider']), len(self.components['full_break']))
        elif self.part == 'spider':
            return len(self.components['spider'])
        elif self.part == 'full_break':
            return len(self.components['full_break'])

@registry.register('model', 'EncDec')
class EncDecModel(torch.nn.Module):
    class Preproc(abstract_preproc.AbstractPreproc):
        def __init__(
                self,
                encoder,
                decoder,
                encoder_preproc,
                decoder_preproc):
            super().__init__()

            self.enc_preproc = registry.lookup('encoder', encoder['name']).Preproc(**encoder_preproc)
            self.dec_preproc = registry.lookup('decoder', decoder['name']).Preproc(**decoder_preproc)

            self.items_raw = collections.defaultdict(list)
            self.data_dir = os.path.abspath(os.path.join(self.enc_preproc.data_dir, "..", "raw"))

        def validate_item(self, item, section):
            enc_result, enc_info = self.enc_preproc.validate_item(item, section)
            dec_result, dec_info = self.dec_preproc.validate_item(item, section, enc_info)
            return np.array(dec_result) * np.array(enc_result), (enc_info, dec_info)

        def add_item(self, item, section, idx_to_add, validation_info):
            enc_info, dec_info = validation_info
            self.enc_preproc.add_item(item, section, idx_to_add, enc_info)
            self.dec_preproc.add_item(item, section, idx_to_add, dec_info)
            self.add_raw_item(item, section)

        def add_raw_item(self, item, section):
            raw_item = attr.evolve(item,
                                   schema=None, # delete schema
                                   text=item.orig_spider_entry['question'] if item.orig_spider_entry is not None else item.text, # give priority to whatever is written in orig_spider_entry
                                   orig_spider_entry=None,
                                   orig_schema=None,
                                   sql_code=None,
            )
            self.items_raw[section].append(raw_item)

        def clear_items(self):
            self.enc_preproc.clear_items()
            self.dec_preproc.clear_items()
            self.items_raw = collections.defaultdict(list)

        def save(self, partition=None):
            self.enc_preproc.save(partition=partition)
            self.dec_preproc.save(partition=partition)
            self.save_raw_items()

        def save_raw_items(self):
            os.makedirs(self.data_dir, exist_ok=True)
            for section, items in self.items_raw.items():
                with open(os.path.join(self.data_dir, section + '.jsonl'), 'w') as f:
                    for item in items:
                        item = attr.evolve(item, grounding=prepare_dict_for_json(item.grounding))
                        f.write(json.dumps(item, cls=ComplexEncoder) + '\n')

        def load_raw_dataset(self, section, paths=None):
            dataset = [BreakItem(**json.loads(line, cls=ComplexDecoder))
                        for line in open(os.path.join(self.data_dir, section + '.jsonl'))]
            # load groundings properly
            if paths:
                schemas, _ = load_tables(paths['tables_path'])
                # load spider data
                spider_data = json.load(open(paths['spider_path']))

            eval_graphs = {}
            for item in dataset:
                if item.grounding != 'None':
                    grnds = load_grounding_list_from_file(None, data={item.full_name: {"GROUNDINGS": item.grounding}})
                    grnds = grnds[item.full_name]["GROUNDINGS"]
                    item.grounding = grnds
                if paths:
                    spider_entry = spider_data[item.subset_idx]
                    item.sql_code = spider_entry['sql']
                    item.schema = schemas[item.db_id]

                    if item.db_id not in eval_graphs:
                        schema_with_data = item.schema.database_schema
                        schema_with_data.load_table_data(paths['db_path'])
                        rdf_graph = RdfGraph(schema_with_data)
                        eval_graphs[item.db_id] = (schema_with_data, rdf_graph)
                    item.eval_graphs = eval_graphs[item.db_id]

                    item.orig_spider_entry = spider_entry
                    item.orig_schema = schemas[item.db_id].orig

                values = item.values
                item.values = []
                for val_list in values:
                    val_list_new = [ValueUnit(**val_dict) for val_dict in val_list]
                    item.values.append(val_list_new)

            return dataset

        def load(self):
            self.enc_preproc.load()
            self.dec_preproc.load()

        def process_in_dataloader(self, item_tuple, section, schemas):
            item = copy.deepcopy(item_tuple[0])

            # load schema
            if item.db_id is not None:
                item = attr.evolve(item, schema=schemas[item.db_id])

            flag_shuffle_values = self.config_data["augment_at_iter_shuffle_values"]
            flag_shuffle_tables = self.config_data["augment_at_iter_shuffle_tables"]
            flag_shuffle_columns = self.config_data["augment_at_iter_shuffle_columns"]
            flag_shuffle_qdmr_ordering = self.config_data["augment_at_iter_shuffle_qdmr_ordering"]
            flag_shuffle_sort_dir = self.config_data["augment_at_iter_shuffle_sort_dir"]
            flag_shuffle_compsup_op = self.config_data["augment_at_iter_shuffle_compsup_op"]
            enc_result, enc_info = self.enc_preproc.validate_item(item, section, shuffle_values=flag_shuffle_values,
                                                                                shuffle_tables=flag_shuffle_tables,
                                                                                shuffle_columns=flag_shuffle_columns,
                                                                                shuffle_sort_dir=flag_shuffle_sort_dir,
                                                                                shuffle_compsup_op=flag_shuffle_compsup_op,
                                                                                shuffle_qdmr_ordering=flag_shuffle_qdmr_ordering)
            dec_result, dec_info = self.dec_preproc.validate_item(item, section, enc_info)

            to_add = np.array(dec_result) * np.array(enc_result)
            if not to_add.any():
                print(f"WARNING: Item was too long at validation: will need to throw it away later")
                to_add[0] = True # hack to process items that are too long

            enc_data_new = self.enc_preproc.preprocess_item(item, to_add, enc_info)
            dec_data_new = self.dec_preproc.preprocess_item(item, to_add, dec_info)

            return (enc_data_new, dec_data_new)

        def dataset(self, section, two_datasets=False, config=None):

            if config is not None and "use_online_data_processing" in config and config["use_online_data_processing"]:
                self.config_data = config['data'][section]
                schemas, _ = load_tables(self.config_data["paths"]["tables_path"])
                raw_dataset = self.load_raw_dataset(section)

                if two_datasets:
                    dataset = TwoZippedDataset(raw_dataset, process_func=lambda item : self.process_in_dataloader(item, section=section, schemas=schemas))
                else:
                    dataset = ZippedDataset(raw_dataset, process_func=lambda item : self.process_in_dataloader(item, section=section, schemas=schemas))

                print("Loaded dataset size:", len(dataset))
                return dataset

            if two_datasets:
                return TwoZippedDataset(self.enc_preproc.dataset(section), self.dec_preproc.dataset(section))
            return ZippedDataset(self.enc_preproc.dataset(section), self.dec_preproc.dataset(section))

    def __init__(self, preproc, device, encoder, decoder):
        super().__init__()
        self.preproc = preproc
        self.encoder = registry.construct(
                'encoder', encoder, device=device, preproc=preproc.enc_preproc)
        self.decoder = registry.construct(
                'decoder', decoder, device=device, preproc=preproc.dec_preproc)

        if getattr(self.encoder, 'batched'):
            self.compute_loss = self._compute_loss_enc_batched
        else:
            self.compute_loss = self._compute_loss_unbatched

    # add regular forward to wrap into DistributedDataParallel
    def forward(self, batch):
        return self.compute_loss(batch)

    def _compute_loss_enc_batched(self, batch, debug=False):
        losses = []
        enc_states = self.encoder([enc_input for enc_input, _ in batch])

        # throw away batch elements that appeared to be too long and were not processed
        batch = [b for b, st_ in zip(batch, enc_states) if st_ is not None]
        enc_states = [st_ for st_ in enc_states if st_ is not None]
        enc_inputs = [b[0] for b in batch]
        dec_outputs = [b[1] for b in batch]

        if len(batch) == 0:
            # all batch elements were thrown away
            return torch.zeros([1])

        decoder_inputs = []
        for enc_state, (enc_input, dec_output) in zip(enc_states, batch):
            decoder_input = self.decoder.compute_decoder_input(enc_input, dec_output, enc_state,
                                                               self.decoder.exclude_rules_loss,
                                                               self.decoder.preproc,
                                                               self.decoder.sup_att,
                                                               self.decoder.attn_type,
                                                               debug)
            decoder_inputs.append(decoder_input)

        batch_size = len(batch)
        self.decoder.state_update.set_dropout_masks(batch_size=batch_size) #(batch_size=1)

        losses_batched = self.decoder.compute_loss_batched(enc_inputs, dec_outputs, enc_states, decoder_inputs, debug)

        losses = losses_batched

        if debug:
            return losses
        else:
            return torch.mean(torch.stack(losses, dim=0), dim=0)

    def _compute_loss_enc_batched2(self, batch, debug=False):
        losses = []
        for enc_input, dec_output in batch:
            enc_state, = self.encoder([enc_input])
            loss = self.decoder.compute_loss(enc_input, dec_output, enc_state, debug)
            losses.append(loss)
        if debug:
            return losses
        else:
            return torch.mean(torch.stack(losses, dim=0), dim=0)

    def _compute_loss_unbatched(self, batch, debug=False):
        losses = []
        for enc_input, dec_output in batch:
            enc_state = self.encoder(enc_input)
            loss = self.decoder.compute_loss(enc_input, dec_output, enc_state, debug)
            losses.append(loss)
        if debug:
            return losses
        else:
            return torch.mean(torch.stack(losses, dim=0), dim=0)

    def eval_on_batch(self, batch):
        mean_loss = self.compute_loss(batch).item()
        batch_size = len(batch)
        result = {'loss': mean_loss * batch_size, 'total': batch_size}
        return result

    def begin_inference(self, preproc_item):
        enc_input, _ = preproc_item
        if getattr(self.encoder, 'batched'):
            enc_state, = self.encoder([enc_input])
        else:
            enc_state = self.encoder(enc_input)
        return self.decoder.begin_inference(enc_state)

max_stars_repo_name: yandex-research/sparqling-queries

max_issues_repo_name: yandex-research/sparqling-queries

max_forks_repo_name: yandex-research/sparqling-queries


text:
"""Mesh tools."""
# pylint: disable=invalid-name
import numpy as np
import numpy.lib.recfunctions as rfn
import random
import scipy.sparse as sp
import plyfile

class Mesh(object):  # pylint: disable=too-few-public-methods

    """An easy to use mesh interface."""

    def __init__(self, filename=None, v=None, vc=None, f=None):
        """Construct a mesh either from a file or with the provided data."""
        if filename is not None:
            assert v is None and f is None and vc is None
        else:
            assert v is not None or f is not None
            if vc is not None:
                assert len(v) == len(vc)

        if filename is not None:
            plydata = plyfile.PlyData.read(filename)
            self.v = np.hstack((np.atleast_2d(plydata['vertex']['x']).T,
                                np.atleast_2d(plydata['vertex']['y']).T,
                                np.atleast_2d(plydata['vertex']['z']).T))
            self.vc = np.hstack((np.atleast_2d(plydata['vertex']['red']).T,
                                 np.atleast_2d(plydata['vertex']['green']).T,
                                 np.atleast_2d(plydata['vertex']['blue']).T)).astype('float') / 255.
            # Unfortunately, the vertex indices for the faces are stored in an
            # object array with arrays as objects. :-/ Work around this.
            self.f = np.vstack([np.atleast_2d(elem) for
                                elem in list(plydata['face']['vertex_indices'])]).astype('uint32')
        else:
            self.v = v
            self.vc = vc
            self.f = f

    def write_ply(self, out_name):
        """Write to a .ply file."""
        vertex = rfn.merge_arrays([
            self.v.view(dtype=[('x', 'f4'), ('y', 'f4'), ('z', 'f4'),]),
            (self.vc * 255.).astype('uint8').view(
                dtype=[('red', 'u1'), ('green', 'u1'), ('blue', 'u1')]),
            ],
                                  flatten=True,
                                  usemask=False)
        face = self.f.view(dtype=[('vertex_indices', 'i4', (3,))])[:, 0]
        vert_el = plyfile.PlyElement.describe(vertex, 'vertex')
        face_el = plyfile.PlyElement.describe(face, 'face')
        plyfile.PlyData([
            vert_el,
            face_el
        ]).write(out_name)

max_stars_repo_name: sg-first/Motion-Capture-CV

max_issues_repo_name: sg-first/Motion-Capture-CV

max_forks_repo_name: sg-first/Motion-Capture-CV

reason: import numpy,import scipy


text:
'''
This code presents an approach for implementing the gravitational field
produced by a sphere by using the analytical formulas.

References

'''

import numpy as np
from numba import jit

#: The gravitational constant in m^3 kg^{-1} s^{-1}
GRAVITATIONAL_CONST = 0.00000000006673

def gravitational(coordinates,spheres,density,field):
    """
    Computing the gravitational field generated by a sphere.
    """

    kernels = {"potential":kernel_potential,
               "g_z": kernel_g_z,
               "g_x": kernel_g_x,
               "g_y": kernel_g_y}

    # Checking input field
    if field not in kernels:
        raise ValueError("Gravitational field {} not recognized".format(field))

    # Checking input parameters
    if coordinates.ndim != 2:
        raise ValueError(
            "coordinates ndim ({}) ".format(coordinates.ndim)
            + "not equal to 2"
        )
    if coordinates.shape[0] != 3:
        raise ValueError(
            "Number of lines in coordinates ({}) ".format(coordinates.shape[0])
            + "not equal to 3"
        )
    if spheres.ndim != 2:
        raise ValueError(
            "prisms ndim ({}) ".format(spheres.ndim)
            + "not equal to 2"
        )
    if spheres.shape[1] != 4:
        raise ValueError(
            "Number of columns in spheres ({}) ".format(spheres.shape[1])
            + "not equal to 6"
        )
    if density.ndim != 1:
        raise ValueError(
            "density ndim ({}) ".format(density.ndim)
            + "not equal to 1"
        )
    if density.size != spheres.shape[0]:
        raise ValueError(
            "Number of elements in density ({}) ".format(density.size)
            + "mismatch the number of spheres ({})".format(spheres.shape[0])
        )

    # Create the array to store the result
    result = np.zeros(coordinates[0].size, dtype="float64")

    # Compute gravitational field
    jit_gravitational(coordinates, spheres, density, kernels[field], result)
    result *= GRAVITATIONAL_CONST
    # Convert from m/s^2 to mGal
    if field in ["potential", "g_z","g_x","g_y"]:
        result *= 1e5
    return result

@jit(nopython=True)
def jit_gravitational(coordinates,spheres,density,kernel,out):
    """
    Compute gravitational field at the observation points
    """

    # Iterate over coordinates
    for l in range(coordinates[0].size):
        # Iterate over spheres:
        for m in range(spheres.shape[0]):
            Y = spheres[m,0] - coordinates[0,l]
            X = spheres[m,1] - coordinates[1,l]
            Z = spheres[m,2] - coordinates[2,l]
            r3 = spheres[m,3]*spheres[m,3]*spheres[m,3]
            Volume = (4/3)*np.pi*r3
            out[l] += (density[m]*Volume*kernel(Y,X,Z))

@jit(nopython=True)
def kernel_potential(Y, X, Z):
    """
    Kernel function for potential gravitational field generated by a sphere
    """

    radius = np.sqrt(Y ** 2 + X ** 2 + Z ** 2)
    kernel = 1/radius

    return kernel

@jit(nopython=True)
def kernel_g_x(Y, X, Z):
    """
    Kernel for x component of gravitational acceleration of a sphere
    """
    radius = np.sqrt(Y ** 2 + X ** 2 + Z ** 2)
    r3 = radius*radius*radius
    kernel = X /r3
    return kernel

@jit(nopython=True)
def kernel_g_y(Y, X, Z):
    """
    Kernel for y component of gravitational acceleration of a sphere
    """
    radius = np.sqrt(Y ** 2 + X ** 2 + Z ** 2)
    r3 = radius*radius*radius
    kernel = Y /r3
    return kernel

@jit(nopython=True)
def kernel_g_z(Y, X, Z):
    """
    Kernel for z component of gravitational acceleration of a sphere
    """
    radius = np.sqrt(Y ** 2 + X ** 2 + Z ** 2)
    r3 = radius*radius*radius
    kernel = Z/r3
    return kernel

max_stars_repo_name: andrelreis/metodos-potenciais

max_issues_repo_name: andrelreis/metodos-potenciais

max_forks_repo_name: andrelreis/metodos-potenciais

reason: import numpy,from numba


text:
#!/usr/bin/env python
# coding: utf-8

# In[1]:

import pandas as pd
import numpy as np

# In[2]:

all_papers = pd.read_csv('all_papers.csv')
all_papers['title'] = all_papers['title'].astype(str)
accepted_papers = pd.read_excel('accepted.xlsx',  header=None)

# In[3]:

accepted_papers.columns = ['paper_id', 'oral_pos', 'paper_type', 'track', 'title', 'abstract', 'authors' ]

# In[4]:

accepted_papers['paper_id'] = accepted_papers['paper_id'].astype(str)

# In[5]:

all_papers['paper_id'] = all_papers.pdf_url.str.split('.').str[-1]

# In[8]:

accepted_papers['pdf_url'] = 'https://www.aclweb.org/anthology/2021.eacl-main.' + accepted_papers.paper_id

# In[9]:

accepted_papers = accepted_papers.drop(['paper_id', 'oral_pos'], axis = 1)

# In[10]:

accepted_papers

# In[ ]:

max_stars_repo_path: scripts/accepted papers csv/accepted papers script.py


text:
from __future__ import print_function

import mxnet as mx
import numpy as np
from mxnet import nd, autograd, gluon

mx.random.seed(1)

# ctx = mx.gpu()
ctx = mx.cpu()

batch_size = 64
num_inputs = 784
num_outputs = 10

def transform(data, label):
    return nd.transpose(data.astype(np.float32), (2, 0, 1)) / 255, label.astype(np.float32)

train_data = gluon.data.DataLoader(gluon.data.vision.MNIST(train=True, transform=transform),
                                   batch_size, shuffle=True)
test_data = gluon.data.DataLoader(gluon.data.vision.MNIST(train=False, transform=transform),
                                  batch_size, shuffle=False)

num_fc = 512
net = gluon.nn.Sequential()
with net.name_scope():
    net.add(gluon.nn.Conv2D(channels=20, kernel_size=5, activation='relu'))
    net.add(gluon.nn.MaxPool2D(pool_size=2, strides=2))
    net.add(gluon.nn.Conv2D(channels=50, kernel_size=5, activation='relu'))
    net.add(gluon.nn.MaxPool2D(pool_size=2, strides=2))
    # The Flatten layer collapses all axis, except the first one, into one axis.
    net.add(gluon.nn.Flatten())
    net.add(gluon.nn.Dense(num_fc, activation="relu"))
    net.add(gluon.nn.Dense(num_outputs))

net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx)

softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()

trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': .1})

def evaluate_accuracy(data_iterator, net):
    acc = mx.metric.Accuracy()
    for i, (data, label) in enumerate(data_iterator):
        data = data.as_in_context(ctx)
        label = label.as_in_context(ctx)
        output = net(data)
        predictions = nd.argmax(output, axis=1)
        acc.update(preds=predictions, labels=label)
    return acc.get()[1]

epochs = 1
smoothing_constant = .01

for e in range(epochs):
    for i, (data, label) in enumerate(train_data):
        data = data.as_in_context(ctx)
        label = label.as_in_context(ctx)
        with autograd.record():
            output = net(data)
            loss = softmax_cross_entropy(output, label)
        loss.backward()
        trainer.step(data.shape[0])

        ##########################
        #  Keep a moving average of the losses
        ##########################
        curr_loss = nd.mean(loss).asscalar()
        moving_loss = (curr_loss if ((i == 0) and (e == 0))
                       else (1 - smoothing_constant) * moving_loss + smoothing_constant * curr_loss)

    test_accuracy = evaluate_accuracy(test_data, net)
    train_accuracy = evaluate_accuracy(train_data, net)
    print("Epoch %s. Loss: %s, Train_acc %s, Test_acc %s" % (e, moving_loss, train_accuracy, test_accuracy))

max_stars_repo_name: foamliu/Learn-MXNet

max_issues_repo_name: foamliu/Learn-MXNet

max_forks_repo_name: foamliu/Learn-MXNet


text:
# Copyright 2018 Iguazio
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from copy import deepcopy
from datetime import datetime
import numpy as np
import uuid
import os

from mlrun.artifacts import ModelArtifact
from .artifacts import ArtifactManager, DatasetArtifact
from .datastore import StoreManager
from .secrets import SecretsStore
from .db import get_run_db
from .utils import run_keys, get_in, dict_to_yaml, logger, dict_to_json, now_date, to_date_str

class MLCtxValueError(Exception):
    pass

class MLClientCtx(object):
    """ML Execution Client Context

    the context is generated using the get_or_create_ctx call (see its doc)
    and provides an interface to use run params, metadata, inputs, and outputs

    base metadata include: uid, name, project, and iteration (for hyper params)
    users can set labels and annotations using set_labels(), set_annotation()
    access parameters and secrets using get_param(), get_secret()
    access input data objects using get_input()
    store results, artifacts, and real-time metrics using log_xx methods

    see doc for the individual params and methods
    """
    kind = 'run'

    def __init__(self, autocommit=False, tmp='', log_stream=None):
        self._uid = ''
        self.name = ''
        self._iteration = 0
        self._project = ''
        self._tag = ''
        self._secrets_manager = SecretsStore()

        # runtime db service interfaces
        self._rundb = None
        self._tmpfile = tmp
        self._logger = log_stream or logger
        self._log_level = 'info'
        self._matrics_db = None
        self._autocommit = autocommit

        self._labels = {}
        self._annotations = {}

        self._function = ''
        self._parameters = {}
        self._in_path = ''
        self._out_path = ''
        self._inputs = {}
        self._outputs = []

        self._results = {}
        self._state = 'created'
        self._error = None
        self._commit = ''
        self._host = None
        self._start_time = now_date()
        self._last_update = now_date()
        self._iteration_results = None

    def set_logger_stream(self, stream):
        handlers = self._logger.handlers
        if len(handlers)>0:
            handlers[0].stream = stream

    def _init_dbs(self, rundb):
        if rundb:
            if isinstance(rundb, str):
                self._rundb = get_run_db(rundb)
                self._rundb.connect(self._secrets_manager)
            else:
                self._rundb = rundb
        self._data_stores = StoreManager(self._secrets_manager, db=self._rundb)
        self._artifacts_manager = ArtifactManager(
            self._data_stores, db=self._rundb, out_path=self._out_path)

    def get_meta(self):
        """Reserved for internal use"""
        uri = f'{self._project}/{self.uid}' if self._project else self.uid
        resp = {
            'name': self.name,
            'kind': 'run',
            'uri': uri,
            'owner': get_in(self._labels, 'owner'),
        }
        if 'workflow' in self._labels:
            resp['workflow'] = self._labels['workflow']
        return resp

    @classmethod
    def from_dict(cls, attrs: dict, rundb='', autocommit=False, tmp='',
                  host=None, log_stream=None):

        self = cls(autocommit=autocommit, tmp=tmp, log_stream=log_stream)

        meta = attrs.get('metadata')
        if meta:
            self._uid = meta.get('uid', self._uid or uuid.uuid4().hex)
            self._iteration = meta.get('iteration', self._iteration)
            self.name = meta.get('name', self.name)
            self._project = meta.get('project', self._project)
            self._annotations = meta.get('annotations', self._annotations)
            self._labels = meta.get('labels', self._labels)
        spec = attrs.get('spec')
        if spec:
            self._secrets_manager = SecretsStore.from_list(
                spec.get(run_keys.secrets))
            self._log_level = spec.get('log_level', self._log_level)
            self._function = spec.get('function', self._function)
            self._parameters = spec.get('parameters', self._parameters)
            self._outputs = spec.get('outputs', self._outputs)
            self._out_path = spec.get(run_keys.output_path, self._out_path)
            self._in_path = spec.get(run_keys.input_path, self._in_path)
            inputs = spec.get(run_keys.inputs)

        self._init_dbs(rundb)

        if spec:
            # init data related objects (require DB & Secrets to be set first)
            self._data_stores.from_dict(spec)
            if inputs and isinstance(inputs, dict):
                for k, v in inputs.items():
                    self._set_input(k, v)

        if host:
            self.set_label('host', host)

        start = get_in(attrs, 'status.start_time')
        if start:
            self._start_time = start
        self._state = 'running'
        self._update_db(commit=True)
        return self

    @property
    def uid(self):
        """Unique run id"""
        if self._iteration:
            return f'{self._uid}-{self._iteration}'
        return self._uid

    @property
    def tag(self):
        """run tag (uid or workflow id if exists)"""
        return self._labels.get('workflow', self._uid)

    @property
    def iteration(self):
        """child iteration index, for hyper parameters """
        return self._iteration

    @property
    def project(self):
        """project name, runs can be categorized by projects"""
        return self._project

    @property
    def logger(self):
        """built-in logger interface"""
        return self._logger

    @property
    def log_level(self):
        """get the logging level, e.g. 'debug', 'info', 'error'"""
        return self._log_level

    @log_level.setter
    def log_level(self, value: str):
        """set the logging level, e.g. 'debug', 'info', 'error'"""
        self._log_level = value
        print(f'changed log level to: {value}')

    @property
    def parameters(self):
        """dictionary of run parameters (read-only)"""
        return deepcopy(self._parameters)

    @property
    def inputs(self):
        """dictionary of input data items (read-only)"""
        return self._inputs

    @property
    def results(self):
        """dictionary of results (read-only)"""
        return deepcopy(self._results)

    @property
    def artifacts(self):
        """dictionary of artifacts (read-only)"""
        return deepcopy(self._artifacts_manager.artifact_list())

    @property
    def in_path(self):
        """default input path for data objects"""
        return self._in_path

    @property
    def out_path(self):
        """default output path for artifacts"""
        logger.info('.out_path will soon be deprecated, use .artifact_path')
        return self._out_path

    @property
    def artifact_path(self):
        """default output path for artifacts"""
        return self._out_path

    @property
    def labels(self):
        """dictionary with labels (read-only)"""
        return deepcopy(self._labels)

    def set_label(self, key: str, value, replace: bool = True):
        """set/record a specific label"""
        if replace or not self._labels.get(key):
            self._labels[key] = str(value)

    @property
    def annotations(self):
        """dictionary with annotations (read-only)"""
        return deepcopy(self._annotations)

    def set_annotation(self, key: str, value, replace: bool = True):
        """set/record a specific annotation"""
        if replace or not self._annotations.get(key):
            self._annotations[key] = str(value)

    def get_param(self, key: str, default=None):
        """get a run parameter, or use the provided default if not set"""
        if key not in self._parameters:
            self._parameters[key] = default
            if default:
                self._update_db()
            return default
        return self._parameters[key]

    def get_secret(self, key: str):
        """get a key based secret e.g. DB password from the context
        secrets can be specified when invoking a run through files, env, ..
        """
        if self._secrets_manager:
            return self._secrets_manager.get(key)
        return None

    def _set_input(self, key, url=''):
        if not url:
            url = key
        if self.in_path and not (url.startswith('/') or '://' in url):
            url = os.path.join(self._in_path, url)
        obj = self._data_stores.object(url, key, project=self._project)
        self._inputs[key] = obj
        return obj

    def get_input(self, key: str, url: str = ''):
        """get an input data object, data objects have methods such as
         .get(), .download(), .url, .. to access the actual data"""
        if key not in self._inputs:
            return self._set_input(key, url)
        else:
            return self._inputs[key]

    def log_result(self, key: str, value, commit=False):
        """log a scalar result value"""
        self._results[str(key)] = _cast_result(value)
        self._update_db(commit=commit)

    def log_results(self, results: dict, commit=False):
        """log a set of scalar result values"""
        if not isinstance(results, dict):
            raise MLCtxValueError(
                '(multiple) results must be in the form of dict')

        for p in results.keys():
            self._results[str(p)] = _cast_result(results[p])
        self._update_db(commit=commit)

    def log_iteration_results(
      self, best, summary: list, task: dict, commit=False):
        """Reserved for internal use"""

        if best:
            self._results['best_iteration'] = best
            for k, v in get_in(task, ['status', 'results'], {}).items():
                self._results[k] = v
            for a in get_in(task, ['status', run_keys.artifacts], []):
                self._artifacts_manager.artifacts[a['key']] = a
                self._artifacts_manager.link_artifact(self.project, self.name, self.tag,
                                                      a['key'], self.iteration,
                                                      a['target_path'],
                                                      link_iteration=best)

        self._iteration_results = summary
        if commit:
            self._update_db(commit=True)

    def log_metric(self, key: str, value, timestamp=None, labels=None):
        """TBD, log a real-time time-series metric"""
        labels = {} if labels is None else labels
        if not timestamp:
            timestamp = datetime.now()
        if self._rundb:
            self._rundb.store_metric({key: value}, timestamp, labels)

    def log_metrics(self, keyvals: dict, timestamp=None, labels=None):
        """TBD, log a set of real-time time-series metrics"""
        labels = {} if labels is None else labels
        if not timestamp:
            timestamp = datetime.now()
        if self._rundb:
            self._rundb.store_metric(keyvals, timestamp, labels)

    def log_artifact(self, item, body=None, local_path=None, artifact_path=None,
                     tag='', viewer=None, target_path='', src_path=None,
                     upload=None, labels=None, format=None, db_key=None, **kwargs):
        """log an output artifact and optionally upload it"""
        local_path = src_path or local_path
        item = self._artifacts_manager.log_artifact(self, item, body=body,
                                                    local_path=local_path,
                                                    artifact_path=artifact_path,
                                                    target_path=target_path,
                                                    tag=tag,
                                                    viewer=viewer,
                                                    upload=upload,
                                                    labels=labels,
                                                    db_key=db_key,
                                                    format=format)
        self._update_db()
        return item

    def log_dataset(self, key, df, tag='', local_path=None,
                    artifact_path=None, upload=True, labels=None,
                    format='', preview=None, stats=False, db_key=None,
                    target_path='', analysis=None, **kwargs):
        """log a dataset artifact and optionally upload it"""

        ds = DatasetArtifact(key, df, preview=preview, analysis=analysis,
                             format=format, stats=stats, **kwargs)

        item = self._artifacts_manager.log_artifact(self, ds, local_path=local_path,
                                                    artifact_path=artifact_path,
                                                    target_path=target_path,
                                                    tag=tag,
                                                    upload=upload,
                                                    db_key=db_key,
                                                    labels=labels)
        self._update_db()
        return item

    def log_model(self, key, body=None, tag='', model_dir=None, model_file=None,
                  metrics=None, parameters=None, artifact_path=None,
                  upload=True, labels=None, inputs=None, outputs=None,
                  extra_data=None, db_key=None):
        """log a model artifact and optionally upload it"""

        model = ModelArtifact(key, body, model_file=model_file,
                              metrics=metrics, parameters=parameters,
                              inputs=inputs, outputs=outputs, extra_data=extra_data)

        item = self._artifacts_manager.log_artifact(self, model, local_path=model_dir,
                                                    artifact_path=artifact_path,
                                                    tag=tag,
                                                    upload=upload,
                                                    db_key=db_key,
                                                    labels=labels)
        self._update_db()
        return item

    def commit(self, message: str = ''):
        """save run state and add a commit message"""
        if message:
            self._annotations['message'] = message
        self._last_update = now_date()
        self._update_db(commit=True, message=message)

    def set_state(self, state: str = None, error: str = None, commit=True):
        """modify and store the run state or mark an error"""
        updates = {'status.last_update': now_date().isoformat()}

        if error:
            self._state = 'error'
            self._error = str(error)
            updates['status.state'] = 'error'
            updates['status.error'] = error
        elif state and state != self._state and self._state != 'error':
            self._state = state
            updates['status.state'] = state
        self._last_update = now_date()

        if self._rundb and commit:
            self._rundb.update_run(updates, self._uid,
                                   self.project, iter=self._iteration)

    def set_hostname(self, host: str):
        """update the hostname"""
        self._host = host
        if self._rundb:
            updates = {'status.host': host}
            self._rundb.update_run(updates, self._uid,
                                   self.project, iter=self._iteration)

    def to_dict(self):
        """convert the run context to a dictionary"""

        def set_if_valid(struct, key, val):
            if val:
                struct[key] = val

        struct = {
            'kind': 'run',
            'metadata':
                {'name': self.name,
                 'uid': self._uid,
                 'iteration': self._iteration,
                 'project': self._project,
                 'labels': self._labels,
                 'annotations': self._annotations},
            'spec':
                {'function': self._function,
                 'log_level': self._log_level,
                 'parameters': self._parameters,
                 'outputs': self._outputs,
                 run_keys.output_path: self._out_path,
                 run_keys.inputs: {k: v.artifact_url for k, v in self._inputs.items()},
                 },
            'status':
                {'state': self._state,
                 'results': self._results,
                 'start_time': to_date_str(self._start_time),
                 'last_update': to_date_str(self._last_update)},
            }

        set_if_valid(struct['status'], 'error', self._error)
        set_if_valid(struct['status'], 'commit', self._commit)

        if self._iteration_results:
            struct['status']['iterations'] = self._iteration_results
        struct['status'][run_keys.artifacts] = \
            self._artifacts_manager.artifact_list()
        self._data_stores.to_dict(struct['spec'])
        return struct

    def to_yaml(self):
        """convert the run context to a yaml buffer"""
        return dict_to_yaml(self.to_dict())

    def to_json(self):
        """convert the run context to a json buffer"""
        return dict_to_json(self.to_dict())

    def _update_db(self, commit=False, message=''):
        self.last_update = now_date()
        if self._tmpfile:
            data = self.to_json()
            with open(self._tmpfile, 'w') as fp:
                fp.write(data)
                fp.close()

        if commit or self._autocommit:
            self._commit = message
            if self._rundb:
                self._rundb.store_run(
                    self.to_dict(), self._uid, self.project, iter=self._iteration)

def _cast_result(value):
    if isinstance(value, (int, str, float)):
        return value
    if isinstance(value, list):
        return [_cast_result(v) for v in value]
    if isinstance(value, dict):
        return {k: _cast_result(v) for k, v in value.items()}
    if isinstance(value, (np.int64, np.integer)):
        return int(value)
    if isinstance(value, (np.floating, np.float64)):
        return float(value)
    if isinstance(value, np.ndarray):
        return value.tolist()
    return str(value)

max_stars_repo_name: orkiguazio/mlrun

max_issues_repo_name: orkiguazio/mlrun

max_forks_repo_name: orkiguazio/mlrun


text:
from fltk import Fl
import torch
from DartDeep.sh_v2.ppo_v2 import PPO
from PyCommon.modules.GUI import hpSimpleViewer as hsv
from PyCommon.modules.Renderer import ysRenderer as yr
import numpy as np

import pydart2 as pydart

def main():
    MOTION_ONLY = False

    pydart.init()

    env_name = 'walk'

    ppo = PPO(env_name, 1, visualize_only=True)
    if not MOTION_ONLY:
        ppo.LoadModel('model/' + env_name + '.pt')
    ppo.env.Resets(False)
    ppo.env.ref_skel.set_positions(ppo.env.ref_motion.get_q(ppo.env.phase_frame))

    # viewer settings
    rd_contact_positions = [None]
    rd_contact_forces = [None]
    dart_world = ppo.env.world
    viewer = hsv.hpSimpleViewer(rect=(0, 0, 1200, 800), viewForceWnd=False)
    viewer.doc.addRenderer('MotionModel', yr.DartRenderer(ppo.env.ref_world, (150,150,255), yr.POLYGON_FILL))
    if not MOTION_ONLY:
        viewer.doc.addRenderer('controlModel', yr.DartRenderer(dart_world, (255,240,255), yr.POLYGON_FILL))
        viewer.doc.addRenderer('contact', yr.VectorsRenderer(rd_contact_forces, rd_contact_positions, (255,0,0)))

    def postCallback(frame):
        ppo.env.ref_skel.set_positions(ppo.env.ref_motion.get_q(frame))

    def simulateCallback(frame):
        state = ppo.env.GetState(0)
        action_dist, _ = ppo.model(torch.tensor(state.reshape(1, -1)).float())
        action = action_dist.loc.detach().numpy()
        res = ppo.env.Steps(action)
        # res = ppo.env.Steps(np.zeros_like(action))
        # print(frame, ppo.env.ref_skel.current_frame, ppo.env.world.time()*ppo.env.ref_motion.fps)
        # print(frame, res[0][0])
        # if res[0][0] > 0.46:
        #     ppo.env.continue_from_now_by_phase(0.2)
        if res[2]:
            print(frame, 'Done')
            ppo.env.reset()

        # contact rendering
        contacts = ppo.env.world.collision_result.contacts
        del rd_contact_forces[:]
        del rd_contact_positions[:]
        for contact in contacts:
            rd_contact_forces.append(contact.f/1000.)
            rd_contact_positions.append(contact.p)

    if MOTION_ONLY:
        viewer.setPostFrameCallback_Always(postCallback)
        viewer.setMaxFrame(len(ppo.env.ref_motion)-1)
    else:
        viewer.setSimulateCallback(simulateCallback)
        viewer.setMaxFrame(3000)
    viewer.startTimer(1./30.)
    viewer.show()

    Fl.run()

if __name__ == '__main__':
    main()

max_stars_repo_name: hpgit/HumanFoot

max_issues_repo_name: hpgit/HumanFoot

max_forks_repo_name: hpgit/HumanFoot


text:
import scipy.io.wavfile
import python_speech_features
import numpy

# Function which can be called to exctract MFCCs
def mfcc_extract(signal, fs=8000):
    mfccs = python_speech_features.mfcc(
        signal=signal, samplerate=fs, winlen=0.025,
        winstep=0.01, numcep=13, nfilt=26, nfft=512,
        lowfreq=0, highfreq=None, preemph=0.97,
        ceplifter=22, appendEnergy=True
    )
    return mfccs

# Extracting MFCCs from the five training sound (.wav) files
fs1, data1 = scipy.io.wavfile.read('training//bass//bass-training.wav')
mfccs_bass = mfcc_extract(signal=data1, fs=fs1)

fs2, data2 = scipy.io.wavfile.read('training//drum//drum-training.wav')
mfccs_drum = mfcc_extract(signal=data2, fs=fs2)

fs3, data3 = scipy.io.wavfile.read('training//guitar//guitar-training.wav')
mfccs_guitar = mfcc_extract(signal=data3, fs=fs3)

fs4, data4 = scipy.io.wavfile.read('training//piano//piano-training.wav')
mfccs_piano = mfcc_extract(signal=data4, fs=fs4)

fs5, data5 = scipy.io.wavfile.read('training//violin//violin-training.wav')
mfccs_violin = mfcc_extract(signal=data5, fs=fs5)

# Saving the MFCCs as a Binary File
numpy.save('training//bass//mfccs_bass.npy', mfccs_bass)
numpy.save('training//drum//mfccs_drum.npy', mfccs_drum)
numpy.save('training//guitar//mfccs_guitar.npy', mfccs_guitar)
numpy.save('training//piano//mfccs_piano.npy', mfccs_piano)
numpy.save('training//violin//mfccs_violin.npy', mfccs_violin)

print("MFCCs have been extracted and saved to disk.")


text:
import matplotlib.pyplot as plt
import numpy as np

###########bayesian_linear_regression#######
##t his can only fit linear data#####

if __name__ == '__main__':
    m = 10
    x = np.linspace(- 2, 2, 10).reshape(10, 1)

    x_intercept = np.ones((m,1))

    y = 2 * x + 1

    Y = y + np.random.normal(0,1, np.shape(x))

    X = np.concatenate((x, x_intercept), axis = 1)

    # plt.scatter(x,y)
    # plt.show()
    x_test = np.linspace(-5 , 5, 100).reshape(100,1)
    x_test_intercept = np.ones((100,1))
    X_test = np.concatenate((x_test, x_test_intercept), axis = 1)

    A = (X.T) @ X + np.identity(2)

    mean = X_test @ np.linalg.inv(A) @ (X.T) @ Y

    cov = X_test @ np.linalg.inv(A) @ (X_test.T) + np.identity(100)

    variance = np.diagonal(cov).reshape(-1, 1)

    stand_deviation = np.sqrt(variance)

    upper_bound = mean + 2 * stand_deviation
    lower_bound = mean - 2 * stand_deviation

    plt.plot(x_test, mean,c = 'r')
    plt.plot(x_test, upper_bound, c = 'r')
    plt.plot(x_test, lower_bound, c = 'r')

    plt.fill_between(x_test.reshape(-1), upper_bound.reshape(-1), lower_bound.reshape(-1), facecolor = 'r', alpha = .25)
    plt.scatter(x, Y)
    ### 1-d array###
    plt.savefig('bayesian_reg.png', dpi=250, bbox_inches='tight')
    plt.show()


text:
import pandas as pd
import numpy as np
import json
import uuid
import pickle
from os.path import abspath
from app.core.objects.FastQData import FastQData
from app.core.preprocessing.calculations import get_sequence_length

class FastQDataframe:

    __df = pd.DataFrame
    __uuid = ""

    # Dictionaries to save the used options in:
    __preprocOptions = {}
    __originalPreprocOptions = {}
    __blastOptions = {}

    def __init__(self, fw_data: FastQData=None, rvc_data: FastQData=None, fw_items: list=None, fw_indices: list=None,
                 rvc_items: list=None, rvc_indices: list=None, df_id: str=None, fw_dict=None, rv_dict=None, from_blast=None):
        """Creates a FastQ dataframe.

        A FastQDataframe can be constructed multiple ways. The most straight forward way is to use FastQData objects,
        But it is also possible to use list items.

        Args:
            fw_data: FastQData.
            rvc_data: FastQData.
            fw_items: List with forward sequence and quality score items.
            fw_indices: List with identifiers of the forward items (fastQ headers).
            rvc_items: List with reverse complement sequence and quality score items.
            rvc_indices: List with identifiers of the reverse complement items (fastQ headers).
            df_id: Unique identifier for a FastQDataframe instance. Is used for writing this object to a pickle.
            fw_dict: ???
            rv_dict: ???
            from_blast: ???

        """
        if df_id:
            self.__uuid = df_id
        else:
            self.__uuid = str(uuid.uuid1())
        if fw_items and fw_indices and rvc_items and rvc_indices:
            self.__from_lists(fw_items, fw_indices, rvc_items, rvc_indices)
        elif fw_data and rvc_data:
            self.__from_fastqdata(fw_data, rvc_data)
        elif fw_dict and rv_dict:
            self.__from_dict(fw_dict, rv_dict)
        elif from_blast:
            self.__from_blast()
        else:
            raise Exception("No input data given, please use either FastQData objects or list objects")

    def __from_blast(self):
        fw_rvc_columns = []
        for prefix in ['fw_', 'rv_']:
            fw_rvc_columns.extend([prefix+'accession', prefix+'full_tax', prefix+'id', prefix+'bit', prefix+'coverage_length',
                       prefix+'Kingdom', prefix+'Phylum', prefix+'Class', prefix+'Order', prefix+'Family', prefix+'Genus', prefix+'Species', prefix + 'Strain'])
        fw_rvc_columns.extend(['paired_flag', 'fw_a_perc_flag', 'fw_t_perc_flag','fw_g_perc_flag', 'fw_c_perc_flag',
                   'rv_a_perc_flag', 'rv_t_perc_flag', 'rv_g_perc_flag', 'rv_c_perc_flag', 'fw_seq_len_flag',
                   'rv_seq_len_flag', 'identity_flag', 'fw_seq', 'rvc_seq' ])
        self.__df = pd.DataFrame(columns=fw_rvc_columns)

    # Cannot find what this function is used for.
    @DeprecationWarning
    def __from_lists(self, items_fw: list, indices_fw: list, items_rvc: list, indices_rvc: list):
        """Creates a pandas dataframe from list items.

        Args:
            items_fw: ?
            indices_fw: ?
            items_rvc: ?
            indices_rvc: ?

        """
        fw_columns = ['fw_seq', 'fw_seq_score']
        fw_df = pd.DataFrame(data=items_fw, columns=fw_columns, index=indices_fw)

        rvc_columns = ['rvc_seq', 'rvc_seq_score']
        rvc_df = pd.DataFrame(data=items_rvc, columns=rvc_columns, index=indices_rvc)

        self.__df = fw_df.join(rvc_df)

    def __from_fastqdata(self, fw_data: FastQData, rvc_data: FastQData):
        """Create a pandas dataframe from FastQData objects.

        Args:
            fw_data: ?
            rvc_data: ?

        """
        fw_columns = ['index_sequence','original_fw_seq', 'original_fw_seq_score',
                      'fw_seq',    'fw_seq_score', 'fw_seq_length',
                      'fw_A_perc', 'fw_T_perc',    'fw_G_perc',
                      'fw_C_perc', 'fw_Q-score']
        fw_df = pd.DataFrame(columns = fw_columns,
                             index   = fw_data.get_index_list())

        if len(fw_data.get_info_index_list()) > 0:
            fw_df['index_sequence'] = fw_data.get_info_index_list()
            fw_df["index_sequence"] = fw_df["index_sequence"].str.split(":").str[3]
        else:
            fw_df.drop(['index_sequence'], axis=1, inplace=True)
        fw_df['fw_seq'] = fw_data.get_sequence_list()
        fw_df['fw_seq_score'] = fw_data.get_q_score_list()
        fw_df['original_fw_seq'] = fw_data.get_sequence_list()
        fw_df['original_fw_seq_score'] = fw_data.get_q_score_list()

        rvc_columns = ['original_rv_seq', 'original_rvc_seq', 'original_rv_seq_score',
                      'rv_seq',       'rvc_seq',   'rv_seq_score',
                      'rv_seq_length', 'rv_A_perc', 'rv_T_perc',
                      'rv_G_perc',     'rv_C_perc', 'rv_Q-score']
        rvc_df = pd.DataFrame(columns = rvc_columns,
                              index   = rvc_data.get_index_list())

        rvc_df['rv_seq'] = rvc_data.get_sequence_list()
        rvc_df['rvc_seq'] = rvc_data.get_complement_list()
        rvc_df['rv_seq_score'] = rvc_data.get_q_score_list()
        rvc_df['original_rv_seq'] = rvc_data.get_sequence_list()
        rvc_df['original_rvc_seq'] = rvc_data.get_complement_list()
        rvc_df['original_rv_seq_score'] = rvc_data.get_q_score_list()

        joined_df = fw_df.join(rvc_df)

        # TODO: Check if new columns need to be added to the table for future features.
        for newColumn in ['flagged','fw_flagged','rv_flagged','paired_flag',
                        'fw_a_perc_flag','fw_t_perc_flag', 'fw_g_perc_flag',
                        'fw_c_perc_flag','rv_a_perc_flag', 'rv_t_perc_flag',
                        'rv_g_perc_flag','rv_c_perc_flag', 'fw_seq_len_flag',
                        'rv_seq_len_flag','identity_flag', 'fw_quality_flag',
                        'rv_quality_flag','original_contains_unknown_symbols']:
            # Fill the rows of the new column with False.
            joined_df[newColumn] = False

        self.__df = joined_df

        self.__df['original_contains_unknown_symbols'] = list(map(self.__has_unknown, self.__df[["original_fw_seq","original_rv_seq"]].values.tolist()))

        self.calcQuality("fw")
        self.calcQuality("rv")

    @staticmethod
    def __has_unknown(row):
        """Checks if the original seq had unknown symbols (these were replaced with N in previous processes)."""
        def check_N(item):
            if not pd.isnull(item):
                if "N" in item.upper():
                    return True
            return False

        for item in row:
            if check_N(item):
                return True
        return False

    def __from_dict(self, fw_data, rv_data):
        """Create a pandas dataframe from FastQData objects.

        Args:
            fw_data: ?
            rvc_data: ?

        """
        fw_columns = ['fw_seq',    'fw_seq_score', 'fw_seq_length',
                      'fw_A_perc', 'fw_T_perc',    'fw_G_perc',
                      'fw_C_perc', 'fw_Q-score']
        fw_df = pd.DataFrame(columns=fw_columns, index=fw_data['index'])

        fw_df['fw_seq'] = fw_data['seq']
        fw_df['fw_seq_score'] = fw_data['qual']

        rv_columns = ['rv_seq',        'rvc_seq',   'rv_seq_score',
                      'rv_seq_length', 'rv_A_perc', 'rv_T_perc',
                      'rv_G_perc',     'rv_C_perc', 'rv_Q-score']
        rv_df = pd.DataFrame(columns=rv_columns, index=rv_data['index'])
        rv_df['rv_seq'] = rv_data['seq']
        rv_df['rvc_seq'] = rv_data['comp']
        rv_df['rv_seq_score'] = rv_data['qual']

        del fw_data
        del rv_data

        joined_df = fw_df.join(rv_df)

        del fw_df
        del rv_df

        for newColumn in ['flagged',        'paired_flag',     'fw_a_perc_flag',
                          'fw_t_perc_flag', 'fw_g_perc_flag',  'fw_c_perc_flag', 'fw_Q-score',
                          'rv_a_perc_flag', 'rv_t_perc_flag',  'rv_g_perc_flag',
                          'rv_c_perc_flag', 'rv_Q-score', 'fw_seq_len_flag', 'rv_seq_len_flag',
                          'identity_flag']:
            joined_df[newColumn] = False  # Fill the rows of the new column with False.

        self.__df = joined_df
        self.calcQuality("fw")
        self.calcQuality("rv")

    def set_df(self, df: pd.DataFrame):
        self.__df = df

    def update_by_list(self, data: list, columns: list):
        # Items of data must be the same length as the number of indexes in the dataframe.
        for i, column in enumerate(columns):
            self.__df[column] = [data][i]

    def update_by_series(self, data: pd.Series):
        self.__df.update(data)

    def update_by_dataframe(self, data: pd.DataFrame):
        self.__df.update(data)

    def get_dataframe(self):
        return self.__df

    def combine_dataframes(self, df: pd.DataFrame):
        self.__df = pd.concat([self.__df, df], axis = 1)

    def get_list_columns(self):
        return self.__df.columns

    def get_column(self, column_name: str):
        return self.__df[column_name]

    def get_columns(self, column_names: list):
        return self.__df[column_names]

    def get_uuid(self):
        return self.__uuid

    def filter_between(self, value1, value2, columns: list):
        filtered_df = self.__df.copy()
        for column in columns:
            filtered_df = filtered_df[filtered_df[column].between(value1, value2, inclusive=True)]
        return filtered_df

    def filter_greater_than(self, value, columns: list):
        filtered_df = self.__df.copy()
        for column in columns:
            filtered_df = filtered_df[filtered_df[column] > value]
        return filtered_df

    def filter_smaller_than(self, value, columns: list):
        filtered_df = self.__df.copy()
        for column in columns:
            filtered_df = filtered_df[filtered_df[column] < value]
        return filtered_df

    def filter_equals(self, value, columns: list):
        filtered_df = self.__df.copy()
        for column in columns:
            filtered_df = filtered_df[filtered_df[column] == value]
        return filtered_df

    @staticmethod
    def is_filtered(row):
        try:
            if row['flagged'] == False:
                return True
        except KeyError:
            return False

    def flag_any(self, fwColumns: list=['paired_flag','fw_seq_len_flag','fw_a_perc_flag', 'fw_t_perc_flag','fw_g_perc_flag', 'fw_c_perc_flag','fw_quality_flag', 'identity_flag'],
                        rvColumns: list=['paired_flag','rv_seq_len_flag','rv_a_perc_flag', 'rv_t_perc_flag','rv_g_perc_flag', 'rv_c_perc_flag','rv_quality_flag', 'identity_flag'],
                        flaggedColumns : list=['fw_flagged', 'rv_flagged'], axis=1,
                        fw_flag_column = 'fw_flagged', rv_flag_column = 'rv_flagged', flagged = 'flagged'):
        self.__df[fw_flag_column] = self.__df[fwColumns].any(axis)
        self.__df[rv_flag_column] = self.__df[rvColumns].any(axis)
        self.__df[flagged] = self.__df[flaggedColumns].any(axis)

    def flag_between(self, filter_on, value1, value2, column='flagged'):
        self.__df[column] = self.__df[filter_on].apply(lambda row: False if value1 <= row <= value2 else True)

    @staticmethod
    def __do_between(row, column, value1, value2):
        if value1 <= row[column] <= value2:
            return False
        else:
            return True

    def flag_smaller_than(self, filter_on, value, column='flagged'):
        self.__df[column] = self.__df[filter_on].apply(lambda row: False if row <= value else True)

    @staticmethod
    def __do_smaller(row, column, value):
        if row[column] <= value:
            return False
        else:
            return True

    def flag_greater_than(self, filter_on, value, column='flagged'):
        self.__df[column] = self.__df[filter_on].apply(lambda row: False if row >= value else True)

    @staticmethod
    def __do_greater(row, column, value):
        if row[column] >= value:
            return False
        else:
            return True

    def flag_reset_column(self, column, value = False):
        self.__df[column] = value

    def flag_equals(self, filter_on, value, column='flagged'):
        self.__df[column] = self.__df[filter_on].apply(lambda row: False if row == value else True)

    def flag_indexes(self,indexList, column = 'flagged'):
        self.__df.loc[indexList,column] = True

    @staticmethod
    def __do_equals(row, column, value):
        if row[column] == value:
            return False
        else:
            return True

    def to_json(self):
        return self.__df.to_json()

    def to_csv(self, path):
        return self.__df.to_csv(path)

    def to_pickle(self, path='data/', filename=None):
        if filename:
            with open(abspath(path+filename+".pkl"), "wb") as f:
                pickle.dump(self, f, pickle.HIGHEST_PROTOCOL)
        else:
            with open(abspath(path+self.__uuid+".pkl"), "wb") as f:
                pickle.dump(self, f, pickle.HIGHEST_PROTOCOL)

    @staticmethod
    def load_pickle(file):
        with open(abspath(file), 'rb') as f:
            df = pickle.load(f)
            if type(df) is FastQDataframe:
                return df
            else:
                raise TypeError('Pickle is not a FastQDataframe')

    def get_preprocessOptions(self):
        return self.__preprocOptions

    def get_originalPreprocessOptionsJSON(self):
        return self.__originalPreprocOptions

    def get_blastOptions(self):
        return self.__blastOptions

    #TODO see what options to delete later on.
    def set_preprocessOptions(self, paired_read_percentages = 0,
                                 min_A_perc  = 0,    min_C_perc     = 0,
                                 min_G_perc     = 0,   min_T_perc  = 0,    max_A_perc     = 100,
                                 max_C_perc     = 100, max_G_perc  = 100,  max_T_perc     = 100):

        min_A_perc = int(np.floor(self.__df[['fw_A_perc','rv_A_perc']].min(axis=1).min()))
        min_T_perc = int(np.floor(self.__df[['fw_T_perc','rv_T_perc']].min(axis=1).min()))
        min_G_perc = int(np.floor(self.__df[['fw_G_perc','rv_G_perc']].min(axis=1).min()))
        min_C_perc = int(np.floor(self.__df[['fw_C_perc','rv_C_perc']].min(axis=1).min()))
        max_A_perc = int(np.ceil(self.__df[['fw_A_perc','rv_A_perc']].max(axis=1).max()))
        max_T_perc = int(np.ceil(self.__df[['fw_T_perc','rv_T_perc']].max(axis=1).max()))
        max_G_perc = int(np.ceil(self.__df[['fw_G_perc','rv_G_perc']].max(axis=1).max()))
        max_C_perc = int(np.ceil(self.__df[['fw_C_perc','rv_C_perc']].max(axis=1).max()))

        self.__preprocOptions = {"paired_read_percentages" : paired_read_percentages,
                                 "min_A_perc"  : min_A_perc,  "min_C_perc" : min_C_perc,
                                 "min_G_perc"   : min_G_perc,   "min_T_perc"  : min_T_perc,  "max_A_perc" : max_A_perc,
                                 "max_C_perc"   : max_C_perc,   "max_G_perc"  : max_G_perc,  "max_T_perc" : max_T_perc}
        self.__originalPreprocOptions = self.__preprocOptions.copy()

    def getForwardStats(self, percentageDict):
        self.__df = get_sequence_length(self.__df, True)

        qualityCutoff = self.__df['fw_Q-score'] >= percentageDict['fw_Q_cutoff']

        minA = self.__df['fw_A_perc'] >= percentageDict['min_A_perc']
        minT = self.__df['fw_T_perc'] >= percentageDict['min_T_perc']
        minG = self.__df['fw_G_perc'] >= percentageDict['min_G_perc']
        minC = self.__df['fw_C_perc'] >= percentageDict['min_C_perc']

        maxA = self.__df['fw_A_perc'] <= percentageDict['max_A_perc']
        maxT = self.__df['fw_T_perc'] <= percentageDict['max_T_perc']
        maxG = self.__df['fw_G_perc'] <= percentageDict['max_G_perc']
        maxC = self.__df['fw_C_perc'] <= percentageDict['max_C_perc']
        df = self.__df[qualityCutoff&minA&minT&minG&minC&maxA&maxT&maxG&maxC]
        fwTotal = df['fw_flagged'][df.fw_flagged == False].count()
        avgLength = self.__df.loc[self.__df['fw_flagged'] == False, 'fw_seq_length'].mean()

        return fwTotal, avgLength

    def getReverseStats(self, percentageDict):
        qualityCutoff = self.__df['rv_Q-score'] >= percentageDict['rv_Q_cutoff']

        minA = self.__df['rv_A_perc'] >= percentageDict['min_A_perc']
        minT = self.__df['rv_T_perc'] >= percentageDict['min_T_perc']
        minG = self.__df['rv_G_perc'] >= percentageDict['min_G_perc']
        minC = self.__df['rv_C_perc'] >= percentageDict['min_C_perc']

        maxA = self.__df['rv_A_perc'] <= percentageDict['max_A_perc']
        maxT = self.__df['rv_T_perc'] <= percentageDict['max_T_perc']
        maxG = self.__df['rv_G_perc'] <= percentageDict['max_G_perc']
        maxC = self.__df['rv_C_perc'] <= percentageDict['max_C_perc']
        df = self.__df[qualityCutoff&minA&minT&minG&minC&maxA&maxT&maxG&maxC]
        rvTotal = df['rv_flagged'][df.rv_flagged == False].count()
        avgLength = self.__df.loc[self.__df['rv_flagged'] == False, 'rv_seq_length'].mean()
        return rvTotal, avgLength

    def getMaxLength(self, strand = ""):
        maxFw = max(self.__df[self.__df["fw_seq_length"].notnull()]["fw_seq_length"])
        # maxFw = self.__df.original_fw_seq.map(len).max()
        if strand == "fw":
            return maxFw
        maxRv = max(self.__df[self.__df["rv_seq_length"].notnull()]["rv_seq_length"])
        # maxRv = self.__df.original_rv_seq.map(len).max()
        if strand == "rv":
            return maxRv
        return max(maxFw,maxRv)

    def calcQuality(self,strand):
        self.__df[strand+"_Q-score"] = list(map(lambda qual:
                                                sum(list(map(lambda s:
                                                                ord(s) - 33,
                                                            [i for i in qual])
                                                        )
                                                    ) / len(qual)  if not pd.isnull(qual) and len(qual) != 0 else 0,
                                              list(self.__df.loc[:,strand+"_seq_score"])
                                              )
                                        )

    def get_read_length(self, strand = "fw"):
        return list(map(lambda x: x if not pd.isnull(x) else 0, self.__df[strand+"_seq_length"]))

    def getMinLength(self, strand = ""):
        minFw = min(self.__df[self.__df["fw_seq_length"].notnull()]["fw_seq_length"])
        if strand == "fw":
            return minFw
        minRv = min(self.__df[self.__df["rv_seq_length"].notnull()]["rv_seq_length"])
        if strand == "rv":
            return minRv
        return min(minFw,minRv)

    def getAvgQuality(self):
        avgFw = round(self.__df.loc[self.__df['fw_flagged'] == False, 'fw_Q-score'].mean(), 2)
        avgRv = round(self.__df.loc[self.__df['rv_flagged'] == False, 'rv_Q-score'].mean(), 2)
        return avgFw, avgRv

    def getMaxQuality(self):
        fwMaxQuality = np.floor(self.__df['fw_Q-score'].max())
        rvMaxQuality = np.floor(self.__df['rv_Q-score'].max())
        return fwMaxQuality, rvMaxQuality

    def flag_qual(self, cutoff, strand):
        self.__df[strand+'_quality_flag'] = (self.__df[strand+'_Q-score'] <= cutoff)

    def flag_test(self, preDict):
        self.__df['fw_quality_flag'] = (self.__df['fw_Q-score'] <= preDict['fw_Q_cutoff'])
        self.__df['rv_quality_flag'] = (self.__df['rv_Q-score'] <= preDict['rv_Q_cutoff'])

    def getTotalReads(self):
        fwTotal = self.__df['fw_flagged'][self.__df.fw_flagged == False].count()
        rvTotal = self.__df['rv_flagged'][self.__df.rv_flagged == False].count()
        return (fwTotal, rvTotal)

    def getUnflaggedReads(self):
        return self.__df['flagged'][self.__df.flagged == False].count()

    def setRangeOptions(self, rangeDict):
        self.__preprocOptions.update(rangeDict)

    def set_blastOptions(self, blastOptions):
        # TODO: If you add, change or remove options please also do this in app/containr/preprocessing/parser.py @ makeOptionsDict()
        self.__blastOptions = {"minIdent" : blastOptions["minIdent"], "reward": blastOptions["reward"],
                            "penalty": blastOptions["penalty"],  "gapOpen": blastOptions["gapOpen"],  "gapExtend": blastOptions["gapExtend"]}

    def merge_blastOptionsDicts(self,blastDict):
        self.__blastOptions.update(blastDict)

    def merge_preprocessOptionsDicts(self,preDict):
        self.__preprocOptions.update(preDict)

max_stars_repo_name: JordanDekker/ContainR

max_issues_repo_name: JordanDekker/ContainR

max_forks_repo_name: JordanDekker/ContainR