text:
import logging
import pprint
from typing import Dict, Optional

import numpy

import torch
import torch.nn.functional as F
from allennlp.common.checks import ConfigurationError
from allennlp.data import Vocabulary
from allennlp.models.model import Model
from allennlp.modules import FeedForward, Seq2VecEncoder, TextFieldEmbedder
from allennlp.modules.seq2seq_encoders import (PytorchSeq2SeqWrapper,
                                               Seq2SeqEncoder)
from allennlp.nn import InitializerApplicator, RegularizerApplicator, util
from allennlp.training.metrics import CategoricalAccuracy
from overrides import overrides

logger = logging.getLogger(__name__)
#logger.setLevel(logging.INFO)
logger.setLevel(logging.DEBUG)

# @Model.register("generalizer")
class ActionGeneralizer(Model):
    def __init__(
            self,
            vocab: Vocabulary,
            field_type2embedder: Dict[str, TextFieldEmbedder],
            field_type2seq2vec_encoder: Dict[str, Seq2VecEncoder],
            field_type2seq2seq_encoder: Dict[str, Seq2SeqEncoder],
            classifier_feedforward: FeedForward,
            initializer: InitializerApplicator = InitializerApplicator(),
            regularizer: Optional[RegularizerApplicator] = None,
            framework2field_type2embedder: Dict[str, Dict[
                str, TextFieldEmbedder]] = None,
    ) -> None:
        super(ActionGeneralizer, self).__init__(vocab, regularizer)
        self.field_types = [
            'word', 'pos', 'resolved', 'token_node_label',
            'token_node_prev_action'
        ]
        self.field_type2embedder = torch.nn.ModuleDict(field_type2embedder)
        assert all(field_type in self.field_type2embedder
                   for field_type in self.field_types)
        self.framework2field_type2embedder = torch.nn.ModuleDict(
            framework2field_type2embedder)
        self.field_type2seq2vec_encoder = torch.nn.ModuleDict(
            field_type2seq2vec_encoder)
        self.classifier_feedforward = classifier_feedforward
        self.action_type_loss = torch.nn.CrossEntropyLoss()
        self.metrics = {
            "accuracy": CategoricalAccuracy(),
        }
        initializer(self)

    @overrides
    def forward(
            self,
            framework,
            parse_node_labels: Dict[str, torch.LongTensor],
            parse_node_lemmas: Dict[str, torch.LongTensor],
            parse_node_uposs: Dict[str, torch.LongTensor],
            parse_node_xposs: Dict[str, torch.LongTensor],
            curr_node_id,
            action_type: torch.LongTensor,
            action_params,
            token_state,
            token_node_resolveds: Dict[str, torch.LongTensor],
            token_node_labels: Dict[str, torch.LongTensor],
            token_node_prev_actions: Dict[str, torch.LongTensor],
            token_node_ids,
            token_node_childs,
    ) -> Dict[str, torch.Tensor]:

        logger.debug(('curr_node_id', curr_node_id[0]))
        logger.debug(('action_types', action_type[0]))
        logger.debug(('action_paramss', action_params[0]))
        logger.debug(('token_states', token_state[0]))
        logger.debug(('token_node_resolveds', token_node_resolveds))
        logger.debug(('token_node_labels', token_node_labels))
        logger.debug(('token_node_prev_actions', token_node_prev_actions))

        seq2vec_fields = [
            # (parse_node_labels, 'word'),
            # (parse_node_lemmas, 'word'),
            # (parse_node_uposs, 'pos'),
            # (parse_node_xposs, 'pos'),
            (token_node_resolveds, 'resolved'),
            (token_node_labels, 'token_node_label'),
            (token_node_prev_actions, 'token_node_prev_action'),
        ]

        embedded_fields = []
        for field, field_type in seq2vec_fields:
            embedded_fields.append(self.field_type2embedder[field_type](field))

        logger.debug(('embedded_fields', embedded_fields[0].shape))

        encoded_fields = []
        for seq2vec_field, embedded_field in zip(seq2vec_fields,
                                                 embedded_fields):
            field, field_type = seq2vec_field
            field_mask = util.get_text_field_mask(field)
            logger.debug(('field_mask', field_mask.shape))
            encoded_field = self.field_type2seq2vec_encoder[field_type](
                embedded_field, field_mask)
            encoded_fields.append(encoded_field)

        logits = self.classifier_feedforward(torch.cat(encoded_fields, dim=-1))

        output_dict = {'logits': logits}
        logger.debug(('output_dict', output_dict))

        if action_type is not None:
            logger.debug(('logits', logits))
            logger.debug(('label', action_type))
            action_type_loss = self.action_type_loss(logits, action_type)
            for metric in self.metrics.values():
                metric(logits, action_type)
            output_dict['loss'] = action_type_loss

        # raise NotImplementedError
        return output_dict

    @overrides
    def decode(self, output_dict: Dict[str, torch.Tensor]
               ) -> Dict[str, torch.Tensor]:
        """
        Does a simple argmax over the class probabilities, converts indices to string labels, and
        adds a ``"label"`` key to the dictionary with the result.
        """
        class_probabilities = F.softmax(output_dict['logits'], dim=-1)
        output_dict['class_probabilities'] = class_probabilities

        predictions = class_probabilities.cpu().data.numpy()
        argmax_indices = numpy.argmax(predictions, axis=-1)
        labels = [
            self.vocab.get_token_from_index(x, namespace="labels")
            for x in argmax_indices
        ]
        output_dict['label'] = labels
        return output_dict

    @overrides
    def get_metrics(self, reset: bool = False) -> Dict[str, float]:
        return {
            metric_name: metric.get_metric(reset)
            for metric_name, metric in self.metrics.items()
        }

# # @Model.register("generalizer")
# class ActionGeneralizerOld(Model):
#     def __init__(
#             self,
#             vocab: Vocabulary,
#             word_embedder: TextFieldEmbedder,
#             pos_embedder: TextFieldEmbedder,
#             seq2vec_encoder: Seq2VecEncoder,
#             classifier_feedforward: FeedForward,
#             initializer: InitializerApplicator = InitializerApplicator(),
#             regularizer: Optional[RegularizerApplicator] = None,
#     ) -> None:
#         super(ActionGeneralizer, self).__init__(vocab, regularizer)
#         self.word_embedder = word_embedder
#         logger.debug(('word_embedder', word_embedder))
#         self.pos_embedder = pos_embedder
#         logger.debug(('pos_embedder', pos_embedder))
#         self.seq2vec_encoder = seq2vec_encoder
#         self.classifier_feedforward = classifier_feedforward
#         self.loss = torch.nn.CrossEntropyLoss()
#         self.metrics = {
#             "accuracy": CategoricalAccuracy(),
#         }
#         initializer(self)

#     @overrides
#     def forward(
#             self,
#             curr_parse_node_label: Dict[str, torch.LongTensor],
#             curr_parse_node_lemma: Dict[str, torch.LongTensor],
#             curr_parse_node_upos: Dict[str, torch.LongTensor],
#             curr_parse_node_xpos: Dict[str, torch.LongTensor],
#             prev_parse_node_labels: Dict[str, torch.LongTensor],
#             prev_parse_node_lemmas: Dict[str, torch.LongTensor],
#             prev_parse_node_uposs: Dict[str, torch.LongTensor],
#             prev_parse_node_xposs: Dict[str, torch.LongTensor],
#             next_parse_node_labels: Dict[str, torch.LongTensor],
#             next_parse_node_lemmas: Dict[str, torch.LongTensor],
#             next_parse_node_uposs: Dict[str, torch.LongTensor],
#             next_parse_node_xposs: Dict[str, torch.LongTensor],
#             action_type: torch.LongTensor,
#     ) -> Dict[str, torch.Tensor]:

#         features = [
#             curr_parse_node_label,
#             curr_parse_node_lemma,
#             curr_parse_node_upos,
#             curr_parse_node_xpos,
#             prev_parse_node_labels,
#             prev_parse_node_lemmas,
#             prev_parse_node_uposs,
#             prev_parse_node_xposs,
#             next_parse_node_labels,
#             next_parse_node_lemmas,
#             next_parse_node_uposs,
#             next_parse_node_xposs,
#         ]

#         parse_features = [
#             ('curr_parse_node_label', curr_parse_node_label, 'word'),
#             ('curr_parse_node_lemma', curr_parse_node_lemma, 'word'),
#             ('curr_parse_node_upos', curr_parse_node_upos, 'pos'),
#             ('curr_parse_node_xpos', curr_parse_node_xpos, 'pos'),
#             ('prev_parse_node_labels', prev_parse_node_labels, 'word'),
#             ('prev_parse_node_lemmas', prev_parse_node_lemmas, 'word'),
#             ('prev_parse_node_uposs', prev_parse_node_uposs, 'pos'),
#             ('prev_parse_node_xposs', prev_parse_node_xposs, 'pos'),
#             ('next_parse_node_labels', next_parse_node_labels, 'word'),
#             ('next_parse_node_lemmas', next_parse_node_lemmas, 'word'),
#             ('next_parse_node_uposs', next_parse_node_uposs, 'pos'),
#             ('next_parse_node_xposs', next_parse_node_xposs, 'pos'),
#         ]

#         embedded_features = []
#         for field_name, field, field_type in parse_features:
#             if field_type == 'word':
#                 embedded_features.append(self.word_embedder(field))
#             elif field_type == 'pos':
#                 embedded_features.append(self.pos_embedder(field))

#         encoded_features = []
#         for feature, embedded_feature in zip(features, embedded_features):
#             feature_mask = util.get_text_field_mask(feature)
#             encoded_feature = self.seq2vec_encoder(embedded_feature,
#                                                    feature_mask)
#             encoded_features.append(encoded_feature)

#         logits = self.classifier_feedforward(
#             torch.cat(encoded_features, dim=-1))

#         output_dict = {'logits': logits}

#         if action_type is not None:
#             logger.debug(('logits', logits))
#             logger.debug(('label', action_type))
#             loss = self.loss(logits, action_type)
#             for metric in self.metrics.values():
#                 metric(logits, action_type)
#             output_dict['loss'] = loss

#         return output_dict

#     @overrides
#     def decode(self, output_dict: Dict[str, torch.Tensor]
#                ) -> Dict[str, torch.Tensor]:
#         """
#         Does a simple argmax over the class probabilities, converts indices to string labels, and
#         adds a ``"label"`` key to the dictionary with the result.
#         """
#         class_probabilities = F.softmax(output_dict['logits'], dim=-1)
#         output_dict['class_probabilities'] = class_probabilities

#         predictions = class_probabilities.cpu().data.numpy()
#         argmax_indices = numpy.argmax(predictions, axis=-1)
#         labels = [
#             self.vocab.get_token_from_index(x, namespace="labels")
#             for x in argmax_indices
#         ]
#         output_dict['label'] = labels
#         return output_dict

#     @overrides
#     def get_metrics(self, reset: bool = False) -> Dict[str, float]:
#         return {
#             metric_name: metric.get_metric(reset)
#             for metric_name, metric in self.metrics.items()
#         }

# # @Model.register("generalizer")
# class Generalizer(Model):
#     def __init__(
#             self,
#             vocab: Vocabulary,
#             word_embedder: TextFieldEmbedder,
#             pos_embedder: TextFieldEmbedder,
#             seq2vec_encoder: Seq2VecEncoder,
#             classifier_feedforward: FeedForward,
#             initializer: InitializerApplicator = InitializerApplicator(),
#             regularizer: Optional[RegularizerApplicator] = None,
#     ) -> None:
#         super(Generalizer, self).__init__(vocab, regularizer)
#         self.word_embedder = word_embedder
#         logger.debug(('word_embedder', word_embedder))
#         self.pos_embedder = pos_embedder
#         logger.debug(('pos_embedder', pos_embedder))
#         self.seq2vec_encoder = seq2vec_encoder
#         self.classifier_feedforward = classifier_feedforward
#         self.loss = torch.nn.CrossEntropyLoss()
#         initializer(self)

#     @overrides
#     def forward(
#             self,
#             parse_label: Dict[str, torch.LongTensor],
#             parse_lemma: Dict[str, torch.LongTensor],
#             parse_upos: Dict[str, torch.LongTensor],
#             parse_xpos: Dict[str, torch.LongTensor],
#             label: torch.LongTensor,
#     ) -> Dict[str, torch.Tensor]:

#         features = [
#             parse_label,
#             parse_lemma,
#             parse_upos,
#             parse_xpos,
#         ]

#         embedded_parse_label = self.word_embedder(parse_label)
#         embedded_parse_lemma = self.word_embedder(parse_lemma)
#         embedded_parse_upos = self.pos_embedder(parse_upos)
#         embedded_parse_xpos = self.pos_embedder(parse_xpos)

#         embedded_features = [
#             embedded_parse_label,
#             embedded_parse_lemma,
#             embedded_parse_upos,
#             embedded_parse_xpos,
#         ]

#         encoded_features = []
#         for feature, embedded_feature in zip(features, embedded_features):
#             feature_mask = util.get_text_field_mask(feature)
#             encoded_feature = self.seq2vec_encoder(embedded_feature,
#                                                    feature_mask)
#             encoded_features.append(encoded_feature)

#         logits = self.classifier_feedforward(
#             torch.cat(encoded_features, dim=-1))

#         output_dict = {'logits': logits}

#         if label is not None:
#             logger.debug(('label', label))
#             loss = self.loss(logits, label)
#             output_dict['loss'] = loss

#         return output_dict

#     @overrides
#     def decode(self, output_dict: Dict[str, torch.Tensor]
#                ) -> Dict[str, torch.Tensor]:
#         """
#         Does a simple argmax over the class probabilities, converts indices to string labels, and
#         adds a ``"label"`` key to the dictionary with the result.
#         """
#         class_probabilities = F.softmax(output_dict['logits'], dim=-1)
#         output_dict['class_probabilities'] = class_probabilities

#         predictions = class_probabilities.cpu().data.numpy()
#         argmax_indices = numpy.argmax(predictions, axis=-1)
#         labels = [
#             self.vocab.get_token_from_index(x, namespace="labels")
#             for x in argmax_indices
#         ]
#         output_dict['label'] = labels
#         return output_dict


text:
import numpy as np

nafe_drake_poly = np.poly1d([0.000106, -0.0043, 0.0671, -0.4721, 1.6612, 0.0])

def nafe_drake(velocity_p):
    """Nafe-Drake's correlation for density."""
    return nafe_drake_poly(velocity_p)

ext: py

max_stars_repo_name: keurfonluu/evodcinv

max_issues_repo_name: keurfonluu/evodcinv

max_forks_repo_name: keurfonluu/evodcinv

reason: import numpy


text:
# #!/usr/bin/env python3

# Copyright (c) Facebook, Inc. and its affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# Download and build the data if it does not exist.

import os
import re
import json
import pandas as pd
import numpy as np
import copy
import random
from parlai.core.build_data import DownloadableFile
import parlai.core.build_data as build_data
import parlai.utils.logging as logging
from parlai.utils.io import PathManager
from typing import Tuple, Dict, List

from math import isclose
from collections import OrderedDict

from icecream import ic
from parlai.scripts.eval_model import eval_model

RESOURCES = {
    'convai2': DownloadableFile(
        'http://parl.ai/downloads/convai2/convai2_fix_723.tgz',
        'convai2_fix_723.tgz',
        'd0ae89defe2fd0b0a4221eaa642a457d7d40cef475f54798119c7f3b8dd9361d',
    ),
    'wizard_of_wikipedia': DownloadableFile(
        'http://parl.ai/downloads/wizard_of_wikipedia/wizard_of_wikipedia.tgz',
        'wizard_of_wikipedia.tgz',
        '2a549627a83fea745efa2076a41d1c0078ad002ab2b54eae6a4e3d3d66ae24b7',
    ),
    'empatheticdialogues': DownloadableFile(
        'http://parl.ai/downloads/empatheticdialogues/empatheticdialogues.tar.gz',
        'empatheticdialogues.tar.gz',
        '56f234d77b7dd1f005fd365bb17769cfe346c3c84295b69bc069c8ccb83be03d',
    )
}

SPLIT_RATIO = OrderedDict({'train': 0.8, 'valid': 0.1, 'test': 0.1})

def build(opt):
    version = 'v0.0'
    dpath = os.path.join(opt['datapath'], 'pbst')

    if not build_data.built(dpath, version):
        subtaskpaths = []

        for subtask in opt['subtasks']:
            subtaskpath = os.path.join(opt['datapath'], subtask)
            subtaskpaths.append(subtaskpath)

        logging.info('building data: ' + dpath)
        if build_data.built(dpath):
            # An older version exists, so remove these outdated files.
            build_data.remove_dir(dpath)
        build_data.make_dir(dpath)

        # Download the data.
        for subtask, subtaskpath in zip(opt['subtasks'], subtaskpaths):
            if not build_data.built(subtaskpath):
                build_data.make_dir(subtaskpath)
                downloadable_file = RESOURCES[subtask]
                downloadable_file.download_file(subtaskpath)

        # Mark the data as built.
        for subtaskpath in subtaskpaths:
            build_data.mark_done(subtaskpath, version)

        if 'empatheticdialogues' in opt['subtasks']:
            # Move empatheticdialogues to parent directory
            # (ED 데이터셋만 내부폴더가 하나 더 생긴다. tar.gz라서 그런듯.)
            from shutil import move
            ed_path = subtaskpaths[opt['subtasks'].index('empatheticdialogues')]
            srcdir = os.path.join(ed_path, 'empatheticdialogues')
            if os.path.isdir(srcdir):
                for filename in os.listdir(srcdir):
                    move(os.path.join(srcdir, filename), os.path.join(ed_path, filename))
                os.rmdir(os.path.join(ed_path, 'empatheticdialogues'))

        context, random_candidates = _build_context_and_response(opt, subtaskpaths)

        blended_context_path = os.path.join(dpath, 'blended_context.jsonl')
        with open(blended_context_path, 'w') as fout:
            # json.dump(context, fout)
            for dic in context:
                json.dump(dic, fout)
                fout.write("\n")

        context_splits = _split(context, dpath, SPLIT_RATIO, randomized=True)

        _create_parlai_format(dpath, opt)

        # Mark the data as built.
        build_data.mark_done(dpath, version)

def _convai_parser(filepath
    ) -> Tuple[List[str], List[str], List[str], List[str]]:

    debug = False

    print('Parsing ConvAI2 on', filepath)
    leading_contexts, following_contexts, seed_list, responses = [], [], [], []

    # Pre-processing
    with open(filepath, 'r') as file:
        lines = file.readlines()
    for i, line in enumerate(lines):
        lines[i] = re.sub(r"^[0-9]+", "", line).strip()

    # Collecting
    persona1, persona2, seed_pair = [], [], None
    episode_done = False
    all_seed_pairs = []

    def roleswap(persona_list):
        for i, persona_sent in enumerate(persona_list):
            if persona_sent.startswith("partner's persona: "):
                persona_list[i] = persona_sent.replace("partner's persona: ", "your persona: ")
            else:
                persona_list[i] = persona_sent.replace("your persona: ", "partner's persona: ")
        return persona_list

    for i, line in enumerate(lines):
        if debug and i < 50:
            print('Line', i, line) # for debug
        if line.startswith("partner's persona: "):
            persona1.append(line)
        elif line.startswith("your persona: "):
            persona2.append(line)
            episode_done = False
        elif not episode_done:
            seed_pair = line.split('\t')
            assert len(seed_pair) == 2
            if seed_pair[0] == '__SILENCE__':
                nextline = lines[i+1]
                nextpair = nextline.split('\t')
                seed_list.append([seed_pair[1], nextpair[0]])
                leading_contexts.append('\n'.join(roleswap(persona2)))
                following_contexts.append('\n'.join(roleswap(persona1)))
                # print(seed_list[-1]) for roleswap debug
                # print(leading_contexts[-1])
                # print(following_contexts[-1])
                # input()
                all_seed_pairs.append([seed_pair[1], nextpair[0]])
            else:
                seed_list.append(seed_pair)
                leading_contexts.append('\n'.join(persona1))
                following_contexts.append('\n'.join(persona2))
                all_seed_pairs.append(seed_pair)
            responses.extend([seed_pair[0], seed_pair[1]])
            episode_done = True
            persona1, persona2, seed_pair = [], [], []
        else:
            utt_pair = line.split('\t')
            responses.extend([utt_pair[0], utt_pair[1]])
            all_seed_pairs.append([utt_pair[0], utt_pair[1]])

    if debug:
        for i, (leading_context, following_context, seed) in enumerate(zip(leading_contexts, following_contexts, seed_list)):
            if i == 2:
                break
            print(leading_context)
            print(following_context)
            print(seed)
            input()
        for i, response in enumerate(responses):
            if i == 20:
                break
            print(response)
            input()

    return leading_contexts, following_contexts, seed_list, responses, all_seed_pairs

def _wizard_of_wikipedia_parser(filepath
    ) -> Tuple[List[str], List[str], List[str], List[str]]:

    debug = False

    print('Parsing wizard_of_wikipedia on', filepath)
    leading_contexts, following_contexts, seed_list, responses = [], [], [], []
    all_seed_pairs = []

    topic_list, passage_list, persona_list = [], [], []
    with open(filepath, 'r') as file:
        wow = json.load(file)
    for i, episode in enumerate(wow):
        # keys: 1 'chosen_topic', 1 'chosen_topic_passage (title?)', X 'persona', 1 'wizard_eval', X 'dialog'
        topic = episode['chosen_topic']
        persona = episode['persona']
        wizard_eval = episode['wizard_eval']
        dialog = episode['dialog']
        chosen_topic_passage = episode['chosen_topic_passage']

        if debug and i < 2:
            print('topic', topic)
            print('persona', persona)
            print('wizard_eval', wizard_eval)
            print('chosen_topic_passage', chosen_topic_passage)
            print('len passage', len(chosen_topic_passage))
            print('len dialog', len(dialog))
            print('utts', [utt['text'] for utt in dialog])
        # # each utt in dialog has keys: 1 speaker, 1 text, 1 checked_passage (title), 1 checked_sentence (article), 7 or 10 retrieved_passages
        # input()

        # 모든 시간대에 사용한 지식 sentence를 전부 제공한다. (o) vs 너무 길기 때문에 처음만 제공한다. (x)
        passage = ' '.join(chosen_topic_passage)
        # passage = chosen_topic_passage[0]
        if dialog[0]['speaker'].endswith('Apprentice'):
            # 1_Apprentice first and 0_Wizard second
            for i in range(0, len(dialog), 2):
                if i+1 < len(dialog):
                    seed_pair = [dialog[i]['text'], dialog[i+1]['text']]
                    all_seed_pairs.append(seed_pair)
            seed_pair = [utt['text'] for utt in dialog[:2]]
        else:
            # 0_Wizard first and 1_Apprentice second
            for i in range(1, len(dialog), 2):
                if i+1 < len(dialog):
                    seed_pair = [dialog[i]['text'], dialog[i+1]['text']]
                    all_seed_pairs.append(seed_pair)
            seed_pair = [utt['text'] for utt in dialog[1:3]]
        topic_list.append(topic)
        passage_list.append(passage)
        persona_list.append(persona)
        seed_list.append(seed_pair)
        responses.extend([utt['text'] for utt in dialog])

        # # for debug
        # for j, utt_dic in enumerate(dialog):
        #     # keys: 'speaker', 'text', 'candidate_responses', 'retrieved_passages', 'retrieved_topics'
        #     speaker = utt_dic['speaker']
        #     text = utt_dic['text']
        #     if 'checked_sentence' in utt_dic and 'checked_passage' in utt_dic: # guiding only
        #         checked_sentence = utt_dic['checked_sentence']
        #         checked_passage = utt_dic['checked_passage']
        #     if 'candidate_responses' in utt_dic: # guided only
        #         candidate_responses = utt_dic['candidate_responses'] # 100 utterance sentences
        #     retrieved_passages = utt_dic['retrieved_passages'] # 7 topics as key with its passage as value
        #     retrieved_topics = utt_dic['retrieved_topics'] # 7 topics same above
        #     if j < 3:
        #         print('Speaker', speaker, text)
        #         input()

    assert  len(topic_list) == len(passage_list) == len(seed_list) == len(persona_list)

    for topic, passage, persona in zip(topic_list, passage_list, persona_list):
        # leading_contexts.append(f'Topic: {topic}\nPersona: {persona}') # Persona 떼어냈다.
        leading_contexts.append(f'topic: {topic}')
        following_contexts.append(f'topic: {topic}\nknowledge: {passage}')

    assert len(leading_contexts) == len(following_contexts) == len(seed_list)

    # for debug
    if debug:
        for i, (leading_context, following_context, seed) in enumerate(zip(leading_contexts, following_contexts, seed_list)):
            if i == 5:
                break
            print(leading_context)
            print(following_context)
            print(seed)
            input()

        for i, response in enumerate(responses):
            if i == 20:
                break
            print(response)
            input()

    return leading_contexts, following_contexts, seed_list, responses, all_seed_pairs

def _empatheticdialogues_parser(filepath
    ) -> Tuple[List[str], List[str], List[str], List[str]]:

    debug = False

    print('Parsing empatheticdialogues on', filepath)
    leading_contexts, following_contexts, seed_list, responses = [], [], [], []
    all_seed_pairs = []

    situation_list, emotion_list, seed_list = [], [], []

    # Preprocessing
    df = pd.read_csv(filepath, usecols=range(8), sep=',', lineterminator='\n', quotechar="`")
    df['prompt'] = df['prompt'].str.replace('_comma_', ',')
    df['utterance'] = df['utterance'].str.replace('_comma_', ',')

    # Collecting
    situation_list = df.groupby('conv_id').agg({'prompt':lambda x: list(x)[0]})['prompt']
    emotion_list = df.groupby('conv_id').agg({'context':lambda x: list(x)[0]})['context']
    seed_list = df.groupby('conv_id').agg({'utterance':lambda x: list(x)[:2]})['utterance']
    responses = df['utterance']

    for i in range(0, len(df['utterance']), 2):
        if i + 1 < len(df['utterance']):
            all_seed_pairs.append([df['utterance'].iloc[i], df['utterance'].iloc[i+1]])

    # Check
    remove_idx = []
    for i, seed in enumerate(seed_list):
        # Some episodes have only one turn! So we drop the such rows,
        # as we cannot get a pair of initial utterances
        if len(seed) != 2:
            remove_idx.append(i)
    situation_list = situation_list.drop(situation_list.index[remove_idx]).tolist()
    emotion_list = emotion_list.drop(emotion_list.index[remove_idx]).tolist()
    seed_list = seed_list.drop(seed_list.index[remove_idx]).tolist()
    responses = responses.tolist() # We don't when we're gathering utterances

    assert len(situation_list) == len(emotion_list) == len(seed_list)

    # Added extra information of emotion labels to the leader. Fair enough?
    for situation, emotion in zip(situation_list, emotion_list):
        leading_contexts.append(f'situation: {situation}\nemotion: {emotion}')
        following_contexts.append(f'')
    assert len(leading_contexts) == len(following_contexts) == len(seed_list)

    if debug:
        for i, (leading_context, following_context, seed) in enumerate(zip(leading_contexts, following_contexts, seed_list)):
            if i == 2:
                break
            print(leading_context)
            print(following_context)
            print(seed)
            input()
        for i, response in enumerate(responses):
            if i == 20:
                break
            print(response)
            input()
    return leading_contexts, following_contexts, seed_list, responses, all_seed_pairs

def parser_switch():
    parser_switch = {
        'convai2': {
            'files': ['train_both_original_no_cands.txt', 'valid_both_original_no_cands.txt'],
            'func': _convai_parser,
        },
        'wizard_of_wikipedia': {
            'files': ['train.json', 'valid_random_split.json', 'test_random_split.json'],
            'func': _wizard_of_wikipedia_parser,
        },
        'empatheticdialogues': {
            'files': ['train.csv', 'valid.csv', 'test.csv'],
            'func': _empatheticdialogues_parser,
        }
    }
    return parser_switch

def _parse_task_dataset(subtask, subtaskpath
    ) -> Tuple[List[str], List[str], List[List[str]]]:
    # Collect the context and the seed utterance pair of each episode
    leading_contextss, following_contextss, seeds, responses = [], [], [], []
    all_seed_pairs = []

    # Identify correct parser and iterate all files to parse
    parser = parser_switch()[subtask]
    for file in parser['files']:
        filepath = os.path.join(subtaskpath, file)
        fleading_contextss, ffollowing_contextss, fseeds, fresponses, fall_seed_pairs = parser['func'](filepath)
        leading_contextss.extend(fleading_contextss)
        following_contextss.extend(ffollowing_contextss)
        seeds.extend(fseeds)
        responses.extend(fresponses)
        all_seed_pairs.extend(fall_seed_pairs)
    return leading_contextss, following_contextss, seeds, responses, all_seed_pairs

def _retrieve_contextual_document(origin_opt, seed_queries, contextual_docs, teacher, origin, target, subtaskpath):
    '''
        retreival: query (seed utterances) -> document (contexts)
    '''
    retrieved_doc = []

    # Semantic Retreival (e.g. poly-encoder, DPR)
    if teacher == 'semantic':
        parlai_data_path = subtaskpath[:subtaskpath.find('pbst')]

        opt = {}
        if target == 'convai2':
            opt['task'] = 'persona_inference:retrieval'
        elif target == 'wizard_of_wikipedia':
            opt['task'] = 'topic_inference:retrieval'
        elif target == 'empatheticdialogues':
            opt['task'] = 'emotion_inference:retrieval'
        else:
            raise RuntimeError('Unimplemented subtask')

        split = opt['task'].split(':')

        # --world-logs true --report-filename ~/bst/convai_generation.json
        # opt['model_file'] = '/home/minju/bst/models/' + split[0] + '/tmp/model'
        opt['model'] = 'transformer/biencoder'
        opt['eval_candidates'] = 'inline'
        opt['fixed_candidates_path'] = parlai_data_path + split[0] + '/fixed_candidates.txt'
        opt['batchsize'] = 256
        opt['datatype'] = 'retrieval'
        opt['world_logs'] = parlai_data_path + split[0] + '/retrieval_report.json'
        opt['report_filename'] = parlai_data_path + split[0] + '/retrieval_report.json'
        opt['log_keep_fields'] = 'all'
        opt['num_examples'] = -1
        opt['display_examples'] = False
        opt['save_format'] = 'conversations'

        eval_list = []

        candidates_path = parlai_data_path + split[0] + '/fixed_candidates.txt'
        f = open(candidates_path, 'r')
        candidates = f.readlines()
        f.close()

        for query in seed_queries:
            # input_dict = {'text': query, 'label_candidates': candidates}
            input_dict = {'text': query, 'labels': candidates[0]}
            eval_list.append(input_dict)

        with open(parlai_data_path + split[0] + '/retrieval.json', "w") as json_file:
            json.dump(eval_list, json_file)
        print("Saved queries to", parlai_data_path + split[0] + '/retrieval.json')

        eval_model(opt)

        # Open retrieval result (jsonl file)
        retrieval_result_path = opt['report_filename'] + 'l'

        with open(retrieval_result_path, 'r') as json_file:
            json_list = list(json_file)
        retrieval_result = []
        for json_str in json_list:
            result = json.loads(json_str)
            retrieval_result.append(result)

        retrieved_doc = []
        for retrieved in retrieval_result:
            retrieved_doc.append(retrieved['dialog'][0][1]['text'])

    # Random Retrieval
    elif teacher == 'random':
        doc_ids = list(range(len(contextual_docs)))
        retrieved_doc_idx = random.choices(doc_ids, k=len(seed_queries))
        retrieved_doc = contextual_docs[retrieved_doc_idx]

    # TODO Manual Retrieval (e.g. BST -> 이 경우 context가 좀 더 단순해져야 한다. 현재 leading/following 불필요)

    # Lexical Retrieval??
    elif teacher == 'lexical_retrieval':

        parlai_data_path = origin_opt['datapath']
        src2trg = f'{origin}->{target}'

        # Loading TF-IDF Retriever Model
        tr_opt = {}
        if target.startswith('convai2'):
            task = 'persona_inference'
        elif target.startswith('wizard_of_wikipedia'):
            task = 'topic_inference'
        elif target.startswith('empatheticdialogues'):
            task = 'emotion_inference'
        else:
            raise RuntimeError('Unimplemented retrieval task')
        tr_opt['task'] = f'{task}:{teacher}'
        tr_opt['model'] = 'tfidf_retriever'
        tr_opt['model_file'] = f'{parlai_data_path}/pbst/contextual_alignment/{task}/{teacher}/model'
        tr_opt['eval_candidates'] = 'inline'
        tr_opt['fixed_candidates_path'] = None
        tr_opt['batchsize'] = 256
        tr_opt['datatype'] = f'{teacher}/{src2trg}/query'
        tr_opt['label_candidates_file'] = f'{parlai_data_path}/pbst/contextual_alignment/{task}/fixed_candidates.txt'
        tr_opt['world_logs'] = f'{parlai_data_path}/pbst/contextual_alignment/{task}/{teacher}/{src2trg}/results.jsonl'
        tr_opt['report_filename'] = f'{parlai_data_path}/pbst/contextual_alignment/{task}/{teacher}/{src2trg}/model_report.json'
        tr_opt['log_keep_fields'] = 'all'
        tr_opt['num_examples'] = -1
        tr_opt['display_examples'] = False
        tr_opt['save_format'] = 'conversations'

        # prepare candidate for the retrieval, which are contextual documents
        # TODO 민주는 fixed candidates을 leading following 다 뭉쳐놨다. 나는 분리할 줄 알았다. 이 둘을 구분하는 것은 안 중요한가?
        with open(tr_opt['label_candidates_file'], 'r') as f:
            candidates = f.readlines()

        # prepare queries of the retireval, which are the seed utterances
        eval_list = []
        if target.startswith('wizard_of_wikipedia'):
            leading_queries = seed_queries[0]
            following_queries = seed_queries[1]
            for i in range(len(leading_queries)):
                input_dict = {'text': leading_queries[i] + ' ' + following_queries[i], 'labels': candidates[0]} # tfidf ranker는 이렇게 붙어있는애에 학습이 안되어있어서 성능이 안나올수도 있다. 이걸 쓰려면 tf-idf도 다시 학습해야할수도 잇음
                eval_list.append(input_dict)
        else:
            for query in seed_queries:
                input_dict = {'text': query, 'labels': candidates[0]}
                eval_list.append(input_dict)

        # save the queries as files
        retrieval_query_path = f'{parlai_data_path}/pbst/contextual_alignment/{task}/{teacher}/{src2trg}/query.json'
        retrieval_dirpath = retrieval_query_path.rsplit('/', 1)[0]
        if os.path.exists(retrieval_dirpath):
            build_data.remove_dir(retrieval_dirpath)
        build_data.make_dir(retrieval_dirpath)
        with open(retrieval_query_path, "w+") as json_file:
            json.dump(eval_list, json_file)
        print("Saved queries to", retrieval_query_path)

        # run parali retreival task, which then saves the retrieval results in as a 'world_logs' file
        eval_model(tr_opt)

        # load retrieval result (as in jsonl file) to read and return the retrieved documents
        with open(tr_opt['world_logs'], 'r') as json_file:
            json_list = list(json_file)

        retrieval_result = []
        for json_str in json_list:
            result = json.loads(json_str)
            if 'text' not in result['dialog'][0][1]:
                result['dialog'][0][1]['text'] = ""
                # result['dialog'][0][1]['candidate_ids'] = []
                # result['dialog'][0][1]['text_candidates'] = []
                # result['dialog'][0][1]['candidate_scores'] = []
            retrieval_result.append(result)

        for retrieved in retrieval_result:
            retrieved_doc.append(retrieved['dialog'][0][1]['text'])

        print('*'*5, "Contextual Alignment Example", '*'*5)
        print("Query:", seed_queries[0])
        print("Retreived Document:", retrieved_doc[0])
        print()

    return retrieved_doc

def _build_context_and_response(opt, subtaskpaths):
    # contexts are different: for leading speaker and following speaker
    subtasks, nsubtasks = opt['subtasks'], len(opt['subtasks'])
    leading_context_dic, following_context_dic, seed_dic = {}, {}, {} # seed pairs are concatenated into a sentence
    response_candidates = {}

    # Collect task-wise contexts and seeds
    for subtask, subtaskpath in zip(subtasks, subtaskpaths):
        leading_contexts, following_contexts, seeds, responses, all_seed_pairs = _parse_task_dataset(subtask, subtaskpath)
        lc = os.path.join(opt['datapath'], 'pbst', f'leading_{subtask}_kb.json')
        fc = os.path.join(opt['datapath'], 'pbst', f'following_{subtask}_kb.json')
        su = os.path.join(opt['datapath'], 'pbst', f'seed_utterance_pairs_{subtask}.json')
        rc = os.path.join(opt['datapath'], 'pbst', f'responses_candidates_{subtask}.json')
        asp = os.path.join(opt['datapath'], 'pbst', f'all_seed_utterance_pairs_{subtask}.json')
        with open(lc, 'w') as f0, open(fc, 'w') as f1, open(su, 'w') as f2, open(rc, 'w') as f3, open(asp, 'w') as f4:
            json.dump(leading_contexts, f0)
            json.dump(following_contexts, f1)
            json.dump(seeds, f2)
            json.dump(responses, f3)
            json.dump(all_seed_pairs, f4)
        assert len(leading_contexts) == len(following_contexts) == len(seeds)
        leading_context_dic[subtask] = leading_contexts
        following_context_dic[subtask] = following_contexts
        seed_dic[subtask] = all_seed_pairs
        response_candidates[subtask] = responses
        print(f'{len(seeds)} contexts were parsed from the {subtask}')
        print(f'{len(all_seed_pairs)} seed utterances pairs were parsed from the {subtask}')
        print(f'Also, {len(responses)} response candidates were parsed from the {subtask}\n')

    # writing the maximum pool of response candidate sampled from each task datasets
    response_candidates = np.array([response.strip() for response_set in response_candidates.values() for response in response_set])
    response_candidates = np.unique(response_candidates).tolist()
    rc = os.path.join(opt['datapath'], 'pbst', f'responses_candidates.json')
    with open(rc, 'w') as f:
        json.dump(response_candidates, f)
    print(f'\nTotal of {len(response_candidates)} response candidates were parsed from the given {", ".join(subtasks)} subtasks\n')

    # Contextual alignment
    lcm = leading_contextual_matrix = [[None]*nsubtasks for _ in range(nsubtasks)]
    fcm = following_contextual_matrix = copy.deepcopy(lcm)

    # Align inter-task relationhip between seeds and contexts
    for i, origin in enumerate(subtasks):
        seed_pairs = np.array(seed_dic[origin])
        leading_seeds, following_seeds = seed_pairs[:,0], seed_pairs[:,1]
        for j, (target, subtaskpath) in enumerate(zip(subtasks, subtaskpaths)):
            leading_contexts = leading_context_dic[target]
            following_contexts = following_context_dic[target]
            # Retrieve contextual document from different task
            # And align the seed with all the other subtask's context
            if target == 'convai2':
                lcm[i][j] = _retrieve_contextual_document(opt, leading_seeds, leading_contexts, 'lexical_retrieval', origin, target+'_1', subtaskpath)
                fcm[i][j] = _retrieve_contextual_document(opt, following_seeds, following_contexts, 'lexical_retrieval', origin, target+'_2', subtaskpath)
            elif target == 'wizard_of_wikipedia':
                lcm[i][j] = _retrieve_contextual_document(opt, [leading_seeds, following_seeds], leading_contexts, 'lexical_retrieval', origin, target+'_1', subtaskpath)
                fcm[i][j] = copy.deepcopy(lcm[i][j]) # 일단은 fcm이 lcm을 따라가도록 해두었음
                for k in range(len(lcm[i][j])):
                    lcm[i][j][k] = lcm[i][j][k].split('\n')[0] # knowledge resource는 잘리도록...
            else:
                lcm[i][j] = _retrieve_contextual_document(opt, leading_seeds, leading_contexts, 'lexical_retrieval', origin, target+'_1', subtaskpath)
                fcm[i][j] = [''] * len(lcm[i][j])

    context = []

    for srctaskid, srctask in enumerate(subtasks):

        context_length = len(lcm[srctaskid][0])
        alignedseed = seed_dic[srctask]

        for context_id in range(context_length):
            episode = {}
            episode['source_task'] = srctask
            episode['leader'] = {}
            episode['follower'] = {}

            episode['leader']['seed'] = alignedseed[context_id][0]
            episode['follower']['seed'] = alignedseed[context_id][1]
            episode['leader']['context'] = {}
            episode['follower']['context'] = {}

            for alignedtaskid, alignedtask in enumerate(subtasks):
                try:
                    episode['leader']['context'][alignedtask] = lcm[srctaskid][alignedtaskid][context_id] # 여기서 오류남
                except:
                    ic(context_id)
                    raise RuntimeError

            for alignedtaskid, alignedtask in enumerate(subtasks):
                episode['follower']['context'][alignedtask] = fcm[srctaskid][alignedtaskid][context_id]

            context.append(episode)

    # for git merge
    # # for debug
    # for i, episode in enumerate(context):
    #     print('Episode', i, '*'*40)
    #     print('1 Source task', episode['source_task'])
    #     print('2 Leader\'s context')
    #     leader = episode['leader']
    #     for task, context in leader['context'].items():
    #         print('2+ Task', task)
    #         print(context)
    #     print('3 Leader\'s seed')
    #     print(leader['seed'])
    #     print()
    #     print('4 Follower\'s context')
    #     follower = episode['follower']
    #     for task, context in follower['context'].items():
    #         print('4+ Task', task)
    #         print(context)
    #     print('5 Follower\'s seed')
    #     print(follower['seed'])
    #     input()

    return context, response_candidates

def _split(json_list, dpath, split_ratio: OrderedDict, randomized=True):
    # sr = split_ratio = OrderedDict({'train': 0.6, 'val': 0.3, 'test': 0.1})

    # filepath = '/home/theorist/data3/ParlAI/data/pbst/pbst.jsonl'
    # with open(filepath, 'r') as json_file:
    #     data = json_list = [json.loads(jline) for jline in json_file.read().splitlines()]

    data = json_list

    # TODO 버그 있을 것만 같은 코드
    ds = dataset_size = len(json_list)

    assert isclose(sum([v for v in split_ratio.values()]), 1) # escape overflow
    ss = splitset_size = {k: round(ds * v) for k, v in split_ratio.items()}

    # Random sampling
    if randomized:
        random.shuffle(data)

    def greedy_split(data, sample_ratio):
        split_index = int(round(len(data) * sample_ratio, 6)) # 6의 자리 반올림
        sampled_data = data[:split_index]
        remained_data = data[split_index:]
        return sampled_data, remained_data

    original_length = len(data)
    sd = split_data = OrderedDict({split_name: None for split_name in split_ratio.keys()})

    remained_ratio = 1
    for i, (split_name, split_ratio) in enumerate(split_ratio.items()):
        sample_ratio = split_ratio / remained_ratio
        sd[split_name], data = greedy_split(data, sample_ratio)
        remained_ratio -= split_ratio

    assert isclose(remained_ratio, 0, abs_tol=1e-5), "Errors in split ratio"
    split_lengths = [len(d) for d in sd.values()]
    assert sum(split_lengths) == original_length, "Some samples in datset is remained after split"

    for split_name, dataset in sd.items():
        bc = os.path.join(dpath, f'blended_context_{split_name}.jsonl')
        pbc = os.path.join(dpath, f'pretty_blended_context_{split_name}.jsonl')
        with open(bc, 'w') as outputfile, open(pbc, 'w') as prettyfile:
            for sample in dataset:
                assert isinstance(sample, Dict)
                json.dump(sample, outputfile)
                outputfile.write('\n')
                json.dump(sample, prettyfile, indent=4)
                prettyfile.write('\n')
    return split_data

def _create_parlai_format(dpath: str, opt: List):
    """
    Copy data into the format read by ParlAIDialogTeacher.

    'text' will be from the free Turker, who speaks first, and 'label' will be from the
    guided Turker.
    """
    datatypes = ['train', 'valid', 'test']
    for datatype in datatypes:

        load_path = os.path.join(dpath, f'blended_context_{datatype}.jsonl')
        save_path = os.path.join(dpath, f'blended_context_{datatype}.txt')

        print(f'Loading {load_path}.')
        # with PathManager.open(load_path, 'r', encoding='utf8') as f_read:
        #     data = json.load(f_read)
        data = []
        with PathManager.open(load_path, 'r', encoding='utf8') as f_read:
            for line in f_read:
                data.append(json.loads(line))

        print(f'Saving to {save_path}')
        subtasks = opt['subtasks']
        with PathManager.open(save_path, 'w', encoding='utf8') as f_write:
            for episode in data:
                num_entries = 1
                entry_idx = 0
                for entry_idx in range(num_entries):
                    line = _get_line(
                        episode, num_entries, entry_idx, subtasks
                    )
                    f_write.write(f'{line} \n')

def _get_line(episode: dict, num_entries: int, entry_idx: int, subtasks: List) -> str:
    """
    Return the line to print in the reformatted file.
    """
    episode_done = entry_idx == num_entries - 1

    if entry_idx == 0:
        leader_context = '\n'.join([f"{episode['leader']['context'][task]}" for task in subtasks])
        follower_context = '\n'.join([f"{episode['follower']['context'][task]}" for task in subtasks])
        context_dataset = f"context dataset: {episode['source_task']}"
        original_context = '\n'.join([leader_context, follower_context, context_dataset]) + '\n'

    else:
        original_context = ''
    input_utterance = episode['leader']['seed']
    model_label = episode['follower']['seed']
    source_task = episode['source_task']

    # Compile into text string
    parts = {
        'text': input_utterance,
        'labels': model_label,
        'source_task': source_task
    }
    assert all([isinstance(part, str) for part in parts.values()])
    line = '\t'.join([f'{key}:{_escape(value)}' for key, value in parts.items()])

    # Add episode_done
    if episode_done:
        line += '\tepisode_done:True'

    return line

def _escape(value: str) -> str:
    return value.replace('\t', '\\t').replace('\n', '\\n').replace('|', '__PIPE__')

max_stars_repo_name: convei-lab/ParlAI

max_issues_repo_name: convei-lab/ParlAI

max_forks_repo_name: convei-lab/ParlAI


text:
#####################################################
#               thresholdFuntions.py                #
#     made by Iván Montalvo & Santiago Buitrago     #
#                                                   #
# The idea behind this definition is to do a        #
# threshold given a limit. The idea is to           #
# binarize the image and reduce the noise.          #
#                                                   #
#####################################################

import cv2 as cv
import numpy as np

def thresh(img):
    # Here we want to do an adaptative threshold.
    # The idea of this threshold is to clean the
    # noise and let the image in a binary format.
    #
    # The input of this function is the paragraph
    # that we want to analyze.
    tLimit = 120

    medianBlur = cv.medianBlur(img, 5)
    th1 = cv.adaptiveThreshold(medianBlur, 255, cv.ADAPTIVE_THRESH_GAUSSIAN_C, \
                               cv.THRESH_BINARY, 11, 2)

    gaussianBlur = cv.GaussianBlur(img, (3, 3), -1)
    _, th2 = cv.threshold(gaussianBlur, 0, 255, cv.THRESH_BINARY + cv.THRESH_OTSU)

    total = th1 + th2

    total[total < tLimit] = 0
    total[total > tLimit] = 255

    tt = total.copy()
    tt[tt > tLimit] = 1

    kernel = np.ones((2, 2))
    cl2 = cv.morphologyEx(tt, cv.MORPH_CLOSE, kernel)
    op2 = cv.morphologyEx(cl2, cv.MORPH_OPEN, kernel)

    return op2


text:
"""This module contains kinematic model of a hand compatible with the glove."""

from dataclasses import dataclass
from typing import Dict, Sequence, Tuple

import numpy as np
from transforms3d import affines, euler

from .data import GloveSample

__all__ = ('HandModel')

Vec3f = Tuple[float, float, float]
Mat4f = np.ndarray

@dataclass(frozen=True)
class Link:
    """Link data helper class."""

    name: str
    parent: str
    euler: Vec3f
    length: float

class HandModel:
    """Kinematic hand model to interpret the glove sensor data."""

    def __init__(self):
        """Initialize the model."""
        self._links = (
            Link('wrist', 'root', (0.0, 0.0, 0.0), 0.060),
            Link('hand', 'wrist', (0.0, 0.0, 0.0), 0.030),
            Link('thumb0', 'hand', (0.0, 5.0, 115.0), 0.045),
            Link('thumb1', 'thumb0', (0.0, 0.0, -70.0), 0.060),
            Link('thumb2', 'thumb1', (0.0, 0.0, 0.0), 0.028),
            Link('thumb3', 'thumb2', (0.0, 0.0, 0.0), 0.025),
            Link('index0', 'hand', (0.0, 0.0, 15.0), 0.090),
            Link('index1', 'index0', (0.0, 0.0, 5.0), 0.040),
            Link('index2', 'index1', (0.0, 0.0, 0.0), 0.028),
            Link('index3', 'index2', (0.0, 0.0, 0.0), 0.020),
            Link('middle0', 'hand', (0.0, 0.0, 0.0), 0.085),
            Link('middle1', 'middle0', (0.0, 0.0, 0.0), 0.035),
            Link('middle2', 'middle1', (0.0, 0.0, 0.0), 0.032),
            Link('middle3', 'middle2', (0.0, 0.0, 0.0), 0.025),
            Link('ring0', 'hand', (0.0, 0.0, -15.0), 0.085),
            Link('ring1', 'ring0', (0.0, 0.0, -5.0), 0.030),
            Link('ring2', 'ring1', (0.0, 0.0, 0.0), 0.028),
            Link('ring3', 'ring2', (0.0, 0.0, 0.0), 0.025),
            Link('little0', 'hand', (0.0, 0.0, -35.0), 0.075),
            Link('little1', 'little0', (0.0, 0.0, -5.0), 0.030),
            Link('little2', 'little1', (0.0, 0.0, 0.0), 0.025),
            Link('little3', 'little2', (0.0, 0.0, 0.0), 0.017),
        )

    @property
    def links(self) -> Sequence[Link]:
        """Get links of the hand model.

        Returns:
            Sequence[Link] -- list of links
        """
        return self._links

    @property
    def tip_names(self) -> Sequence[str]:
        """Get names of tip links.

        Returns:
            Sequence[str] -- list of names
        """
        return ['thumb3', 'index3', 'middle3', 'ring3', 'little3']

    def angles(self, sample: GloveSample) -> Dict[str, Vec3f]:
        """Convert sensor data to euler angles in joints.

        Arguments:
            sample {GloveSample} -- data from the glove

        Returns:
            Dict[str, Vec3f] -- euler angles (in degrees) by link name
        """
        # pylint: disable=no-self-use
        w_roll, w_pitch, w_yaw = np.rad2deg(euler.quat2euler(sample.wrist_quat, 'sxyz'))
        _, h_pitch, _ = np.rad2deg(euler.quat2euler(sample.hand_quat, 'sxyz'))
        return {
            'wrist': (-w_roll, -w_pitch, w_yaw),
            'hand': (0.0, w_pitch-h_pitch, 0.0),
            'thumb0': (0.0, 57.5 * (sample.palm_arch + sample.thumb_cross_over), 0.0),
            'thumb1': (0.0, 20.0 * sample.pip_joints[0], -25.0 * sample.abductions[0]),
            'thumb2': (0.0, 30.0 * sample.pip_joints[0], 0.0),
            'thumb3': (0.0, 85.0 * sample.dip_joints[0], 0.0),
            'index0': (0.0, 0.0, 0.0),
            'index1': (0.0, 70.0 * sample.pip_joints[1], -25.0 * sample.abductions[1]),
            'index2': (0.0, 100.0 * sample.dip_joints[1], 0.0),
            'index3': (0.0, 30.0 * sample.dip_joints[1], 0.0),
            'middle0': (0.0, 0.0, 0.0),
            'middle1': (0.0, 70.0 * sample.pip_joints[2], 0.0),
            'middle2': (0.0, 100.0 * sample.dip_joints[2], 0.0),
            'middle3': (0.0, 30.0 * sample.dip_joints[2], 0.0),
            'ring0': (0.0, 0.0, 0.0),
            'ring1': (0.0, 70.0 * sample.pip_joints[3], 25.0 * sample.abductions[2]),
            'ring2': (0.0, 100.0 * sample.dip_joints[3], 0.0),
            'ring3': (0.0, 30.0 * sample.dip_joints[3], 0.0),
            'little0': (0.0, 0.0, 0.0),
            'little1': (0.0, 70.0 * sample.pip_joints[4], 25.0 * sample.abductions[3]),
            'little2': (0.0, 100.0 * sample.dip_joints[4], 0.0),
            'little3': (0.0, 30.0 * sample.dip_joints[4], 0.0),
        }

    def frames(self,
               sample: GloveSample,
               fixed_frame: Tuple[str, Mat4f] = None) -> Dict[str, Mat4f]:
        """Convert sensor data to link frames.

        Arguments:
            sample {GloveSample} -- data from the glove

        Keyword Arguments:
            fixed_frame {Tuple[str, Mat4f]} -- known link frame transformation (default: {None})

        Returns:
            Dict[str, Mat4f] -- dict of link frame transformation matrix (4x4) by link name
        """
        joints = self.angles(sample)
        frames = {'root': np.eye(4)}
        for link in self._links:
            rotate = euler.euler2mat(*np.deg2rad(np.add(link.euler, joints[link.name])), 'sxyz')
            frame = affines.compose(np.dot(rotate, [link.length, 0, 0]), rotate, [1, 1, 1])
            frames[link.name] = np.dot(frames[link.parent], frame)

        if fixed_frame is not None:
            link_name, frame = fixed_frame
            shift = np.dot(frame, np.linalg.inv(frames[link_name]))
            for link_name in frames:
                frames[link_name] = np.dot(shift, frames[link_name])

        return frames

    def points(self,
               sample: GloveSample,
               fixed_frame: Tuple[str, Mat4f] = None) -> Dict[str, Vec3f]:
        """Convert sensor data to link positions.

        Arguments:
            sample {GloveSample} -- data from the glove

        Keyword Arguments:
            fixed_frame {Tuple[str, Mat4f]} -- known frame transformation (default: {None})

        Returns:
            Dict[str, Vec3f] -- dict of link frame position by link name
        """
        frames = self.frames(sample, fixed_frame)
        points = {link: (*m44[:3, 3],) for link, m44 in frames.items()}

        return points


text:
import numpy as np
import cv2
from funcy import print_durations
import itertools

def read_file(file):
    data = []
    with open(file, "r") as f:
        for line in f:
            data.append(list(line.strip()))

    return np.array(data, dtype=int)

def step(octos):

    any_has_flashed = True

    has_flashed_this_step = np.zeros_like(octos, dtype=bool)
    energy_kernel = np.ones((3, 3), np.uint8)

    # first, the energy level of each octopus increases by 1
    octos += 1

    while any_has_flashed:

        # then, any octopus with an energy level greater than 9 flashes
        flashing = octos > 9

        # an octopus can only flash at most once per step
        flashing &= ~has_flashed_this_step
        any_has_flashed = flashing.any()

        # this increases the energy level of all adjacent octopuses by 1
        # including octopuses that are diagonally adjacent

        # wm: that's wrong - an octopuses energy can increase by more than 1
        #     per loop, if it has more than 1 neighbor flashing
        # energy_increase = cv2.dilate(flashing.astype(np.uint8), kernel)

        energy_increase = cv2.filter2D(src=flashing.astype(np.uint8), ddepth=-1, kernel=energy_kernel, borderType=cv2.BORDER_ISOLATED)
        octos += energy_increase

        # if this causes an octopus to have an energy level greater than 9, it also flashes
        # This process continues as long as new octopuses keep having their energy level increased beyond 9

        has_flashed_this_step |= flashing

    # finally, any octopus that flashed during this step has its energy level set to 0
    # as it used all of its energy to flash
    octos[has_flashed_this_step] = 0

    return has_flashed_this_step.sum()

@print_durations
def day11(file):

    octos = read_file(file)

    total_flashes = 0

    octos = read_file(file)
    for i in itertools.count(start=1):
        num_flashes = step(octos)
        total_flashes += num_flashes

        # puzzle a
        if i == 100:
            yield total_flashes

        # puzzle b
        if num_flashes == np.prod(octos.shape):
            assert i > 100
            yield i
            break

if __name__ == "__main__":
    test_a, test_b = day11("test_input.txt")
    solution_a, solution_b = day11("input.txt")

    print(f"Day 11a test: {test_a}")
    print(f"Day 11b test: {test_b}")

    print(f"Day 11a solution: {solution_a}")
    print(f"Day 11b solution: {solution_b}")

    assert test_a == 1656
    assert test_b == 195

    assert solution_a == 1585
    assert solution_b == 382


text:
import os
import unittest
from pprint import pprint

import numpy as np

import anago
from anago.reader import load_data_and_labels, load_glove
from anago.utils import download

get_path = lambda path: os.path.join(os.path.dirname(__file__), path)
DATA_ROOT = get_path('../data/conll2003/en/ner')
SAVE_ROOT = get_path('models')  # trained model
LOG_ROOT = get_path('logs')     # checkpoint, tensorboard
EMBEDDING_PATH = get_path('../data/glove.6B/glove.6B.100d.txt')

class TrainerTest(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        if not os.path.exists(LOG_ROOT):
            os.mkdir(LOG_ROOT)

        if not os.path.exists(SAVE_ROOT):
            os.mkdir(SAVE_ROOT)

        train_path = os.path.join(DATA_ROOT, 'train.txt')
        valid_path = os.path.join(DATA_ROOT, 'valid.txt')
        test_path = os.path.join(DATA_ROOT, 'test.txt')

        cls.x_train, cls.y_train = load_data_and_labels(train_path)
        cls.x_valid, cls.y_valid = load_data_and_labels(valid_path)
        cls.x_test, cls.y_test = load_data_and_labels(test_path)

        cls.embeddings = load_glove(EMBEDDING_PATH)

        cls.words = 'President Obama is speaking at the White House.'.split()

        cls.dir_path = 'models'

    def test_train(self):
        # Not use pre-trained word embeddings
        model = anago.Sequence(max_epoch=1)
        model.train(self.x_train, self.y_train, self.x_valid, self.y_valid)

        # Use pre-trained word embeddings
        model = anago.Sequence(max_epoch=1, embeddings=self.embeddings)
        model.train(self.x_train, self.y_train, self.x_valid, self.y_valid)

    def test_eval(self):
        model = anago.Sequence(max_epoch=1, embeddings=self.embeddings)
        model.train(self.x_train, self.y_train, self.x_valid, self.y_valid)
        model.eval(self.x_test, self.y_test)

    def test_analyze(self):
        model = anago.Sequence(max_epoch=1, embeddings=self.embeddings)
        model.train(self.x_train, self.y_train, self.x_valid, self.y_valid)
        res = model.analyze(self.words)
        pprint(res)

        self.assertIn('words', res)
        self.assertIn('entities', res)

    def test_save(self):
        model = anago.Sequence(max_epoch=1, embeddings=self.embeddings)
        model.train(self.x_train, self.y_train, self.x_valid, self.y_valid)
        model.save(dir_path=self.dir_path)

        config_file = os.path.join(self.dir_path, model.config_file)
        weight_file = os.path.join(self.dir_path, model.weight_file)
        preprocessor_file = os.path.join(self.dir_path, model.preprocessor_file)

        self.assertTrue(os.path.exists(config_file))
        self.assertTrue(os.path.exists(weight_file))
        self.assertTrue(os.path.exists(preprocessor_file))

    def test_load(self):
        model = anago.Sequence(max_epoch=1, embeddings=self.embeddings)
        model.train(self.x_train, self.y_train, self.x_valid, self.y_valid)
        model.eval(self.x_test, self.y_test)
        model.save(dir_path=self.dir_path)

        model = anago.Sequence.load(self.dir_path)
        model.eval(self.x_test, self.y_test)

    def test_train_vocab_init(self):
        vocab = set()
        for words in np.r_[self.x_train, self.x_valid, self.x_test]:
            for word in words:
                vocab.add(word)
        model = anago.Sequence(max_epoch=15, embeddings=self.embeddings, log_dir='logs')
        model.train(self.x_train, self.y_train, self.x_test, self.y_test, vocab_init=vocab)
        model.save(dir_path=self.dir_path)

    def test_train_all(self):
        x_train = np.r_[self.x_train, self.x_valid, self.x_test]
        y_train = np.r_[self.y_train, self.y_valid, self.y_test]
        model = anago.Sequence(max_epoch=15, embeddings=self.embeddings, log_dir='logs')
        model.train(x_train, y_train, self.x_test, self.y_test)
        model.save(dir_path=self.dir_path)

    def test_download(self):
        dir_path = 'test_dir'
        url = 'https://storage.googleapis.com/chakki/datasets/public/models.zip'
        download(url, dir_path)
        model = anago.Sequence.load(dir_path)
        model.eval(self.x_test, self.y_test)

max_stars_repo_name: slouvan/anago

max_issues_repo_name: slouvan/anago


text:
#!/usr/bin/python3.7

from tkinter import *
from models import cube
from models.cube import n_triangles
import camera, time, math
from geometry import Triangle, rotate, translate
from numpy import arange

def edgeFunction(x1, y1, x2, y2, p1, p2):
    cond = ((p1-x1)*(y2-y1)-(p2-y1)*(x2-x1))
    if cond > 0:
        return True
    else:
        return False

def rasterize(v, c, res):
    colors = [
        "#e42256",
        "#ff8370",
        "#00b1b0",
        "#fec84d",
    ]

    for i in range(-500, 200, res):
        for j in range(-300, 300, res):
            if (edgeFunction(v[0][0], v[0][1], v[1][0], v[1][1], i, j) and
                edgeFunction(v[2][0], v[2][1], v[0][0], v[0][1], i, j) and
                edgeFunction(v[1][0], v[1][1], v[2][0], v[2][1], i, j)):
                canvas.create_oval(w/2+i, h/2+j, w/2+i, h/2+j, width=6, outline=colors[c%4])

def drawLine(x1, y1, x2, y2):
        canvas.create_line(w/2+x1, h/2+y1, w/2+x2, h/2+y2, fill="#F64C72", width=2.5)

def drawFrame():
    for i, t in enumerate(triangles):
        vertices2d.clear()
        # for each triangle vertex compute the trasformation from 3d space to 2d space
        for v in t:
            x = (camera.x - v[0])/((-camera.z + v[2] + 100) * 0.001)
            y = (camera.y - v[1])/((-camera.z + v[2] + 100) * 0.001)
            vertices2d.append((x, y))

        if wireframeOn:
            drawLine(vertices2d[0][0], vertices2d[0][1], vertices2d[1][0], vertices2d[1][1])
            drawLine(vertices2d[0][0], vertices2d[0][1], vertices2d[2][0], vertices2d[2][1])
            drawLine(vertices2d[1][0], vertices2d[1][1], vertices2d[2][0], vertices2d[2][1])

        rasterize(vertices2d, i, 8)

def animateRotation():
    canvas.delete("all")
    rotate(triangles)
    #translate(triangles, 0.02, 0.02, 0.0)
    drawFrame()
    canvas.after(20, animateRotation)

def loadModel():
    for t in n_triangles:
        triangles.append(t)

if __name__ == '__main__':
    root = Tk()
    w = root.winfo_screenwidth()/1.5
    h = root.winfo_screenheight()/1.5
    root.wm_attributes("-type", "splash")
    canvas = Canvas(root, width=w, height=h, bg="#ffffff")
    canvas.pack()

    camX = 3 ; camY = 5 ; camZ = 65
    camera = camera.Camera(camX, camY, camZ)

    wireframeOn = False

    #vertices = [ origin ]
    vertices2D = []
    triangles = []
    edges = []

    vertices3d = [] # not used for now
    vertices2d = []

    loadModel()
    animateRotation()

    root.mainloop()

max_stars_repo_name: git-audo/poli-renderer

max_issues_repo_name: git-audo/poli-renderer

max_forks_repo_name: git-audo/poli-renderer

reason: from numpy


text:
import cv2
import numpy as np
from dg_util.python_utils import misc_util
from dg_util.python_utils import pytorch_util as pt_util

def draw_attention(image, attention, height_channel=0, width_channel=1):
    dtype = image.dtype
    image = pt_util.to_numpy(image)
    attention = pt_util.to_numpy(attention)
    attention = np.clip(attention, 0, 1)
    im_width = image.shape[width_channel]
    im_height = image.shape[height_channel]
    attention = misc_util.resize(
        attention,
        (im_width, im_height),
        interpolation=cv2.INTER_LINEAR,
        height_channel=height_channel,
        width_channel=width_channel,
    )
    image, attention = pt_util.fix_broadcast(image, attention)
    image = (image * (1 - attention) + 255 * attention).astype(dtype)
    return image

max_stars_repo_name: gabrielsluz/vince

max_issues_repo_name: gabrielsluz/vince

max_forks_repo_name: gabrielsluz/vince


text:
import unittest
import os
import datetime

import numpy as np

from gpm_repo import gpm_wrapper

class TestGMPImergWrapper(unittest.TestCase):
    def setUp(self):
        package_dir = os.path.dirname(os.path.dirname(__file__))
        repo_dir = os.path.dirname(package_dir)
        test_dir = os.path.join(repo_dir, 'data', 'gpm_sample_data')
        onefile = os.listdir(test_dir)[0]
        abs_onefile = os.path.join(test_dir, onefile)
        self.gpm = gpm_wrapper.GPMImergeWrapper(abs_onefile)

    def test_start_dt(self):
        self.assertIsInstance(self.gpm.start_dt, datetime.datetime)

    def test_end_dt(self):
        self.assertIsInstance(self.gpm.end_dt, datetime.datetime)

    def test_str_attrib(self):
        self.assertIsInstance(self.gpm.abspath, str)
        self.assertIsInstance(self.gpm.basename, str)
        self.assertTrue(self.gpm.abspath.endswith(self.gpm.basename))

    def test_precipCal(self):
        self.assertIsInstance(self.gpm.precipCal, np.ndarray)
        self.assertEqual(self.gpm.precipCal.shape[0], 3600)
        self.assertEqual(self.gpm.precipCal.dtype, np.float32)

max_stars_repo_name: sbsimo/gpm-accumul

max_issues_repo_name: sbsimo/gpm-accumul

max_forks_repo_name: sbsimo/gpm-accumul


text:
from __future__ import (absolute_import, division, print_function)
# from future.utils import iteritems

from collections import namedtuple
import netCDF4 as nc
import numpy as np
import xarray as xr

class ModelOutput(object):
    def __init__(self, filename, varname, startdate=None, enddate=None, nhm_hrus=None, nhm_segs=None):
        self.__filename = filename
        self.__varname = varname
        self.__stdate = startdate
        self.__endate = enddate
        self.__nhm_hrus = nhm_hrus
        self.__nhm_segs = nhm_segs
        self.__data = None
        self.__dataset = None

        self.read_netcdf()

    @property
    def data(self):
        return self.__data

    @property
    def dataset(self):
        return self.__dataset

    def nearest(self, items, pivot):
        return min(items, key=lambda x: abs(x - pivot))

    def get_var(self, varname):
        data = None

        if self.__stdate is not None and self.__endate is not None:
            if self.__nhm_hrus:
                data = self.__dataset[varname].loc[self.__stdate:self.__endate, self.__nhm_hrus].to_pandas()
            elif self.__nhm_segs:
                data = self.__dataset[varname].loc[self.__stdate:self.__endate, self.__nhm_segs].to_pandas()
        else:
            if self.__nhm_hrus:
                data = self.__dataset[varname].loc[:, self.__nhm_hrus].to_pandas()
            elif self.__nhm_segs:
                data = self.__dataset[varname].loc[:, self.__nhm_segs].to_pandas()

        return data

    def read_netcdf(self):
        """Read model output file stored in netCDF format"""
        if self.__nhm_hrus:
            # print('\t\tOpen dataarray')

            self.__dataset = xr.open_dataset(self.__filename, chunks={'hru': 1000})
            # self.__dataset = xr.open_dataarray(self.__filename, chunks={'hru': 1000})

            # print('\t\tConvert subset to pandas dataframe')
            # self.__data = ds.loc[:, self.__nhm_hrus].to_pandas()
            #
            # # print('\t\tRestrict to date range')
            # if self.__stdate is not None and self.__endate is not None:
            #     # Restrict dataframe to the given date range
            #     self.__data = self.__data[self.__stdate:self.__endate]
            #
            # self.__dataarray = ds.loc[:, self.__nhm_hrus]
            #
        elif self.__nhm_segs:
            self.__dataset = xr.open_dataset(self.__filename, chunks={'segment': 1000})
        # print('\t\tInsert date info')
        # self.__data['year'] = self.__data.index.year
        # self.__data['month'] = self.__data.index.month
        # self.__data['day'] = self.__data.index.day
        # self.ds = ds
        # self.__data['hour'] = 0
        # self.__data['minute'] = 0
        # self.__data['second'] = 0

    def write_csv(self, pathname=None, fileprefix=None):

        data = self.get_var(self.__varname)

        if self.__nhm_hrus:
            data.to_csv(f'{pathname}/{self.__varname}.csv', columns=self.__nhm_hrus,
                        sep=',', index=True, header=True, chunksize=50)
        elif self.__nhm_segs:
            data.to_csv(f'{pathname}/{self.__varname}.csv', columns=self.__nhm_segs,
                        sep=',', index=True, header=True, chunksize=50)

    def write_netcdf(self, pathname=None):
        # Write output variable to a netcdf file

        if self.__nhm_hrus:
            ss = self.__dataset[self.__varname].loc[self.__stdate:self.__endate, self.__nhm_hrus]
            ss.to_netcdf(f'{pathname}/{self.__varname}.nc', mode='w', format='NETCDF4',
                         encoding = {'time': {'dtype': 'float32', 'calendar': 'standard', '_FillValue': None},
                                     'hru': {'_FillValue': None}})
        elif self.__nhm_segs:
            ss = self.__dataset[self.__varname].loc[self.__stdate:self.__endate, self.__nhm_segs]
            ss.to_netcdf(f'{pathname}/{self.__varname}.nc', mode='w', format='NETCDF4',
                         encoding={'time': {'dtype': 'float32', 'calendar': 'standard', '_FillValue': None},
                                   'segment': {'_FillValue': None}})

max_stars_repo_name: paknorton/Band

max_issues_repo_name: paknorton/Band

max_forks_repo_name: paknorton/Bandit


text:
"""

"""

import numpy as np
import scipy as sp
import pylab as py
from .kmeans import KMEANSC
from .spectralCluster import SCC
from .k_medoids import KmedoidsC
__all__ = ['KMEANSC','SCC','KmedoidsC'
]

reason: import numpy,import scipy


text:
import pdb

import copy
import numpy

class DynamicSystem(object):
    def __init__(
        self,
        X0,
        dt,
        sys_dynamics_fcn,
        output_fcn=None,
        input_fcn=None):
        """ Constructor

        Arguments:
            X0: initial state
            dt: time step size
            sys_dynamics_fcn: function which defines the system dynamics. It
                takes in X and U and returns Xdot
            output_fcn: output function, takes X and returns Y
            input_fcn: a function to determine input from the state
                for example a controller

        Returns:
            object instance
        """
        self._X = X0
        self._Xdot = X0 * 0.0
        self._U = None
        self.Y = 0

        self._dynamics_fcn = sys_dynamics_fcn
        self._output_fcn = output_fcn
        self._input_fcn = input_fcn

        self._dt = dt

    def rk4(self, U=None):
        """ Integrate the system forward one timestep using an RK4 method

        Arguments:
            U: input, optional. If an input function was specified and this is
                not specified, then the input function will be used instead

        Returns:
            no returns
        """
        if U is None and self._input_fcn is not None:
            U = self._input_fcn(self._X)
        assert U is not None, 'Either U or an input function must be specified'
        k1 = self._dynamics_fcn(self._X, U)
        k2 = self._dynamics_fcn(self._X + self._dt/2.0 * k1, U)
        k3 = self._dynamics_fcn(self._X + self._dt/2.0 * k2, U)
        k4 = self._dynamics_fcn(self._X + self._dt * k3, U)
        self._Xdot = 1.0 / 6.0 * (k1 + 2.0 * k2 + 2.0 * k3 + k4)
        self._X += self._dt * self._Xdot
        self._U = U

    def euler(self, U=None):
        """ Integrate the system forward one timestep using an euler method

        Arguments:
            U: input, optional. If an input function was specified and this is
                not specified, then the input function will be used instead

        Returns:
            no returns
        """
        if U is None and self._input_fcn is None:
            U = self._input_fcn(self._X)
        assert U is not None, 'Either U or an input function must be specified'
        self._Xdot = self._dynamics_fcn(self._X, U)
        self._X += self._dt * self._Xdot
        self._U = U

    def output(self):
        """ Compute the system output

        Arguments:
            no arguments

        Returns:
            Y: the system output
        """
        if self._output_fcn:
            return self._output_fcn(self._X)

    @property
    def state(self):
        """Getter for the state
        Arguments:
            no arguments

        Returns:
            state
        """
        return copy.deepcopy(self._X)

    @property
    def state_dot(self):
        """Getter for state derivative from last step

        Arguments:
            no arguments

        Returns:
            state_dot: state time derivative
        """
        return self._Xdot

    @property
    def input(self):
        """Get the last input
        """
        return self._U

max_stars_repo_name: aivian/robots

max_issues_repo_name: aivian/robots

max_forks_repo_name: aivian/robots


text:
__author__ = "Christian Kongsgaard"
__license__ = 'MIT'

# -------------------------------------------------------------------------------------------------------------------- #
# IMPORTS

# Modules
import os
import pytest
import numpy as np
import platform

# RiBuild Modules
from delphin_6_automation.sampling import sampling
from delphin_6_automation.database_interactions import sampling_interactions
from delphin_6_automation.database_interactions import delphin_interactions
from delphin_6_automation.database_interactions import simulation_interactions
from delphin_6_automation.database_interactions.db_templates import delphin_entry
from delphin_6_automation.database_interactions.db_templates import sample_entry
from delphin_6_automation.backend import sampling_worker
from delphin_6_automation.file_parsing import delphin_parser

# -------------------------------------------------------------------------------------------------------------------- #
# RIBuild

@pytest.fixture(params=[5, 15], ids=['5 dimensions ', '15 dimensions '])
def add_mock_strategy(empty_database, request):
    sampling_settings = {'initial samples per set': 1,
                         'add samples per run': 1,
                         'max samples': 500,
                         'sequence': 10,
                         'standard error threshold': 0.01,
                         'raw sample size': 2 ** 9}
    distributions = {str(i): 'test distribution'
                     for i in range(request.param)}
    strategy = {'design': {'Test Design': ''}, 'scenario': {'Test Scenario': ''},
                'distributions': distributions, 'settings': sampling_settings}
    strategy_id = sampling_interactions.upload_sampling_strategy(strategy)

    return strategy_id

@pytest.fixture()
def mock_sampling_distribution(monkeypatch):
    def mockreturn(sampling_strategy, samples_raw, used_samples_per_set):
        new_samples_per_set = sampling_strategy['settings']['initial samples per set']
        samples_raw = samples_raw[used_samples_per_set:used_samples_per_set + new_samples_per_set, :]

        return samples_raw

    monkeypatch.setattr(sampling, 'compute_sampling_distributions', mockreturn)

@pytest.fixture()
def mock_create_delphin(monkeypatch, delphin_file_path, add_two_materials):
    def mockreturn(sampling_strategy: dict, samples: dict):
        delphin_ids = []
        for sequence in samples.keys():
            delphin_id = delphin_interactions.upload_delphin_dict_to_database(
                delphin_parser.dp6_to_dict(delphin_file_path),
                1)
            delphin_interactions.add_sampling_dict(delphin_id, {'input': samples[sequence][0].tolist(),
                                                                'output': None,
                                                                'design_option': 'Test Design',
                                                                'sequence': sequence})
            delphin_ids.append(delphin_id)

        return delphin_ids

    monkeypatch.setattr(sampling, 'create_delphin_projects', mockreturn)

def sobol_test_function1(array: np.ndarray) -> np.ndarray:
    return np.prod(1 + (array ** 2 - array + 1 / 6))

def sobol_test_function2(array: np.ndarray) -> np.ndarray:
    return np.prod(1 + (array ** 6 - 3 * array ** 5 + 5 / 2 * array ** 4 - 1 / 2 * array ** 2 + 1 / 42))

@pytest.fixture(params=[1, 2], ids=[' 2nd Bernoulli', ' 6th Bernoulli'])
def mock_wait_until_finished(monkeypatch, request):
    def mock_return(delphin_ids):
        for delphin_id in delphin_ids:
            delphin_doc = delphin_entry.Delphin.objects(id=delphin_id).first()
            input_array = np.array(delphin_doc.sample_data['input'])

            if request.param == 1:
                output_array = sobol_test_function1(input_array)
            elif request.param == 2:
                output_array = sobol_test_function2(input_array)

            delphin_doc.update(set__sample_data__output=float(output_array))

    monkeypatch.setattr(simulation_interactions, 'wait_until_simulated', mock_return)

@pytest.fixture()
def mock_calculate_sample_output(monkeypatch):

    def mock_return(sample_strategy: dict, sampling_id: str):
        sample_mean = dict()
        sample_std = dict()
        sequence = sample_strategy['settings']['sequence']
        for sequence_index in range(sequence):
            sample_mean[str(sequence_index)] = dict()
            sample_std[str(sequence_index)] = dict()

            for design in sample_strategy['design']:
                projects_given_design = delphin_entry.Delphin.objects(sample_data__design_option=design,
                                                                      sample_data__sequence=str(sequence_index))
                mould = []
                heat_loss = []

                for project in projects_given_design:
                    mould.append(project.sample_data['output'])
                    heat_loss.append(project.sample_data['output'])

                sample_mean[str(sequence_index)][design] = {'mould': np.mean(mould),
                                                            'heat_loss': np.mean(heat_loss)}
                sample_std[str(sequence_index)][design] = {'mould': np.std(mould),
                                                           'heat_loss': np.std(heat_loss)}

        sampling_interactions.upload_sample_mean(sampling_id, sample_mean)
        sampling_interactions.upload_sample_std(sampling_id, sample_std)

    monkeypatch.setattr(sampling, 'calculate_sample_output', mock_return)

@pytest.mark.skipif(platform.system() == 'Linux', reason='Test should only run locally')
def test_sampling_worker(add_mock_strategy, mock_sampling_distribution, mock_create_delphin,
                         mock_wait_until_finished, mock_calculate_sample_output,
                         ):

    with pytest.raises(SystemExit) as exc_info:
        sampling_worker.sampling_worker(add_mock_strategy)

    strategy_doc = sample_entry.Strategy.objects(id=add_mock_strategy).first()
    threshold = strategy_doc.strategy['settings']['standard error threshold']
    keys = list(strategy_doc.standard_error.keys())
    number_of_runs = len(strategy_doc.standard_error[keys[0]]['mould'])

    assert number_of_runs < 1/threshold ** 2
    assert number_of_runs < 1/threshold


text:
import os
import h5py
import numpy as np
import pickle as pkl
import tensorflow as tf
from tqdm import tqdm

from gcn.train import train_model
from gcn.utils import load_data_pkl, load_data_h5

# Set random seed
seed = 123
np.random.seed(seed)
tf.set_random_seed(seed)

# Settings
flags = tf.app.flags
FLAGS = flags.FLAGS
flags.DEFINE_bool('use_h5', True, 'use h5 feature or not')
flags.DEFINE_string('data_dir', './data', 'Data root dir')
flags.DEFINE_string('save_dir', './genrated_textvqa_gcn_sg', 'dir to save generated scene graph')
flags.DEFINE_integer('epochs', 300, 'Number of epochs to train.')
flags.DEFINE_integer('hidden1', 600, 'Number of units in hidden layer, equal to the dim of output scene graph')
flags.DEFINE_string('tiers', 'train_val_test', 'what tier of textvqa sg to use: train, val, or test')
flags.DEFINE_integer('start_index', 0, 'image start index')

flags.DEFINE_float('learning_rate', 0.01, 'Initial learning rate.')
flags.DEFINE_float('dropout', 0.2, 'Dropout rate (1 - keep probability).')
flags.DEFINE_float('weight_decay', 5e-4, 'Weight for L2 loss on embedding matrix.')
flags.DEFINE_bool('use_dummy', False, 'use dummy data for test')

if not os.path.exists(FLAGS.save_dir):
    os.mkdir(FLAGS.save_dir)

tiers = FLAGS.tiers.split('_')

for tier in tiers:
    data_root = os.path.join(FLAGS.data_dir, 'textvqa_{}'.format(tier))

    if FLAGS.use_h5:
        node_feature_h5 = h5py.File(os.path.join(data_root, 'node_features.h5'), "r")
        adj_matrix_h5 = h5py.File(os.path.join(data_root, 'adjacent_matrix.h5'), "r")
        target_h5 = h5py.File(os.path.join(data_root, 'targets.h5'), "r")
        mask_h5 = h5py.File(os.path.join(data_root, 'mask.h5'), "r")

        n_images = mask_h5['masks'].shape[0]

        save_h5 = h5py.File(os.path.join(FLAGS.save_dir, '{}_sg.h5'.format(tier)), "w")
        save_h5.create_dataset("gcn_scene_graphs",
                               (n_images,
                                node_feature_h5['object_name_embeddings'].shape[1] +
                                node_feature_h5['ocr_token_embeddings'].shape[1],
                                FLAGS.hidden1),
                               dtype='float32')

        for image_index in tqdm(range(FLAGS.start_index, n_images),
                                unit='image',
                                desc='Generating gcn sg for {}'.format(tier)):
            data = load_data_h5(image_index, node_feature_h5, adj_matrix_h5, target_h5, mask_h5)
            hidden_state = train_model(data)
            save_h5['gcn_scene_graphs'][image_index] = hidden_state
        node_feature_h5.close()
        adj_matrix_h5.close()
        target_h5.close()
        mask_h5.close()
        save_h5.close()

    else:
        target_dir = os.path.join(data_root, 'targets')
        n_images = len(os.listdir(target_dir))
        for image_index in tqdm(range(FLAGS.start_index, n_images),
                                unit='image',
                                desc='Generating gcn sg for {}'.format(tier)):
            data = load_data_pkl(data_root, image_index, FLAGS.use_dummy)
            hidden_state = train_model(data)

            # save 2nd layer hidden state
            save_dir = os.path.join(FLAGS.save_dir, '{}_sg'.format(tier))
            if not os.path.exists(save_dir):
                os.mkdir(save_dir)
            with open(os.path.join(save_dir, '{}.p'.format(image_index)), 'wb') as f:
                pkl.dump(hidden_state, f)

max_stars_repo_name: zean-wen/gcn

max_issues_repo_name: zean-wen/gcn

max_forks_repo_name: zean-wen/gcn


text:
import pandas as pd
import numpy as np
import os

def split(data, propotion):
    """
    :param data: data matrix
    :param propotion: as a fraction e.g. 0.2 for 20% test
    :return: train set, test set
    """
    N = data.shape[0]
    all_indx = np.arange(N)
    test_indx = np.random.choice(all_indx, np.int(N*propotion), replace=False)
    train_indx = np.in1d(all_indx, test_indx, invert=True)
    print(' Orig size={}, Train size={}, Test size={}'.format(data.shape[0], train_indx.shape[0], test_indx.shape[0]))

    return data[train_indx, :], data[test_indx, :]

def split_n_save(fn, propotion=0.2):
    """
    :param fn: file name of the raw dataset
    :param propotion: as a fraction e.g. 0.2 for 20% test
    """
    print(' Train-test {} splitting '.format(propotion) + fn + '.csv...')
    data = pd.read_csv(fn+'.csv', delimiter=',')
    train_set, test_set = split(data.values, propotion)

    header = data.columns.values
    header = ', '.join(header)
    with open(fn + '_train.csv', 'w') as f_handle:  # try 'a'
        np.savetxt(f_handle, train_set, delimiter=',', header=header, comments='')
    with open(fn + '_test.csv', 'w') as f_handle:  # try 'a'
        np.savetxt(f_handle, test_set, delimiter=',', header=header, comments='')

    print(' Train-test datasets saved.')

files = \
    ['1_toy/1_toy1_vel',
     '2_carla/2_carla1_frame_250',
     '3_astyx/3_astyx1/normalized/000002',
     '4_nuscenes/4_nuscenes1/samples/normalized/RADAR_FRONT/n008-2018-08-01-15-16-36-0400__RADAR_FRONT__1533151603555991',
     '5_airsim/5_airsim1/5_airsim1_pos',
     '5_airsim/5_airsim1/5_airsim1_vel',
     '6_jfk/6_jfk_partial1_pos',
     '6_jfk/6_jfk_partial1_vel'
     ]

for f in files:
    split_n_save('../datasets/kyle_ransalu/' + f, 0.2)

max_stars_repo_name: RansML/SpaTUn

max_issues_repo_name: RansML/SpaTUn

max_forks_repo_name: RansML/SpaTUn


text:
#A module to read raw data from source for further analysis.
#Data are stored in the raw_data folder for the original dataset and processed_data for the cleaned dataset.

from enum import Enum
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler, PolynomialFeatures
import imblearn
from imblearn import over_sampling
from imblearn.over_sampling import SMOTE

class NBARawData(Enum):
    '''List of members to identify the name of each dataset stored in the raw_data folder.
       One enumerator member represents one file in the folder.
    '''
    TRAIN = 0
    TEST = 1

class DataReader:
    def __init__(self, relative_path = "../"):

        self.filepath = {}
        #raw data
        self.filepath.update({NBARawData.TRAIN : relative_path + "data/raw/train.csv"})
        self.filepath.update({NBARawData.TEST : relative_path + "data/raw/test.csv"})

    def read_data(self, source, relative_path = "../"):
        '''Read the CSV file and load it into a data frame.

           Argument:
               source (enum): the member name of the dataset that will be loaded, e.g., NBARawData.TRAIN

           Return:
               a data frame that store the dataset
        '''
        if (not isinstance(source,NBARawData)):
            raise Exception("argument should be filled with NBARawData "
                            "Try NBARawData.TRAIN and/or NBARawData.TEST")

        #read the data and load them into a dataframe
        data = pd.read_csv(self.filepath[source])
        #for consistency purposes, change the case of the column names into a lower case and remove extra spaces
        data = data.rename(columns = lambda x: x.strip(" "))
        return(data)

    def split_data(self, data, relative_path = "../"):
        ''' Split the given dataset into two sets randlomly
            such as Train and Validation sets by 80:20 ratio and
            save the split data into the project's /data/processed directory
            for later reuse during the experiment.

        Argument:
        ----------
        data : a panda dataframe
            A dataframe to split
        relative_path: a relative path string
            A relative path string to save the splited data

        Returns
        -------
        X_train: A panda dataframe with all the independent variables without the target column.
                It is 80% of the original dataset which is meant for model training
        X_val : A panda dataframe with all the independent variables.
                It is 20% of the original dataset which is meant for model validation
        y_train: A separate array of target column values from training set(X_train)
        y_val:  A separate array of target column values from validation set(X_val)
        '''
        data = pd.DataFrame(data)
        target = data.pop('TARGET_5Yrs')
        X_train, X_val, y_train, y_val = train_test_split(data, target, test_size = 0.2, random_state=8, shuffle=True )

        # Save the splited data
        np.save(relative_path+ "data/processed/X_train", X_train)
        np.save(relative_path+ "data/processed/X_train", X_val)

        np.save(relative_path+ "data/processed/X_train", y_train)
        np.save(relative_path+ "data/processed/X_train", y_val)
        return(X_train, X_val, y_train, y_val)

    def select_feature_by_correlation(self, data, columns_to_drop):
        '''This function generates the correlation heat map
            select the features according to the correlation result.
            The features which have correlation > 0.9 are filter out.

        Argument:
        ----------
        data: a panda dataframe
            to examine the correlation among the features
        columns_to_drop: a list of columns
            to exclude in the correlation analysis

        Returns
        -------
        selected_columns: An array of selected columns which have correlation < 0.9

        '''

        data.drop(columns_to_drop, axis=1, inplace=True )
        corr = data.corr()
        sns.heatmap(corr)

        columns = np.full((corr.shape[0],), True, dtype=bool)
        for i in range(corr.shape[0]):
            for j in range(i+1, corr.shape[0]):
                if corr.iloc[i,j] >= 0.9:
                    if columns[j]:
                        columns[j] = False
        selected_columns = data.columns[columns]
        return selected_columns

    def scale_features_by_standard_scaler(self, df):
        '''
        This function scales all the features included in the dataframe using Standard Scaler

        Arguments:
        ----------
        df: a panda dataframe with all the features to be scaled

        Return:
        -------
        scaled_df: a panda dataframe with all the scaled features keeping the original column names
        '''
        scaler = StandardScaler()
        df_scaled = scaler.fit_transform(df)

        # Set as pd.dataframe and Re-apply column names and
        scaled_df = pd.DataFrame(df_scaled)
        scaled_df.columns = df.columns
        return scaled_df

    def polynomialize_data(self, df, degree):

        '''
        This function transfrom and Generate a new feature matrix consisting of all polynomial combinations of the features with
        degree less than or equal to the specified degree given in the second argument

        Arguments:
        ----------
        df: a panda dataframe with all the features to be polynomialised
        degree: integer
                It is a degree to be applyed polynomial such as power 2 or 3

        Returns:
        -------
        data_poly: a panda dataframe
                 a dataframe with the polynomial combinations of the features
        '''
        # Polynomialise
        poly = PolynomialFeatures(degree)
        data_poly = poly.fit_transform(df)

        data_poly = pd.DataFrame(data_poly)
        return data_poly

    def plot_class_balance(self, df):
        '''
        This function helps to visualises how balance the target class is
        by plotting the bar graph on the target's value count

        Argument:
        --------
        df: a panda dataframe with the target feature

        Return:
        -------
        The bar plot

        '''

        pl = pd.DataFrame(df)
        pl.columns = ['Target']
        pl.Target.value_counts().plot(kind="bar", title="Count Target")

    def resample_data_upsample_smote(self, X, y):
        '''
        This function does oversampling the minority class using SMOTE from imblearn and
        then fit and apply it in one step.

        Arguments:
        ----------
        X: a panda dataframe with the features only
        y: an array of target variable

        Returns:
        -------
        X_res:  a panda dataframe
                a transformed version of the dataset with all features after upsampling
        y_res: an array of target values after upsampling

        '''

        sm = SMOTE(random_state = 23, sampling_strategy = 1.0)
        X_res, y_res = sm.fit_resample(X, y.ravel())
        return( X_res, y_res)

    def clean_negatives(self, strategy, df):
        '''
        This function does imputation to the dataset accorting to the strategy given in the arguemnt

        Arguments:
        ---------
        strategy: a string
                It is a strategy to apply to the imputation of the null / negative data.
                The strategy must be 'abs' to make the absolute value, 'null' to replace with 0 or
                'mean' to replace the negative with mean values.
        Returns:
        --------
        df: a panda dataframe
            A dataframe with features imputed

        '''

        if strategy=='abs':
            df = abs(df)
        if strategy=='null':
            df[df < 0] = None
        if strategy=='mean':
            df[df < 0] = None
            df.fillna(df.mean(), inplace=True)
        if strategy=='median':
            df[df < 0] = None
            df.fillna(df.median(), inplace=True)

        return(df)

# end of DataReader

def confusion_matrix(true, pred):
    #'''
    #This function plots confusion matrix
    #
    #Arguments:
    #---------
    #true: an array of target's original value
    #pred: an array of predicted values on target

    #Result:
    #-------
    #cmtx: ndarray of shape (n_classes, n_classes) in a dataframe format
    #'''

    import numpy as np
    import pandas as pd
    from sklearn import metrics

    unique_label = np.unique([true, pred])
    cmtx = pd.DataFrame(
        metrics.confusion_matrix(true, pred, labels=unique_label),
        index=['true:{:}'.format(x) for x in unique_label],
        columns=['pred:{:}'.format(x) for x in unique_label]
    )
    return(cmtx)

def plot_roc(true, pred):
    #'''
    #This function plots confusion matrix
    #
    #Arguments:
    #---------
    #true: an array of target's original value
    #pred: an array of predicted values on target

    #Result:
    #-------
    #cmtx: ndarray of shape (n_classes, n_classes) in a dataframe format
    #'''
    from sklearn.metrics import roc_curve, auc
    import matplotlib.pyplot as plt

    fpr, tpr, thresh = roc_curve(true, pred)
    roc_auc = auc(fpr, tpr)
    print('AUC = %0.3f' % roc_auc)
    plt.plot(fpr, tpr, 'b', label='AUC = %0.3f' % roc_auc)
    plt.plot([0,1],[0,1], 'r--')
    plt.xlim([-0.1,1.0])
    plt.xlim([-0.1,1.01])
    return(plt)

def eval_report(true, pred):
    # '''
    #This function generates confusion matrix, classification report and ROC curve
    #
    #Arguments:
    #---------
    #true: an array of target's original value
    #pred: an array of predicted values on target

    #Prints:
    #-------
    #matrix: ndarray of shape (n_classes, n_classes)
    #matrix: Classification report in string format
    #plot: ROC curve displaying the accuracy of the model
    #'''
    import numpy as np
    import pandas as pd
    from sklearn import metrics

    unique_label = np.unique([true, pred])
    cmtx = pd.DataFrame(
        metrics.confusion_matrix(true, pred, labels=unique_label),
        index=['true:{:}'.format(x) for x in unique_label],
        columns=['pred:{:}'.format(x) for x in unique_label]
    )
    print("Confusion Matrix:")
    print(cmtx)
    print("")
    print("Classification Report:")
    print(metrics.classification_report(true, pred))
    print("")
    print("ROC Curve:")

    import matplotlib.pyplot as plt
    plot_roc(true, pred)
    plt.show()

max_stars_repo_name: ajduncanson/nba-career-prediction

max_issues_repo_name: ajduncanson/nba-career-prediction

max_forks_repo_name: ajduncanson/nba-career-prediction


text:
from abc import ABC, abstractmethod
import collections

import numpy as np
import torch
from torch.utils import data

def _worker_init(_):
    return np.random.seed(torch.initial_seed() % np.iinfo(np.uint32).max)

class Reader(ABC):

    def __init__(self, splits):
        """Initialize the reader.

        splits - dict
            A dictionary of {split_name: int} which maps the name of the split to the number of samples in that split
        """
        self.splits = splits

    def get_size(self, split=None):
        if split is not None:
            return self.splits[split]
        else:
            return sum(self.splits.values())

    @abstractmethod
    def iter_batches(self, split_name, batch_size, shuffle=True, partial_batching=False, threads=1, epochs=1,
                     max_batches=-1):
        pass

class DatasetReader(Reader):
    """Creates a Reader wrapper around PyTorch datasets."""

    def __init__(self, dataset_splits):
        """dataset_splits is a dictionary of {split_name: Dataset}"""
        self.dataset_splits = dataset_splits
        super().__init__(collections.OrderedDict([(k, len(v)) for k, v in dataset_splits.items()]))

    def process_batch(self, batch):
        """Process batch before yielding from iter_batches."""
        return batch

    def iter_batches(self, split_name, batch_size, shuffle=True, partial_batching=False, threads=1, epochs=1,
                     max_batches=-1):
        loader = data.DataLoader(self.dataset_splits[split_name], batch_size=batch_size, shuffle=shuffle,
                                 num_workers=threads, drop_last=not partial_batching, worker_init_fn=_worker_init)
        for _ in range(epochs):
            for i, batch in enumerate(loader):
                yield self.process_batch(batch)
                if i + 1 == max_batches:
                    break

max_issues_repo_name: gallego-posada/gait

max_forks_repo_name: gallego-posada/gait