text:
import sparse_numeric_table as spt
import numpy as np
import pandas as pd
import tempfile
import os

EXAMPLE_TABLE_STRUCTURE = {
    'elementary_school': {
        'lunchpack_size': {'dtype': '<f8'},
        'num_friends': {'dtype': '<i8'},
    },
    'high_school': {
        'time_spent_on_homework': {'dtype': '<f8'},
        'num_best_friends': {'dtype': '<i8'},
    },
    'university': {
        'num_missed_classes': {'dtype': '<i8'},
        'num_fellow_students': {'dtype': '<i8'},
    },
}

def _make_example_table(prng, size, start_index=0):
    """
    Children start in elementary school. 10% progress to high school, and 10%
    of those progress to university.
    At each point in their career statistics are collected that can be put to
    columns, while every child is represented by a line.
    Unfortunately, a typical example of a sparse table.
    """

    t = {}
    t['elementary_school'] = spt.dict_to_recarray(
        {
            spt.IDX: start_index + np.arange(size).astype(spt.IDX_DTYPE),
            'lunchpack_size': prng.uniform(size=size).astype('<f8'),
            'num_friends': prng.uniform(
                low=0,
                high=5,
                size=size).astype('<i8'),
        }
    )
    high_school_size = size//10
    t['high_school'] = spt.dict_to_recarray(
        {
            spt.IDX: prng.choice(
                t['elementary_school'][spt.IDX],
                size=high_school_size,
                replace=False),
            'time_spent_on_homework': 100 + 100*prng.uniform(
                size=high_school_size).astype('<f8'),
            'num_best_friends': prng.uniform(
                low=0,
                high=5,
                size=high_school_size).astype('<i8'),
        }
    )
    university_size = high_school_size//10
    t['university'] = spt.dict_to_recarray(
        {
            spt.IDX: prng.choice(
                t['high_school'][spt.IDX],
                size=university_size,
                replace=False),
            'num_missed_classes': 100*prng.uniform(
                size=university_size).astype('<i8'),
            'num_fellow_students': prng.uniform(
                low=0,
                high=5,
                size=university_size).astype('<i8'),
        }
    )
    spt.assert_structure_keys_are_valid(structure=EXAMPLE_TABLE_STRUCTURE)
    spt.assert_table_has_structure(table=t, structure=EXAMPLE_TABLE_STRUCTURE)
    return t

def test_from_records():
    prng = np.random.Generator(np.random.MT19937(seed=0))
    rnd = prng.uniform

    # define what your table will look like
    # -------------------------------------
    structure = {
        "A": {
            "a": {"dtype": '<f8'},
            "b": {"dtype": '<f8'},
        },
        "B": {
            "c": {"dtype": '<f8'},
            "d": {"dtype": '<f8'},
        },
        "C": {
            "e": {"dtype": '<f8'},
        },
    }

    # populate the table using records
    # --------------------------------
    with tempfile.TemporaryDirectory(prefix='test_sparse_table') as tmp:

        num_jobs = 100
        n = 5
        job_result_paths = []
        for j in range(num_jobs):

            # map the population of the sparse table onto many jobs
            # -----------------------------------------------------
            i = j*n
            table_records = {}

            table_records["A"] = []
            table_records["A"].append({spt.IDX: i+0, "a": rnd(), "b": rnd()})
            table_records["A"].append({spt.IDX: i+1, "a": rnd(), "b": rnd()})
            table_records["A"].append({spt.IDX: i+2, "a": rnd(), "b": rnd()})
            table_records["A"].append({spt.IDX: i+3, "a": rnd(), "b": rnd()})
            table_records["A"].append({spt.IDX: i+4, "a": rnd(), "b": rnd()})

            table_records["B"] = []
            table_records["B"].append({spt.IDX: i+0, "c": rnd(), "d": 5*rnd()})
            table_records["B"].append({spt.IDX: i+3, "c": rnd(), "d": 5*rnd()})

            table_records["C"] = []
            if rnd() > 0.9:
                table_records["C"].append({spt.IDX: i+3, "e": -rnd()})

            table = spt.table_of_records_to_sparse_numeric_table(
                table_records=table_records,
                structure=structure)

            path = os.path.join(tmp, '{:06d}.tar'.format(j))
            job_result_paths.append(path)
            spt.write(path=path, table=table, structure=structure)

        # reduce
        # ------
        full_table = spt.concatenate_files(
            list_of_table_paths=job_result_paths,
            structure=structure)

    spt.assert_table_has_structure(
        table=full_table,
        structure=structure)

def test_write_read_full_table():
    prng = np.random.Generator(np.random.MT19937(seed=1337))

    my_table = _make_example_table(prng=prng, size=1000*1000)
    with tempfile.TemporaryDirectory(prefix='test_sparse_table') as tmp:
        path = os.path.join(tmp, 'my_table.tar')
        spt.write(
            path=path,
            table=my_table,
            structure=EXAMPLE_TABLE_STRUCTURE)
        my_table_back = spt.read(
            path=path,
            structure=EXAMPLE_TABLE_STRUCTURE)
        spt.assert_tables_are_equal(my_table, my_table_back)

        # no structure
        path_nos = os.path.join(tmp, 'my_table_no_structure.tar')
        spt.write(path=path_nos, table=my_table)
        my_table_back_nos = spt.read(path=path_nos)
        spt.assert_tables_are_equal(my_table, my_table_back_nos)
        spt.assert_table_has_structure(
            table=my_table_back_nos,
            structure=EXAMPLE_TABLE_STRUCTURE)

def test_write_read_empty_table():
    prng = np.random.Generator(np.random.MT19937(seed=1337))

    empty_table = _make_example_table(prng=prng, size=0)
    with tempfile.TemporaryDirectory(prefix='test_sparse_table') as tmp:
        path = os.path.join(tmp, 'my_empty_table.tar')
        spt.write(
            path=path,
            table=empty_table,
            structure=EXAMPLE_TABLE_STRUCTURE)
        my_table_back = spt.read(
            path=path,
            structure=EXAMPLE_TABLE_STRUCTURE)
        spt.assert_tables_are_equal(empty_table, my_table_back)

        # no structure
        path_nos = os.path.join(tmp, 'my_empty_table_no_structure.tar')
        spt.write(path=path_nos, table=empty_table)
        my_table_back_nos = spt.read(path=path_nos)
        spt.assert_tables_are_equal(empty_table, my_table_back_nos)
        spt.assert_table_has_structure(
            table=my_table_back_nos,
            structure=EXAMPLE_TABLE_STRUCTURE)

def test_merge_common():
    prng = np.random.Generator(np.random.MT19937(seed=1337))

    my_table = _make_example_table(prng=prng, size=1000*1000)

    common_indices = spt.intersection(
        [my_table[lvl][spt.IDX] for lvl in my_table]
    )

    my_common_table = spt.cut_table_on_indices(
        table=my_table,
        common_indices=common_indices
    )
    my_sorted_common_table = spt.sort_table_on_common_indices(
        table=my_common_table,
        common_indices=common_indices
    )

    np.testing.assert_array_equal(
        my_sorted_common_table['elementary_school'][spt.IDX],
        my_sorted_common_table['high_school'][spt.IDX]
    )

    np.testing.assert_array_equal(
        my_sorted_common_table['elementary_school'][spt.IDX],
        my_sorted_common_table['university'][spt.IDX]
    )

    my_common_df = spt.make_rectangular_DataFrame(table=my_sorted_common_table)

    np.testing.assert_array_equal(
        my_sorted_common_table['elementary_school'][spt.IDX],
        my_common_df[spt.IDX]
    )

def test_merge_across_all_levels_random_order_indices():
    prng = np.random.Generator(np.random.MT19937(seed=1337))

    size = 1000*1000
    my_table = _make_example_table(prng=prng, size=size)

    has_elementary_school = my_table['elementary_school'][spt.IDX]
    has_high_school = my_table['high_school'][spt.IDX]
    has_university = my_table['university'][spt.IDX]
    has_big_lunchpack = my_table['elementary_school'][spt.IDX][
        my_table['elementary_school']['lunchpack_size'] > 0.5]
    has_2best_friends = my_table['high_school'][spt.IDX][
        my_table['high_school']['num_best_friends'] >= 2]

    cut_indices = np.intersect1d(has_elementary_school, has_high_school)
    cut_indices = np.intersect1d(cut_indices, has_university)
    cut_indices = np.intersect1d(cut_indices, has_big_lunchpack)
    cut_indices = np.intersect1d(cut_indices, has_2best_friends)
    np.random.shuffle(cut_indices)

    cut_table = spt.cut_table_on_indices(
        table=my_table,
        common_indices=cut_indices,
        level_keys=['elementary_school', 'high_school', 'university'])
    sorted_cut_table = spt.sort_table_on_common_indices(
        table=cut_table,
        common_indices=cut_indices
    )

    np.testing.assert_array_equal(
        sorted_cut_table['elementary_school'][spt.IDX],
        sorted_cut_table['high_school'][spt.IDX]
    )
    np.testing.assert_array_equal(
        sorted_cut_table['elementary_school'][spt.IDX],
        sorted_cut_table['university'][spt.IDX]
    )
    np.testing.assert_array_equal(
        sorted_cut_table['elementary_school'][spt.IDX],
        cut_indices
    )

def test_merge_random_order_indices():
    prng = np.random.Generator(np.random.MT19937(seed=1337))

    size = 1000*1000
    my_table = _make_example_table(prng=prng, size=size)

    has_elementary_school = my_table['elementary_school'][spt.IDX]
    has_high_school = my_table['high_school'][spt.IDX]
    has_big_lunchpack = my_table['elementary_school'][spt.IDX][
        my_table['elementary_school']['lunchpack_size'] > 0.5]
    has_2best_friends = my_table['high_school'][spt.IDX][
        my_table['high_school']['num_best_friends'] >= 2]

    cut_indices = np.intersect1d(has_elementary_school, has_high_school)
    cut_indices = np.intersect1d(cut_indices, has_big_lunchpack)
    cut_indices = np.intersect1d(cut_indices, has_2best_friends)
    np.random.shuffle(cut_indices)

    cut_table = spt.cut_table_on_indices(
        table=my_table,
        common_indices=cut_indices,
        level_keys=['elementary_school', 'high_school']
    )
    sorted_cut_table = spt.sort_table_on_common_indices(
        table=cut_table,
        common_indices=cut_indices
    )

    assert 'university' not in sorted_cut_table
    assert 'elementary_school' in sorted_cut_table
    assert 'high_school' in sorted_cut_table

    np.testing.assert_array_equal(
        sorted_cut_table['elementary_school'][spt.IDX],
        sorted_cut_table['high_school'][spt.IDX]
    )
    np.testing.assert_array_equal(
        sorted_cut_table['elementary_school'][spt.IDX],
        cut_indices
    )

def test_concatenate_several_tables():
    prng = np.random.Generator(np.random.MT19937(seed=1337))

    block_size = 10*1000
    num_blocks = 100

    with tempfile.TemporaryDirectory(prefix='test_sparse_table') as tmp:
        paths = []
        for i in range(num_blocks):
            table_i = _make_example_table(
                prng=prng,
                size=block_size,
                start_index=i*block_size)
            paths.append(os.path.join(tmp, "{:06d}.tar".format(i)))
            spt.write(
                path=paths[-1],
                table=table_i,
                structure=EXAMPLE_TABLE_STRUCTURE)
        output_path = os.path.join(tmp, "full.tar")
        full_table = spt.concatenate_files(
            list_of_table_paths=paths,
            structure=EXAMPLE_TABLE_STRUCTURE,)
    spt.assert_table_has_structure(
        table=full_table,
        structure=EXAMPLE_TABLE_STRUCTURE)

    assert (
        full_table['elementary_school'][spt.IDX].shape[0] ==
        num_blocks*block_size)
    assert (
        len(set(full_table['elementary_school'][spt.IDX])) ==
        num_blocks*block_size), "The indices must be uniqe"
    assert (
        full_table['high_school'][spt.IDX].shape[0] ==
        num_blocks*block_size//10)
    assert (
        len(set(full_table['high_school'][spt.IDX])) ==
        num_blocks*block_size//10)
    assert (
        full_table['university'][spt.IDX].shape[0] ==
        num_blocks*block_size//100)
    assert (
        len(set(full_table['university'][spt.IDX])) ==
        num_blocks*block_size//100)

def test_concatenate_empty_list_of_paths():
    with tempfile.TemporaryDirectory(prefix='test_sparse_table') as tmp:
        output_path = os.path.join(tmp, 'empty_table.tar')
        empty_table = spt.concatenate_files(
            list_of_table_paths=[],
            structure=EXAMPLE_TABLE_STRUCTURE)
    assert empty_table['elementary_school'][spt.IDX].shape[0] == 0

def test_only_index_in_level():
    prng = np.random.Generator(np.random.MT19937(seed=1337))

    structure = {
        "A": {"height": {"dtype": "<i8"}},
        "B": {},
    }

    table = {}
    table["A"] = spt.dict_to_recarray({
        spt.IDX: np.arange(10).astype(spt.IDX_DTYPE),
        "height": np.ones(10, dtype='<i8'),
    })
    table["B"] = spt.dict_to_recarray({
        spt.IDX: prng.choice(table["A"][spt.IDX], 5),
    })

    spt.assert_table_has_structure(
        table=table,
        structure=structure)

    with tempfile.TemporaryDirectory(prefix='test_sparse_table') as tmp:
        path = os.path.join(tmp, 'table_with_index_only_level.tar')
        spt.write(
            path=path,
            table=table,
            structure=structure)
        table_back = spt.read(path=path, structure=structure)
        spt.assert_tables_are_equal(table, table_back)


text:
# -*- coding: utf-8 -*-

from collections import Counter, defaultdict

import regex
import torch
import torch.nn as nn
from pytorch_pretrained_bert import BertTokenizer
import numpy as np
import unicodedata
import os

class Vocab(object):
    PAD = '<PAD>'

    def __init__(self, words, chars, rels):
        self.pad_index = 0

        self.words = [self.PAD] + sorted(words)
        self.chars = [self.PAD] + sorted(chars)
        self.rels = sorted(rels)

        self.word_dict = {word: i for i, word in enumerate(self.words)}
        self.char_dict = {char: i for i, char in enumerate(self.chars)}
        self.rel_dict = {rel: i for i, rel in enumerate(self.rels)}

        # ids of punctuation that appear in words
        self.puncts = set(sorted(i for word, i in self.word_dict.items()
                                if regex.match(r'\p{P}+$', word)))

        self.n_words = len(self.words)
        self.n_chars = len(self.chars)
        self.n_rels = len(self.rels)
        self.n_train_words = self.n_words

        self.tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased', do_lower_case=False)
        # self.tokenizer = BertTokenizer.from_pretrained('bert-base-cased')

    def __repr__(self):
        info = f"{self.__class__.__name__}(\n"
        info += f"  num of words: {self.n_words}\n"
        info += f"  num of chars: {self.n_chars}\n"
        info += f"  num of rels: {self.n_rels}\n"
        info += f")"

        return info

    def rel2id(self, sequence):
        return torch.tensor([self.rel_dict.get(rel, 0)
                             for rel in sequence])

    def id2rel(self, ids):
        return [self.rels[i] for i in ids]

    def extend(self, words):
        self.words += sorted(set(words).difference(self.word_dict))
        self.chars += sorted(set(''.join(words)).difference(self.char_dict))
        self.word_dict = {w: i for i, w in enumerate(self.words)}
        self.char_dict = {c: i for i, c in enumerate(self.chars)}
        self.puncts = set(sorted(i for word, i in self.word_dict.items()
                                if regex.match(r'\p{P}+$', word)))
        self.n_words = len(self.words)
        self.n_chars = len(self.chars)

    def numericalize(self, corpus, save_name=None):
        words_numerical = []
        arcs_numerical = []
        rels_numerical = []
        token_start_mask = []
        attention_mask = []
        offending_set = set()
        symbol_set = set()
        empty_words = set()
        len_dict = defaultdict(int)
        sent_count = 0
        exceeding_count = 0
        for words, arcs, rels in zip(corpus.words, corpus.heads, corpus.rels):
            sentence_token_ids = []
            sentence_arc_ids = []
            sentence_rel_ids = []
            token_starts = []
            attentions = []
            words = ['[CLS]'] + words + ['[SEP]']
            arcs = [0] + arcs + [0]
            rels = ['<ROOT>'] + rels + ['<ROOT>']
            for word, arc, rel in zip(words, arcs, rels):
                # skip <ROOT>
                if word == '<ROOT>':
                    continue

                # take care of some idiosyncracies
                if word == '`':
                    word = "'"
                if word == '``':
                    word = '"'
                if word == "''":
                    word = '"'
                if word == "non-``":
                    word = 'non-"'
                word = word.replace('“', '"')
                word = word.replace('”', '"')
                word = word.replace("`", "'")
                word = word.replace("’", "'")
                word = word.replace("‘", "'")
                word = word.replace("'", "'")
                word = word.replace("´", "'")
                word = word.replace("…", "...")
                word = word.replace("–", "-")
                word = word.replace('—', '-')

                tokens = self.tokenizer.tokenize(word)
                if tokens:
                    ids = self.tokenizer.convert_tokens_to_ids(tokens)

                    # take care of punctuation
                    if regex.match(r'\p{P}+$', word):
                        for token_id in ids:
                            self.puncts.add(token_id)

                    # log any unknown words
                    if '[UNK]' in tokens:
                        # print('words: ', words)
                        # print('offending word: ', word)
                        # print('offending chars: ')
                        for offending_char in word:
                            token = self.tokenizer.tokenize(offending_char)
                            if '[UNK]' in token:
                                if unicodedata.category(offending_char) != 'So':
                                    offending_set.add(offending_char)
                                else:
                                    symbol_set.add(offending_char)

                    # main thing to do
                    sentence_token_ids.extend(ids)
                    sentence_arc_ids.extend([arc] * len(tokens))
                    sentence_rel_ids.extend([self.rel_dict.get(rel, 0)] * len(tokens))
                    token_starts.extend([1] + [0] * (len(tokens) - 1))
                    attentions.extend([1] * len(tokens))

                # take care of empty tokens
                else:
                    # print('\noffending word: ', word)
                    # print('empty words: ', ' '.join(words))
                    empty_words.add(word)
                    continue

            # error checking for lengths
            len_sentence_token_ids = len(sentence_token_ids)
            len_sentence_arc_ids = len(sentence_arc_ids)
            len_sentence_rel_ids = len(sentence_rel_ids)
            len_token_starts = len(token_starts)
            len_attentions = len(attentions)
            if not (len_sentence_token_ids == len_sentence_arc_ids == len_sentence_rel_ids == len_token_starts == len_attentions):
                print(words)
                print(arcs)
                print(rels)
                print('len_sentence_token_ids: ', len_sentence_token_ids)
                print('len_sentence_arc_ids', len_sentence_arc_ids)
                print('len_sentence_rel_ids', len_sentence_rel_ids)
                print('len_token_starts', len_token_starts)
                print('len_attentions', len_attentions)
                raise RuntimeError("Lengths don't match up.")

            # Skip too long sentences
            len_dict[len_sentence_token_ids] += 1
            if len_sentence_token_ids > 128:
                exceeding_count += 1
                continue
            sent_count += 1

            words_numerical.append(torch.tensor(sentence_token_ids))
            arcs_numerical.append(torch.tensor(sentence_arc_ids))
            rels_numerical.append(torch.tensor(sentence_rel_ids))
            token_start_mask.append(torch.ByteTensor(token_starts))
            attention_mask.append(torch.ByteTensor(attentions))

        if offending_set:
            print('WARNING: The following non-symbol characters are unknown to BERT:')
            print(offending_set)
        if symbol_set:
            print('WARNING: The following symbol characters are unknown to BERT:')
            print(symbol_set)
        if empty_words:
            print('WARNING: The following characters are empty after going through tokenizer:')
            print(empty_words)
        if save_name:
            try:
                index = save_name.rfind('/')
                if index > -1:
                    save_dir = save_name[:index]
                    os.makedirs(save_dir)
            except FileExistsError:
                # directory already exists
                pass
            torch.save((words_numerical, attention_mask, token_start_mask, arcs_numerical, rels_numerical), save_name)

        print('Total number of sentences: {}'.format(sent_count))
        print('Number of sentences exceeding max seq length of 128: {}'.format(exceeding_count))

        return words_numerical, attention_mask, token_start_mask, arcs_numerical, rels_numerical

    def numericalize_sentences(self, sentences):
        words_numerical = []
        token_start_mask = []
        attention_mask = []
        offending_set = set()
        symbol_set = set()
        empty_words = set()
        exceeding_count = 0
        for sentence in sentences:
            sentence_token_ids = []
            token_starts = []
            attentions = []
            sentence = ['[CLS]'] + sentence + ['[SEP]']
            for word in sentence:
                # skip <ROOT>
                if word == '<ROOT>':
                    continue

                # take care of some idiosyncracies
                if word == '`':
                    word = "'"
                if word == '``':
                    word = '"'
                if word == "''":
                    word = '"'
                if word == "non-``":
                    word = 'non-"'
                word = word.replace('“', '"')
                word = word.replace('”', '"')
                word = word.replace("`", "'")
                word = word.replace("’", "'")
                word = word.replace("‘", "'")
                word = word.replace("'", "'")
                word = word.replace("´", "'")
                word = word.replace("…", "...")
                word = word.replace("–", "-")
                word = word.replace('—', '-')

                tokens = self.tokenizer.tokenize(word)
                if tokens:
                    ids = self.tokenizer.convert_tokens_to_ids(tokens)

                    # Keep track of punctuation
                    if regex.match(r'\p{P}+$', word):
                        for token_id in ids:
                            self.puncts.add(token_id)

                    # log any unknown words
                    if '[UNK]' in tokens:
                        for offending_char in word:
                            token = self.tokenizer.tokenize(offending_char)
                            if unicodedata.category(offending_char) != 'So':
                                offending_set.add(offending_char)
                            else:
                                symbol_set.add(offending_char)

                    sentence_token_ids.extend(ids)
                    token_starts.extend([1] + [0] * (len(tokens) - 1))
                    attentions.extend([1] * len(tokens))

                # take care of empty tokens
                else:
                    empty_words.add(word)
                    continue

            # Skip too long sentences
            len_sentence_token_ids = len(sentence_token_ids)
            if len_sentence_token_ids > 128:
                exceeding_count += 1
                continue

            words_numerical.append(torch.tensor(sentence_token_ids))
            attention_mask.append(torch.ByteTensor(attentions))
            token_start_mask.append(torch.ByteTensor(token_starts))

        return words_numerical, attention_mask, token_start_mask

    def numericalize_tags(self, corpus):
        words_numerical = []
        words_total = []
        tags_total = []
        token_start_mask = []
        attention_mask = []
        offending_set = set()
        symbol_set = set()
        empty_words = set()
        exceeding_count = 0
        for sentence, words, tags in zip(corpus.words, corpus.words, corpus.tags):
            # skip <ROOT>
            words = words[1:]
            tags = tags[1:]

            sentence_token_ids = []
            token_starts = []
            attentions = []
            sentence = ['[CLS]'] + sentence + ['[SEP]']
            for word in sentence:
                # skip <ROOT>
                if word == '<ROOT>':
                    continue

                # take care of some idiosyncracies
                if word == '`':
                    word = "'"
                if word == '``':
                    word = '"'
                if word == "''":
                    word = '"'
                if word == "non-``":
                    word = 'non-"'
                word = word.replace('“', '"')
                word = word.replace('”', '"')
                word = word.replace("`", "'")
                word = word.replace("’", "'")
                word = word.replace("‘", "'")
                word = word.replace("'", "'")
                word = word.replace("´", "'")
                word = word.replace("…", "...")
                word = word.replace("–", "-")
                word = word.replace('—', '-')

                tokens = self.tokenizer.tokenize(word)
                if tokens:
                    ids = self.tokenizer.convert_tokens_to_ids(tokens)

                    # Keep track of punctuation
                    if regex.match(r'\p{P}+$', word):
                        for token_id in ids:
                            self.puncts.add(token_id)

                    # log any unknown words
                    if '[UNK]' in tokens:
                        for offending_char in word:
                            token = self.tokenizer.tokenize(offending_char)
                            if unicodedata.category(offending_char) != 'So':
                                offending_set.add(offending_char)
                            else:
                                symbol_set.add(offending_char)

                    sentence_token_ids.extend(ids)
                    token_starts.extend([1] + [0] * (len(tokens) - 1))
                    attentions.extend([1] * len(tokens))

                # take care of empty tokens
                else:
                    empty_words.add(word)
                    continue

            # Skip too long sentences
            len_sentence_token_ids = len(sentence_token_ids)
            if len_sentence_token_ids > 128:
                exceeding_count += 1
                continue

            words_numerical.append(torch.tensor(sentence_token_ids))
            attention_mask.append(torch.ByteTensor(attentions))
            token_start_mask.append(torch.ByteTensor(token_starts))
            words_total.append(words)
            tags_total.append(tags)

        return words_numerical, attention_mask, token_start_mask, words_total, tags_total

    @classmethod
    def from_corpus(cls, corpus, min_freq=1):
        words = Counter(word for seq in corpus.words for word in seq)
        words = list(word for word, freq in words.items() if freq >= min_freq)
        chars = list({char for seq in corpus.words for char in ''.join(seq)})
        rels = list({rel for seq in corpus.rels for rel in seq})
        vocab = cls(words, chars, rels)

        return vocab

ext: py

reason: import numpy


text:
# Copyright (c) 2018-2020, NVIDIA CORPORATION.

from __future__ import division

import numpy as np
import pytest

from cudf.core import Series
from cudf.tests.utils import NUMERIC_TYPES

supported_types = NUMERIC_TYPES

@pytest.mark.parametrize("dtype", supported_types)
def test_applymap(dtype):

    size = 500

    lhs_arr = np.random.random(size).astype(dtype)
    lhs_col = Series(lhs_arr)._column

    def generic_function(a):
        return a ** 3

    out_col = lhs_col.applymap(generic_function)

    result = lhs_arr ** 3

    np.testing.assert_almost_equal(result, out_col.to_array())

max_stars_repo_name: isVoid/cudf

max_issues_repo_name: isVoid/cudf

max_forks_repo_name: isVoid/cudf


text:
import time

import numpy as np
import torch.optim as optim
from torch.utils.data import DataLoader

from augmentations import transforms_train, transforms_val
from config import *
from dataset import DC_Dataset, seed_worker
from model import B1_ns
from strategy import GradualWarmupSchedulerV2
from train_valid import train_epoch, valid_epoch

def run(fold):
    content = 'Fold: ' + str(fold)
    print(content)
    with open(log_file, 'a') as appender:appender.write(content + '\n')
    train_ = df_train.query(f'fold!={fold}').copy()
    valid_ = df_train.query(f'fold=={fold}').copy()

    dataset_train = DC_Dataset(train_, 'train', transform=transforms_train)
    dataset_valid = DC_Dataset(valid_, 'valid', transform=transforms_val)
    train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size, shuffle=True, num_workers=num_workers, worker_init_fn=seed_worker)
    valid_loader = torch.utils.data.DataLoader(dataset_valid, batch_size=batch_size, shuffle=False, num_workers=num_workers, worker_init_fn=seed_worker)

    model = B1_ns(enet_type, num_classes)

    if is_resume_train:
        model_file = os.path.join(model_dir, f'{kernel_type}_best_fold{fold}_epo{resume_epoch - 1}.pth')
        model.load_state_dict(torch.load(model_file))

    model = model.to(device)
    aucs_max = 0

    optimizer = optim.Adam(model.parameters(), lr=init_lr)
    scheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, cosine_epo)
    scheduler_warmup = GradualWarmupSchedulerV2(optimizer, multiplier=10, total_epoch=warmup_epo,
                                                after_scheduler=scheduler_cosine)
    for epoch in range(1 if not is_resume_train else resume_epoch, n_epochs + 1):
        print('Epoch:', epoch)
        scheduler_warmup.step(epoch - 1)

        train_loss = train_epoch(model, train_loader, optimizer)
        val_loss, aucs = valid_epoch(model, valid_loader)

        content = f'Fold {fold} Epoch {epoch}, lr: {optimizer.param_groups[0]["lr"]:.7f}, train loss: {train_loss:.4f}, valid loss: {(val_loss):.4f}, aucs: {np.mean(aucs):.4f}.'
        content += '\n' + ' '.join([f'{x:.4f}' for x in aucs])
        print(content)
        with open(log_file, 'a') as appender:
            appender.write(content + '\n')

        if aucs_max < np.mean(aucs):
            model_file = os.path.join(model_dir, f'{kernel_type}_best_fold{fold}_epo{epoch}.pth')
            torch.save(model.state_dict(), model_file)
            aucs_max = np.mean(aucs)

    torch.save(model.state_dict(), os.path.join(model_dir, f'{kernel_type}_best_fold{fold}.pth'))


text:
#!/usr/bin/env python
import os,sys,time,re
import matplotlib.pyplot as plt
import numpy as np
import scipy.stats as stats

## create doc beginning
class LatexReportCreator():
    def __init__(self,docName,docType='report',docDir="report",verbose=False,extraPackages=[],generatedBy="Python"):
        self.docName = docName
        self.docType = docType
        self.docDir = docDir
        self.verbose = verbose
        self.generatedBy = generatedBy
        self.extraPackages = extraPackages
        self.fontSizeList = ["tiny","scriptsize","footnotesize","small","normalsize","large","Large","LARGE","huge","Huge"]

        ## error check
        if type(self.extraPackages) != type([]):
            print("INPUT ERROR: extra packages must be in list form")

        if os.path.isdir(self.docDir) == False:
            if self.verbose == True:
                print("...making report directory")
            os.mkdir(self.docDir)

        self.fid = open(os.path.join(self.docDir,self.docName + ".tex"),'w')

    def write_beginning(self,title,author,shortTitle='report',shortAuthor='',toc=False):
        self.fid.write("\documentclass[letterpaper,10pt]{%s}\n"%self.docType)
        self.fid.write("\\usepackage{pgf,graphics,color,hyperref,fullpage,natbib,algorithm,algorithmic,amsmath,ulem,bm,morefloats}\n")
        for package in self.extraPackages:
            self.fid.write("\\usepackage{%s}\n"%package)
        self.fid.write("\definecolor{darkblue}{rgb}{0.0,0.0,0.50}\n")
        self.fid.write("\definecolor{darkgreen}{rgb}{0.0,0.35,0.0}\n")
        self.fid.write("\hypersetup{colorlinks=true, linkcolor=darkblue, citecolor=darkblue, urlcolor=darkblue}\n")
        self.fid.write("\hypersetup{pdfauthor={%s}, pdftitle={%s}}\n"%(shortAuthor,shortTitle))
        self.fid.write("\\usepackage{titlesec}\\titleformat{\chapter}{\centering\\normalfont\Large\\bfseries}{\\thechapter}{1em}{}\n")

        ## beginning materials
        self.fid.write("\\begin{document}\n")
        self.fid.write("%%%%%\n")
        self.fid.write("\\begin{flushleft}\n")
        self.fid.write("\\textbf{%s}\\\ \n"%title)
        self.fid.write("Created by: %s\\\ \n"%author)
        self.fid.write("Created on: %s\n"%time.asctime())
        self.fid.write("\end{flushleft}\n")
        self.fid.write("\medskip\hrule height 1pt\n")
        self.fid.write("\\vspace{7pt}\n")

        if toc == True:
            self.fid.write("\\tableofcontents\n")

    ## convenience functions
    def begin(self,item):
        if item in self.fontSizeList:
            self.fid.write("\%s\n"%item)
        else:
            self.fid.write("\\begin{%s}\n"%item)

    def end(self,item):
        if item in self.fontSizeList:
            self.fid.write("\\normalsize\n")
        else:
            self.fid.write("\end{%s}\n"%item)

    def section(self,sectionName,sectionType="section",numbers=True,toc=False):
        '''
        toc - used to manually add a section to the table of contents -- must also call toc = True in 'write_beginning'

        '''

        if numbers == True:
            self.fid.write("\%s{%s}\n"%(sectionType,sectionName))
        else:
            self.fid.write("\%s*{%s}\n"%(sectionType,sectionName))

        if toc == True:
            self.fid.write("\\addcontentsline{toc}{%s}{%s}\n"%(sectionType,sectionName))

    ## main functions
    def write_paragraph(self,body):
        self.fid.write(body+"\n\n")

    def write_end(self):
        self.fid.write("\n\end{document}")
        self.fid.close()

    def compile(self):
        currentDir = os.getcwd()
        os.chdir(self.docDir)
        os.system("pdflatex %s"%self.docName+".tex")
        os.chdir(currentDir)

    def include_figure(self,figPath,caption=None,label=None,scale=0.65):
        if os.path.isfile(figPath) == False:
            print("ERROR: could not include figure does not exist", figPath)

        self.fid.write("\\begin{figure}[!ht]\n")
        self.begin("center")
        self.fid.write("\includegraphics[scale = %s]{%s}\n"%(scale,figPath))
        if caption != None:
            self.fid.write("\caption{%s}\n"%caption)
        if label != None:
            self.fid.write("\label{fig:%s}\n"%label)
        self.end("center")
        self.end("figure")

    def include_table(self,rowDict,colHeader=None,customHeader=None,justifications=None,caption=None,label=None,fontSize='normalsize',ordered=False):
        '''
        rowDict - which has row header elements as keys and lists to fill out the cols as values
        colHeader - are the header elements for the columns
        '''

        ## if keys are only numeric
        numericKeys = False
        if not re.search('\D',str(rowDict.keys()[0])):
            numericKeys = True

        ## error checking
        if type(rowDict) != type({}):
            print("ERROR: rowDict for include_table must be of type dict")
        if type(rowDict[rowDict.keys()[0]]) != type([]):
            print("ERROR: the values in rowDict for included_table must be of type list")

        numCols = len(rowDict[rowDict.keys()[0]])
        if justifications == None and numericKeys == False:
            justifications = '|l|'+'c'*numCols+"|"
        elif justifications == None and numericKeys == True:
            justifications = '|l|'+'c'*(numCols-1)+"|"

        self.fid.write('\\begin{table}[!h]\n')
        self.begin('center')
        self.begin(fontSize)
        self.fid.write("\\begin{tabular}{%s}\n"%justifications)
        self.fid.write("\\hline\n")

        if customHeader != None:
            self.fid.write(customHeader)
            self.fid.write("\\hline\n")

        if colHeader != None:
            header = "".join([i + "&" for i in colHeader])[:-1]+"\\\ \n"
            self.fid.write(header)
            self.fid.write("\\hline\n")

        if numericKeys == True:
            sk = [int(k) for k in rowDict.keys()]
            sk.sort()
            for key in sk:
                row = rowDict[str(key)]
                if row[0] == "hline":
                    self.fid.write("\\hline\n")
                else:
                    row = "".join([i + "&" for i in row])[:-1]+"\\\ \n"
                    self.fid.write(row)
        elif type(ordered) == type([]):
            for rowHead in ordered:
                row = rowDict[rowHead]
                if rowHead == 'hline':
                    self.fid.write("\\hline\n")
                else:
                    row = rowHead + "&" + "".join([i + "&" for i in row])[:-1]+"\\\ \n"
                    self.fid.write(row)
        elif ordered == True:
            it = iter(sorted(rowDict.iteritems()))
            for rowHead, row in it:
                if row[0] == 'hline':
                    self.fid.write("\\hline\n")
                else:
                    row = rowHead + "&" + "".join([i + "&" for i in row])[:-1]+"\\\ \n"
                    self.fid.write(row)
        else:
            for rowHead, row in rowDict.iteritems():
                if row[0] == 'hline':
                    self.fid.write("\\hline\n")
                else:
                    row = rowHead + "&" + "".join([i + "&" for i in row])[:-1]+"\\\ \n"
                    self.fid.write(row)

        self.fid.write("\\hline\n")
        self.end('tabular')
        self.end(fontSize)
        self.end('center')
        if caption != None:
            self.fid.write("\caption{%s}\n"%caption)
        if label != None:
            self.fid.write("\label{tab:%s}\n"%label)
        self.end('table')

## run a small test
if __name__ == '__main__':

    ## create a preambble
    report = LatexReportCreator('test',verbose=True)
    report.write_beginning("Test Report Title","Ender Wiggin",toc=True)

    ## create the body
    report.section("About",numbers=False,toc=True)

    report.write_paragraph("this is a paragraph blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah "+
                           "blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah")
    report.write_paragraph("this is a paragraph blah blah blah blah blah blah blah blah blah blah blah blah "+
                           "blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah")

    report.section("Results",numbers=False,toc=True)
    report.write_paragraph("this is a paragraph blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah "+
                           "blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah")
    report.write_paragraph("this is a paragraph blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah "+
                           "blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah")

    colHeader = ["I", "II", "III", "IV"]
    rowDict = {"a":["1","2","3"],
               "b":["4","5","6"],
               "c":["7","8","9"]
               }

    caption = "\\textbf{an example table}. this is a table caption"
    label = "example-table"
    report.include_table(rowDict,colHeader,caption=caption,label=label)

    ## create a test figure
    fig = plt.figure()
    ax = fig.add_subplot(111)
    n = 5000
    alpha,beta = 2.0,3.0
    samples = np.random.beta(alpha,beta,size=n)
    p1 =ax.hist(samples,normed=1,alpha=0.7)
    pdfX = np.linspace(0,1,300)
    pdfY = stats.beta.pdf(pdfX,alpha,beta)
    p2 = ax.plot(pdfX,pdfY,color='orange',linewidth=2.5)
    ax.set_xlim((0,1))

    ax.set_title(r"Beta distribution: $a$ = %s, $b$ = %s, $n$ = %s"%(alpha,beta,n))
    figPath = os.path.join(report.docDir,"examplefig.pdf")
    ax.legend([p2],["pdf"])
    plt.savefig(figPath)

    ## add in test figure
    caption = "\\textbf{The beta distribution}. this is a figure caption"
    label = "beta-distn"
    report.include_figure("examplefig.pdf",caption=caption,label=label,scale=0.5)

    ## add another paragraph and close report
    report.write_paragraph("This is an example of how to make a reference to Figure~\\ref{fig:beta-distn} blah blah blah blah blah "+
                           "blah blah blah blah blah blah blah blah blah blah blah blah blah blah blah")

    report.write_end()
    report.compile()

max_stars_repo_name: ajrichards/hts-integrate

max_issues_repo_name: ajrichards/htsint

max_forks_repo_name: ajrichards/hts-integrate

reason: import numpy,import scipy


text:
# stdlib
from typing import List

# third party
import numpy as np
import pytest
import torch

from sympc.session import Session
from sympc.session import SessionManager
from sympc.tensor import MPCTensor
from sympc.tensor.grads.grad_functions import GradAdd
from sympc.tensor.grads.grad_functions import GradConv2d
from sympc.tensor.grads.grad_functions import GradFlatten
from sympc.tensor.grads.grad_functions import GradFunc
from sympc.tensor.grads.grad_functions import GradMatMul
from sympc.tensor.grads.grad_functions import GradMul
from sympc.tensor.grads.grad_functions import GradPow
from sympc.tensor.grads.grad_functions import GradReshape
from sympc.tensor.grads.grad_functions import GradSigmoid
from sympc.tensor.grads.grad_functions import GradSub
from sympc.tensor.grads.grad_functions import GradSum
from sympc.tensor.grads.grad_functions import GradT
from sympc.utils import parallel_execution

def test_grad_func_abstract_forward_exception() -> None:
    with pytest.raises(NotImplementedError):
        GradFunc.forward({})

def test_grad_func_abstract_backward_exception() -> None:
    with pytest.raises(NotImplementedError):
        GradFunc.backward({})

def test_grad_transpose_forward(get_clients) -> None:
    secret = torch.Tensor([[1, 2, 3], [4, 5, 6]])
    mpc_tensor = secret.share(parties=get_clients(4))

    ctx = {}
    res_mpc = GradT.forward(ctx, mpc_tensor)

    res = res_mpc.reconstruct()
    expected = secret.t()

    assert (res == expected).all()

def test_grad_transpose_backward(get_clients) -> None:
    parties = get_clients(4)
    grad = torch.Tensor([[1, 2, 3], [4, 5, 6]])
    grad_mpc = grad.t().share(parties=parties)

    ctx = {}
    res_mpc = GradT.backward(ctx, grad_mpc)

    res = res_mpc.reconstruct()
    expected = grad

    assert (res == expected).all()

def test_grad_add_different_dims_forward(get_clients) -> None:
    parties = get_clients(4)
    x = torch.Tensor([[1, 2, 3], [4, 5, 6]])
    y = torch.Tensor([1, 2, 3])

    x_mpc = x.share(parties=parties)
    y_mpc = y.share(parties=parties)

    ctx = {}
    res_mpc = GradAdd.forward(ctx, x_mpc, y_mpc)

    res = res_mpc.reconstruct()
    expected = x + y

    assert (res == expected).all()

def test_grad_add_different_dims_backward(get_clients) -> None:
    parties = get_clients(4)

    grad = torch.Tensor([[[2, 4, 6], [5, 7, 9]]])
    grad_x = grad
    grad_y = torch.Tensor([[7, 11, 15]])
    grad_mpc = grad.share(parties=parties)

    ctx = {"x_shape": (2, 3), "y_shape": (1, 3)}
    res_mpc_x, res_mpc_y = GradAdd.backward(ctx, grad_mpc)

    assert (res_mpc_x.reconstruct() == grad_x).all()
    assert (res_mpc_y.reconstruct() == grad_y).all()

def test_grad_add_forward(get_clients) -> None:
    parties = get_clients(4)
    x = torch.Tensor([[1, 2, 3], [4, 5, 6]])
    y = torch.Tensor([[1, 4, 6], [8, 10, 12]])

    x_mpc = x.share(parties=parties)
    y_mpc = y.share(parties=parties)

    ctx = {}
    res_mpc = GradAdd.forward(ctx, x_mpc, y_mpc)

    res = res_mpc.reconstruct()
    expected = x + y

    assert (res == expected).all()

def test_grad_add_backward(get_clients) -> None:
    parties = get_clients(4)

    grad = torch.Tensor([1, 2, 3, 4])
    grad_mpc = grad.share(parties=parties)

    ctx = {"x_shape": (4,), "y_shape": (4,)}
    res_mpc_x, res_mpc_y = GradAdd.backward(ctx, grad_mpc)

    assert (res_mpc_x.reconstruct() == grad).all()
    assert (res_mpc_y.reconstruct() == grad).all()

def test_grad_sum_forward(get_clients) -> None:
    parties = get_clients(4)
    x = torch.Tensor([[1, 2, 3], [4, 5, 6]])

    x_mpc = x.share(parties=parties)

    ctx = {}
    res_mpc = GradSum.forward(ctx, x_mpc)

    assert ctx["x_shape"] == (2, 3)

    res = res_mpc.reconstruct()
    expected = x.sum()

    assert (res == expected).all()

def test_grad_sum_backward(get_clients) -> None:
    parties = get_clients(4)
    grad = torch.tensor(420)

    grad_mpc = grad.share(parties=parties)

    shape = (2, 3)
    ctx = {"x_shape": shape}

    res_mpc = GradSum.backward(ctx, grad_mpc)

    res = res_mpc.reconstruct()
    expected = torch.ones(size=shape) * grad

    assert (res == expected).all()

def test_grad_sub_forward(get_clients) -> None:
    parties = get_clients(4)
    x = torch.Tensor([[1, 2, 3], [4, 5, 6]])
    y = torch.Tensor([[1, 4, 6], [8, 10, 12]])

    x_mpc = x.share(parties=parties)
    y_mpc = y.share(parties=parties)

    ctx = {}
    res_mpc = GradSub.forward(ctx, x_mpc, y_mpc)

    res = res_mpc.reconstruct()
    expected = x - y

    assert (res == expected).all()

def test_grad_sub_backward(get_clients) -> None:
    parties = get_clients(4)

    grad = torch.Tensor([1, 2, 3, 4])
    grad_mpc = grad.share(parties=parties)

    ctx = {"x_shape": (4,), "y_shape": (4,)}
    res_mpc_x, res_mpc_y = GradSub.backward(ctx, grad_mpc)

    assert (res_mpc_x.reconstruct() == grad).all()
    assert (res_mpc_y.reconstruct() == -grad).all()

def test_grad_sub_different_dims_forward(get_clients) -> None:
    parties = get_clients(4)
    x = torch.Tensor([[1, 2, 3], [4, 5, 6]])
    y = torch.Tensor([1, 2, 3])

    x_mpc = x.share(parties=parties)
    y_mpc = y.share(parties=parties)

    ctx = {}
    res_mpc = GradSub.forward(ctx, x_mpc, y_mpc)

    res = res_mpc.reconstruct()
    expected = x - y

    assert (res == expected).all()

def test_grad_sub_different_dims_backward(get_clients) -> None:
    parties = get_clients(4)

    grad = torch.Tensor([[[2, 4, 6], [5, 7, 9]]])
    grad_x = grad
    grad_y = -torch.Tensor([[7, 11, 15]])
    grad_mpc = grad.share(parties=parties)

    ctx = {"x_shape": (2, 3), "y_shape": (1, 3)}
    res_mpc_x, res_mpc_y = GradSub.backward(ctx, grad_mpc)

    assert (res_mpc_x.reconstruct() == grad_x).all()
    assert (res_mpc_y.reconstruct() == grad_y).all()

def test_grad_sigmoid_forward(get_clients) -> None:
    # We need Function Secret Sharing (only for 2 parties) for
    # comparing
    parties = get_clients(2)
    x = torch.Tensor([7, 10, 12])

    x_mpc = x.share(parties=parties)

    ctx = {}
    res_mpc = GradSigmoid.forward(ctx, x_mpc)

    assert "probabilities" in ctx

    res = res_mpc.reconstruct()
    expected = x.sigmoid()

    assert np.allclose(res, expected, rtol=1e-2)

def test_grad_sigmoid_backward(get_clients) -> None:
    parties = get_clients(4)
    grad = torch.tensor([0.3, 0.4, 0.7])

    grad_mpc = grad.share(parties=parties)

    ctx = {"probabilities": grad}

    res_mpc = GradSigmoid.backward(ctx, grad_mpc)

    res = res_mpc.reconstruct()
    expected = grad * grad * (1 - grad)

    assert np.allclose(res, expected, rtol=1e-2)

def test_grad_mul_forward(get_clients) -> None:
    parties = get_clients(4)
    x = torch.Tensor([[1, 2], [3, -4]])
    y = torch.Tensor([[1, -4], [8, 9]])

    x_mpc = x.share(parties=parties)
    y_mpc = y.share(parties=parties)

    ctx = {}
    res_mpc = GradMul.forward(ctx, x_mpc, y_mpc)

    assert "x" in ctx
    assert "y" in ctx

    res = res_mpc.reconstruct()
    expected = x * y

    assert np.allclose(res, expected, rtol=1e-3)

def test_grad_mul_backward(get_clients) -> None:
    parties = get_clients(4)

    grad = torch.Tensor([[1, 2], [3, 4]])
    x = torch.Tensor([[1, 2], [3, -4]])
    y = torch.Tensor([[1, -4], [8, 9]])

    x_mpc = x.share(parties=parties)
    y_mpc = y.share(parties=parties)
    grad_mpc = grad.share(parties=parties)

    ctx = {"x": x_mpc, "y": y_mpc}

    res_mpc_x, res_mpc_y = GradMul.backward(ctx, grad_mpc)

    assert np.allclose(res_mpc_x.reconstruct(), y * grad, rtol=1e-3)
    assert np.allclose(res_mpc_y.reconstruct(), x * grad, rtol=1e-3)

def test_grad_conv2d_forward(get_clients) -> None:
    parties = get_clients(4)
    input_secret = torch.ones(1, 1, 4, 4)
    weight_secret = torch.ones(1, 1, 2, 2)
    input = input_secret.share(parties=parties)
    weight = weight_secret.share(parties=parties)

    kwargs = {"bias": None, "stride": 1, "padding": 0, "dilation": 1, "groups": 1}

    ctx = {}
    res_mpc = GradConv2d.forward(ctx, input, weight, **kwargs)

    assert "input" in ctx
    assert "weight" in ctx
    assert "stride" in ctx
    assert "padding" in ctx
    assert "dilation" in ctx
    assert "groups" in ctx

    res = res_mpc.reconstruct()
    expected = torch.nn.functional.conv2d(input_secret, weight_secret, **kwargs)

    assert np.allclose(res, expected, rtol=1e-3)

def test_grad_conv2d_backward(get_clients) -> None:
    parties = get_clients(4)

    grad = torch.Tensor([[[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]]]])

    input = torch.Tensor(
        [
            [
                [
                    [-2.1805, -1.3338, -0.9718, -0.1335],
                    [-0.5632, 1.2667, 0.9994, -0.0627],
                    [-0.9563, 0.5861, -1.4422, -0.4825],
                    [0.2732, -1.1900, -0.6624, -0.7513],
                ]
            ]
        ]
    )

    weight = torch.Tensor([[[[0.3257, -0.7538], [-0.5773, -0.7619]]]])

    x_mpc = input.share(parties=parties)
    y_mpc = weight.share(parties=parties)
    grad_mpc = grad.share(parties=parties)

    ctx = {
        "input": x_mpc,
        "weight": y_mpc,
        "stride": 1,
        "padding": 0,
        "dilation": 1,
        "groups": 1,
    }

    res_mpc_input, res_mpc_weight = GradConv2d.backward(ctx, grad_mpc)
    expected_input = torch.nn.functional.grad.conv2d_input(input.size(), weight, grad)
    expected_weight = torch.nn.functional.grad.conv2d_weight(input, weight.size(), grad)

    assert np.allclose(res_mpc_input.reconstruct(), expected_input, rtol=1e-3)
    assert np.allclose(res_mpc_weight.reconstruct(), expected_weight, rtol=1e-3)

@pytest.mark.parametrize(
    "common_args", [[(6, 6), 2, 1, (3, 3), 1], [(4, 4), 1, 0, (2, 2), 1]]
)
@pytest.mark.parametrize("nr_parties", [2, 3, 4])
def test_get_grad_input_padding(get_clients, common_args: List, nr_parties) -> None:
    clients = get_clients(2)
    session = Session(parties=clients)
    SessionManager.setup_mpc(session)

    grad = torch.Tensor([[[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]]]])
    grad_mpc = MPCTensor(secret=grad, session=session)

    input_size, stride, padding, kernel_size, dilation = common_args

    expected_padding = torch.nn.functional.grad._grad_input_padding(
        grad,
        input_size,
        (stride, stride),
        (padding, padding),
        kernel_size,
        (dilation, dilation),
    )

    args = [[el] + common_args + [session] for el in grad_mpc.share_ptrs]
    shares = parallel_execution(
        GradConv2d.get_grad_input_padding, grad_mpc.session.parties
    )(args)
    grad_input_padding = MPCTensor(shares=shares, session=grad_mpc.session)
    output_padding_tensor = grad_input_padding.reconstruct()
    output_padding_tensor /= grad_mpc.session.nr_parties
    calculated_padding = tuple(output_padding_tensor.to(torch.int).tolist())

    assert calculated_padding == expected_padding

def test_grad_reshape_forward(get_clients) -> None:
    parties = get_clients(4)
    x = torch.Tensor([[1, 2], [3, -4], [-9, 0]])

    x_mpc = x.share(parties=parties)

    ctx = {}
    shape = (3, 2)
    res_mpc = GradReshape.forward(ctx, x_mpc, shape)

    assert "x_shape" in ctx

    res_shape = res_mpc.shape

    assert res_shape == shape
    assert np.allclose(res_mpc.reconstruct(), x.reshape(shape), rtol=1e-3)

def test_grad_reshape_backward(get_clients) -> None:
    parties = get_clients(4)

    grad = torch.Tensor([[1, 2, 3], [3, 4, 7]])
    x = torch.Tensor([[1, 2], [3, -4], [5, 8]])

    x_mpc = x.share(parties=parties)
    grad_mpc = grad.share(parties=parties)

    ctx = {"x_shape": x_mpc.shape}

    res_mpc_grad = GradReshape.backward(ctx, grad_mpc)
    res_mpc_grad_shape = res_mpc_grad.shape

    assert res_mpc_grad_shape == x_mpc.shape
    assert np.allclose(res_mpc_grad.reconstruct(), grad.reshape(x_mpc.shape), rtol=1e-3)

@pytest.mark.parametrize("args", [[0, -1], [1, -1], [0, 1]])
def test_grad_flatten_forward(get_clients, args: list) -> None:
    parties = get_clients(4)
    x = torch.tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])

    x_mpc = x.share(parties=parties)

    ctx = {}
    start_dim, end_dim = args
    res_mpc = GradFlatten.forward(ctx, x_mpc, start=start_dim, end=end_dim)

    assert "x_shape" in ctx

    expected = torch.flatten(x, start_dim=start_dim, end_dim=end_dim)
    assert np.allclose(res_mpc.reconstruct(), expected, rtol=1e-3)

def test_grad_flatten_backward(get_clients) -> None:
    parties = get_clients(4)

    grad = torch.Tensor([1, 2, 3, 4, 5, 6, 7, 8])
    x = torch.Tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])

    x_mpc = x.share(parties=parties)
    grad_mpc = grad.share(parties=parties)

    ctx = {"x_shape": x_mpc.shape}

    res_mpc_grad = GradFlatten.backward(ctx, grad_mpc)

    assert np.allclose(res_mpc_grad.reconstruct(), x, rtol=1e-3)

@pytest.mark.parametrize("power", [2, 4, 5])
def test_grad_pow_forward(get_clients, power) -> None:
    parties = get_clients(4)
    x = torch.Tensor([[1, 2, 3], [4, 5, 6]])

    x_mpc = x.share(parties=parties)

    ctx = {}

    res_mpc = GradPow.forward(ctx, x_mpc, power)

    assert "x" in ctx
    assert "y" in ctx

    res = res_mpc.reconstruct()
    expected = x ** power

    assert np.allclose(res, expected, rtol=1e-3)

@pytest.mark.parametrize("power", [1.0, torch.tensor([1, 3])])
def test_grad_pow_forward_exception(get_clients, power) -> None:
    parties = get_clients(4)
    x = torch.Tensor([[1, 2, 3], [4, 5, 6]])

    x_mpc = x.share(parties=parties)

    ctx = {}

    with pytest.raises(TypeError):
        GradPow.forward(ctx, x_mpc, power)

@pytest.mark.parametrize("power", [2, 4, 5])
def test_grad_pow_backward(get_clients, power) -> None:
    parties = get_clients(4)

    grad = torch.Tensor([1, 2, 3, 4])
    grad_mpc = grad.share(parties=parties)

    x = torch.Tensor([1, 4, 9, 16])
    x_mpc = x.share(parties=parties)

    ctx = {"x": x_mpc, "y": power}
    res_mpc = GradPow.backward(ctx, grad_mpc)
    res = res_mpc.reconstruct()

    expected = power * x ** (power - 1) * grad

    assert np.allclose(res, expected, rtol=1e-3)

def test_grad_matmul_forward(get_clients) -> None:
    parties = get_clients(4)
    x = torch.Tensor([[1, 2], [3, -4]])
    y = torch.Tensor([[1, -4], [8, 9]])

    x_mpc = x.share(parties=parties)
    y_mpc = y.share(parties=parties)

    ctx = {}
    res_mpc = GradMatMul.forward(ctx, x_mpc, y_mpc)

    assert "x" in ctx
    assert "y" in ctx

    res = res_mpc.reconstruct()
    expected = x @ y

    assert np.allclose(res, expected, rtol=1e-3)

def test_grad_matmul_backward(get_clients) -> None:
    parties = get_clients(4)

    grad = torch.Tensor([[1, 2], [3, 4]])
    x = torch.Tensor([[1, 2], [3, -4]])
    y = torch.Tensor([[1, -4], [8, 9]])

    x_mpc = x.share(parties=parties)
    y_mpc = y.share(parties=parties)
    grad_mpc = grad.share(parties=parties)

    ctx = {"x": x_mpc, "y": y_mpc}

    res_mpc_x, res_mpc_y = GradMatMul.backward(ctx, grad_mpc)

    assert np.allclose(res_mpc_x.reconstruct(), grad @ y.T, rtol=1e-3)
    assert np.allclose(res_mpc_y.reconstruct(), x.T @ grad, rtol=1e-3)

    # Test Value Error
    y = torch.Tensor([[1, -4], [8, 9], [10, 11]])
    y_mpc = y.share(parties=parties)

    ctx = {"x": x_mpc, "y": y_mpc}

    with pytest.raises(ValueError):
        GradMatMul.backward(ctx, grad_mpc)

max_stars_repo_name: Boluwatifeh/SyMPC

max_issues_repo_name: Boluwatifeh/SyMPC

max_forks_repo_name: Boluwatifeh/SyMPC


text:
import numpy as np
import pickle
import pandas as pd
from gradientboostingmodel import GrientBoostingModel
from knnmodel import KNNModel
from preprocess.splittrainvalidation import HoldoutSplitMethod
from xgboostmodel import DidiXGBoostModel
import logging
import sys

class ForwardFeatureSel:
    """Implement greedy forward feature selection
       it requries the model to implement run_croos_validation method
       it support two ways to find best feature combinations: manual and automatic
    """
    def __init__(self):
        root = logging.getLogger()
        root.setLevel(logging.DEBUG)
        root.addHandler(logging.StreamHandler(sys.stdout))
        root.addHandler(logging.FileHandler('logs/forwardfeatureselection_knn.log', mode='w'))
        clfDict = {1: GrientBoostingModel, 2:KNNModel, 3: DidiXGBoostModel}
        self.clf =  clfDict[2]()
#         self.result = []
#         self.featureList = [101,102, 201,502]
        self.featureList =  [101,102,103,104,105,106,107,
                             201, 203,204,205,206,
                             301,
                             401,402,
                             501,502,503,504,505,506,507,
                             601,602,603,604,605,606,
                             8801,8802
                             ]

        return

    def selectBestFeaturList(self, featureLists):
        result_combinations = []
        for fealist in featureLists:
                try:
                    self.clf.usedFeatures = fealist
                    logging.info( "Feature combination: {}, {}".format(self.clf.usedFeatures, self.clf.get_used_features()))

                    res =  self.clf.run_croos_validation()
                    result_combinations.append([res, fealist, self.clf.get_used_features()])
                except Exception as inst:
                    logging.info("XXXXXXX Ignore this combinationXXXXXX {}, features used: {}".format(inst,  fealist))
        return self.disResult(result_combinations)

    def run(self):
        base_feature_list = [603, 101,602,103,203,606]
        featureLists =  self.generateFeatureList(base_feature_list)
        self.selectBestFeaturList(featureLists)

        return
    def auto_run(self):
        base_feature_list = []
        res = []
        while len(base_feature_list) < len(self.featureList):
            featureLists =  self.generateFeatureList(base_feature_list)
            base_feature_list, best_score = self.selectBestFeaturList(featureLists)
            res.append((best_score, base_feature_list, self.clf.get_used_features()))
        df = pd.DataFrame(res, columns=['scores', 'featlist_id', 'feature_list'])
        logging.info("Final Result: {}".format(df))

        df2 = df.sort(columns = ['scores'], ascending=False).reset_index(drop = True)
        logging.info("################Final Best combinations: {}".format(df2[0:1].values))
        return
    def generateFeatureList(self, basefeature):
        res = []
        for fea in self.featureList:
            if fea in basefeature:
                continue
            tempList = []
            tempList.append(fea)
            res.append(basefeature + tempList)
        if(len(res) == 0):
            res = [basefeature]
        return res
    def disResult(self, result_combinations):
        df = pd.DataFrame(result_combinations, columns= ['scores', 'featlist_id','feature_list'])
        df2 = df.sort(columns = ['scores'], ascending=False).reset_index(drop = True)
        logging.info("result for above feature combination: {}".format(df2))
        logging.info("@@@@@@@best combinations: {}".format(df2[0:1].values))
        return df2[0:1].featlist_id[0], df2[0:1].scores[0]

if __name__ == '__main__':
    test = ForwardFeatureSel()
#     test.run()
    test.auto_run()


text:
"""
https://pythonprogramming.net/template-matching-python-opencv-tutorial/
"""
import cv2
import numpy as np

img_bgr = cv2.imread('opencv-template-matching-python-tutorial.jpg')
img_gray = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY)

template = cv2.imread('opencv-template-for-matching.jpg', 0)
w, h = template.shape[::-1]

res = cv2.matchTemplate(img_gray, template, cv2.TM_CCOEFF_NORMED)
threshold = 0.75
loc = np.where(res >= threshold)

for pt in zip(*loc[::-1]):
    color = np.random.randint(0, 256, 3)
    color = color.tolist()  # https://stackoverflow.com/a/16192589/1123955
    cv2.rectangle(
        img_bgr,
        pt, (pt[0] + w, pt[1] + h),
        color,
        # (0,255,0),
        2)

cv2.imshow('detected', img_bgr)

cv2.waitKey(0)
cv2.destroyAllWindows()

max_stars_repo_name: zixia/python-facenet

max_issues_repo_name: zixia/python-facenet

max_forks_repo_name: zixia/python-facenet


text:
from djitellopy import Tello
import cv2
import pygame
from pygame.locals import *
import numpy as np
import time
import math

# Speed of the drone
S = 60
# Frames per second of the pygame window display
FPS = 25
#initial coordinates
py = 0.0
#pz = 0.0
px = 0.0
vx = 0.0
vy = 0.0
i = 1
#vz = 0.0
class FrontEnd(object):
    """ Maintains the Tello display and moves it through the keyboard keys.
        Press escape key to quit.
        The controls are:
            - T: Takeoff
            - L: Land
            - Arrow keys: Forward, backward, left and right.
            - A and D: Counter clockwise and clockwise rotations
            - W and S: Up and down.
    """

    def __init__(self):
        # Init pygame
        pygame.init()

        # Creat pygame window
        pygame.display.set_caption("Tello video stream")
        self.screen = pygame.display.set_mode([960, 720])

        # Init Tello object that interacts with the Tello drone
        self.tello = Tello()
        #initital r,p,y
        self.initial_roll=0.0
        self.initial_pitch=0.0
        self.initial_yaw=0.0
        #initial accelerations
        self.initial_ax = 0.0
        self.initial_ay = 0.0
        self.initial_az = 9999.0

        # Drone velocities between -100~100
        self.for_back_velocity = 0
        self.left_right_velocity = 0
        self.up_down_velocity = 0
        self.yaw_velocity = 0
        self.speed = 10

        self.send_rc_control = False
        self.ptx = 0.0
        self.pty = 0.0
        #self.ptz = 0.0
        self.vx1 = 0.0
        self.vy1 = 0.0
        #self.vz1 = 0.0
        self.ax = 0.0
        self.ay = 0.0
        #self.az = 0.0
        # create update timer
        pygame.time.set_timer(USEREVENT + 1, 50)
    def get_acc(self):
        roll = math.radians(self.tello.get_roll()-self.initial_roll)
        pitch = math.radians(self.tello.get_pitch()-self.initial_pitch)
        yaw = -math.radians(self.tello.get_yaw()-self.initial_yaw)
        Tz = np.matrix([[math.cos(yaw), math.sin(yaw), 0], [-math.sin(yaw), math.cos(yaw), 0], [0, 0, 1]])
        Ty = np.matrix([[math.cos(pitch), 0, math.sin(pitch)], [0, 1, 0], [-math.sin(pitch), 0, math.cos(pitch)]])
        Tx = np.matrix([[1, 0, 0], [0, math.cos(roll), -math.sin(roll)], [0, math.sin(roll), math.cos(roll)]])
        T_n = Tz*Ty*Tx
        A = np.matrix([[self.tello.get_agx()], [self.tello.get_agy()], [self.tello.get_agz()]])
        A_correct = T_n*A
        A_correct[0] = A_correct[0]-self.initial_ax
        A_correct[1] = A_correct[1]-self.initial_ay
        A_correct[2] = A_correct[2]-self.initial_az
        return A_correct

    def run(self):

        if not self.tello.connect():
            print("Tello not connected")
            return

        if not self.tello.set_speed(self.speed):
            print("Not set speed to lowest possible")
            return

        # In case streaming is on. This happens when we quit this program without the escape key.
        if not self.tello.streamoff():
            print("Could not stop video stream")
            return

        if not self.tello.streamon():
            print("Could not start video stream")
            return

        frame_read = self.tello.get_frame_read()

        should_stop = False
        while not should_stop:

            for event in pygame.event.get():
            	global px,py,pz,vx,vy,vz,i
            	start = time.time()
            	#v_x =self.tello.get_vgx()
            	#v_y =self.tello.get_vgy()
            	#v_z =self.tello.get_vgz()
            	#ax = self.tello.get_agx()
            	#ay = self.tello.get_agy()
            	#az = self.tello.get_agz()
                A = self.get_acc()

                end = time.time()
                #start1 = time.time()
                self.ptx = px
                self.pty = py
                #self.ptz = pz
                self.vx1 = A[0]*(end-start)
                self.vy1 = A[1]*(end-start)
                #self.vz1 = self.az*(end-start)
                vx = vx + self.vx1
                vy = vy + self.vy1
                #vz = vz + self.vz1
                px = self.ptx + (vx*1000)*((end-start)*1000)/1000000.0
                py = self.pty + (vy*1000)*((end-start)*1000)/1000000.0
            	#pz = self.ptz + vz*(end-start)
            	print("VX :   "+ str(vx) + "    VY :   " + str(vy))
            	print("X :   "+ str(px) + "    Y :   " + str(py))
            	print("AX :   "+ str(A[0]) + "    AY :   " + str(A[1])+"    AZ :    " + str(A[2]))
                if event.type == USEREVENT + 1:
                	#end1 = time.time()
                	#print("Extra time lag : "+str(end1-start1))
                	self.update()
                elif event.type == QUIT:
                	#end1 = time.time()
                	#print("Extra time lag : "+str(end1-start1))
                	should_stop = True
                elif event.type == KEYDOWN:
                    if event.key == K_ESCAPE:
                    	#end1 = time.time()
                    	#print("Extra time lag : "+str(end1-start1))
                    	should_stop = True
                    else:
                    	#end1 = time.time()
                    	#print("Extra time lag : "+str(end1-start1))
                        self.keydown(event.key)
                elif event.type == KEYUP:
                	#end1 = time.time()
                	#print("Extra time lag : "+str(end1-start1))
                	self.keyup(event.key)

            if frame_read.stopped:
                frame_read.stop()
                break

            self.screen.fill([0, 0, 0])
            frame = cv2.cvtColor(frame_read.frame, cv2.COLOR_BGR2RGB)
            frame = np.rot90(frame)
            frame = np.flipud(frame)
            frame = pygame.surfarray.make_surface(frame)
            self.screen.blit(frame, (0, 0))
            pygame.display.update()

            time.sleep(1 / FPS)

        # Call it always before finishing. I deallocate resources.
        self.tello.end()

    def keydown(self, key):
        """ Update velocities based on key pressed
        Arguments:
            key: pygame key
        """
        if key == pygame.K_UP:  # set forward velocity
            self.for_back_velocity = S
        elif key == pygame.K_DOWN:  # set backward velocity
            self.for_back_velocity = -S
        elif key == pygame.K_LEFT:  # set left velocity
            self.left_right_velocity = -S
        elif key == pygame.K_RIGHT:  # set right velocity
            self.left_right_velocity = S
        elif key == pygame.K_w:  # set up velocity
            self.up_down_velocity = S
        elif key == pygame.K_s:  # set down velocity
            self.up_down_velocity = -S
        elif key == pygame.K_a:  # set yaw clockwise velocity
            self.yaw_velocity = -S
        elif key == pygame.K_d:  # set yaw counter clockwise velocity
            self.yaw_velocity = S

    def keyup(self, key):
        """ Update velocities based on key released
        Arguments:
            key: pygame key
        """
        if key == pygame.K_UP or key == pygame.K_DOWN:  # set zero forward/backward velocity
            self.for_back_velocity = 0
        elif key == pygame.K_LEFT or key == pygame.K_RIGHT:  # set zero left/right velocity
            self.left_right_velocity = 0
        elif key == pygame.K_w or key == pygame.K_s:  # set zero up/down velocity
            self.up_down_velocity = 0
        elif key == pygame.K_a or key == pygame.K_d:  # set zero yaw velocity
            self.yaw_velocity = 0
        elif key == pygame.K_t:  # takeoff
            self.tello.takeoff()
            self.send_rc_control = True
        elif key == pygame.K_l:  # land
            self.tello.land()
            self.send_rc_control = False

    def update(self):
        """ Update routine. Send velocities to Tello."""
        if self.send_rc_control:
            self.tello.send_rc_control(self.left_right_velocity, self.for_back_velocity, self.up_down_velocity,
                                       self.yaw_velocity)

def main():
    frontend = FrontEnd()

    # run frontend
    frontend.run()

if __name__ == '__main__':
    main()


text:
import numpy as np
from mvn import *
from utils import kernel_covariance

class FactorGP:
    """
    Latent factor Gaussian process model for multivariate time series
    Data: n epochs, t time points, q dimensional, r latent factors
    Parameters: loading matrix (r x q), variance vector (q), length scale (r)
    Priors: Conjugate Normal, inverse-Gamma, and Gamma (needs to be informative)
    """

    def __init__(self, dims, mu_var=[0, 1], inverse_gamma=[1, 1], gamma=[10, 1], F=None):
        n, t, q, r = dims
        self.dims = dims
        self.x = np.linspace(1, t, t)  # time points are indexed by intergers from 1 to t
        self.loading_prior_params = mu_var  # prior mean and variance for loading coeffcients
        self.variance_prior_params = inverse_gamma  # inverse Gamma prior for variance
        self.length_prior_params = gamma  # Gamma prior on length scale
        # self.kernel_type = 'default'
        self.loading, self.variance, self.theta = self.__initiate_params(dims, mu_var, inverse_gamma, gamma)
        self.F = F

    def __initiate_params(self, dims, mu_var, inverse_gamma, gamma):
        n, t, q, r = dims
        loading = np.random.normal(mu_var[0], np.sqrt(mu_var[1]), [r, q])
        variance = np.random.normal(0, 0.5, q) ** 2  # TODO: implement inverse-Gamma prior
        theta = np.repeat(gamma[0] * gamma[1], r)  # set length scale to gamma mean
        return loading, variance, theta

    def update_conditional_latent(self, Y):
        n, t, q, r = self.dims
        covs = []
        for l in self.theta:
            covs.append(kernel_covariance(self.x, l, 1.0))
        prod, covariance = conditional_F_dist(covs, self.loading, self.variance)  # only invert covariance once
        F = np.zeros((n * t, r))
        for i in range(n):  # sample from F conditional distribution for each epoch independently
            F[(i * t):(i * t + t), :] = sample_conditonal_F_dist(Y[(i * t):(i * t + t), :], prod, covariance)
        self.F = F

    def predict(self):
        return np.matmul(self.F, self.loading)

class IterFactorGP:
    """
    Update latent factors iteratively.
    """

    def __init__(self, dims, mu_var=[0, 1], inverse_gamma=[1, 1], gamma=[10, 1], F=None):
        n, t, q, r = dims
        self.dims = dims
        self.x = np.linspace(1, t, t)  # time points are indexed by intergers from 1 to t
        self.loading_prior_params = mu_var  # prior mean and variance for loading coeffcients
        self.variance_prior_params = inverse_gamma  # inverse Gamma prior for variance
        self.length_prior_params = gamma  # Gamma prior on length scale
        # self.kernel_type = 'default'
        self.loading, self.variance, self.theta = self.__initiate_params(dims, mu_var, inverse_gamma, gamma)
        self.F = F

    def __initiate_params(self, dims, mu_var, inverse_gamma, gamma):
        n, t, q, r = dims
        loading = np.random.normal(mu_var[0], np.sqrt(mu_var[1]), [r, q])
        variance = np.random.normal(0, 0.5, q) ** 2  # TODO: implement inverse-Gamma prior
        theta = np.repeat(gamma[0] * gamma[1], r)  # set length scale to gamma mean
        return loading, variance, theta

    def update_conditional_latent(self, Y):
        n, t, q, r = self.dims
        covs = []
        for l in self.theta:
            covs.append(kernel_covariance(self.x, l, 1.0))
        F = np.zeros((n * t, r))
        residuals = Y.copy()
        for j in range(r):  # update factors iteratively
            prod, covariance = conditional_factor_dist(covs, self.loading, self.variance, j)
            for i in range(n):  # sample from F conditional distribution for each epoch independently
                F[(i * t):(i * t + t), j] = sample_conditonal_factor_dist(residuals[(i * t):(i * t + t), :], prod, covariance)
            hat = np.matmul(F[:, j].reshape((n * t, 1)), self.loading[j, :].reshape((1, q)))
            residuals = residuals - hat
        self.F = F

    def predict(self):
        return np.matmul(self.F, self.loading)


text:
import time
import os
import json
import sys
from pprint import pformat

import joblib
from joblib import Parallel, delayed

import rasterio
from rasterio.plot import reshape_as_image
from rasterio.mask import mask
from rasterio.merge import merge
import geopandas as gpd
import pandas as pd
import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, cohen_kappa_score

from shapely.geometry import mapping
from geoRpro.utils import NumpyEncoder, to_json, json_to_disk, load_json, gen_sublist
import geoRpro.raster as rst

import logging
import pdb

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

def cohen_kappa(confusion):
    """
    Simplified version of: cohen_kappa_score
    https://github.com/scikit-learn/scikit-learn/blob/main/sklearn/metrics/_classification.py
    """
    n_classes = confusion.shape[0]
    sum0 = np.sum(confusion, axis=0)
    sum1 = np.sum(confusion, axis=1)
    expected = np.outer(sum0, sum1) / np.sum(sum0)

    w_mat = np.ones([n_classes, n_classes], dtype=int)
    w_mat.flat[:: n_classes + 1] = 0
    k = np.sum(w_mat * confusion) / np.sum(w_mat * expected)
    return 1 - k

def train_RandomForestClf(X, y, estimators):
    """
    Use train_test split to get accuacy of a Random forerst classifier
    """
    logger.info(f'Start to train the model with estimators: {estimators}')
    data = {}

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, stratify=y)

    clf = RandomForestClassifier(n_estimators=estimators)
    clf.fit(X_train, y_train)

    # Acc on X_train (Internal)
    y_pred_train = clf.predict(X_train)
    cm_train = confusion_matrix(y_train,y_pred_train)

    ## NOTE!! This is a new confusion matrix without dummy classes
    ## Valid ONLY for a specific case. To be removed or generalized !!
    cm_train = cm_train[:-2, :-2]
    strg_cm_train = np.array2string(cm_train)
    data['cm_train'] = cm_train
    internal_accuracy=sum(np.diagonal(cm_train))/np.sum(cm_train)
    internal_kappa=cohen_kappa(cm_train)
    data['train_oa'] = internal_accuracy
    data['train_ck'] = internal_kappa
    logger.info(f"Train overall accuracy: {internal_accuracy}")
    logger.info(f"Train cohen_kappa accuracy: {internal_kappa}")

    # Acc on X_test (External)
    y_pred = clf.predict(X_test)
    cm_test = confusion_matrix(y_test,y_pred)
    ## NOTE!! This is a new confusion matrix without dummy classes
    ## Valid ONLY for a specific case. To be removed or generalized !!
    cm_test = cm_test[:-2, :-2]
    strg_cm_test = np.array2string(cm_test)
    data['cm_test'] = cm_test
    external_accuracy=sum(np.diagonal(cm_test))/np.sum(cm_test)
    external_kappa=cohen_kappa(cm_test)
    data['test_oa'] = external_accuracy
    data['test_ck'] = external_kappa
    logger.info(f"Test overall accuracy: {external_accuracy}")
    logger.info(f"Test cohen_kappa accuracy: {external_kappa}")

    diag = np.diagonal(cm_test, offset=0)
    label_ids = np.unique(y)
    data['commission_error'] = {}
    data['omission_error'] = {}
    data['user_accuracy'] = {}
    data['producer_accuracy'] = {}

    logger.info(f"Commission error:")
    for idx, row in enumerate(cm_test):
        commission_error  =  round( sum(np.delete(row, idx)) / sum(row), 2)
        label = label_ids[idx]
        data['commission_error'][label] = commission_error
        logger.info(f"label_id: {label_ids[idx]}, commission error {commission_error}")

    logger.info(f"Omission error:")
    for idx, col in enumerate(cm_test.T):
        omission_error  =  round( sum(np.delete(col, idx)) / sum(col), 2)
        label = label_ids[idx]
        data['omission_error'][label] = omission_error
        logger.info(f"label_id: {label_ids[idx]}, omission error {omission_error}")

    logger.info(f"User accuracy:")
    for idx, row in enumerate(cm_test):
        user_accuracy  =  round( diag[idx] / sum(row), 2)
        label = label_ids[idx]
        data['user_accuracy'][label] = user_accuracy
        logger.info(f"label_id: {label_ids[idx]}, user accuracy {user_accuracy}")

    logger.info(f"Producer accuracy:")
    for idx, col in enumerate(cm_test.T):
        producer_accuracy  = round( diag[idx] / sum(col), 2)
        label = label_ids[idx]
        data['producer_accuracy'][label] = producer_accuracy
        logger.info(f"label_id: {label_ids[idx]}, producer accuracy {producer_accuracy}")

    ##print(classification_report(y_test,y_pred))
    return clf, data

def predict_arr(patch, classifier):
    """
    Use a classifier to predict on an array

    params:
    ----------

    patch : nd numpy array

    classifier: optimized classifier

    return:
        2D numpy array of int with
        the same size as the window
    """
    patch = reshape_as_image(patch)
    np.nan_to_num(patch, copy=False,nan=0.0, posinf=0,neginf=0)
    # Generate a prediction array
    # new rows: rows * cols
    # new cols: bands
    class_prediction = classifier.predict(patch.reshape(-1, patch.shape[2]))
    # Reshape our classification map back into a 2D matrix so we can visualize it
    class_prediction = class_prediction.reshape(patch[:,:,0].shape)

    return class_prediction

def predict_parallel(src, classifier, write=False, fpath=None):
    """
    Use a classifier to predict on an entire raster. A number of patches, usually
    equal to the number_of_cpu are processed in parallel

    params:
    ----------

    src : nd numpy array

    classifier: optimized classifier

    number_of_cpu: int

    write: bool

    fpath: str

    return:
        2D numpy array of int with
        the same size as the window

    """
    # Allocate a numpy array for strings
    logger.info('Start predictions on real data...')
    class_final = np.empty((src.height, src.width), dtype = np.int64)
    logger.debug('Numpy arr allocated for classname of type {}'.format(class_final.dtype))

    windows = rst.get_windows(src)
    number_of_cpu = joblib.cpu_count()
    print(f"cpus: {number_of_cpu}")

    for blocks in gen_sublist(windows, number_of_cpu):
        patches =  [rst.load_window(src, w) for w in blocks]

        with joblib.parallel_backend(backend="threading"):
            parallel = Parallel(n_jobs=number_of_cpu, verbose=5)
            results = parallel([delayed(predict_arr)(patch, classifier)
                for patch, _ in patches])

        for r, w in zip(results, blocks): # fill in the class_final with class id
            class_final[ w.row_off:w.row_off+w.height,
                    w.col_off:w.col_off+w.width] = r

    logger.info(f'All patches processed')
    if write:
        logger.info('Writing class prediction to disk')
        json_arr = to_json({'class_prediction':class_final}, encoder=NumpyEncoder)
        json_to_disk(json_arr, fpath)
    return class_final

def predict(src, classifier, write=False, fpath=None):
    """
    Use an already optimized skitlearn classifier to classify pixels of
    a raster object.
    ************
    params:
        r_obj -> An istance of the Raster class
        classifier -> optimised classifier
        outdir -> full path to output directory
    return:
        class_final: A 2D numpy array of classnames with
                     the same height and width of src
    """
    # Allocate a numpy array for strings
    logger.info('Start predictions on real data...')
    class_final = np.empty((src.height, src.width), dtype = np.int64)
    logger.debug('Numpy arr allocated for classname of type {}'.format(class_final.dtype))

    for idx, w in enumerate(rst.gen_windows(src), start=1):
        logger.info(f'Processing patch: {idx}')
        logger.info(f'Window: {w}')
        patch, meta = rst.load_window(src, w)
        patch = reshape_as_image(patch)
        np.nan_to_num(patch, copy=False,nan=0.0, posinf=0,neginf=0)
        # Generate a prediction array
        # new rows: rows * cols
        # new cols: bands
        class_prediction = classifier.predict(patch.reshape(-1, patch.shape[2]))
        # Reshape our classification map back into a 2D matrix so we can visualize it
        class_prediction = class_prediction.reshape(patch[:,:,0].shape)
        # fill in the class_final with class names
        class_final[ w.row_off:w.row_off+w.height, w.col_off:w.col_off+w.width] = class_prediction
    logger.info(f'All patches processed')
    if write:
        logger.info('Writing class prediction to disk')
        json_arr = to_json({'class_prediction':class_final}, encoder=NumpyEncoder)
        json_to_disk(json_arr, fpath)
    return class_final

max_stars_repo_name: ESS-uzh/geoRpro

max_issues_repo_name: ESS-uzh/geoRpro

max_forks_repo_name: ESS-uzh/geoRpro


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Course :
    GTI770 — Systèmes intelligents et apprentissage machine

Project :
    Lab # X - Lab's name

Students :
    Names — Permanent Code

Group :
    GTI770-H18-0X
"""

import cv2
import numpy as np
import os

class DataSet(object):
    """
        An object for storing data set elements.
    """

    def __init__(self):
        self._epochs_done = 0
        self._index_in_epoch = 0

    @property
    def get_images(self):
        return self._images

    @property
    def get_features(self):
        return self._features

    @property
    def get_labels(self):
        return self._labels

    @property
    def get_num_examples(self):
        return self._num_examples

    @property
    def get_epochs_done(self):
        return self._epochs_done

    def withLabels(self, labels):
        self._labels = labels
        self._num_examples = labels.shape[0]
        return self

    def withImg_names(self, img_names):
        self._img_names = img_names
        self._num_examples = img_names.shape[0]
        return self

    def withFeatures(self, features):
        self._features = features
        self._num_examples = features.shape[0]
        return self

    def withImages(self, images):
        self._images = images
        self._num_examples = images.shape[0]
        return self

    def next_feature_batch(self, batch_size):
        """
        Return the next `batch_size` examples from this data set.

        Args:
            batch_size: the number of element in the batch.

        Returns:
            A tuple containing a list of feature vectors and the associated labels.
        """
        start = self._index_in_epoch
        self._index_in_epoch += batch_size
        if self._index_in_epoch > self._num_examples:
            # Finished epoch
            self._epochs_done += 1
            # Shuffle the data
            perm = np.arange(self._num_examples)
            self._features = self._features[perm]
            self._labels = self._labels[perm]
            # Start next epoch
            start = 0
            self._index_in_epoch = batch_size
            assert batch_size <= self._num_examples
        end = self._index_in_epoch
        return self._features[start:end], self._labels[start:end]

    def next_image_batch(self, batch_size):
        """
        Return the next `batch_size` examples from this data set.

        Args:
            batch_size: the number of element in the batch.

        Returns:
            A tuple containing a list of img_names (i.e. 1000742) and the associated labels.
        """
        start = self._index_in_epoch
        self._index_in_epoch += batch_size
        if self._index_in_epoch > self._num_examples:
            # Finished epoch
            self._epochs_done += 1
            # Shuffle the data
            perm = np.arange(self._num_examples)
            self._img_names = self._img_names[perm]
            self._labels = self._labels[perm]
            # Start next epoch
            start = 0
            self._index_in_epoch = batch_size
            assert batch_size <= self._num_examples
        end = self._index_in_epoch
        return self._img_names[start:end], self._labels[start:end]

    def load_images(self, batch):
        """ Load a training image data set.

        Args:
            batch: A list of IDs used in the image file names to load.

        Return:
             The loaded images.
        """

        # Declare a list for storing OpenCV images.
        images = list()

        for sample in batch:
            # Create path to image's file.

            path = (os.environ["VIRTUAL_ENV"] + "/data/images/" + str(sample[0]) + ".jpg")

            # Use OpenCV to read the image.
            image = cv2.imread(path)

            # Transform image as a 32-bit numpy array.
            image = image.astype(np.float32)

            # Normalize the image.
            image = np.multiply(image, 1.0 / 255.0)

            # Append the image's name.
            images.append(image)

        # Put all the loaded images into a numpy array.
        images = np.array(images)

        self._images = images

        return self._images

max_stars_repo_name: iraqforlife/Smart-classification-system

max_issues_repo_name: iraqforlife/Smart-classification-system

max_forks_repo_name: iraqforlife/Smart-classification-system


text:
import json
import logging
import numpy as np
import os
import pandas as pd
import pycountry
import pycountry_convert as pc
import yaml
from collections import namedtuple

from jmetal.core.observer import Observer
from jmetal.core.solution import BinarySolution
from jmetal.util.comparator import DominanceComparator
from jmetal.util.termination_criterion import TerminationCriterion

continent_codes = {'AF': 100, 'AN': 101, 'AS': 102, 'EU': 103, 'NA': 104, 'OC': 105, 'SA': 106}

LOGGER = logging.getLogger('jmetal')

class Parser:

    def __init__(self, input_data: str):
        self.x = self.file2obj(input_data)

    def object_hook(self, d):
        return namedtuple('X', d.keys())(*d.values())

    def file2obj(self, data):
        pass

class Infrastructure():

    def __init__(self, file_location: str):
        self.infrastructure = pd.read_csv(file_location)

        x = lambda txt : np.fromstring(txt[1:-1], sep=',')
        self.infrastructure.consumption = self.infrastructure.consumption.apply(x)
        self.infrastructure.parallelization = self.infrastructure.parallelization.apply(x)

        y = lambda row : int(pycountry.countries.get(alpha_2=row['country_code']).numeric)
        self.infrastructure['country'] = self.infrastructure.apply(y, axis=1)

        # convert alpha2 country_code to continent name
        z = lambda row : continent_codes[pc.country_alpha2_to_continent_code(row['country_code'])]
        self.infrastructure['continent'] = self.infrastructure.apply(z, axis=1)

    def load(self):
        return self.infrastructure

class Pipeline(Parser):

    def file2obj(self, data):
        y = yaml.load(data, Loader=yaml.FullLoader)
        return json.loads(json.dumps(y), object_hook=self.object_hook)

    def load(self):

        columns = ['cpus','memory', 'country', 'privacy_type', 'continent']
        data = {columns[0]: [], columns[1]: [], columns[2]: [], columns[3]: [], columns[4]: []}
        for i in range(len(self.x.pipeline)):
            data[columns[0]].append(float(self.x.pipeline[i].resources.cpus))
            data[columns[1]].append(float(self.x.pipeline[i].resources.memory))
            # get numeric from alpha_2 country code
            country_numeric = pycountry.countries.get(alpha_2=self.x.pipeline[i].privacy.location).numeric
            data[columns[2]].append(int(country_numeric))
            data[columns[3]].append(int(self.x.pipeline[i].privacy.type))
            # get continent from alpha_2 country code
            data[columns[4]].append(continent_codes[pc.country_alpha2_to_continent_code(self.x.pipeline[i].privacy.location)])
        b = pd.DataFrame(data=data, columns=columns)

        return b

class ParetoTools():

    def __init__(self, front):
        self.front = front

    def save(self):
        filename = f"/tmp/pareto"
        with open(filename, "w") as pareto_file:
            for s in self.front:
                pareto_file.write(f"{abs(s.objectives[0])},{abs(s.objectives[1])},{s.objectives[2]}\n")

class Objectives():

    def get_consumption(self, pipe: Pipeline, infra: Infrastructure, solution: BinarySolution) -> int:
        s = np.asfarray(solution.variables, dtype=np.bool)

        threads_required = s.transpose().dot(pipe.cpus.to_numpy()).astype(int)

        consumption = [m[min(n, len(m)-1)] for m,n in zip(infra.consumption, threads_required)]
        return sum(consumption)

    def get_performance(self, pipe: Pipeline, infra: Infrastructure, solution: BinarySolution) -> int:
        s = np.asfarray(solution.variables, dtype=np.bool)

        base_performance = s*infra.performance.to_numpy()

        threads_required = s.transpose().dot(pipe.cpus.to_numpy()).astype(int)

        coefficient = [m[min(n-1, len(m)-1)] for m,n in zip(infra.parallelization, threads_required)]

        return (base_performance*coefficient).max(1).sum()
        #return (base_performance*coefficient).mean(1).sum()

    def get_resilience(self, infra: Infrastructure, solution: BinarySolution) -> int:
        s = np.asfarray(solution.variables, dtype=np.bool)
        if s.sum() == 0:
            return 0
        x = s*infra.resillience.to_numpy()
        x = np.ma.masked_array(x, mask=s==0)
        return np.nanmin(x,axis=1).sum()

class WriteObjectivesToFileObserver(Observer):

    def __init__(self) -> None:
        self.filename = '/tmp/fitnesses'
        if os.path.exists(self.filename):
            os.remove(self.filename)

    def update(self, *args, **kwargs):
        evaluations = kwargs['EVALUATIONS']
        LOGGER.info(f"Evaluations: {evaluations}")

        #solutions = np.array([s.objectives for s in kwargs['SOLUTIONS'][:10]])
        s = np.array([s.objectives for s in kwargs['SOLUTIONS']])

        #means = s.mean(0)
        objective0 = s[np.argsort(s[:,0])][:3].mean(0)[0]
        #objective0 = s[np.argsort(s[:,0])][0][0]
        objective1 = s[np.argsort(s[:,1])][:3].mean(0)[1]
        #objective1 = s[np.argsort(s[:,1])][0][1]
        objective2 = s[np.argsort(s[:,2])][:3].mean(0)[2]
        #objective2 = s[np.argsort(s[:,2])][0][2]

        with open(self.filename, 'a') as out_file:
            #out_file.write(f"{abs(means[0])},{abs(means[1])},{abs(means[2])}\n")
            out_file.write(f"{abs(objective0)},{abs(objective1)},{abs(objective2)}\n")

class StoppingByNonDominance(TerminationCriterion):

    def __init__(self, idle_evaluations: int):
        super(StoppingByNonDominance, self).__init__()
        self.idle_evaluations = idle_evaluations
        self.evaluations = 0
        self.best_solution = None

    def update(self, *args, **kwargs):
        current_solution = kwargs['SOLUTIONS'][0]
        if self.best_solution is None:
            self.best_solution = current_solution
        else:
            result = DominanceComparator().compare(self.best_solution, current_solution)
            if result != 1:
                self.evaluations += 1
                LOGGER.info(f"{self.evaluations} evaluations without improvement.")
            else:
                self.evaluations = 0
            self.best_solution = current_solution

    @property
    def is_met(self):
        return self.evaluations >= self.idle_evaluations

class StoppingByTotalDominance(TerminationCriterion):

    def __init__(self, idle_evaluations: int):
        super(StoppingByTotalDominance, self).__init__()
        self.idle_evaluations = idle_evaluations
        self.evaluations = 0
        self.best_objectives = None
        self.seconds = 0.0

    def update(self, *args, **kwargs):
        self.seconds = kwargs['COMPUTING_TIME']

        s = np.array([s.objectives for s in kwargs['SOLUTIONS']])
        objective0 = s[np.argsort(s[:,0])][0][0]
        objective1 = s[np.argsort(s[:,1])][0][1]
        objective2 = s[np.argsort(s[:,2])][0][2]
        current_objectives = [objective0, objective1, objective2]

        s = kwargs['SOLUTIONS'][0]
        if self.best_objectives is None:
            self.best_objectives = current_objectives
            #LOGGER.info(self.best_objectives)
        else:
            #LOGGER.info(self.best_objectives)
            #LOGGER.info(current_objectives)
            if all([x<=y for x,y in zip(self.best_objectives, current_objectives)]):
                self.evaluations += 1
            else:
                self.best_objectives = [min(x,y) for x,y in zip(self.best_objectives, current_objectives)]
                self.evaluations = 0
        LOGGER.info(f"{self.evaluations} evaluations without improvement")

    @property
    def is_met(self):
        return self.evaluations >= (self.idle_evaluations - (self.seconds/10)**2)

class StoppingByFullPareto(TerminationCriterion):

    def __init__(self, offspring_size: int):
        super(StoppingByFullPareto, self).__init__()
        self.offspring_size = offspring_size

    def update(self, *args, **kwargs):
        self.current_offspring_size = len(kwargs['SOLUTIONS'])
        LOGGER.info(f"Current size = {self.current_offspring_size}.")

    @property
    def is_met(self):
        return self.offspring_size <= self.current_offspring_size

max_stars_repo_name: josu-arcaya/orfeon

max_issues_repo_name: josu-arcaya/orfeon

max_forks_repo_name: josu-arcaya/orfeon


text:
import os

import numpy as np
from torch.utils.data import Dataset

from ConSSL.utils import _PIL_AVAILABLE
from ConSSL.utils.warnings import warn_missing_pkg

if _PIL_AVAILABLE:
    from PIL import Image
else:  # pragma: no cover
    warn_missing_pkg('PIL', pypi_name='Pillow')

DEFAULT_VOID_LABELS = (0, 1, 2, 3, 4, 5, 6, 9, 10, 14, 15, 16, 18, 29, 30, -1)
DEFAULT_VALID_LABELS = (7, 8, 11, 12, 13, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 31, 32, 33)

class KittiDataset(Dataset):
    """
    Note:
        You need to have downloaded the Kitti dataset first and provide the path to where it is saved.
        You can download the dataset here: http://www.cvlibs.net/datasets/kitti/eval_semseg.php?benchmark=semantics2015

    There are 34 classes, however not all of them are useful for training (e.g. railings on highways). These
    useless classes (the pixel values of these classes) are stored in `void_labels`. Useful classes are stored
    in `valid_labels`.

    The `encode_segmap` function sets all pixels with any of the `void_labels` to `ignore_index`
    (250 by default). It also sets all of the valid pixels to the appropriate value between 0 and
    `len(valid_labels)` (since that is the number of valid classes), so it can be used properly by
    the loss function when comparing with the output.
    """
    IMAGE_PATH = os.path.join('training', 'image_2')
    MASK_PATH = os.path.join('training', 'semantic')

    def __init__(
        self,
        data_dir: str,
        img_size: tuple = (1242, 376),
        void_labels: list = DEFAULT_VOID_LABELS,
        valid_labels: list = DEFAULT_VALID_LABELS,
        transform=None
    ):
        """
        Args:
            data_dir (str): where to load the data from path, i.e. '/path/to/folder/with/data_semantics/'
            img_size: image dimensions (width, height)
            void_labels: useless classes to be excluded from training
            valid_labels: useful classes to include
        """
        if not _PIL_AVAILABLE:  # pragma: no cover
            raise ModuleNotFoundError('You want to use `PIL` which is not installed yet.')

        self.img_size = img_size
        self.void_labels = void_labels
        self.valid_labels = valid_labels
        self.ignore_index = 250
        self.class_map = dict(zip(self.valid_labels, range(len(self.valid_labels))))
        self.transform = transform

        self.data_dir = data_dir
        self.img_path = os.path.join(self.data_dir, self.IMAGE_PATH)
        self.mask_path = os.path.join(self.data_dir, self.MASK_PATH)
        self.img_list = self.get_filenames(self.img_path)
        self.mask_list = self.get_filenames(self.mask_path)

    def __len__(self):
        return len(self.img_list)

    def __getitem__(self, idx):
        img = Image.open(self.img_list[idx])
        img = img.resize(self.img_size)
        img = np.array(img)

        mask = Image.open(self.mask_list[idx]).convert('L')
        mask = mask.resize(self.img_size)
        mask = np.array(mask)
        mask = self.encode_segmap(mask)

        if self.transform:
            img = self.transform(img)

        return img, mask

    def encode_segmap(self, mask):
        """
        Sets void classes to zero so they won't be considered for training
        """
        for voidc in self.void_labels:
            mask[mask == voidc] = self.ignore_index
        for validc in self.valid_labels:
            mask[mask == validc] = self.class_map[validc]
        # remove extra idxs from updated dataset
        mask[mask > 18] = self.ignore_index
        return mask

    def get_filenames(self, path):
        """
        Returns a list of absolute paths to images inside given `path`
        """
        files_list = list()
        for filename in os.listdir(path):
            files_list.append(os.path.join(path, filename))
        return files_list