text:
"""
General utility class for accessing pickle data files and for simple transformations

Generally data is stored as dict of arrays, as dumped and compressed file.
using pytables is probably the best approach but we may have problems with portability
(or easy installation in different environments)
so here we use simpler approach with more standard packages zlib and pickle
"""
import gzip
from io import BufferedReader
import io
import os
import pickle
import zlib
import logging

import numpy as np

from config import DATA_DIR, PUBLISH_DIR, MEAN_MARKERS, BED_TO_BIG_BED, BED_GRAPH_TO_BIG_WIG, CHROM_SIZES
import re

__author__ = 'eran'

def wig_transform(wig_file, smoothing=100, output=True):
    """
    Gets a wig file and transforms it to dictionary (chr1 -> [scores], ch2 -> [scores]...)
    and saves it with pickle
    @param output: Whether to output the transformed wig to npz file (name.smoothing.npz)
    @param smoothing: bin size for smoothing (common raw data can be 20bp)
    @param wig_file: name of wig file (or wig.gz)
    @return: a dictionary where key are chromosomes and values are scores

    @note it is recommended to use use bed graph files and use load_bg which works much faster
    """
    written_dict = dict()  # dictionary keys: chromosomes [ score, score], and chromosome-position: start position

    if wig_file.endswith('.gz'):
        in_stream = BufferedReader(gzip.open(wig_file, 'rb'))
        in_file = (in_line.decode('ascii') for in_line in in_stream)
    else:
        in_stream = io.open(wig_file, 'r')

    prev_data = []
    l_prev_data = 0
    chrm_score = []
    chrom = None
    rlines = 0
    frag_size = 20
    cur_smoothing = smoothing / frag_size
    prev_pos = 0
    for r in in_file:
        if r[0] == 'v':  # new chromosome, variable step
            if chrom is not None:
                chrm_score.append(sum(prev_data))
                written_dict[chrom] = chrm_score

            chrm_score = []
            prev_data = []
            print(r.strip())
            chrom_track = r.rstrip().split(' ')
            frag_size = int(chrom_track[2].split('=')[1].strip())
            cur_smoothing = int(smoothing / frag_size)
            chrom = chrom_track[1].split('=')[1]
            prev_pos = 1
            pos, score = next(in_file).split('\t')
        elif r[0] == 't':
            continue  # track line
        else:
            pos, score = r.split('\t')
        pos = int(pos)
        score = int(score)
        if pos != prev_pos:
            prev_data += [0] * int(np.floor((pos - prev_pos) / frag_size))
            l_prev_data = len(prev_data)
        prev_data += [score]
        l_prev_data += 1
        prev_pos = pos + frag_size
        if l_prev_data > cur_smoothing:
            for smooth_bin in zip(*[iter(prev_data)] * cur_smoothing):
                chrm_score.append(sum(smooth_bin))
            remaining = (len(prev_data) % cur_smoothing)
            prev_data = prev_data[-remaining:] if remaining > 0 else []
            l_prev_data = len(prev_data)
        rlines += 1

    if len(prev_data) != 0:
        chrm_score.append(sum(prev_data))
    written_dict[chrom] = chrm_score

    in_stream.close()
    print('Finished reading and down-sampling')

    if output:
        # save to same file but with different extensions
        wig_file = wig_file.replace('.gz', '')  # remove .gz extension for compressed file
        output_filename = wig_file.replace('.wig', '.%i.npz' % smoothing)
        save_result_dict(output_filename, output_filename)
        #with open(output_filename, 'wb') as output:
        #    output.write(zlib.compress(pickle.dumps(written_dict, pickle.HIGHEST_PROTOCOL)))
        print('Finished writing file')

    return written_dict

def chrom_sizes(bin_sizes=20):
    """
    Get chromosome sizes of hg19 as dictionary
    @rtype : dict
    @return: dictionary with key as chromosome name and value as size (for bins of size of 20)
    """
    with open(CHROM_SIZES) as chrom_size_fd:
        chromosomes = (r.split('\t') for r in chrom_size_fd.readlines())
    chrom_dict = dict(((chrom[0], int(chrom[1]) // bin_sizes) for chrom in chromosomes))
    return chrom_dict

def load_bg(bg_file):
    """
    Loads bed graph
    should be faster then the above and directly load it
    @param bg_file: bg file to be loaded
    @return:  dictionary of chromosomes keys and values as wig like arrays
    """

    def load_bg_slow():
        """
        Fallback - if pandas doesnt exist and you get into trouble with cython compilations
        """
        chr_to_ind = dict()
        chr_to_ind_inv = dict()
        for i in range(1, 26):
            chr_name = 'chr%i' % i
            if i == 23:
                chr_name = 'chrX'
            if i == 24:
                chr_name = 'chrY'
            if i == 25:
                chr_name = 'chrM'
            chr_name = chr_name.encode('ascii')
            chr_to_ind[chr_name] = i
        for k, v in chr_to_ind.items():
            chr_to_ind_inv[v] = k.decode('utf-8')
        # 0 - chromosome, 1 - start, 2 - end
        bed_matrix = np.loadtxt(bg_file, delimiter='\t', usecols=(0, 1, 2, 3),
                                converters={
                                    0: lambda x: chr_to_ind[x]
                                }, dtype=int)
        chromosomes_ind = set(bed_matrix[:, 0])
        res_dict = dict()
        for chromosome in chromosomes_ind:
            selector = bed_matrix[:, 0] == chromosome
            chrom_matrix = np.array(bed_matrix[selector, [1, 2, 3]])
            # TO BIN SIZE
            chrom_matrix[:, [0, 1]] = chrom_matrix[:, [0, 1]] / 20
            last_end = max(chrom_matrix[:, 1])
            long_rep = np.zeros(last_end)
            for line in chrom_matrix:
                long_rep[line[0]:line[1]] = line[2]
            res_dict[chr_to_ind_inv[chromosome]] = long_rep
        return res_dict

    def load_bg_pandas():
        """
        Load bed graph using pandas
        """
        import pandas as pd

        data = pd.read_csv(bg_file, sep='\t', header=None)
        group_by_chr = data.groupby(0)
        res_dict = dict()
        for k, v in group_by_chr:
            chrom_matrix = v[[1, 2, 3]].values
            # TO BIN SIZE
            chrom_matrix[:, [0, 1]] = chrom_matrix[:, [0, 1]] / 20
            last_end = max(chrom_matrix[:, 1])
            long_rep = np.zeros(last_end)
            rows_sel = chrom_matrix[:, 0] > 0
            sig_len = 0
            indics_arr = np.array(chrom_matrix, dtype=int)
            while np.any(rows_sel):
                indics = indics_arr[:, 0] + sig_len
                long_rep[indics] = indics_arr[:, 2]
                sig_len += 1
                rows_sel = (indics_arr[:, 1] - indics_arr[:, 0]) > sig_len
                indics_arr = indics_arr[rows_sel, :]

            res_dict[k] = long_rep
        return res_dict

    try:
        from pyx import BedGraphReader

        return BedGraphReader.load_bedgraph(bg_file, chrom_sizes())
    except ImportError:
        logging.warn("couldn't load BedGraphReader. reading bg files could be slow. Compile pyx directory/get pandas")
        try:
            return load_bg_pandas(bg_file)
        except ImportError:
            return load_bg_slow(bg_file)

def save_result_dict_old(output_filename, written_dict):
    """
    Saves a result dictionary to file for later use
    @param output_filename: filename for the saved dictionary
    @param written_dict: dictionary to save
    @return:
    """
    with open(output_filename, 'wb') as output:
        output.write(zlib.compress(pickle.dumps(written_dict, pickle.HIGHEST_PROTOCOL)))

def save_result_dict(output_filename, written_dict):
    """
    Saves a result dictionary to file for later use
    @param output_filename: filename for the saved dictionary
    @param written_dict: dictionary to save
    @return:
    """
    np.savez_compressed(output_filename, **written_dict)

def load_result_dict(output_filename):
    """
    Saves a result dictionary to file for later use
    @param output_filename: filename for the saved dictionary
    @return: result dictionary
    """
    if output_filename.endswith('.npz'):
        return np.load(output_filename)
    else:
        with open(output_filename, 'rb') as file:
            data = zlib.decompress(file.read())
            sequence_dict = pickle.loads(data)
            return sequence_dict

def available_experiments(cell_type, experiments=None):
        """
        @param experiments: name of experiments to check for availability
        @param cell_type: cell type to locate experiments for
        @return: mapping between available experiments and the file path
        @rtype: dict
        @raise Exception:
        """
        warn = experiments is not None
        experiments_to_load = os.listdir(MEAN_MARKERS) if experiments is None else experiments
        available_ex = dict()
        for ex in experiments_to_load:
            ex_dir = os.path.join(MEAN_MARKERS, ex)
            if not os.path.exists(ex_dir):
                raise Exception("Data for experiment %s isn\'t available - you should download & transform it" % ex)
            ex_cell_type_dir = os.path.join(ex_dir, cell_type)
            # fallback to find similar cell type
            if not os.path.exists(ex_cell_type_dir):
                # may be different naming fetal_brain => brain_fetal
                ex_cell_type_dir = os.path.join(ex_dir, re.sub('fetal_(.+)', '\\1_fetal', cell_type))
                if not os.path.exists(ex_cell_type_dir):
                    if warn:
                        logging.warning('Cell type %s not found for experiment %s' % (cell_type, ex))
                    continue
            mean_file = os.path.join(ex_cell_type_dir, 'mean.npz')
            available_ex[ex] = mean_file
        return available_ex

def load_experiments(cell_type, experiments=None, chromosomes=None, resolution=20):
    """
    Generic function to access experiments data of "markers", such as methylations and acetylations.
    Since it is common that we don't have the experiment data for the specific sample
    (e.g. same tissue, same person) we workaround it with mean scores of various experiments on same tissue
    but different people.
    {MEAN_MARKERS} directory can be populated by data_provider/createMeanMarkers script

    @param resolution: resolution (number of bp) to load experiments. should be multiple of 20
    @param cell_type: cell type
    @param experiments: name of experiments
    @param chromosomes:
    @return: Dictionary with key as chromosomes and values as matrix, where rows are different experiments
             and values are columns
    @raise Exception: In case of data not available in MEAN_MARKERS
    @rtype : dict
    """
    from scipy.sparse import lil_matrix, vstack, csr_matrix

    res_dict = dict()
    experiment_mapping = available_experiments(cell_type, experiments)
    existing_experiments = experiment_mapping.keys()
    #n_expiremnts = len(list(experiment_mapping.keys()))
    for ex_i, mean_file in enumerate(experiment_mapping.values()):
        mean_data = load_result_dict(mean_file)

        chromosomes_to_load = mean_data.keys() if chromosomes is None else chromosomes
        for chromosome_key in chromosomes_to_load:
            chromosome_value = mean_data[chromosome_key]
            if resolution != 20:
                chromosome_value = down_sample(chromosome_value, resolution//20)
            # add to combined dictionary
            if chromosome_key not in res_dict:
                #combined = np.zeros((n_expiremnts, chromosome_value.shape[0]))
                #combined[ex_i, :] = chromosome_value
                combined = csr_matrix(chromosome_value)
            else:
                existing_val = res_dict[chromosome_key]
                # extend number of columns if necessary
                if existing_val.shape[1] < chromosome_value.shape[0]:
                    #combined = np.zeros((n_expiremnts, chromosome_value.shape[0]))
                    existing_val = existing_val.todense()
                    combined = np.zeros((existing_val.shape[0], chromosome_value.shape[0]))
                    combined[:, 0:existing_val.shape[1]] = existing_val
                    combined = csr_matrix(combined)
                    #combined[:ex_i, 0:existing_val.shape[1]] = existing_val[:ex_i, :]
                else:
                    combined = existing_val
                if chromosome_value.shape[0] < existing_val.shape[1]:
                    chromosome_value = np.append(chromosome_value,
                                                 np.zeros(existing_val.shape[1]-chromosome_value.shape[0]))
                combined = vstack([combined, csr_matrix(chromosome_value)])
                #combined[ex_i, 0:chromosome_value.shape[0]] = chromosome_value

            res_dict[chromosome_key] = combined

    # for now don't spread sparse matrix around out of the function scope
    for k in res_dict.keys():
        res_dict[k] = res_dict[k].todense()
    return res_dict, existing_experiments

def load_dict(name, resolution=20, transform=None, directory=DATA_DIR, chromosome=None, orig_resolution=20):
    """
    Loads transformed dictionary from DATA_DIR.
    @param orig_resolution: original resolution
    @param directory: directory to load the data from
    @param transform: A transform function/callable object after reading the sequence
    @param resolution: resolution for genome default is 20. can be any number that can be divided by 20
    @param name: name fo file in DATA_DIR
    @param chromosome: chromosomes to load (other chromosomes are ignored)
    @return: dictionary of chromosomes scores and start positions
    """
    if os.path.exists(os.path.join(directory, '%s.%i.npz' % (name, orig_resolution))):
        sequence_dict = load_result_dict(os.path.join(directory, '%s.%i.npz' % (name, orig_resolution)))
    elif os.path.exists(os.path.join(directory, name)):
        sequence_dict = load_result_dict(os.path.join(directory, name))
        logging.warning('loading file %s with no resolution in its name. Assuming 20' % name)
    else:
        file_name = os.path.join(directory, '%s.%i.pkl' % (name, orig_resolution))
        try:
            with open(file_name, 'rb') as file:
                decompress = zlib.decompress(file.read())
                sequence_dict = pickle.loads(decompress)
            # select relevant chromosomes
            dict_keys = [key for key in sequence_dict.keys() if
                         '-position' not in key and (not isinstance(chromosome, list) or key in chromosome)
                         and (not isinstance(chromosome, str) or chromosome == key)]
            # remove the rest
            orig_keys = list(sequence_dict.keys())
            for k in orig_keys:
                if k not in dict_keys:
                    del sequence_dict[k]
        except IOError as err:
            print('Please check: ', os.path.abspath(file_name))
            raise err
    dict_keys = [k for k in sequence_dict.keys() if chromosome is None or k in chromosome]
    res_dict = dict()
    for key in dict_keys:
        seq = sequence_dict[key]
        logging.debug('Preparing %s' % key)
        if resolution != orig_resolution:  # Down-sample sequence
            seq = down_sample(seq, resolution / orig_resolution)  # TODO: would be nice to create lazy evaluated
        if transform is not None:
            seq = transform(seq)
        res_dict[key] = seq

    return res_dict

def continuous_transform(seq):
    """
    Transform the sequence to ln(1+x)
    @param seq: sequence to transform
    """
    return np.log(1 + np.array(seq))

def down_sample(seq, new_sampling_factor=1):
    """
    down samples a sequence with defined factor.
    @param new_sampling_factor: number of bins to combine together
    @param seq: sequence to down sample
    """
    if new_sampling_factor != int(new_sampling_factor):
        raise Exception("Bad sampling factor - validate your resolution")

    seq = np.array(seq)
    # either cut the end (here) or extend with zero (but may later damage the max chromosome length)
    seq = seq[0:seq.shape[0] - (seq.shape[0] % new_sampling_factor)]
    return seq.reshape(seq.shape[0] // new_sampling_factor, new_sampling_factor).sum(1)

def build_bedgraph(classified_seq, resolution, output_file):
    """
    Creates bed graph file from dictionary
    @param classified_seq: Dictionary to transform
    @param resolution: bin resolution
    @param output_file: file or path to file
    """
    if isinstance(output_file, str):
        output = open(output_file, 'w')
    else:
        output = output_file
    for chromosome, chromosome_seq in classified_seq.items():
        prev_val = None
        start = str(1)
        chromosome_seq = np.around(chromosome_seq, decimals=3)
        for i, val in enumerate(chromosome_seq):
            if prev_val is None:
                prev_val = val
            if val != prev_val or i - 1 == len(chromosome_seq):
                prev_val = '%.3f' % prev_val
                pos = str((i + 1) * resolution)
                to_write = '\t'.join([chromosome, start, pos, prev_val]) + '\n'
                output.write(to_write)
                start = pos
            prev_val = val
    if isinstance(output_file, str):
        output.close()

def bg_to_bigwig(bed_graph, big_wig=None):
    """
    Transforms bed graph to big wig
    @param bed_graph: file to transform
    @param big_wig: full output file or default to PUBLISH_DIR
    """
    if big_wig is None:
        big_wig = os.path.split(bed_graph)[-1].replace('.bg', '.bw')
        big_wig = os.path.join(PUBLISH_DIR, big_wig)
        print('Saving to ', big_wig)
    import subprocess

    subprocess.call([BED_GRAPH_TO_BIG_WIG, bed_graph, CHROM_SIZES, big_wig])
    print('Created bw')

def build_bed(classified_seq, resolution, output_file_name):
    """
    Writes a bed file
    @param classified_seq: dictionary like object where keys are chromosomes and values are their data
    @param resolution: bin size in bp
    @param output_file_name: name of file to write
    @return:
    """
    from pyx import BedGraphReader
    resolution = int(resolution)
    BedGraphReader.write_bed(classified_seq, resolution, output_file_name)

def bed_to_bigbed(bed_graph, big_bed=None):
    """
    Transforms bed to big bed
    @param bed_graph: file to transform
    @param big_bed: full output file or default to PUBLISH_DIR
    """
    if big_bed is None:
        big_bed = os.path.split(bed_graph)[-1].replace('.bed', '.bb')
        big_bed = os.path.join(PUBLISH_DIR, big_bed)
        print('Saving to ', big_bed)
    import subprocess

    subprocess.call([BED_TO_BIG_BED, bed_graph, CHROM_SIZES, big_bed])

max_stars_repo_name: eranroz/dnase

max_issues_repo_name: eranroz/dnase

max_forks_repo_name: eranroz/dnase


text:
# Copyright (c) 1996-2015 PSERC. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

"""Construct linear constraints for constant power factor var loads.
"""

from sys import stderr

from numpy import array, zeros, arange, sin, cos, arctan2, r_
from numpy import flatnonzero as find
from scipy.sparse import csr_matrix as sparse

from pypower.idx_gen import PG, QG, PMIN, QMIN, QMAX

from pypower.isload import isload

def makeAvl(baseMVA, gen):
    """Construct linear constraints for constant power factor var loads.

    Constructs parameters for the following linear constraint enforcing a
    constant power factor constraint for dispatchable loads::

         lvl <= Avl * [Pg, Qg] <= uvl

    C{ivl} is the vector of indices of generators representing variable loads.

    @author: Ray Zimmerman (PSERC Cornell)
    @author: Carlos E. Murillo-Sanchez (PSERC Cornell & Universidad
    Autonoma de Manizales)
    """
    ## data dimensions
    ng = gen.shape[0]      ## number of dispatchable injections
    Pg   = gen[:, PG] / baseMVA
    Qg   = gen[:, QG] / baseMVA
    Pmin = gen[:, PMIN] / baseMVA
    Qmin = gen[:, QMIN] / baseMVA
    Qmax = gen[:, QMAX] / baseMVA

    # Find out if any of these "generators" are actually dispatchable loads.
    # (see 'help isload' for details on what constitutes a dispatchable load)
    # Dispatchable loads are modeled as generators with an added constant
    # power factor constraint. The power factor is derived from the original
    # value of Pmin and either Qmin (for inductive loads) or Qmax (for
    # capacitive loads). If both Qmin and Qmax are zero, this implies a unity
    # power factor without the need for an additional constraint.

    ivl = find( isload(gen) & ((Qmin != 0) | (Qmax != 0)) )
    nvl = ivl.shape[0]  ## number of dispatchable loads

    ## at least one of the Q limits must be zero (corresponding to Pmax == 0)
    if any( (Qmin[ivl] != 0) & (Qmax[ivl] != 0) ):
        stderr.write('makeAvl: either Qmin or Qmax must be equal to zero for '
                     'each dispatchable load.\n')

    # Initial values of PG and QG must be consistent with specified power
    # factor This is to prevent a user from unknowingly using a case file which
    # would have defined a different power factor constraint under a previous
    # version which used PG and QG to define the power factor.
    Qlim = (Qmin[ivl] == 0) * Qmax[ivl] + (Qmax[ivl] == 0) * Qmin[ivl]
    if any( abs( Qg[ivl] - Pg[ivl] * Qlim / Pmin[ivl] ) > 1e-6 ):
        stderr.write('makeAvl: For a dispatchable load, PG and QG must be '
                     'consistent with the power factor defined by PMIN and '
                     'the Q limits.\n')

    # make Avl, lvl, uvl, for lvl <= Avl * [Pg Qg] <= uvl
    if nvl > 0:
        xx = Pmin[ivl]
        yy = Qlim
        pftheta = arctan2(yy, xx)
        pc = sin(pftheta)
        qc = -cos(pftheta)
        ii = r_[ arange(nvl), arange(nvl) ]
        jj = r_[ ivl, ivl + ng ]
        Avl = sparse((r_[pc, qc], (ii, jj)), (nvl, 2 * ng))
        lvl = zeros(nvl)
        uvl = lvl
    else:
        Avl = zeros((0, 2*ng))
        lvl = array([])
        uvl = array([])

    return Avl, lvl, uvl, ivl

ext: py

max_stars_repo_name: Bengt/PYPOWER

max_issues_repo_name: Bengt/PYPOWER

max_forks_repo_name: Bengt/PYPOWER

reason: from numpy,from scipy


text:
import numpy
import random
import itertools
from torch.utils.data import DataLoader
from torch.utils.data.sampler import Sampler

class LAHeart(DataLoader):
    def __init__(self, dataset, config):

        def worker_init_fn(worker_id):
            random.seed(config.seed+worker_id)

        self.init_kwags = {
            'dataset': dataset,
            'drop_last': config.drop_last,
            'batch_size': config.batch_size,
            'shuffle': config.shuffle,
            'num_workers': config.num_workers,
            'pin_memory': True,
            'worker_init_fn': worker_init_fn
        }

        super(LAHeart, self).__init__(**self.init_kwags)

class TwoStreamBatchSampler(Sampler):
    """Iterate two sets of indices

    An 'epoch' is one iteration through the primary indices.
    During the epoch, the secondary indices are iterated through
    as many times as needed.
    """
    def __init__(self, primary_indices, secondary_indices, batch_size, secondary_batch_size):
        self.primary_indices = primary_indices
        self.secondary_indices = secondary_indices
        self.secondary_batch_size = secondary_batch_size
        self.primary_batch_size = batch_size - secondary_batch_size

        assert len(self.primary_indices) >= self.primary_batch_size > 0
        assert len(self.secondary_indices) >= self.secondary_batch_size > 0

    def __iter__(self):
        primary_iter = iterate_once(self.primary_indices)
        secondary_iter = iterate_eternally(self.secondary_indices)
        return (
            primary_batch + secondary_batch
            for (primary_batch, secondary_batch)
            in zip(grouper(primary_iter, self.primary_batch_size),
                   grouper(secondary_iter, self.secondary_batch_size))
        )

    def __len__(self):
        return len(self.primary_indices) // self.primary_batch_size

def iterate_once(iterable):
    return numpy.random.permutation(iterable)

def iterate_eternally(indices):
    def infinite_shuffles():
        while True:
            yield numpy.random.permutation(indices)
    return itertools.chain.from_iterable(infinite_shuffles())

def grouper(iterable, n):
    " Collect data into fixed-length chunks or blocks "
    # grouper('ABCDEFG', 3) --> ABC DEF"
    args = [iter(iterable)] * n
    return zip(*args)

reason: import numpy


text:
# people = ["Lisa","Pam","Phil","John"]
# unique pairs where order is irrelevant
# result = [print(people[p1], people[p2]) for p1 in range(len(people)) for p2 in range(p1+1,len(people))]
# All possible pairs, excluding duplicates:
# result = [print(p1, p2) for p1 in people for p2 in people if p1 != p2]

import spacy
import numpy as np
import pickle
from sklearn.svm import LinearSVC, SVC
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from dataset_analysis import analyse_dataset
import csv
from sklearn.metrics import confusion_matrix
from matplotlib import pyplot as plt
from imblearn.under_sampling import RandomUnderSampler
from sklearn.metrics import accuracy_score
from features_relations import structural_number_tokens_source, relationship, structural_difference_number_tokens, structural_sentence_distance, structural_target_before_source, lexical_number_common_terms, structural_number_punctuation_marks_source, structural_difference_number_punctuation_marks, lexical_source_contains_modal_verb, indicators_source_contains_discourse_markers

nlp = spacy.load('de_core_news_sm')

# TODO operate on clausal level!? maybe...

def create_train_test_split(df):
    y = df.loc[:, ['relationship']] # y = dataset.label
    X = df.drop(['relationship'], axis=1)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) #random_state=1234

    return X_train, X_test, y_train, y_test

def train_linear_SVC():
    df = encode_features()
    print(df)

    classifier = LinearSVC()
    X_train, X_test, y_train, y_test = create_train_test_split(df)

    rus = RandomUnderSampler(return_indices=True)
    X_rus, y_rus, id_rus = rus.fit_sample(X_train, y_train) # TODO other sampling techniques, e.g. ros = RandomOverSampler() X_ros, y_ros = ros.fit_sample(X, y)

    print('Removed indexes:', len(X_train), len(id_rus), id_rus)

    classifier.fit(X_rus, y_rus) # X_train, y_train.values.ravel()

    pickle.dump(classifier, open("linear_SVC_relations.pkl", 'wb'))

    # Predict Class
    y_pred = classifier.predict(X_test)
    print(y_pred)

    # Accuracy
    accuracy = accuracy_score(y_test, y_pred)
    print("acc: ", accuracy)

    #X_unseen = create_data_set(995)

    #pred = classifier.predict(X_unseen)
    #print(pred)

def encode_features():
    f = generate_pairs()

    df = pd.DataFrame.from_dict(f)

    df = pd.get_dummies(df, columns=['label_source', 'label_target']) # 'relationship',
    # print(df['number_tokens_target'])
    scaled_features = df.copy()

    col_names = ['number_tokens_source', 'number_tokens_target', 'difference_number_tokens', 'sentence_distance', 'number_of_common_terms', 'number_of_punctuation_marks_source', 'number_of_punctuation_marks_target', 'difference_number_punctuation_marks']
    features = scaled_features[col_names]
    scaler = StandardScaler().fit(features.values)
    features = scaler.transform(features.values)

    scaled_features[col_names] = features
    #print(scaled_features['number_tokens_target'])

    #print(df)
    #print("df: ", df.columns.values)

    pickle.dump(scaler, open("standard_scaler_linear_SVC.pkl", 'wb'))

    print("scaled: ", scaled_features.columns.values)

    return scaled_features

def generate_features(pair1, pair2):
    features = {}
    features["relationship"] = relationship.get_relationship(pair1, pair2) # 'support' 'non-support' categorical
    features["label_source"] = pair1['label'] # claim, premise, None categorical
    features["label_target"] = pair2['label'] # claim, premise, None categorical
    features["number_tokens_source"] = structural_number_tokens_source.get_number_of_tokens(pair1['text']) # int
    features["number_tokens_target"] = structural_number_tokens_source.get_number_of_tokens(pair2['text']) # int
    features["difference_number_tokens"] = structural_difference_number_tokens.get_difference_number_tokens(pair1, pair2) # int
    features["sentence_distance"] = structural_sentence_distance.get_sentence_distance(pair1, pair2) # int
    features["target_before_source"] = structural_target_before_source.is_target_before_source(pair1, pair2) # bool
    features["number_of_common_terms"] = lexical_number_common_terms.get_number_of_common_terms(pair1, pair2) # int
    features["number_of_punctuation_marks_source"] = structural_number_punctuation_marks_source.get_number_of_punctuation_marks(pair1['text']) # int
    features["number_of_punctuation_marks_target"] = structural_number_punctuation_marks_source.get_number_of_punctuation_marks(pair2['text']) # int
    features["difference_number_punctuation_marks"] = structural_difference_number_punctuation_marks.get_difference_number_of_punctuation_marks(pair1, pair2) # int
    features["contains_modal_verb_source"] = lexical_source_contains_modal_verb.contains_modal_verb(pair1['text']) # bool
    features["contains_modal_verb_target"] = lexical_source_contains_modal_verb.contains_modal_verb(pair2['text']) # bool
    features["contains_argumentative_discourse_markers_source"] = indicators_source_contains_discourse_markers.contains_argumentative_markers(pair1['text']) # bool
    features["contains_argumentative_discourse_markers_target"] = indicators_source_contains_discourse_markers.contains_argumentative_markers(pair2['text']) # bool
    return features

def generate_pairs():
    f = []
    features = {}

    annotations = create_csv(True)

    counter = 0

    number_of_docs = annotations[-1]['docID'] + 1
    #print(number_of_docs)

    while counter < number_of_docs:
        to_compare = []
        for a in annotations:
            if a['docID'] == counter:
                to_compare.append(a)

        #print(to_compare)

        n_pair1 = 0

        while n_pair1 < len(to_compare)-1:
            n_pair2 = 0 #n_pair1 + 1
            while n_pair2 < len(to_compare):
                pair1 = to_compare[n_pair1]
                pair2 = to_compare[n_pair2]
                print("1: ", pair1)
                print("2: ", pair2)

                feat = generate_features(pair1, pair2)
                f.append(feat)

                n_pair2 += 1

            n_pair1 += 1

        counter += 1

    #print(f)

    with open('sentence_level_annotations_with_relations2.csv', mode='w') as csv_file:
        fieldnames = ['relationship', 'label_source', 'label_target', 'number_tokens_source', 'number_tokens_target', 'difference_number_tokens', 'sentence_distance', 'target_before_source', 'number_of_common_terms', 'number_of_punctuation_marks_source', 'number_of_punctuation_marks_target', 'difference_number_punctuation_marks', 'contains_modal_verb_source', 'contains_modal_verb_target', 'contains_argumentative_discourse_markers_source', 'contains_argumentative_discourse_markers_target']
        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)

        writer.writeheader()
        for entry in f:
            writer.writerow(entry)

    return f

    # people = ["Lisa", "Pam", "Phil", "John"]
    # unique pairs where order is irrelevant
    # result = [print(people[p1], people[p2]) for p1 in range(len(people)) for p2 in range(p1 + 1, len(people))]

    # All possible pairs, excluding duplicates:

   # pairs = [(p1, p2) for p1 in sentences for p2 in sentences if p1 != p2]
   # return pairs

def create_csv(three_classes):
    f = []

    global_id = 0
    doc_id = 0

    final_annot = []

    for i in range(0,990):
        doc_annotations = []
        f_in = open("../Corpus/%s.txt" % i, "r")
        text = f_in.read()

        doc = nlp(text)
        sentences = [sent.string.strip() for sent in doc.sents]

        f_ann = open("../Corpus/%s.ann" % i, "r")
        line = f_ann.readline()

        annotated_sentences = []
        related_sentences = []
        while line:
            supports = 'None'
            l = line.split("\t")
            if l[0].startswith("T"):
                annot_id = l[0]
                annot_text = l[2].strip()
                label = l[1].split(" ")[0]
                if not three_classes:
                    if label in ('Claim', 'Premise'):
                        label = 'Argumentative'

                annotated_sentences.append({
                    'annotID': annot_id,
                    'annot_text': annot_text,
                    'label': label,
                    'supports': supports,
                })

            elif l[0].startswith("R"):
                arg1 = l[1].split(" ")[1].split(":")[1]
                arg2 = l[1].split(" ")[2].split(":")[1]

                related_sentences.append({
                    'arg1': arg1,
                    'arg2': arg2,
                })

            line = f_ann.readline()

        for r in related_sentences:
            for a in annotated_sentences:
                if r['arg1'] == a['annotID']:
                    a['supports'] = r['arg2']

        for s in sentences:
            annotated = False
            for a in annotated_sentences:
                if a['annot_text'] in s:
                    annotated = True
                    final_annot.append({
                        'globalID': global_id,
                        'docID': doc_id,
                        'annotID': a['annotID'],
                        'text': s.strip(),
                        'label': a['label'],
                        'supports': a['supports'],
                    })
                    doc_annotations.append({
                        'globalID': global_id,
                        'docID': doc_id,
                        'annotID': a['annotID'],
                        'text': s.strip(),
                        'label': a['label'],
                        'supports': a['supports'],
                    })
            if not annotated:
                final_annot.append({
                    'globalID': global_id,
                    'docID': doc_id,
                    'annotID': 'None',
                    'text': s.strip(),
                    'label': 'None',
                    'supports': 'None',
                })
                doc_annotations.append({
                    'globalID': global_id,
                    'docID': doc_id,
                    'annotID': 'None',
                    'text': s.strip(),
                    'label': 'None',
                    'supports': 'None',
                })
            global_id += 1

        doc_id += 1

    with open('sentence_level_annotations_with_relations.csv', mode='w') as csv_file:
        fieldnames = ['globalID', 'docID', 'annotID', 'text', 'label', 'supports']
        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)

        writer.writeheader()
        for f in final_annot:
            writer.writerow(f)

    return final_annot

#create_csv(True)
#generate_pairs()
#encode_features()
train_linear_SVC()

max_stars_repo_name: cniklaus/argumentation-learning

max_issues_repo_name: cniklaus/argumentation-learning

max_forks_repo_name: cniklaus/argumentation-learning


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
This script show how to find neighboring nearest simplices to a given simplex in 2 dimensions.

This file is part of Fieldosophy, a toolkit for random fields.

Copyright (C) 2021 Anders Gunnar Felix Hildeman <fieldosophySPDEC@gmail.com>

This Source Code is subject to the terms of the BSD 3-Clause License.
If a copy of the license was not distributed with this file, you can obtain one at https://opensource.org/licenses/BSD-3-Clause.

"""

# Import package
from fieldosophy.GRF import FEM
from fieldosophy.GRF import GRF
from fieldosophy import mesh as mesher

from matplotlib import pyplot as plt
import numpy as np

print("Running two-dimensional FEM test case")
print("")

plt.figure(1)
plt.clf()

# %% Create 2D mesh

# Limits of coordinates
coordinateLims = np.array( [ [0,1], [0,1] ] )

print("Compute Mesh in 2D")
meshPlane = mesher.regularMesh.meshInPlaneRegular( coordinateLims, 0.1 )

print("Plot mesh")

plt.figure(1)
ax = plt.subplot(221)
plt.cla()
ax.set_title( "Mesh" )
meshPlotter = mesher.MeshPlotter( meshPlane )
edges = meshPlotter.getLines()
plt.plot(edges[0], edges[1], color="blue")
edges = meshPlotter.getBoundaryLines()
plt.plot(edges[0], edges[1], color="red")

# %%

# Get edges
edges = mesher.Mesh.getEdges( meshPlane.triangles, meshPlane.topD, meshPlane.topD, libInstance = meshPlane._libInstance )
# Get neighbors if simplices
neighs = mesher.Mesh.getSimplexNeighbors( edges["simplicesForEdges"], edges["edgesForSimplices"], libInstance = meshPlane._libInstance )

# Plot original simplex and its neighbors
chosenSimplex = np.random.randint( 0, meshPlane.NT )
edges = meshPlotter.getSimplicesLines( neighs[chosenSimplex,:] )
plt.plot(edges[0], edges[1], color="green", linewidth=6)
edges = meshPlotter.getSimplicesLines( np.array([chosenSimplex]) )
plt.plot(edges[0], edges[1], color="black", linewidth=6)

max_stars_repo_name: andyGFHill/fieldosophy

max_issues_repo_name: andyGFHill/fieldosophy

max_forks_repo_name: andyGFHill/fieldosophy


text:
"""
This script evaluates a saved model (will crash if nothing is saved).
"""
import os
import time

import numpy as np
import scipy.stats as st
import torch
from torch.nn import functional as F
from torch.utils.data import DataLoader

import utils
import arguments
from dataset_miniimagenet import MiniImagenet

def evaluate(args, model, logger, dataloader, mode, num_grad_steps):

    for c, batch in enumerate(dataloader):

        support_x = batch[0].to(args.device)
        support_y = batch[1].to(args.device)
        query_x = batch[2].to(args.device)
        query_y = batch[3].to(args.device)

        for inner_batch_idx in range(support_x.shape[0]):

            # reset context parameters
            model.reset_context_params()

            logger.log_pre_update(support_x[inner_batch_idx], support_y[inner_batch_idx],
                                  query_x[inner_batch_idx], query_y[inner_batch_idx],
                                  model, mode)

            for _ in range(num_grad_steps):
                # forward train data through net
                pred_train = model(support_x[inner_batch_idx])

                # compute loss
                loss_inner = F.cross_entropy(pred_train, support_y[inner_batch_idx])

                # compute gradient for context parameters
                grad_inner = torch.autograd.grad(loss_inner, model.context_params)[0]

                # set context parameters to their updated values
                model.context_params = model.context_params - args.lr_inner * grad_inner

            logger.log_post_update(support_x[inner_batch_idx], support_y[inner_batch_idx],
                                   query_x[inner_batch_idx], query_y[inner_batch_idx],
                                   model, mode)

    # reset context parameters
    model.reset_context_params()

class Logger:
    def __init__(self, args):

        self.args = args

        # initialise dictionary to keep track of accuracies/losses
        self.train_stats = {
            'train_accuracy_pre_update': [],
            'test_accuracy_pre_update': [],
            'train_accuracy_post_update': [],
            'test_accuracy_post_update': [],
        }
        self.valid_stats = {
            'train_accuracy_pre_update': [],
            'test_accuracy_pre_update': [],
            'train_accuracy_post_update': [],
            'test_accuracy_post_update': [],
        }
        self.test_stats = {
            'train_accuracy_pre_update': [],
            'test_accuracy_pre_update': [],
            'train_accuracy_post_update': [],
            'test_accuracy_post_update': [],
        }

        # keep track of how long the experiment takes
        self.start_time = time.time()

    def log_pre_update(self, support_x, support_y, query_x, query_y, model, mode):
        if mode == 'train':
            self.train_stats['train_accuracy_pre_update'].append(self.get_accuracy(support_x, support_y, model))
            self.train_stats['test_accuracy_pre_update'].append(self.get_accuracy(query_x, query_y, model))
        elif mode == 'val':
            self.valid_stats['train_accuracy_pre_update'].append(self.get_accuracy(support_x, support_y, model))
            self.valid_stats['test_accuracy_pre_update'].append(self.get_accuracy(query_x, query_y, model))
        elif mode == 'test':
            self.test_stats['train_accuracy_pre_update'].append(self.get_accuracy(support_x, support_y, model))
            self.test_stats['test_accuracy_pre_update'].append(self.get_accuracy(query_x, query_y, model))

    def log_post_update(self, support_x, support_y, query_x, query_y, model, mode):
        if mode == 'train':
            self.train_stats['train_accuracy_post_update'].append(self.get_accuracy(support_x, support_y, model))
            self.train_stats['test_accuracy_post_update'].append(self.get_accuracy(query_x, query_y, model))
        elif mode == 'val':
            self.valid_stats['train_accuracy_post_update'].append(self.get_accuracy(support_x, support_y, model))
            self.valid_stats['test_accuracy_post_update'].append(self.get_accuracy(query_x, query_y, model))
        elif mode == 'test':
            self.test_stats['train_accuracy_post_update'].append(self.get_accuracy(support_x, support_y, model))
            self.test_stats['test_accuracy_post_update'].append(self.get_accuracy(query_x, query_y, model))

    def print_header(self):
        print(
            '||------------------------------------------------------------------------------------------------------------------------------------------------------||')
        print(
            '||------------- TRAINING ------------------------|---------------------------------------- EVALUATION --------------------------------------------------||')
        print(
            '||-----------------------------------------------|------------------------------------------------------------------------------------------------------||')
        print(
            '||-----------------|     observed performance    |          META_TRAIN         |           META_VALID         |           META_TEST                     ||')
        print(
            '||    selection    |-----------------------------|-----------------------------|------------------------------|-----------------------------------------||')
        print(
            '||    criterion    |    train     |     valid    |    train     |     test     |    train     |     test      |    train     |     test                 ||')
        print(
            '||-----------------|--------------|--------------|--------------|--------------|--------------|---------------|--------------|--------------------------||')

    def print_logs(self, selection_criterion, logged_perf=None):
        if logged_perf is None:
            logged_perf = [' ', ' ']
        else:
            logged_perf = [np.round(logged_perf[0], 3), np.round(logged_perf[1], 3)]

        avg_acc = np.mean(self.test_stats['test_accuracy_post_update'])
        conf_interval = st.t.interval(0.95,
                                      len(self.test_stats['test_accuracy_post_update']) - 1,
                                      loc=avg_acc,
                                      scale=st.sem(self.test_stats['test_accuracy_post_update']))

        print(
            '||   {:<11}   |    {:<5}     |     {:<5}    | {:<5}->{:<5} | {:<5}->{:<5} | {:<5}->{:<5} | {:<5}->{:<5}  | {:<5}->{:<5} | {:<5}->{:<5} (+/- {}) ||'.format(
                selection_criterion,
                # performance we observed during training
                logged_perf[0],
                logged_perf[1],
                # meta-valid, task-test
                np.round(np.mean(self.train_stats['train_accuracy_pre_update']), 3),
                np.round(np.mean(self.train_stats['train_accuracy_post_update']), 3),
                np.round(np.mean(self.train_stats['test_accuracy_pre_update']), 3),
                np.round(np.mean(self.train_stats['test_accuracy_post_update']), 3),
                #
                np.round(np.mean(self.valid_stats['train_accuracy_pre_update']), 3),
                np.round(np.mean(self.valid_stats['train_accuracy_post_update']), 3),
                np.round(np.mean(self.valid_stats['test_accuracy_pre_update']), 3),
                np.round(np.mean(self.valid_stats['test_accuracy_post_update']), 3),
                #
                np.round(np.mean(self.test_stats['train_accuracy_pre_update']), 3),
                np.round(np.mean(self.test_stats['train_accuracy_post_update']), 3),
                np.round(np.mean(self.test_stats['test_accuracy_pre_update']), 2),
                np.round(100 * avg_acc, 3),
                #
                np.round(100 * np.mean(np.abs(avg_acc - conf_interval)), 2),
            ))

    def get_accuracy(self, x, y, model):
        predictions = model(x)
        num_correct = torch.argmax(F.softmax(predictions, dim=1), 1).eq(y).sum().item()
        return num_correct / len(y)

if __name__ == '__main__':

    args = arguments.parse_args()
    log_interval = 100

    utils.set_seed(args.seed)

    # --- settings ---

    args.k_shot = 1
    args.lr_inner = 1.0
    args.lr_meta = 'decay'
    args.num_grad_steps_inner = 2
    args.num_grad_steps_eval = 2
    args.model = 'cnn'
    args.num_context_params = 100

    if args.k_shot == 1:
        args.tasks_per_metaupdate = 4
    else:
        args.tasks_per_metaupdate = 2

    path = os.path.join(utils.get_base_path(), 'result_files', utils.get_path_from_args(args))

    try:
        training_stats, validation_stats = np.load(path + '.npy')
    except FileNotFoundError:
        print('You need to run the experiments first and make sure the results are saved at {}'.format(path))
        raise FileNotFoundError

    # initialise logger
    logger = Logger(args)
    logger.print_header()

    for num_grad_steps in [2]:

        print('\n --- ', num_grad_steps, '--- \n')

        # initialise logger
        logger = Logger(args)

        for selection_criterion in ['valid']:

            logger = Logger(args)

            for dataset in ['train', 'val', 'test']:
                # load model and its performance during training
                model = torch.load(path + '_best_{}'.format(selection_criterion))
                best_performances = np.load(path + '_best_{}.npy'.format(selection_criterion))

                # initialise dataloader
                mini_test = MiniImagenet(mode=dataset, n_way=args.n_way,
                                         k_shot=args.k_shot, k_query=args.k_query,
                                         batchsz=500, verbose=False, imsize=args.imsize)
                db_test = DataLoader(mini_test, batch_size=1, shuffle=True, num_workers=1, pin_memory=True)

                # evaluate the model
                evaluate(args, model, logger, db_test, mode=dataset, num_grad_steps=num_grad_steps)

            logger.print_logs(selection_criterion, best_performances)

max_stars_repo_name: luanagbmartins/cavia

max_issues_repo_name: luanagbmartins/cavia

max_forks_repo_name: luanagbmartins/cavia

reason: import numpy,import scipy


text:
#!/usr/bin/env python3

import sys
import operator
import numpy as np
from cyvcf2 import VCF

#mean_coverage input file should have <sample_id><tab><meanDepth>

vcf = VCF(sys.argv[1])
mean_coverage = sys.argv[2]
mean_coverage_dict = {}
with open(mean_coverage, 'rt') as inhandle:
    for lines in inhandle:
        sample, depth = lines.strip().split('\t')
        mean_coverage_dict[sample.replace('.breseq','').replace('.minimap2','').replace('.bwa','')] = float(depth)
samples_split = [x.split("_") for x in vcf.samples]
samples_split = [x for x in samples_split if x[3] != "c"]
splitted = [[x[0],int(x[1]),int(x[2]),x[3]] for x in samples_split]
splitted.sort(key = operator.itemgetter(1, 2))
samples_ordered = ['_'.join([str(x) for x in x]) for x in splitted]
samples_index = [samples_ordered.index(x) for x in vcf.samples if x in samples_ordered]
samples_timepoints = ' '.join([str(x) for x in range(len(samples_ordered))])
samples_zero = ' '.join([str(0) for x in range(len(samples_ordered))])
samples_coverage = ' '.join([str(mean_coverage_dict[x]) for x in samples_ordered])
#chromosome, pos, depth, timepoints <0 1 2 3 ...>, alternative depth <int int int ...>, mean depth <int int int ...>
var_out_header = ', '.join(['chromosome', '0', 'Depth', samples_timepoints, samples_zero, samples_coverage])
var_out = []
for variant in vcf:
    #chromosome, pos, depth, timepoints <0 1 2 3 ...>, alternative depth <int int int ...>, total depth <int int int ...>
    if variant.var_type == 'snp':
        var_out.append(', '.join([variant.CHROM, str(variant.POS), variant.REF+'->'+variant.ALT[0],samples_timepoints,' '.join([str(x) for x in variant.gt_alt_depths]),' '.join([str(x) for x in np.array(variant.gt_depths) - np.array(variant.gt_alt_depths)])]))
    if variant.var_type == 'deletion':
        var_out.append(', '.join([variant.CHROM, str(variant.POS), 'deletion',samples_timepoints,' '.join([str(x) for x in variant.gt_alt_depths]),' '.join([str(x) for x in np.array(variant.gt_depths) - np.array(variant.gt_alt_depths)])]))
    if variant.var_type == 'indel':
        var_out.append(', '.join([variant.CHROM, str(variant.POS), 'indel',samples_timepoints,' '.join([str(x) for x in variant.gt_alt_depths]),' '.join([str(x) for x in np.array(variant.gt_depths) - np.array(variant.gt_alt_depths)])]))
sys.stdout.write(var_out_header+'\n')
[sys.stdout.write(x+'\n') for x in var_out]

max_stars_repo_name: kullrich/snpless-nf

max_issues_repo_name: kullrich/snpless-nf

max_forks_repo_name: kullrich/snpless-nf


text:
# CameraManager
# Singleton class for keeping an observable coordinates of a ball, found via camera input

import cv2
import threading
import random
import time

class CameraManager:

	# Make this private
	def __init__(self):
		self.ballTrackingEnabled = False
		self.xBallPosition = 0
		self.yBallPosition = 0
		self.lastUpdated = time.localtime()
		self.ballRadius = 3  # NOTE: This is a magic number and should be moved to a "physical properties" class
		self.observers = []

	# TODO: Make singleton
	# def getInstance(self):

	def start_ball_tracking(self):
		# TODO: Handle case where already started

		self.ballTrackingEnabled = True

		# Start ball tracking thread
		t1 = threading.Thread(target=self.track_ball)
		t1.daemon = True
		t1.start()

		# Listen for keyboard input to manually trigger ball tracking commands
		# t2 = threading.Thread(target = self.readKeyboardInput())
		# t2.daemon = True
		# t2.start()

	def stop_ball_tracking(self):
		self.ballTrackingEnabled = False

	# Only to be run on its own thread
	def track_ball(self):
		while self.ballTrackingEnabled:
			self.xBallPosition = random.randint(0, 10)  # temp placeholder for vision tracking results
			self.yBallPosition = random.randint(0, 10)  # temp placeholder for vision tracking results
			self.lastUpdated = time.clock()
			self.update_observers("Location Updated:", x=self.xBallPosition, y=self.yBallPosition, updated_at=self.lastUpdated)

		print ("Ball tracking stopped.")

	def read_keyboard_input(self):
		key = raw_input('q = Exit, p = Get Ball Position \n')
		print ("Key entered: " + key)

		if key == 'q':
			self.stop_ball_tracking()

		elif key == 'p':
			print (self.get_ball_position())
			self.read_keyboard_input()

		else:
			self.read_keyboard_input()

	def get_ball_position(self):
		return [self.xBallPosition, self.yBallPosition, self.lastUpdated]

	def get_ball_radius(self):
		return self.ballRadius

	# Observer Functions
	def register(self, observer):
		if observer not in self.observers:
			self.observers.append(observer)

	def unregister(self, observer):
		if observer in self.observers:
			self.observers.remove(observer)

	def unregister_all(self):
		if self.observers:
			del self.observers[:]

	def update_observers(self, *args, **keywordargs):
		for observer in self.observers:
			observer.update(*args, **keywordargs)

	def initialize_camera(self, path):
		# TODO: Might need to add code here to gracefully handle a failed camera initialization
		if path is None:
			camera = cv2.VideoCapture(0)
		else:
			camera = cv2.VideoCapture(path)

# __________________________________________________

# # USAGE
# # python ball_tracking_demo.py --video ball_tracking_example.mp4
# # python ball_tracking_demo.py

# # import the necessary packages
# from collections import deque
# import numpy as np
# # import ArgumentParser # To be removed
# import imutils

# # construct the argument parse and parse the arguments  # To be removed
# # ap = argparse.ArgumentParser()
# # ap.add_argument("-v", "--video",
# # 	help="path to the (optional) video file")
# # ap.add_argument("-b", "--buffer", type=int, default=64,
# # 	help="max buffer size")
# # args = vars(ap.parse_args())

# # define the lower and upper boundaries of the "green"
# # ball in the HSV color space, then initialize the
# # list of tracked points
# # For HSV, Hue range is [0,179], Saturation range is [0,255] and Value range is [0,255]
# # HSV Info: http://infohost.nmt.edu/tcc/help/pubs/colortheory/web/hsv.html
# yellowLowerThreshold = (160, 100, 140)
# yellowUpperThreshold = (179, 255, 255)
# redLowerThreshold = (0, 100, 140)
# redUpperThreshold = (20, 255, 255)
# pts = deque(maxlen=args["buffer"])

# # keep looping
# while True:
# 	# grab the current frame
# 	(grabbed, frame) = camera.read()

# 	# if we are viewing a video and we did not grab a frame,
# 	# then we have reached the end of the video
# 	if args.get("video") and not grabbed:
# 		break

# 	# resize the frame, blur it, and convert it to the HSV
# 	# color space
# 	frame = imutils.resize(frame, width=600)
# 	# blurred = cv2.GaussianBlur(frame, (11, 11), 0)
# 	hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

# 	# construct a mask for the color "orange", then perform
# 	# a series of dilations and erosions to remove any small
# 	# blobs left in the mask
# 	maskRed = cv2.inRange(hsv, redLowerThreshold, redUpperThreshold)
# 	maskYellow = cv2.inRange(hsv, yellowLowerThreshold, yellowUpperThreshold)
# 	mask = maskRed + maskYellow
# 	mask = cv2.erode(mask, None, iterations=2)
# 	mask = cv2.dilate(mask, None, iterations=2)

# 	# find contours in the mask and initialize the current
# 	# (x, y) center of the ball
# 	cnts = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL,
# 		cv2.CHAIN_APPROX_SIMPLE)[-2]
# 	center = None

# 	# only proceed if at least one contour was found
# 	if len(cnts) > 0:
# 		# find the largest contour in the mask, then use
# 		# it to compute the minimum enclosing circle and
# 		# centroid
# 		c = max(cnts, key=cv2.contourArea)
# 		((x, y), radius) = cv2.minEnclosingCircle(c)
# 		M = cv2.moments(c)
# 		center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"]))

# 		# only proceed if the radius meets a minimum size
# 		if radius > 1:
# 			# draw the circle and centroid on the frame,
# 			# then update the list of tracked points
# 			cv2.circle(frame, (int(x), int(y)), int(radius + 20),
# 				(0, 255, 255), 2)
# 			cv2.circle(frame, center, 5, (0, 0, 255), -1)

# 	# update the points queue
# 	pts.appendleft(center)

# 	# loop over the set of tracked points
# 	for i in xrange(1, len(pts)):
# 		# if either of the tracked points are None, ignore
# 		# them
# 		if pts[i - 1] is None or pts[i] is None:
# 			continue

# 		# otherwise, compute the thickness of the line and
# 		# draw the connecting lines
# 		thickness = int(np.sqrt(args["buffer"] / float(i + 1)) * 0.5)
# 		cv2.line(frame, pts[i - 1], pts[i], (0, 255, 0), thickness)

# 	# show the frame to our screen
# 	cv2.imshow("Frame", frame)
# 	key = cv2.waitKey(1) & 0xFF

# 	# if the 'q' key is pressed, stop the loop
# 	if key == ord("q"):
# 		break

# # cleanup the camera and close any open windows
# camera.release()
# cv2.destroyAllWindows()

max_stars_repo_name: Jaribeau/handle

max_issues_repo_name: Jaribeau/handle

max_forks_repo_name: Jaribeau/handle


text:
import datetime
import os

import numpy as np

from wildfire.data.goes_level_1 import utilities

def test_parse_filename():
    filepath = (
        "/ABI-L1b-RadM/2019/300/20/"
        "OR_ABI-L1b-RadM1-M6C14_G17_s20193002048275_e20193002048332_c20193002048405.nc"
    )
    actual = utilities.parse_filename(filename=filepath)
    assert actual[0] == "M1"
    assert actual[1] == 14
    assert actual[2] == "G17"
    assert actual[3] == datetime.datetime(2019, 10, 27, 20, 48, 27, 500000)

def test_group_filepaths_into_scans(goes_level_1_filepaths_no_wildfire):
    actual = utilities.group_filepaths_into_scans(
        filepaths=goes_level_1_filepaths_no_wildfire
    )
    assert len(actual) == 1
    assert len(actual[0]) == 16

def test_decide_fastest_glob_patterns():
    # no end_time
    actual = utilities.decide_fastest_glob_patterns(
        directory="test_dir",
        satellite="noaa-goes17",
        region="M1",
        start_time=datetime.datetime(2019, 1, 1, 1, 1),
        end_time=None,
    )
    assert len(actual) == 1
    assert actual[0].count("*") == 1

    # differce 1 minunte
    actual = utilities.decide_fastest_glob_patterns(
        directory="test_dir",
        satellite="noaa-goes17",
        region="M1",
        start_time=datetime.datetime(2019, 1, 1, 1, 1),
        end_time=datetime.datetime(2019, 1, 1, 1, 2),
    )
    assert len(actual) == 1
    assert actual[0].count("*") == 2

    # difference 1 hour
    actual = utilities.decide_fastest_glob_patterns(
        directory="test_dir",
        satellite="noaa-goes17",
        region="M1",
        start_time=datetime.datetime(2019, 1, 1, 1, 1),
        end_time=datetime.datetime(2019, 1, 1, 2, 1),
    )
    assert len(actual) == 2
    assert actual[0].count("*") == 2

    # difference 1 day
    actual = utilities.decide_fastest_glob_patterns(
        directory="test_dir",
        satellite="noaa-goes17",
        region="M1",
        start_time=datetime.datetime(2019, 1, 1, 1, 1),
        end_time=datetime.datetime(2019, 1, 2, 1, 1),
    )
    assert len(actual) == 2
    assert actual[0].count("*") == 3

    # difference 1 month
    actual = utilities.decide_fastest_glob_patterns(
        directory="test_dir",
        satellite="noaa-goes17",
        region="M1",
        start_time=datetime.datetime(2019, 1, 1, 1, 1),
        end_time=datetime.datetime(2019, 2, 1, 1, 1),
    )
    assert len(actual) == 32
    assert actual[0].count("*") == 3

    # difference 1 year
    actual = utilities.decide_fastest_glob_patterns(
        directory="test_dir",
        satellite="noaa-goes17",
        region="M1",
        start_time=datetime.datetime(2019, 1, 1, 1, 1),
        end_time=datetime.datetime(2020, 1, 1, 1, 1),
    )
    assert len(actual) == 2
    assert actual[0].count("*") == 4

def test_list_local_files(goes_level_1_filepaths_no_wildfire):
    actual = utilities.list_local_files(
        local_directory=os.path.join(
            "tests", "resources", "goes_level_1_scan_no_wildfire"
        ),
        satellite="noaa-goes17",
        region="M1",
        start_time=datetime.datetime(2019, 12, 1, 10, 27, 27),
    )
    np.testing.assert_array_equal(actual, goes_level_1_filepaths_no_wildfire)

    actual = utilities.list_local_files(
        local_directory=os.path.join(
            "tests", "resources", "goes_level_1_scan_no_wildfire"
        ),
        satellite="noaa-goes17",
        region="M1",
        start_time=datetime.datetime(2019, 12, 1, 10, 27),
        end_time=datetime.datetime(2019, 12, 1, 10, 28),
    )
    np.testing.assert_array_equal(actual, goes_level_1_filepaths_no_wildfire)

max_stars_repo_name: Ferrumofomega/goes

max_issues_repo_name: joyprojects/wildfire

max_forks_repo_name: Ferrumofomega/wildfire


text:
import numpy as np
import pytest

from thgsp.convert import get_array_module
from thgsp.sampling._utils import (
    construct_dia,
    construct_hth,
    construct_sampling_matrix,
)

from ..utils4t import devices, float_dtypes, sparse_formats

@pytest.mark.parametrize("dtype", float_dtypes)
@pytest.mark.parametrize("layout", sparse_formats)
@pytest.mark.parametrize("dv", devices)
def test_construct_sampling_matrix(dtype, dv, layout):
    on_gpu = "cuda" == dv.type
    xp, xcipy, _ = get_array_module(on_gpu)
    S = [5, 4, 1]
    N = 6
    H = construct_sampling_matrix(N, S, dtype, dv, layout)
    assert H.format == layout
    assert H.shape == (len(S), N)

    D = xcipy.sparse.diags(xp.arange(1, N + 1)[S])
    HtHD0 = H.T @ D @ H
    HtH0 = H.T @ H

    print("\n", H.A)
    print(HtH0.A)
    print(HtHD0.A)
    HtH1 = construct_hth(N, S, dtype=dtype, device=dv, layout=layout)
    HtHD1 = construct_hth(
        N, S, xp.arange(1, N + 1)[S], dtype=dtype, device=dv, layout=layout
    )
    # print(HtH1.A)
    # print(HtHD1.A)
    print(type(HtH1), HtH1.shape, "on_gpu: ", on_gpu)
    print(type(HtHD1), HtHD1.shape, "on_gpu: ", on_gpu)
    assert np.allclose(HtH0.A, HtH1.A)
    assert np.allclose(HtHD0.A, HtHD1.A)

@pytest.mark.parametrize("dt", float_dtypes)
@pytest.mark.parametrize("layout", sparse_formats)
@pytest.mark.parametrize("dv", devices)
@pytest.mark.parametrize("inverse", [True, False])
@pytest.mark.parametrize("narrow", [True, False])
def test_construct_dia(dt, dv, layout, inverse, narrow):
    on_gpu = "cuda" == dv.type
    xp, xcipy, _ = get_array_module(on_gpu)

    N = 7
    S = [5, 1, 3]
    M = len(S)
    diag_data = xp.arange(N)
    P = construct_dia(
        S, diag_data=diag_data, ps=narrow, inverse=inverse, dtype=dt, layout=layout
    )
    if narrow:
        assert P.shape == (M, M)
        if inverse:
            assert xp.allclose(
                P.diagonal(),
                1 / diag_data[S],
            )
        else:
            assert xp.allclose(P.diagonal(), diag_data[S])

    else:
        assert P.shape == (N, N)
        if inverse:
            assert xp.allclose(P.diagonal(), 1 / diag_data)
        else:
            assert xp.allclose(P.diagonal(), diag_data)


text:
"""Summation density example using the cell based NNPS partitioner."""

import mpi4py.MPI as mpi

import numpy
from numpy import random

# Carray from PyZoltan
from cyarray.carray import UIntArray

# PySPH imports
from pysph.base.nnps import BoxSortNNPS
from pysph.parallel.parallel_manager import ZoltanParallelManagerGeometric
from pysph.base.utils import get_particle_array_wcsph

from pysph.base.kernels import CubicSpline, get_compiled_kernel
"""Utility to compute summation density"""

dim = 3

def sd_evaluate(nnps, pm, mass, src_index, dst_index):
    # the destination particle array
    dst = nnps.particles[dst_index]
    src = nnps.particles[src_index]

    # particle coordinates
    dx, dy, dz, dh, drho = dst.get(
        'x', 'y', 'z', 'h', 'rho', only_real_particles=True)
    sx, sy, sz, sh, srho = src.get(
        'x', 'y', 'z', 'h', 'rho', only_real_particles=False)

    neighbors = UIntArray()
    cubic = get_compiled_kernel(CubicSpline(dim=dim))

    # compute density for each destination particle
    num_particles = dst.num_real_particles

    # the number of local particles should have tag Local
    assert (num_particles == pm.num_local[dst_index])

    for i in range(num_particles):

        hi = dh[i]

        nnps.get_nearest_particles(src_index, dst_index, i, neighbors)

        nnbrs = neighbors.length

        rho_sum = 0.0
        for indexj in range(nnbrs):
            j = neighbors[indexj]

            wij = cubic.kernel(dx[i], dy[i], dz[i], sx[j], sy[j], sz[j], hi)

            rho_sum = rho_sum + mass * wij

        drho[i] += rho_sum

def main():
    # Initialize MPI and find out number of local particles
    comm = mpi.COMM_WORLD
    rank = comm.Get_rank()
    size = comm.Get_size()

    # number of particles per array
    numMyPoints = 1 << 10
    numGlobalPoints = size * numMyPoints

    avg_vol = 1.0 / numGlobalPoints
    dx = numpy.power(avg_vol, 1.0 / dim)
    mass = avg_vol
    hdx = 1.3

    if numGlobalPoints % size != 0:
        raise RuntimeError("Run with 2^n num procs!")

    # everybody creates two particle arrays with numMyPoints
    x1 = random.random(numMyPoints)
    y1 = random.random(numMyPoints)
    z1 = random.random(numMyPoints)
    h1 = numpy.ones_like(x1) * hdx * dx
    rho1 = numpy.zeros_like(x1)

    x2 = random.random(numMyPoints)
    y2 = random.random(numMyPoints)
    z2 = random.random(numMyPoints)
    h2 = numpy.ones_like(x2) * hdx * dx
    rho2 = numpy.zeros_like(x2)

    # z1[:] = 1.0
    # z2[:] = 0.5

    # local particle arrays
    pa1 = get_particle_array_wcsph(x=x1, y=y1, h=h1, rho=rho1, z=z1)
    pa2 = get_particle_array_wcsph(x=x2, y=y2, h=h2, rho=rho2, z=z2)

    # gather the data on root
    X1 = numpy.zeros(numGlobalPoints)
    Y1 = numpy.zeros(numGlobalPoints)
    Z1 = numpy.zeros(numGlobalPoints)
    H1 = numpy.ones_like(X1) * hdx * dx
    RHO1 = numpy.zeros_like(X1)

    gathers = (numpy.ones(size) * numMyPoints, None)

    comm.Gatherv(sendbuf=[x1, mpi.DOUBLE], recvbuf=[X1, gathers, mpi.DOUBLE])
    comm.Gatherv(sendbuf=[y1, mpi.DOUBLE], recvbuf=[Y1, gathers, mpi.DOUBLE])
    comm.Gatherv(sendbuf=[z1, mpi.DOUBLE], recvbuf=[Z1, gathers, mpi.DOUBLE])
    comm.Gatherv(
        sendbuf=[rho1, mpi.DOUBLE], recvbuf=[RHO1, gathers, mpi.DOUBLE]
    )

    X2 = numpy.zeros(numGlobalPoints)
    Y2 = numpy.zeros(numGlobalPoints)
    Z2 = numpy.zeros(numGlobalPoints)
    H2 = numpy.ones_like(X2) * hdx * dx
    RHO2 = numpy.zeros_like(X2)

    comm.Gatherv(sendbuf=[x2, mpi.DOUBLE], recvbuf=[X2, gathers, mpi.DOUBLE])
    comm.Gatherv(sendbuf=[y2, mpi.DOUBLE], recvbuf=[Y2, gathers, mpi.DOUBLE])
    comm.Gatherv(sendbuf=[z2, mpi.DOUBLE], recvbuf=[Z2, gathers, mpi.DOUBLE])
    comm.Gatherv(
        sendbuf=[rho2, mpi.DOUBLE], recvbuf=[RHO2, gathers, mpi.DOUBLE]
    )

    # create the particle arrays and PM
    PA1 = get_particle_array_wcsph(x=X1, y=Y1, z=Z1, h=H1, rho=RHO1)
    PA2 = get_particle_array_wcsph(x=X2, y=Y2, z=Z2, h=H2, rho=RHO2)

    # create the parallel manager
    PARTICLES = [PA1, PA2]
    PM = ZoltanParallelManagerGeometric(
        dim=dim, particles=PARTICLES, comm=comm
    )

    # create the local NNPS object with all the particles
    Nnps = BoxSortNNPS(dim=dim, particles=PARTICLES)
    Nnps.update()

    # only root computes summation density
    if rank == 0:
        assert numpy.allclose(PA1.rho, 0)
        sd_evaluate(Nnps, PM, mass, src_index=1, dst_index=0)
        sd_evaluate(Nnps, PM, mass, src_index=0, dst_index=0)
        RHO1 = PA1.rho

        assert numpy.allclose(PA2.rho, 0)
        sd_evaluate(Nnps, PM, mass, src_index=0, dst_index=1)
        sd_evaluate(Nnps, PM, mass, src_index=1, dst_index=1)
        RHO2 = PA2.rho

    # wait for the root...
    comm.barrier()

    # create the local particle arrays
    particles = [pa1, pa2]

    # create the local nnps object and parallel manager
    pm = ZoltanParallelManagerGeometric(
        dim=dim, comm=comm, particles=particles
    )
    nnps = BoxSortNNPS(dim=dim, particles=particles)

    # set the Zoltan parameters (Optional)
    pz = pm.pz
    pz.set_lb_method("RCB")
    pz.Zoltan_Set_Param("DEBUG_LEVEL", "0")

    # Update the parallel manager (distribute particles)
    pm.update()

    # update the local nnps
    nnps.update()

    # Compute summation density individually on each processor
    sd_evaluate(nnps, pm, mass, src_index=0, dst_index=1)
    sd_evaluate(nnps, pm, mass, src_index=1, dst_index=1)

    sd_evaluate(nnps, pm, mass, src_index=0, dst_index=0)
    sd_evaluate(nnps, pm, mass, src_index=1, dst_index=0)

    # gather the density and global ids
    rho1 = pa1.rho
    tmp = comm.gather(rho1)
    if rank == 0:
        global_rho1 = numpy.concatenate(tmp)
        assert (global_rho1.size == numGlobalPoints)

    rho2 = pa2.rho
    tmp = comm.gather(rho2)
    if rank == 0:
        global_rho2 = numpy.concatenate(tmp)
        assert (global_rho2.size == numGlobalPoints)

    # gather global x1 and y1
    x1 = pa1.x
    tmp = comm.gather(x1)
    if rank == 0:
        global_x1 = numpy.concatenate(tmp)
        assert (global_x1.size == numGlobalPoints)

    y1 = pa1.y
    tmp = comm.gather(y1)
    if rank == 0:
        global_y1 = numpy.concatenate(tmp)
        assert (global_y1.size == numGlobalPoints)

    z1 = pa1.z
    tmp = comm.gather(z1)
    if rank == 0:
        global_z1 = numpy.concatenate(tmp)
        assert (global_z1.size == numGlobalPoints)

    # gather global x2 and y2
    x2 = pa2.x
    tmp = comm.gather(x2)
    if rank == 0:
        global_x2 = numpy.concatenate(tmp)
        assert (global_x2.size == numGlobalPoints)

    y2 = pa2.y
    tmp = comm.gather(y2)
    if rank == 0:
        global_y2 = numpy.concatenate(tmp)
        assert (global_y2.size == numGlobalPoints)

    z2 = pa2.z
    tmp = comm.gather(z2)
    if rank == 0:
        global_z2 = numpy.concatenate(tmp)
        assert (global_z2.size == numGlobalPoints)

    # gather global indices
    gid1 = pa1.gid
    tmp = comm.gather(gid1)
    if rank == 0:
        global_gid1 = numpy.concatenate(tmp)
        assert (global_gid1.size == numGlobalPoints)

    gid2 = pa2.gid
    tmp = comm.gather(gid2)
    if rank == 0:
        global_gid2 = numpy.concatenate(tmp)
        assert (global_gid2.size == numGlobalPoints)

    # check rho1
    if rank == 0:
        # make sure the arrays are of the same size
        assert (global_x1.size == X1.size)
        assert (global_y1.size == Y1.size)
        assert (global_z1.size == Z1.size)

        for i in range(numGlobalPoints):

            # make sure we're chacking the right point
            assert abs(global_x1[i] - X1[global_gid1[i]]) < 1e-14
            assert abs(global_y1[i] - Y1[global_gid1[i]]) < 1e-14
            assert abs(global_z1[i] - Z1[global_gid1[i]]) < 1e-14

            diff = abs(global_rho1[i] - RHO1[global_gid1[i]])
            condition = diff < 1e-14
            assert condition, "diff = %g" % (diff)

    # check rho2
    if rank == 0:
        # make sure the arrays are of the same size
        assert (global_x2.size == X2.size)
        assert (global_y2.size == Y2.size)
        assert (global_z2.size == Z2.size)

        for i in range(numGlobalPoints):

            # make sure we're chacking the right point
            assert abs(global_x2[i] - X2[global_gid2[i]]) < 1e-14
            assert abs(global_y2[i] - Y2[global_gid2[i]]) < 1e-14
            assert abs(global_z2[i] - Z2[global_gid2[i]]) < 1e-14

            diff = abs(global_rho2[i] - RHO2[global_gid2[i]])
            condition = diff < 1e-14
            assert condition, "diff = %g" % (diff)

    if rank == 0:
        print("Summation density test: OK")

if __name__ == '__main__':
    main()

max_stars_repo_name: nauaneed/pysph

max_issues_repo_name: nauaneed/pysph

max_forks_repo_name: nauaneed/pysph

reason: import numpy,from numpy


text:
import kachery_p2p as kp
import spikeextractors as se
import numpy as np

class Bin1RecordingExtractor(se.RecordingExtractor):
    extractor_name = 'Bin1RecordingExtractor'
    is_writable = False
    def __init__(self, *, raw, raw_num_channels, num_frames, samplerate, channel_ids, channel_map, channel_positions, p2p):
        se.RecordingExtractor.__init__(self)

        self._raw = raw
        self._num_frames = num_frames
        self._samplerate = samplerate
        self._raw_num_channels = raw_num_channels
        self._channel_ids = channel_ids
        self._channel_map = channel_map
        self._channel_positions = channel_positions
        self._p2p = p2p

        for id in self._channel_ids:
            pos = self._channel_positions[str(id)]
            self.set_channel_property(id, 'location', pos)

    def get_channel_ids(self):
        return self._channel_ids

    def get_num_frames(self):
        return self._num_frames

    def get_sampling_frequency(self):
        return self._samplerate

    def get_traces(self, channel_ids=None, start_frame=None, end_frame=None):
        if start_frame is None:
            start_frame = 0
        if end_frame is None:
            end_frame = self._num_frames
        if channel_ids is None:
            channel_ids = self._channel_ids
        M = len(channel_ids)
        N = end_frame - start_frame

        i1 = start_frame * 2 * self._raw_num_channels
        i2 = end_frame * 2 * self._raw_num_channels

        buf = kp.load_bytes(self._raw, start=i1, end=i2, p2p=self._p2p)
        X = np.frombuffer(buf, dtype=np.int16).reshape((end_frame - start_frame, self._raw_num_channels))

        ret = np.zeros((M, N))
        for ii, ch_id in enumerate(channel_ids):
            ret[ii, :] = X[:, self._channel_map[str(ch_id)]]

        return ret


text:
from __future__ import division, print_function

from builtins import object, zip

import pandas as pd
import xarray as xr
from dask.diagnostics import ProgressBar
from numpy import array
from past.utils import old_div

# This file is to deal with CAMx code - try to make it general for CAMx 4.7.1 --> 5.1

ProgressBar().register()

class CAMx(object):
    def __init__(self):
        self.objtype = 'CAMX'
        self.coarse = array(
            ['NA', 'PSO4', 'PNO3', 'PNH4', 'PH2O', 'PCL', 'PEC', 'FPRM', 'FCRS', 'CPRM', 'CCRS', 'SOA1', 'SOA2', 'SOA3',
             'SOA4'])
        self.fine = array(
            ['NA', 'PSO4', 'PNO3', 'PNH4', 'PH2O', 'PCL', 'PEC', 'FPRM', 'FCRS', 'SOA1', 'SOA2', 'SOA3',
             'SOA4'])
        self.noy_gas = array(
            ['NO', 'NO2', 'NO3', 'N2O5', 'HONO', 'HNO3', 'PAN', 'PANX', 'PNA', 'NTR', 'CRON', 'CRN2', 'CRNO',
             'CRPX', 'OPAN'])
        self.poc = array(['SOA1', 'SOA2', 'SOA3', 'SOA4'])
        self.dset = None
        self.grid = None  # gridcro2d obj
        self.fname = None
        self.metcrofnames = None
        self.aerofnames = None
        self.dates = None
        self.keys = None
        self.indexdates = None
        self.metindex = None
        self.latitude = None
        self.longitude = None
        self.map = None

    def get_dates(self):
        print('Reading CAMx dates...')
        print(self.dset)
        tflag1 = array(self.dset['TFLAG'][:, 0], dtype='|S7')
        tflag2 = array(old_div(self.dset['TFLAG'][:, 1], 10000), dtype='|S6')
        date = pd.to_datetime([i + j.zfill(2) for i, j in zip(tflag1, tflag2)], format='%Y%j%H')
        indexdates = pd.Series(date).drop_duplicates(keep='last').index.values
        self.dset = self.dset.isel(time=indexdates)
        self.dset['time'] = date[indexdates]

    def open_camx(self, file):
        from glob import glob
        from numpy import sort
        dropset = ['layer', 'longitude_bounds', 'latitude_bounds',
                   'x', 'y', 'level', 'lambert_conformal_conic']
        nameset = {'COL': 'x', 'ROW': 'y', 'TSTEP': 'time', 'LAY': 'z'}
        if type(file) == str:
            fname = sort(array(glob(file)))
        else:
            fname = sort(array(file))
        if fname.shape[0] >= 1:
            if self.dset is None:
                self.dset = xr.open_mfdataset(
                    fname.tolist(), concat_dim='TSTEP', engine='pnc').drop(dropset).rename(nameset).squeeze()
                self.load_conus_basemap(res='l')
                self.get_dates()
            else:
                dset = xr.open_mfdataset(fname.tolist(), concat_dim='TSTEP',
                                         engine='pnc').drop(dropset).rename(nameset).squeeze()
                self.dset = xr.merge([self.dset, dset])
        else:
            print('Files not found')
        self.keys = list(self.dset.keys())

    def check_z(self, varname):
        if pd.Series(self.dset[varname].dims).isin('z').max():
            return True
        else:
            return False

    def get_nox(self, lay=None):
        if self.check_z('NO'):
            if lay is not None:
                var = self.dset['NO'][:, 0, :, :].squeeze().copy()
                var += self.dset['NO2'][:, 0, :, :].squeeze().copy()
            else:
                var = self.dset['NO'][:, :, :, :].copy()
                var += self.dset['NO2'][:, :, :, :].copy()
        else:
            var = self.dset['NO'][:, :, :].copy()
            var += self.dset['NO2'][:, :, :].copy()
        return var

    def get_pm25(self, lay=None):
        keys = list(self.dset.keys())
        allvars = self.fine
        index = pd.Series(allvars).isin(keys)
        newkeys = allvars[index]
        if self.check_z(newkeys[0]):
            if lay is not None:
                var = self.dset[newkeys[0]][:, 0, :, :].squeeze()
                for i in newkeys[1:]:
                    var += self.dset[i][:, 0, :, :].squeeze()
            else:
                var = self.dset[newkeys[0]][:, :, :, :].squeeze()
                for i in newkeys[1:]:
                    var += self.dset[i][:, :, :, :].squeeze()
        else:
            var = self.dset[newkeys[0]][:, :, :].copy()
            for i in newkeys[1:]:
                var += self.dset[i][:, :, :].squeeze()
        return var

    def get_pm10(self, lay=None):
        keys = list(self.dset.keys())
        allvars = self.coarse
        index = pd.Series(allvars).isin(keys)
        newkeys = allvars[index]
        if self.check_z(newkeys[0]):
            if lay is not None:
                var = self.dset[newkeys[0]][:, 0, :, :].squeeze()
                for i in newkeys[1:]:
                    var += self.dset[i][:, 0, :, :].squeeze()
            else:
                var = self.dset[newkeys[0]][:, :, :, :].squeeze()
                for i in newkeys[1:]:
                    var += self.dset[i][:, :, :, :].squeeze()
        else:
            var = self.dset[newkeys[0]][:, :, :].copy()
            for i in newkeys[1:]:
                var += self.dset[i][:, :, :].squeeze()
        return var

    def get_var(self, param='O3', lay=None):
        p = param.upper()
        print(param)
        if p == 'PM25':
            var = self.get_pm25(lay=lay)
        elif p == 'PM10':
            var = self.get_pm10(lay=lay)
        elif p == 'NOX':
            var = self.get_nox(lay=lay)
        elif p == 'OC':
            var = self.get_oc(lay=lay)
        elif p == 'VOC':
            if lay is not None:
                var = self.dset['VOC'][:, 0, :, :].copy().squeeze()
            else:
                var = self.dset['VOC'][:, :, :, :].copy().squeeze()
        else:
            if self.check_z(param):
                if lay is None:
                    var = self.dset[param][:, :, :, :].copy()
                else:
                    var = self.dset[param][:, lay, :, :].copy().squeeze()
            else:
                var = self.dset[param]
        return var

    def load_conus_basemap(self, res='l'):
        from mpl_toolkits.basemap import Basemap
        if self.map is None:
            lat1 = self.dset.P_ALP
            lat2 = self.dset.P_BET
            lon1 = self.dset.P_GAM
            lon0 = self.dset.XCENT
            lat0 = self.dset.YCENT
            m = Basemap(projection='lcc', resolution=res, lat_1=lat1, lat_2=lat2, lat_0=lat0, lon_0=lon0,
                        lon_1=lon1,
                        llcrnrlat=self.dset.latitude[0, 0], urcrnrlat=self.dset.latitude[-1, -1],
                        llcrnrlon=self.dset.longitude[0, 0],
                        urcrnrlon=self.dset.longitude[-1, -1], rsphere=6371200.,
                        area_thresh=50.)
            self.map = m
        else:
            m = self.map
        return self.map

max_stars_repo_name: barronh/MONET

max_issues_repo_name: barronh/MONET

max_forks_repo_name: barronh/MONET

reason: from numpy


text:
import pydicom  # for loading dicom
import SimpleITK as sitk  # for loading mhd/raw
import os
import numpy as np
import scipy.ndimage

#DICOM: send path to any *.dcm file where containing dir has the other slices (dcm files), or path to the dir itself
#MHD/RAW: send path to the *.mhd file where containing die has the coorisponding *.raw file
def load_scan(path2scan):
    if (path2scan.split('.')[-1] == 'mhd') or (path2scan.split('.')[-1] == 'raw'):
        return load_mhd(path2scan)
    elif path2scan.split('.')[-1] == 'dcm':
        return load_dicom(os.path.split(path2scan)[0]) #pass containing directory
    elif os.listdir(path2scan)[0].split('.')[-1] == 'dcm':
        return load_dicom(path2scan)
    else:
        raise Exception('No valid scan [series] found in given file/directory')

def load_mhd(path2scan):
    if not os.path.exists(path2scan):
        print('no such file, skipping')
        return
    itkimage = sitk.ReadImage(path2scan)
    scan = sitk.GetArrayFromImage(itkimage)
    spacing = np.flip(np.array(itkimage.GetSpacing()), axis=0)
    orientation = np.transpose(np.array(itkimage.GetDirection()).reshape((3, 3)))
    origin = np.flip(np.array(itkimage.GetOrigin()), axis=0)  # origionally in yxz format (read xy in viewers but sanved as yx)
    return scan, spacing, orientation, origin, None  # output in zyx format

def load_dicom(path2scan_dir):
    dicom_folder = path2scan_dir
    dcms = os.listdir(dicom_folder)
    first_slice_data = pydicom.read_file(path2scan_dir + '\\' + dcms[0])
    first_slice = first_slice_data.pixel_array
    orientation = np.transpose(first_slice_data.ImageOrientationPatient) #zyx format
    spacing_xy = np.array(first_slice_data.PixelSpacing, dtype=float)
    spacing_z = np.float(first_slice_data.SliceThickness)
    spacing = np.array([spacing_z, spacing_xy[1], spacing_xy[0]]) #zyx format

    scan = np.zeros((len(dcms),first_slice.shape[0],first_slice.shape[1]))
    raw_slices=[]
    indexes = []
    for dcm in dcms:
        slice_data = pydicom.read_file(dicom_folder + '\\' + dcm)
        slice_data.filename = dcm
        raw_slices.append(slice_data)
        indexes.append(float(slice_data.ImagePositionPatient[2]))
    indexes = np.array(indexes,dtype=float)

    raw_slices = [x for _, x in sorted(zip(indexes, raw_slices))]
    origin = np.array(raw_slices[0][0x00200032].value) #origin is assumed to be the image location of the first slice
    if origin is None:
        origin = np.zeros(3)
    else:
        origin = np.array([origin[2],origin[1],origin[0]]) #change from x,y,z to z,y,x

    for i, slice in enumerate(raw_slices):
        scan[i, :, :] = slice.pixel_array
    # for i, index in enumerate(indexes):
    #     for slice in raw_slices:
    #         if int(slice.InstanceNumber) == index:
    #             scan[i,:,:] = slice._pixel_data_numpy()
    return scan, spacing, orientation, origin, raw_slices

#point to directory of folders conting dicom scans only (subdirs only), runs aon all folders..
# ref directory is used to copy the m
def save_dicom(scan, origional_raw_slices, dst_directory): # \\dfds\\ format
    os.makedirs(dst_directory,exist_ok=True)
    for i, slice in enumerate(origional_raw_slices):
        slice.pixel_array.flat = scan[i,:,:].flat
        slice.PixelData = slice.pixel_array.tobytes()
        slice.save_as(os.path.join(dst_directory,slice.filename))#.dcmwrite(os.path.join(dst_directory,slice.filename),slice)

#img_array: 3d numpy matrix, z,y,x
def toDicom(save_dir, img_array,  pixel_spacing, orientation):
    ref_scan = pydicom.read_file('utils/ref_scan.dcm') #slice from soem real scan so we can copy the meta data

    # write dcm file for each slice in scan
    for i, slice in enumerate(img_array):
        ref_scan.pixel_array.flat = img_array[i,:,:].flat
        ref_scan.PixelData = ref_scan.pixel_array.tobytes()
        ref_scan.RefdSOPInstanceUID = str(i)
        ref_scan.SOPInstanceUID = str(i)
        ref_scan.InstanceNumber = str(i)
        ref_scan.SliceLocation = str(i)
        ref_scan.ImagePositionPatient[2] = str(i*pixel_spacing[0])
        ref_scan.RescaleIntercept = 0
        ref_scan.Rows = img_array.shape[1]
        ref_scan.Columns = img_array.shape[2]
        ref_scan.PixelSpacing = [str(pixel_spacing[2]),str(pixel_spacing[1])]
        ref_scan.SliceThickness = pixel_spacing[0]
        #Pixel Spacing                       DS: ['0.681641', '0.681641']
        #Image Position (Patient)            DS: ['-175.500000', '-174.500000', '49']
        #Image Orientation (Patient)         DS: ['1.000000', '0.000000', '0.000000', '0.000000', '1.000000', '0.000000']
        #Rows                                US: 512
        #Columns                             US: 512
        os.makedirs(save_dir,exist_ok=True)
        ref_scan.save_as(os.path.join(save_dir, str(i)+'.dcm'))

def scale_scan(scan,spacing,factor=1):
    resize_factor = factor * spacing
    new_real_shape = scan.shape * resize_factor
    new_shape = np.round(new_real_shape)
    real_resize_factor = new_shape / scan.shape
    new_spacing = spacing / real_resize_factor
    scan_resized = scipy.ndimage.interpolation.zoom(scan, real_resize_factor, mode='nearest')
    return scan_resized, resize_factor

# get the shape of an orthotope in 1:1:1 ratio AFTER scaling to the given spacing
def get_scaled_shape(shape, spacing):
    new_real_shape = shape * spacing
    return np.round(new_real_shape).astype(int)

def scale_vox_coord(coord, spacing, factor=1):
    resize_factor = factor * spacing
    return (coord*resize_factor).astype(int)

def world2vox(world_coord, spacing, orientation, origin):
    world_coord = np.dot(np.linalg.inv(np.dot(orientation, np.diag(spacing))), world_coord - origin)
    if orientation[0, 0] < 0:
        vox_coord = (np.array([world_coord[0], world_coord[2], world_coord[1]])).astype(int)
    else:
        vox_coord = (np.array([world_coord[0], world_coord[1], world_coord[2]])).astype(int)
    return vox_coord


text:
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 14 17:25:22 2015

@author: Wasit
"""
import numpy as np
x=np.arange(1e6)
y=0
for i in x:
    y+=i


text:
# -*- coding: utf-8 -*-
import tensorflow as tf
import matplotlib.pyplot as plt
from scipy.misc import imsave
from scripts.loss_functions import content_loss, gram_matrix, style_loss, tv_loss
from scripts.image_utils import load_image, preprocess_image, deprocess_image
from scripts.squeezenet import SqueezeNet

def get_session():
    """
    创建一个动态分配内存的会话
    """
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    session = tf.Session(config=config)
    return session

tf.reset_default_graph() # 删除图中的所有变量
sess = get_session() # 开始一个新会话

# 载入训练好的SqueezeNet模型
PATH = 'squeezenet/squeezenet.ckpt'
model = SqueezeNet(save_path=PATH, sess=sess)

def transfer(content_image, style_image,content_layer, content_weight,
             style_layers, style_weights, tv_weight,
             initial_lr = 3.0, decayed_lr = 0.1, decay_lr_at = 180, max_iter = 200,
             model = model, sess = sess, image_size = None, style_size = None,
             init_random = False):
    """Run style transfer!

    Inputs:
    - content_image: filename of content image
    - style_image: filename of style image
    - image_size: size of smallest image dimension (used for content loss and generated image)
    - style_size: size of smallest style image dimension
    - content_layer: layer to use for content loss
    - content_weight: weighting on content loss
    - style_layers: list of layers to use for style loss
    - style_weights: list of weights to use for each layer in style_layers
    - tv_weight: weight of total variation regularization term
    - init_random: initialize the starting image to uniform random noise
    """
    # Extract features from the content image
    content_img = preprocess_image(load_image(content_image, size=image_size))
    feats = model.extract_features(model.image)
    content_target = sess.run(feats[content_layer],
                              {model.image: content_img[None]})

    # Extract features from the style image
    style_img = preprocess_image(load_image(style_image, size=style_size))
    style_feat_vars = [feats[idx] for idx in style_layers]
    style_target_vars = []
    # Compute list of TensorFlow Gram matrices
    for style_feat_var in style_feat_vars:
        style_target_vars.append(gram_matrix(style_feat_var))
    # Compute list of NumPy Gram matrices by evaluating the TensorFlow graph on the style image
    style_targets = sess.run(style_target_vars, {model.image: style_img[None]})

    # Initialize generated image to content image

    if init_random:
        img_var = tf.Variable(tf.random_uniform(content_img[None].shape, 0, 1), name="image")
    else:
        img_var = tf.Variable(content_img[None], name="image")

    # Extract features on generated image
    feats = model.extract_features(img_var)
    # Compute loss
    c_loss = content_loss(content_weight, feats[content_layer], content_target)
    s_loss = style_loss(feats, style_layers, style_targets, style_weights)
    t_loss = tv_loss(img_var, tv_weight)
    loss = c_loss + s_loss + t_loss

    # Create and initialize the Adam optimizer
    lr_var = tf.Variable(initial_lr, name="lr")
    # Create train_op that updates the generated image when run
    with tf.variable_scope("optimizer") as opt_scope:
        train_op = tf.train.AdamOptimizer(lr_var).minimize(loss, var_list=[img_var])
    # Initialize the generated image and optimization variables
    opt_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=opt_scope.name)
    sess.run(tf.variables_initializer([lr_var, img_var] + opt_vars))
    # Create an op that will clamp the image values when run
    clamp_image_op = tf.assign(img_var, tf.clip_by_value(img_var, -1.5, 1.5))

    f, axarr = plt.subplots(1,2)
    axarr[0].axis('off')
    axarr[1].axis('off')
    axarr[0].set_title('Content Source Img.')
    axarr[1].set_title('Style Source Img.')
    axarr[0].imshow(deprocess_image(content_img))
    axarr[1].imshow(deprocess_image(style_img))
    plt.show()
    plt.figure()

    # Hardcoded handcrafted
    for t in range(max_iter):
        # Take an optimization step to update img_var
        sess.run(train_op)
        if t < decay_lr_at:
            sess.run(clamp_image_op)
        if t == decay_lr_at:
            sess.run(tf.assign(lr_var, decayed_lr))
        if t % 100 == 0:
            print('Iteration {}'.format(t))
            img = sess.run(img_var)
            plt.imshow(deprocess_image(img[0], rescale=True))
            plt.axis('off')
            plt.show()
    print('Iteration {}'.format(t))
    img = sess.run(img_var)
    plt.imshow(deprocess_image(img[0], rescale=True))
    plt.axis('off')
    plt.show()
    imsave('./images/transfered.jpg', deprocess_image(img[0], rescale=True))

reason: from scipy


text:
import numpy as np
import scipy as sp
import pandas as pd
from tqdm import tqdm
import copy
import time
from sklearn.model_selection import train_test_split

from jax.config import config
#config.update("jax_enable_x64", True)

import jax.numpy as jnp
from jax import random
from surv_copula.main_copula_survreg_gaussian import fit_copula_survival,predict_copula_survival,check_convergence_pr

def main_copula_survreg(dataset):

	#Import data
	data = pd.read_csv("./data/{}.csv".format(dataset))
	t = np.array(data['t'])
	delta = np.array(data['delta'])
	x = np.array(data['x'])
	n = np.shape(t)[0]

	rep_cv = 10
	if dataset =="melanoma":
		n_train = int(n/2)
	elif dataset == "kidney":
		#n_train = 100
		n_train = int(n/2)

	n_test = n-n_train

	test_ll_cv = np.zeros(rep_cv)
	seed = 100

	for i in tqdm(range(rep_cv)):
		#Train-test split and save for R
		train_ind,test_ind = train_test_split(np.arange(n),test_size = n_test,train_size = n_train,random_state = seed+i)

		t_train = t[train_ind]
		delta_train = delta[train_ind]
		x_train = x[train_ind]

		t_test = t[test_ind]
		delta_test = delta[test_ind]
		x_test = x[test_ind]

		#normalize
		scale = (np.sum(t_train)/np.sum(delta_train))
		t_train = t_train/scale
		t_test = t_test/scale

		mean_x = np.mean(x_train)
		std_x = np.std(x_train)

		x_train = (x_train - mean_x)/std_x
		x_test = (x_test - mean_x)/std_x

		#save for R
		np.savetxt("data/{}_t_train{}.csv".format(dataset,i),t_train,delimiter = ',')
		np.savetxt("data/{}_delta_train{}.csv".format(dataset,i),delta_train,delimiter = ',')
		np.savetxt("data/{}_x_train{}.csv".format(dataset,i),x_train,delimiter = ',')
		np.savetxt("data/{}_t_test{}.csv".format(dataset,i),t_test,delimiter = ',')
		np.savetxt("data/{}_delta_test{}.csv".format(dataset,i),delta_test,delimiter = ',')
		np.savetxt("data/{}_x_test{}.csv".format(dataset,i),x_test,delimiter = ',')

		#Initialize plot and sample number
		B = 2000 #number of posterior samples

		rho_grid = np.array([0.5,0.6,0.7,0.8,0.9])
		rho_,rho_x_ = np.meshgrid(rho_grid,rho_grid)
		hyperparam_grid = np.vstack([rho_.ravel(), rho_x_.ravel()]).transpose()

		#Pass grid of a_values
		cop_surv_obj  = fit_copula_survival(t_train,delta_train,x_train, B,hyperparam_grid = hyperparam_grid)

		print('Optimal rho is {}'.format(cop_surv_obj.rho_opt))
		print('Optimal rho_x is {}'.format(cop_surv_obj.rho_x_opt))

		logcdf_av, logpdf_av = predict_copula_survival(cop_surv_obj,t_test,x_test)
		test_ll = (delta_test)*(logpdf_av) + (1-delta_test)*(np.log1p(-np.exp(logcdf_av)))
		print(np.mean(test_ll))
		test_ll_cv[i] = np.mean(test_ll)

	print("{}: {} +- {}".format(dataset,np.mean(test_ll_cv),np.std(test_ll_cv)/np.sqrt(rep_cv)))

main_copula_survreg("melanoma")
main_copula_survreg("kidney")

reason: import numpy,import scipy,import jax,from jax


text:
# Implementation of pairs.txt from lfw dataset
# Section f: http://vis-www.cs.umass.edu/lfw/lfw.pdf
# More succint, less explicit: http://vis-www.cs.umass.edu/lfw/README.txt

import glob
import io
import os
import random
from argparse import ArgumentParser, Namespace
from typing import List, Optional, Set, Tuple, cast

import numpy as np

from facenet_sandberg.utils import transform_to_lfw_format

Mismatch = Tuple[str, int, str, int]
Match = Tuple[str, int, int]
CommandLineArgs = Namespace

def write_pairs_to_file(fname: str,
                        match_folds: List[List[Match]],
                        mismatch_folds: List[List[Mismatch]],
                        num_folds: int,
                        num_matches_mismatches: int) -> None:
    metadata = '{}\t{}\n'.format(num_folds, num_matches_mismatches)
    with io.open(fname,
                 'w',
                 io.DEFAULT_BUFFER_SIZE,
                 encoding='utf-8') as fpairs:
        fpairs.write(metadata)
        for match_fold, mismatch_fold in zip(match_folds, mismatch_folds):
            for match in match_fold:
                line = '{}\t{}\t{}\n'.format(match[0], match[1], match[2])
                fpairs.write(line)
            for mismatch in mismatch_fold:
                line = '{}\t{}\t{}\t{}\n'.format(
                    mismatch[0], mismatch[1], mismatch[2], mismatch[3])
                fpairs.write(line)
        fpairs.flush()

def _split_people_into_folds(image_dir: str,
                             num_folds: int) -> List[List[str]]:
    names = [d for d in os.listdir(image_dir)
             if os.path.isdir(os.path.join(image_dir, d))]
    random.shuffle(names)
    return [list(arr) for arr in np.array_split(names, num_folds)]

def _make_matches(image_dir: str,
                  people: List[str],
                  total_matches: int) -> List[Match]:
    matches = cast(Set[Match], set())
    curr_matches = 0
    while curr_matches < total_matches:
        person = random.choice(people)
        images = _clean_images(image_dir, person)
        if len(images) > 1:
            img1, img2 = sorted(
                [images.index(random.choice(images)) + 1,
                 images.index(random.choice(images)) + 1])
            match = (person, img1, img2)
            if (img1 != img2) and (match not in matches):
                matches.add(match)
                curr_matches += 1
    return sorted(list(matches), key=lambda x: x[0].lower())

def _make_mismatches(image_dir: str,
                     people: List[str],
                     total_matches: int) -> List[Mismatch]:
    mismatches = cast(Set[Mismatch], set())
    curr_matches = 0
    while curr_matches < total_matches:
        person1 = random.choice(people)
        person2 = random.choice(people)
        if person1 != person2:
            person1_images = _clean_images(image_dir, person1)
            person2_images = _clean_images(image_dir, person2)
            if person1_images and person2_images:
                img1 = person1_images.index(random.choice(person1_images)) + 1
                img2 = person2_images.index(random.choice(person2_images)) + 1
                if person1.lower() > person2.lower():
                    person1, img1, person2, img2 = person2, img2, person1, img1
                mismatch = (person1, img1, person2, img2)
                if mismatch not in mismatches:
                    mismatches.add(mismatch)
                    curr_matches += 1
    return sorted(list(mismatches), key=lambda x: x[0].lower())

def _clean_images(base: str, folder: str):
    images = os.listdir(os.path.join(base, folder))
    images = [image for image in images if image.endswith(
        ".jpg") or image.endswith(".png") or image.endswith(".jpeg")]
    return images

def generate_pairs(
        image_dir: str,
        num_folds: int,
        num_matches_mismatches: int,
        write_to_file: bool=False,
        pairs_file_name: str="") -> None:
    transform_to_lfw_format(image_dir)
    people_folds = _split_people_into_folds(image_dir, num_folds)
    matches = []
    mismatches = []
    for fold in people_folds:
        matches.append(_make_matches(image_dir,
                                     fold,
                                     num_matches_mismatches))
        mismatches.append(_make_mismatches(image_dir,
                                           fold,
                                           num_matches_mismatches))
    if write_to_file:
        write_pairs_to_file(pairs_file_name,
                            matches,
                            mismatches,
                            num_folds,
                            num_matches_mismatches)
    return matches, mismatches

def _cli() -> None:
    args = _parse_arguments()
    generate_pairs(
        args.image_dir,
        args.num_folds,
        args.num_matches_mismatches,
        True,
        args.pairs_file_name)

def _parse_arguments() -> Namespace:
    parser = ArgumentParser()
    parser.add_argument('--image_dir',
                        type=str,
                        required=True,
                        help='Path to the image directory.')
    parser.add_argument('--pairs_file_name',
                        type=str,
                        required=True,
                        help='Filename of pairs.txt')
    parser.add_argument('--num_folds',
                        type=int,
                        required=True,
                        help='Number of folds for k-fold cross validation.')
    parser.add_argument('--num_matches_mismatches',
                        type=int,
                        required=True,
                        help='Number of matches/mismatches per fold.')
    return parser.parse_args()

if __name__ == '__main__':
    _cli()