text:
import os
import glob
import numpy as np
from PIL import Image
import argparse

# split list
def split(data_list, ratio):
    data_len = len(data_list)
    max_idx = int(data_len * ratio)

    random = np.random.permutation(data_len)

    train_list = []
    test_list = []

    for i in range(data_len):
        sampled = random[i]
        if i < max_idx:
            train_list.append(data_list[sampled])
        else:
            test_list.append(data_list[sampled])

    return train_list, test_list

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--number', type=int, help='cross validation set')
    args = parser.parse_args()

    number = args.number

    data_dir = '../dataset/'
    train_dir = f'../dataset/train_{number}/'
    test_dir = f'../dataset/test_{number}/'
    os.makedirs(train_dir, exist_ok = True)
    os.makedirs(test_dir, exist_ok = True)

    ratio = 0.7
    ##################################################
    # good data
    output_dir = '../dataset/'
    good_list = glob.glob('../dataset/good/*.png')
    good_list += glob.glob('../dataset/good/*.jpg')

    os.makedirs(f'../dataset/train_{number}/good/',exist_ok = True)
    os.makedirs(f'../dataset/test_{number}/good/', exist_ok = True)

    train_list, test_list = split(good_list, ratio)
    for i in range(len(train_list)):
        img = Image.open(train_list[i]).convert('RGB')
        img.save(f'../dataset/train_{number}/good/{i}.png')
    for i in range(len(test_list)):
        img = Image.open(test_list[i]).convert('RGB')
        img.save(f'../dataset/test_{number}/good/{i}.png')

    ##################################################
    #bad plank
    bad_plank = glob.glob('../dataset/bad/bad_plank/*.png')
    bad_plank += glob.glob('../dataset/bad/bad_plank/*.jpg')

    train_list, test_list = split(bad_plank, ratio)
    os.makedirs(f'../dataset/train_{number}/bad/bad_plank/', exist_ok=True)
    os.makedirs(f'../dataset/test_{number}/bad/bad_plank/', exist_ok=True)
    for i in range(len(train_list)):
        img = Image.open(train_list[i]).convert('RGB')
        img.save(f'../dataset/train_{number}/bad/bad_plank/{i}.png')
    for i in range(len(test_list)):
        img = Image.open(test_list[i]).convert('RGB')
        img.save(f'../dataset/test_{number}/bad/bad_plank/{i}.png')
    ##################################################
    #bad non plank
    bad_not_plank = glob.glob('../dataset/bad/bad_not_plank/*.png')
    bad_not_plank += glob.glob('../dataset/bad/bad_not_plank/*.jpg')
    bad_not_plank += glob.glob('../dataset/additional/*.jpg')
    os.makedirs(f'../dataset/train_{number}/bad/bad_not_plank/', exist_ok=True)
    os.makedirs(f'../dataset/test_{number}/bad/bad_not_plank/', exist_ok=True)

    train_list, test_list = split(bad_not_plank, ratio)
    for i in range(len(train_list)):
        img = Image.open(train_list[i]).convert('RGB')
        img.save(f'../dataset/train_{number}/bad/bad_not_plank/{i}.png')
    for i in range(len(test_list)):
        img = Image.open(test_list[i]).convert('RGB')
        img.save(f'../dataset/test_{number}/bad/bad_not_plank/{i}.png')


text:
import numpy as np

def get_insertions_and_reads(coordinates, tn_coordinates, readnumb_array):
    """This function computes the total number of transposons per gene , the number of reads per gene and
    the distribution  of transposons along the gene.

    Parameters
    ----------
    coordinates : dict
        This is the output of the function add_chromosome_length(coordinates, chr_lengths_cumsum, ref_tid_roman)
    tn_coordinates : numpy.array
        This is the output of the function add_chromosome_length_inserts(coordinates, ref_names, chr_lengths)
    readnumb_array : numpy.array
        This is the 1st output of the function get_reads(bam)

    Returns
    -------
     dict
        A dict which every key corresponds with each gene and each value with the total number of transposons found in that gene

     dict
        A dict which every key corresponds with each gene and each value with the total number of reads for all the transposons found in that gene

     dict
        A dict which every key corresponds with each gene and each value with a  list of 4 elements:
            - the chromosome number
            - gene start position
            - gene end position
            - distribution of reads per transposon found inside the gene

    """

    tn_per_gene = {}
    reads_per_gene = {}
    tn_coordinates_per_gene = {}

    for gene in coordinates:
        xx = np.where(np.logical_and(tn_coordinates[:,1] >= coordinates.get(gene)[1], tn_coordinates[:,1] <= coordinates.get(gene)[2])) #get all insertions within range of current gene
        tn_per_gene[gene] = np.size(xx)
        reads_per_gene[gene] = sum(readnumb_array[xx]) - max(readnumb_array[xx], default=0) #REMOVE LARGEST VALUE TO REDUCE NOISE

        if np.size(xx) > 0:
            tn_coordinates_per_gene[gene] = [coordinates.get(gene)[0], coordinates.get(gene)[1], coordinates.get(gene)[2], list(tn_coordinates[xx[0][0]:xx[0][-1]+1, 1]), list(readnumb_array[xx])]
        else:
            tn_coordinates_per_gene[gene] = [coordinates.get(gene)[0], coordinates.get(gene)[1], coordinates.get(gene)[2], [], []]

    return tn_per_gene, reads_per_gene, tn_coordinates_per_gene

ext: py

max_stars_repo_name: EKingma/Transposonmapper

max_issues_repo_name: EKingma/Transposonmapper

max_forks_repo_name: EKingma/Transposonmapper

reason: import numpy


text:
import numpy as np
import dgl
import torch
from torch.utils.data import IterableDataset, DataLoader

def compact_and_copy(frontier, seeds):
    block = dgl.to_block(frontier, seeds)
    for col, data in frontier.edata.items():
        if col == dgl.EID:
            continue
        block.edata[col] = data[block.edata[dgl.EID]]
    return block

class ItemToItemBatchSampler(IterableDataset):
    def __init__(self, g, user_type, item_type, batch_size):
        self.g = g
        self.user_type = user_type
        self.item_type = item_type
        self.user_to_item_etype = list(g.metagraph()[user_type][item_type])[0]
        self.item_to_user_etype = list(g.metagraph()[item_type][user_type])[0]
        self.batch_size = batch_size

    def __iter__(self):
        while True:
            heads = torch.randint(0, self.g.number_of_nodes(self.item_type), (self.batch_size,))
            tails = dgl.sampling.random_walk(
                self.g,
                heads,
                metapath=[self.item_to_user_etype, self.user_to_item_etype])[0][:, 2]
            neg_tails = torch.randint(0, self.g.number_of_nodes(self.item_type), (self.batch_size,))

            mask = (tails != -1)
            yield heads[mask], tails[mask], neg_tails[mask]

class NeighborSampler(object):
    def __init__(self, g, user_type, item_type, random_walk_length, random_walk_restart_prob,
                 num_random_walks, num_neighbors, num_layers):
        self.g = g
        self.user_type = user_type
        self.item_type = item_type
        self.user_to_item_etype = list(g.metagraph()[user_type][item_type])[0]
        self.item_to_user_etype = list(g.metagraph()[item_type][user_type])[0]
        self.samplers = [
            dgl.sampling.PinSAGESampler(g, item_type, user_type, random_walk_length,
                random_walk_restart_prob, num_random_walks, num_neighbors)
            for _ in range(num_layers)]

    def sample_blocks(self, seeds, heads=None, tails=None, neg_tails=None):
        blocks = []
        for sampler in self.samplers:
            frontier = sampler(seeds)
            if heads is not None:
                eids = frontier.edge_ids(torch.cat([heads, heads]), torch.cat([tails, neg_tails]), return_uv=True)[2]
                if len(eids) > 0:
                    old_frontier = frontier
                    frontier = dgl.remove_edges(old_frontier, eids)
                    #print(old_frontier)
                    #print(frontier)
                    #print(frontier.edata['weights'])
                    #frontier.edata['weights'] = old_frontier.edata['weights'][frontier.edata[dgl.EID]]
            block = compact_and_copy(frontier, seeds)
            seeds = block.srcdata[dgl.NID]
            blocks.insert(0, block)
        return blocks

    def sample_from_item_pairs(self, heads, tails, neg_tails):
        # Create a graph with positive connections only and another graph with negative
        # connections only.
        pos_graph = dgl.graph(
            (heads, tails),
            num_nodes=self.g.number_of_nodes(self.item_type),
            ntype=self.item_type)
        neg_graph = dgl.graph(
            (heads, neg_tails),
            num_nodes=self.g.number_of_nodes(self.item_type),
            ntype=self.item_type)
        pos_graph, neg_graph = dgl.compact_graphs([pos_graph, neg_graph])
        seeds = pos_graph.ndata[dgl.NID]

        blocks = self.sample_blocks(seeds, heads, tails, neg_tails)
        return pos_graph, neg_graph, blocks

def assign_simple_node_features(ndata, g, ntype, assign_id=False):
    """
    Copies data to the given block from the corresponding nodes in the original graph.
    """
    for col in g.nodes[ntype].data.keys():
        if not assign_id and col == dgl.NID:
            continue
        induced_nodes = ndata[dgl.NID]
        ndata[col] = g.nodes[ntype].data[col][induced_nodes]

def assign_textual_node_features(ndata, textset, ntype):
    """
    Assigns numericalized tokens from a torchtext dataset to given block.

    The numericalized tokens would be stored in the block as node features
    with the same name as ``field_name``.

    The length would be stored as another node feature with name
    ``field_name + '__len'``.

    block : DGLHeteroGraph
        First element of the compacted blocks, with "dgl.NID" as the
        corresponding node ID in the original graph, hence the index to the
        text dataset.

        The numericalized tokens (and lengths if available) would be stored
        onto the blocks as new node features.
    textset : torchtext.data.Dataset
        A torchtext dataset whose number of examples is the same as that
        of nodes in the original graph.
    """
    node_ids = ndata[dgl.NID].numpy()

    for field_name, field in textset.fields.items():
        examples = [getattr(textset[i], field_name) for i in node_ids]

        tokens, lengths = field.process(examples)

        if not field.batch_first:
            tokens = tokens.t()

        ndata[field_name] = tokens
        ndata[field_name + '__len'] = lengths

def assign_features_to_blocks(blocks, g, textset, ntype):
    # For the first block (which is closest to the input), copy the features from
    # the original graph as well as the texts.
    assign_simple_node_features(blocks[0].srcdata, g, ntype)
    assign_textual_node_features(blocks[0].srcdata, textset, ntype)
    assign_simple_node_features(blocks[-1].dstdata, g, ntype)
    assign_textual_node_features(blocks[-1].dstdata, textset, ntype)

class PinSAGECollator(object):
    def __init__(self, sampler, g, ntype, textset):
        self.sampler = sampler
        self.ntype = ntype
        self.g = g
        self.textset = textset

    def collate_train(self, batches):
        heads, tails, neg_tails = batches[0]
        # Construct multilayer neighborhood via PinSAGE...
        pos_graph, neg_graph, blocks = self.sampler.sample_from_item_pairs(heads, tails, neg_tails)
        assign_features_to_blocks(blocks, self.g, self.textset, self.ntype)

        return pos_graph, neg_graph, blocks

    def collate_test(self, samples):
        batch = torch.LongTensor(samples)
        blocks = self.sampler.sample_blocks(batch)
        assign_features_to_blocks(blocks, self.g, self.textset, self.ntype)
        return blocks

max_stars_repo_name: gvvynplaine/dgl

max_issues_repo_name: gvvynplaine/dgl

max_forks_repo_name: gvvynplaine/dgl


text:
import os
import numpy as np
import cv2

path = os.path.dirname(os.path.realpath('__file__'))

def leerImagenDeRequest(infile=None):
    datos = []

    img = leerImagen(infile)
    #debug
    #filename = "{}.png".format(os.getpid())
    #cv2.imwrite(filename, img)
    img_pre = preProcesarImagen(img)

    for n, xr, yr, wr, hr in getRegiones():
        region = img_pre[yr:yr + hr, xr:xr + wr]
        datos.append(getDato(region))

    return datos

def leerImagen(infile):
    if infile is None:
        with open(getAbsPath('test/test0002.png'), 'rb') as infile2:
            file = infile2.read()
    else:
        file = infile.read()
        infile.close()

    file = np.fromstring(file, dtype='uint8')
    return cv2.imdecode(file, cv2.IMREAD_UNCHANGED)

def getAbsPath(dir):
    return os.path.join(path, dir)

def getRegiones():
    regiones = []
    x1 = 281 * 2
    h1 = 50 * 2
    h2 = 44 * 2
    regiones.append(['dni', 120, 300, 300, 66])
    regiones.append(['prenombres', x1, 252 * 2, 184 * 2, h1])
    regiones.append(['primer apellido', x1, 121 * 2, 141 * 2, h1])
    regiones.append(['segundo apellido', x1, 187 * 2, 310, h1])
    regiones.append(['sexo', x1, 322 * 2, 144 * 2, h2])
    regiones.append(['estado civil', 469 * 2, 322 * 2, 141 * 2, h2])
    regiones.append(['fecha de nacimiento', x1, 368 * 2, 141 * 2, h2])

    return regiones

def preProcesarImagen(img):
    img_std = cv2.resize(img, (1858, 1180))
    gray = cv2.cvtColor(img_std, cv2.COLOR_BGR2GRAY)
    blur = cv2.medianBlur(gray, 3)
    return blur

def getModelos():
    modelos = []
    modelos.append(['R', (255, 255, 0), cv2.CascadeClassifier(getAbsPath('model/ml_R/cascade.xml'))])
    modelos.append(['A', (255, 0, 255), cv2.CascadeClassifier(getAbsPath('model/ml_A/cascade.xml'))])
    modelos.append(['C', (0, 255, 255), cv2.CascadeClassifier(getAbsPath('model/ml_C/cascade.xml'))])
    modelos.append(['E', (255, 100, 100), cv2.CascadeClassifier(getAbsPath('model/ml_E/cascade.xml'))])
    return modelos

def getDato(region):
    dato = ''
    letras = []
    #regionx = region
    for letter, color, model in getModelos():
        for (x, y, w, h) in model.detectMultiScale(region):
            # print letter+'-'+str(x)+','+str(y)+'-'+str(w)+','+str(h)
            if (w > 25 and h > 25 and y > 22):
                #print letter+'-'+str(x)+','+str(y)+'-'+str(w)+','+str(h)
                #cv2.rectangle(regionx, (x, y), (x + w, y + h), color, 2)
                letras.append((letter,x))
    #cv2.imshow('img', regionx)
    #cv2.waitKey(0)
    #cv2.destroyAllWindows()

    letras = sorted(letras, key=lambda letra: letra[1])
    for letra in letras:
        dato+=letra[0]

    return dato

#per = leerImagenDeRequest(None)
#print per