text:
# -*- coding: utf-8 -*-

"""Helper functions when working with networks in CoXPath"""

import os
from collections import Counter
from itertools import chain, product
from typing import Iterable, List, Dict, Union, Tuple, Set

import matplotlib.pyplot as plt
import networkx as nx
import pandas as pd
import pyobo
from matplotlib_venn import venn2

homedir = os.path.expanduser("~")
network_cache_dir = os.path.join(homedir, ".coexp_nets")
os.makedirs(network_cache_dir, exist_ok=True)

doid_mappings = pyobo.get_id_name_mapping('doid')
hgnc_mappings = pyobo.get_id_name_mapping('hgnc')
hgnc_id_to_ncbigene_id = pyobo.get_filtered_xrefs('hgnc', 'ncbigene')
entrez_mappings = {
    ncbigene_id: hgnc_mappings[hgnc_id]
    for hgnc_id, ncbigene_id in hgnc_id_to_ncbigene_id.items()
}

class InputError(Exception):
    """Exception raised for errors in the input.

    Attributes
    ----------
    message : str
        explanation of the error
    """

    def __init__(self, message):
        self.message = message

def edge_file_path(data_dir: str, doid: str) -> str:
    """Return the path for the edge list file for a given doid

    Parameters
    ----------
    data_dir : str
        path to the directory which contains co-expression network edges.
    doid : str
        disease DOID

    Returns
    -------
        path to edge file
    """
    return os.path.join(data_dir, doid, "coexp_network_edges.tsv")

def create_network_from_edge_file(file: str, doid: str) -> nx.Graph:
    """Extract the necessary columns to create a weighted undirected graph

    Parameters
    ----------
    file : str
        path to edge list life
    doid : str
        disease DOID

    Returns
    -------
        network as nx.Graph object
    """
    try:  # see if it's already cached
        G = nx.read_gpickle(os.path.join(network_cache_dir, f"{doid}.gpickle"))
    except FileNotFoundError:  # if not, create and cache the graph for next time
        edge_df = pd.read_csv(file, sep="\t", usecols=["from", "to", "weight"])
        G = nx.from_pandas_edgelist(edge_df, source="from", target="to", edge_attr="weight")
        nx.write_gpickle(G, os.path.join(network_cache_dir, f"{doid}.gpickle"))
    return G

def counter(nets: Iterable, comparison: str) -> Counter:
    """Create a counter objects of nodes/edges/genes from a collection of Networks.

    Should not be called by user directly- only by other methods.
    """
    if comparison == "nodes":
        comp = (net.nodes() for net in nets)
    elif comparison == "edges":
        comp = (net.edges() for net in nets)
    elif comparison == "genes":
        comp = nets
    else:
        raise InputError("Can only compare 'nodes' , 'edges', or 'genes'")
    counter_obj = Counter(chain.from_iterable(comp))
    return counter_obj

def most_common(nets: Iterable, comparison: str) -> List[Tuple[Union[str, tuple], Tuple[str, str]]]:
    """Find most common nodes or edges in a collection of networks

    Paramaters
    ----------
    nets : Iterable
        a collection containing the Networkx objects to analyze or of lists of gene names
    comparison : str
        type of object being compared. only accepts 'nodes', 'edges', or 'genes'

    Returns
    -------
    List[Tuple[Union[str,tuple],Tuple[str,str]]]
        list of (comparison_object, (frequency_count, percent)) tuples in order of most common
        comparison_object being nodes(str), edges(tuple of str), or genes(str)
        note - will contain all nodes/edges/genes in this list, check the counts for the info you need
    """
    counter_obj = counter(nets, comparison)
    return [(obj, (f"{count}/{len(nets)}", f"{count / len(nets):.2%}")) for obj, count in counter_obj.most_common()]

def least_common(nets: Iterable, comparison: str) -> List[Tuple[Union[str, tuple], Tuple[str, str]]]:
    """Find least common nodes or edges in a collection of networks

    Paramaters
    ----------
    nets : Iterable
        a collection containing the Networkx objects to analyze or of lists of gene names
    comparison : str
        comparison type. only accepts 'nodes', 'edges', or 'genes'

    Returns
    -------
    List[Tuple[Union[str,tuple],Tuple[str,str]]]
        list of (comparison_object, (frequency_count, percent)) tuples in order of least common
        comparison_object being nodes(str), edges(tuple of str), or genes(str)
        note - will contain all nodes/edges/genes in this list, check the counts for the info you need

    """
    counter_obj = counter(nets, comparison)
    return [(obj, (f"{count}/{len(nets)}", f"{count / len(nets):.2%}")) for obj, count in
            counter_obj.most_common()[::-1]]

def count_in_freq(freq_tuple: Tuple[str, str]) -> int:
    """Return the count from the statistic half of the tuple returned from most_common or least_common

    Parameters
    ----------
    freq_tuple : Tuple[str, str]
        tuple containing stats given as (count/total, percent) from most_common or least_common

    Returns
    -------
        the count from the statistic as an int

    """
    return int(freq_tuple[0].split('/')[0])

def above_cutoff(gene_freq_tup_list: List[Tuple[Union[str, tuple], Tuple[str, str]]], cutoff: int) -> List[str]:
    """Return the genes/edges that are are in at least the given cutoff's networks

    Parameters
    ----------
    gene_freq_tup_list : List[Tuple[Union[str, tuple], Tuple[str, str]]]
        list of (comparison_object, (frequency_count, percent)) tuples in order of most common
        should be return from most_common()
    cutoff : int
        number to be used as minimum for how many networks the object must be present in to be returned

    Returns
    -------
        list of objects that were in at least as many networks as the cutoff given
    """
    above = []
    for gene, freq in gene_freq_tup_list:
        if count_in_freq(freq) >= cutoff:
            above.append(gene)
        else:
            break  # since it's ordered, no need wasting time checking the rest
    return above

def below_cutoff(gene_freq_tup_list: List[Tuple[Union[str, tuple], Tuple[str, str]]], cutoff: int) -> List[str]:
    """Return the genes/edges that are are in less than the given cutoff's networks

    Parameters
    ----------
    gene_freq_tup_list : List[Tuple[Union[str, tuple], Tuple[str, str]]]
        list of (comparison_object, (frequency_count, percent)) tuples in order of least common
        should be return from least_common()
    cutoff : int
        number to be used as maximum for how many networks the object must be present in to be returned

    Returns
    -------
        list of objects that were in at most as many networks as the cutoff given
    """
    below = []
    for gene, freq in gene_freq_tup_list:
        if count_in_freq(freq) <= cutoff:
            below.append(gene)
        else:
            break  # since it's ordered, no need wasting time checking the rest
    return below

def identify_hub_genes(net: nx.Graph) -> List[Tuple[str, int]]:
    """Identify the nodes with the highest connections

    Parameters
    ----------
    net : nx.Graph
        network object

    Returns
    -------
        list of all nodes and their degrees, ordered by highest number of connections first

    """
    degree_sequence = sorted(net.degree, key=lambda x: x[1], reverse=True)
    return degree_sequence

def load_interactome(path_to_interactome_file: str) -> nx.DiGraph:
    """Load the interactome located at the given file location

    Parameters
    ----------
    path_to_interactome_file: str
        path to the location of the interactome file

    Returns
    -------
        interactome given as nextworkx directed graph object

    """
    interactome_df = pd.read_csv(path_to_interactome_file, sep="\t")
    G = nx.DiGraph()
    for index, row in interactome_df.iterrows():
        G.add_edge(hgnc_mappings[str(row.source)], hgnc_mappings[str(row.target)])
    return G

def get_genes_in_overlap(genes, overlap_list) -> Set[str]:
    """Select members from a gene list which are also in a given overlap(second list of genes)

        Parameters
        ----------
        genes : list
            ordered list of genes from which to select from
        overlap_list : list
            list of genes from which selected genes must be in
        cutoff

        Returns
        -------
            list of genes that are also present in the overlap list
        """
    top_genes = set()
    for gene in genes:
        if gene in overlap_list:
            top_genes.add(gene)
    return top_genes

def get_top_x_genes_in_overlap(genes: list, overlap_list: list, topx: int) -> Set[str]:
    """Select a certain number of genes from an ordered list which are also in a given overlap(second list of genes)

    Parameters
    ----------
    genes : list
        list of (genes, stats) from which to take the top x genes from
    overlap_list : list
        list of genes
    topx : int
        number of desired genes in resulting list

    Returns
    -------
        list of genes of length (topx) that are also present in the overlap list
    """
    index = 0
    top_genes = set()
    while len(top_genes) < topx:
        gene, _ = genes[index]
        if gene in overlap_list:
            top_genes.add(gene)
        index += 1
    return top_genes

def venn_d(left: set, right: set, left_name: str, right_name: str, title: str = None,
           output_filename: str = None) -> venn2:
    """Analyze node overlap in two NetworkX objects

    Parameters
    ----------
    left : set
        set of objects for left side of venn diagram
    right : set
        set of objects for right side of venn diagram
    left_name : str
        name of collection on left side of venn diagram
    right_name : str
        name of collection on right side of venn diagram
    title : str
        title of venn diagram, optional
    output_filename : str
        path to desired location of saved figure, optional

    Returns
    -------
        venn diagram object

    """
    v = venn2([left, right], (left_name, right_name))
    v.get_patch_by_id('10').set_color('#5cb85c')
    v.get_patch_by_id('01').set_color('#df3f18')
    v.get_patch_by_id('10').set_edgecolor('none')
    v.get_patch_by_id('01').set_edgecolor('none')
    v.get_patch_by_id('10').set_alpha(.3)
    v.get_patch_by_id('01').set_alpha(.3)
    if v.get_patch_by_id('11'):
        v.get_patch_by_id('11').set_color('#f3ac1f')
        v.get_patch_by_id('11').set_edgecolor('none')
        v.get_patch_by_id('11').set_alpha(.3)
    for text in v.set_labels:
        text.set_fontsize("small")
    for x in range(len(v.subset_labels)):
        if v.subset_labels[x] is not None:
            v.subset_labels[x].set_fontsize("small")
    if title:
        plt.title(title)
    if output_filename:
        plt.savefig(output_filename)
    return v

def node_intersection_from_doid_list(doid_list: List[str], network_d: Dict[str, nx.Graph]) -> Set[str]:
    """Find node intersection in the networks of the given doid list

    Parameters
    ----------
    doid_list: List[str]
        list of doids
    network_d: Dict[str, nx.Graph]
        dictionary mapping doids to their network

    Returns
    -------
        set of nodes
    """
    nodes = [set(network_d[doid].nodes()) for doid in doid_list]
    return nodes[0].intersection(*nodes[1:])

def remove_stats(obj_stat_list: List[Tuple[Union[str, tuple], Tuple[str, str]]]) -> List[Tuple[Union[str, tuple]]]:
    """Given the return from most_common or least_common, remove the stats and just return a list of the nodes/edges

    Parameters
    ----------
    obj_stat_list:
        list of (comparison_object, (frequency_count, percent)) tuples in which you only want the comparison objects

    Returns
    -------
        list of objects (can either be edges or nodes)
    """
    return [obj for obj, stat in obj_stat_list]

def percent_edges_in_other_edges(edge_list: List[Tuple[str, str]], edge_list2: List[Tuple[str, str]]) -> float:
    """Calculate what proportion of the first list of edges can be found in the second list.

    checks edges in both node orders to account for directed edges

    Parameters
    ----------
    edge_list:  List[Tuple[str, str]]
        list of query edges
    edge_list2:  List[Tuple[str, str]]
        list of edges to search in

    Returns
    -------
        proportion of first list that is in second list, accounting for order differences
    """
    count = 0
    for edge in edge_list:
        if (edge in edge_list2) or (edge[::-1] in edge_list2):
            count += 1
    return count / len(edge_list)

def top_edges_subgraph(num_edges: int, sorted_edges: List[Tuple[str, str]]) -> nx.Graph:
    """Given an ordered list(by weight) of edges from a network, create a subgraph containing only a given number of edges

    Parameters
    ----------
    num_edges : int
        desired number of edges to be in resulting subgraph
    sorted_edges: List[Tuple[str, str]]
        list of sorted edges by weight

    Returns
    -------
        subgraph networkx object

    """
    subG = nx.Graph()
    subG.add_edges_from(sorted_edges[:num_edges])
    return subG

def pathway_similarity(gene_list_in_pathway: List[str], network: nx.Graph()) -> float:
    """Calculate pathway similarity.

    Given a list of genes in a pathway, looks for edges between these genes in the given network

    Parameters
    ----------
    gene_list_in_pathway: List[str]
        list of genes in a pathway
    network: nx.Graph
        network to search for pathway similarity

    Returns
    -------
        similarity value
    """
    edges_found = 0
    i = 0
    for gene1, gene2 in product(gene_list_in_pathway, repeat=2):
        i += 1
        if network.has_edge(gene1, gene2):
            edges_found += 1
    return edges_found / i

def pathway_similarity_via_interactome(gene_list_in_pathway: List[str], network: nx.Graph(),
                                       interactome: nx.DiGraph()) -> float:
    """Calculate pathway similarity.

    Given a list of genes in a pathway, looks for edges between these genes that are also in the interactome in the given network

    Parameters
    ----------
    gene_list_in_pathway: List[str]
        list of genes in a pathway
    network: nx.Graph
        network to search for pathway similarity
    interactome: nx.DiGraph

    Returns
    -------
        similarity value
    """
    interactome_subgraph_with_pathway = interactome.subgraph(
        [node for node in gene_list_in_pathway if node in interactome.nodes()])
    edges_found = 0
    if not interactome_subgraph_with_pathway.edges():
        return 0
    for n1, n2 in interactome_subgraph_with_pathway.edges():
        if network.has_edge(n1, n2):
            edges_found += 1
    return edges_found / len(interactome_subgraph_with_pathway.edges())

def largest_subgraph(G1: nx.Graph, G2: nx.Graph) -> nx.Graph:
    """Return largest connected subgraph that is overlapping fully between two graphs

    Parameters
    ----------
    G1 : nx.Graph
        first network
    G2 : nx.Graph
        second network

    Returns
    -------
        largest common subgraph
    """
    intersectionG = nx.intersection(G1.subgraph(G2.nodes), G2.subgraph(G1.nodes))  # still contains many "empty" nodes
    subG = nx.Graph()
    subG.add_edges_from(intersectionG.edges)

    # not necessarily connected yet
    if subG:
        max_comp = max(nx.connected_components(subG), key=len)
        subG = subG.subgraph(max_comp)
    return subG

def largest_subgraph_from_list(nets: List[nx.Graph]) -> nx.Graph:
    """Return largest connected subgraph that is overlapping fully between many graphs

    Parameters
    ----------
    nets : List[nx.Graph]
        list of networks

    Returns
    -------
        largest common subgraph
    """
    for i, net in enumerate(nets):
        if i == 0:
            subg = net
            continue
        subg = largest_subgraph(subg, net)
    return subg

def which_nets_have(item: Union[str, tuple], item_type: str, network_dict: Dict[str, nx.Graph]) -> List[str]:
    """Identify which network contains a given node/edge

    Parameters
    ----------
    item: Union[str,tuple]
        node(str) or edge(tuple) that you want to check which networks have
    item_type : str
        whether to search for a node or edge. only accepts 'node' or 'edge'
    network_dict

    Returns
    -------
        list of doids of the networks that have the query

    """
    if item_type == "node":
        search = {doid: net.nodes() for doid, net in network_dict.items()}
    elif item_type == "edge":
        search = {doid: net.edges() for doid, net in network_dict.items()}
    else:
        raise InputError("Can only find 'node' and 'edge'")
    doids_that_have_item = []
    for doid, group in search.items():
        if item in group:
            doids_that_have_item.append(doid)
    return doids_that_have_item

def ensp_to_hgnc_mappings(URL = "https://stringdb-static.org/download/protein.info.v11.0/9606.protein.info.v11.0.txt.gz"):
    """Map Ensembl protein ids to HGNC symbols

    Parameters
    ----------
    URL: str
        location of the chosen version's STRING protein info file. taken from: https://string-db.org/cgi/download?sessionId=%24input-%3E%7BsessionId%7D&species_text=Homo+sapiens
        version 11 human protein info is given as the default, optional

    Returns
    -------
        dictionary mapping ensembl ids to hgnc gene symbols

    """
    ensembl_hgnc_mappings_df = pd.read_table(URL, compression ="gzip")

    ensp_gene_mappings = {
        row["protein_external_id"] : row["preferred_name"]
        for i, row in ensembl_hgnc_mappings_df.iterrows()
    }
    return ensp_gene_mappings

def load_STRING(URL = "https://stringdb-static.org/download/protein.physical.links.v11.0/9606.protein.physical.links.v11.0.txt.gz"):
    """Load the STRING PPI network located at the given URL

    Parameters
    ----------
    URL: str
        location of the chosen version's STRING PPI physical links file. taken from: https://string-db.org/cgi/download?sessionId=%24input-%3E%7BsessionId%7D&species_text=Homo+sapiens
        version 11 human links are given as the default, optional

    Returns
    -------
        PPI network given as nextworkx directed graph object

    """
    STRING_edges = pd.read_table(URL, compression ="gzip", sep=" ")
    STRING = nx.Graph()
    gene_mappings = ensp_to_hgnc_mappings()
    for i, row in STRING_edges.iterrows():
        source = gene_mappings[row["protein1"]]
        target = gene_mappings[row["protein2"]]
        STRING.add_edge(source, target, weight = row["combined_score"])
    return STRING

def load_HIPPIE(URL = "http://cbdm-01.zdv.uni-mainz.de/~mschaefer/hippie/hippie_current.txt"):
    """Load the HIPPIE PPI network located at the given URL

    Parameters
    ----------
    URL: str
        location of the chosen version's HIPPIE's PPI tab format file. taken from: http://cbdm-01.zdv.uni-mainz.de/~mschaefer/hippie/download.php
        the link automatically uses the current version (2.2  at the time this was conducted) as the default, optional

    Returns
    -------
        PPI network given as nextworkx directed graph object

    """
    HIPPIE_interactions = pd.read_table(URL, header=None)
    HIPPIE= nx.Graph()
    skipped = 0
    for i, row in HIPPIE_interactions.iterrows():
        try:
            source = entrez_mappings[str(row[1])]
            target = entrez_mappings[str(row[3])]
        except KeyError:
            skipped += 1
            continue
        HIPPIE.add_edge(source, target, weight = row[4])
    print(f"{skipped} edges skipped due to discontinued gene ids")
    return HIPPIE

max_stars_repo_name: CoXPath/CoXPath

max_issues_repo_name: YojanaGadiya/CoXPath

max_forks_repo_name: YojanaGadiya/CoXPath

reason: import networkx


text:
import numpy as np
import torch
import torchvision
from torch.utils.data import Dataset, TensorDataset
import torchvision.datasets as datasets
from torchvision import transforms
from sklearn.metrics.pairwise import distance_metrics
from typing import Any, Callable, Optional, Tuple
import PIL

class ConvexHullDataset(Dataset):
    """Takes in a map-style torch.utils.data.Dataset and generates a new map-style Dataset that
    consists of samples from the convex hull of points within a certain distance of each other. This
    class can either store the samples, or call the source Dataset each time it is called. """

    def __init__(self,
                 sourcedataset: Dataset,
                 alpha: Optional[float] = .5,
                 length: Optional[str] = "auto",
                 batchsize: Optional[int] = 5000,
                 store: Optional[bool] = True,
                 seed: Optional[int] = 0,
                 averageneighbors: Optional[int] = 2,
                 maxneighbors: Optional[int] = 100,
                 distmetric: Optional[str] = "l2",
                 transform: Optional[Callable] = None,
                 limitdata: Optional[int] = None):
        """
        Args:
            sourcedataset : an iterable Dataset
            alpha (float): concentration parameter for the Dirichlet distribution
            length (integer or "auto") : length of this Dataset; auto will create averageneighbors*(length of input dataset) size Dataset
            batchsize (integer): number of points per batch to use in computing the interpoint distances
            store (binary) : whether to store the data or not
            seed (integer): seed used to generate this Dataset
            averageneighbors (integer): number of neighbors to average over for each sample
            maxneighbors (integer): maximum number of nearest neighbors to store for each point
            distmetric (function): pairwise distance function used to select nearby points, must be a key in sklearn.metrics.pairwise.distance_metrics()
            transform (function): transform for image data sets
            limitdata (integer): only works with this initial poriton of the source dataset
        """

        self.sourcedataset = sourcedataset

        if limitdata is None:
            self.sourcelen = len(self.sourcedataset)
        else:
            self.sourcelen = limitdata

        self.alpha = alpha
        if length=="auto":
            self.len = averageneighbors*self.sourcelen
        else:
            self.len = length

        self.batchsize=batchsize
        self.store = store
        self.seed = seed
        self.averageneighbors = averageneighbors
        self.maxneighbors = maxneighbors
        self.distmetric=distance_metrics()[distmetric]
        self.transform=transform

        self.testpoint = sourcedataset[0][0]
        self.testtarget = sourcedataset[0][1]

        self.computedistances()
        self.generator = torch.manual_seed(self.seed)
        self.computesamples(self.store)

        # copied from the VisionDataset class to handle transforms
        has_separate_transform = transform is not None

    def __len__(self):
        return self.len

    def __getitem__(self, idx):
        if self.store:
            obj = self.mydata[idx]
        else:
            obj = self.genitem(idx)

        if self.transform is None:
            return obj
        else:
            return (self.transform(obj[0]), obj[1], obj[2])

    def save(self, fname):
        del self.generator
        with open(fname, 'wb') as fout:
            torch.save(self, fout)

    def setaverageneighbors(self, num):
        self.averageneighbors = num

    def getpoint(self, data):
        """converts Tensors or PIL Images to numpy arrays"""
        if type(data) == PIL.Image.Image:
            return transforms.PILToTensor()(data).numpy().astype(np.float32)
        else:
            return data.numpy()

    def givepoint(self,x):
        """converts numpy array back to a PIL Image or a Tensor"""
        if type(self.testpoint) == PIL.Image.Image:
            return transforms.ToPILImage()(torch.Tensor(x))
        else:
            return torch.Tensor(x)

    def genitem(self, idx):
        x = 0*self.getpoint(self.testpoint)
        y = torch.zeros(self.averageneighbors,1, dtype=torch.long)
        weights = torch.tensor(self.mixcoeffs[idx])
        for i in np.arange(self.averageneighbors):
            (sx, sy) = self.sourcedataset[self.mixindices[idx, i]]
            sx = self.getpoint(sx)
            x += self.mixcoeffs[idx, i]*sx
            y[i] = sy
        return (self.givepoint(x), y, weights)

    def getcomponents(self, idx):
        components = []
        for i in np.arange(self.averageneighbors):
            components.append(self.sourcedataset[self.mixindices[idx, i]])
        return components

    def computedistances(self):
        """Computes arrays:
                self.distances of size (sourcelen, maxneighbors) whose ith row contains the distance of the ith datapoint in the source to its nearest neighbors
                self.indices of size (sourcelen, maxneighbors) whose entries are the corresponding indices of the neighbors
        """
        print("Datatype of feature vectors: %s" % (str(type(self.testpoint))))
        testdatapoint = self.getpoint(self.testpoint)
        datasize = np.prod(testdatapoint.shape)
        datatype = testdatapoint.dtype

        self.distances = np.ones((self.sourcelen, self.maxneighbors))*np.Inf
        self.indices = np.zeros((self.sourcelen, self.maxneighbors), dtype=np.int)
        self.maxdistances = np.array([1.0]*self.sourcelen)*np.Inf
        self.maxindices = np.array([0]*self.sourcelen, dtype=np.int)

        def replacenearest(row, col, dist):
            maxindex = self.maxindices[row]
            self.indices[row, maxindex] = col
            self.distances[row, maxindex] = dist
            self.maxdistances[row] = np.amax(self.distances[row, :])
            self.maxindices[row] = np.argmax(self.distances[row, :])

        def mergedistances(neighbordistances, neighborcolindices, rowindices):
            for rowidx in np.arange(neighbordistances.shape[0]):
                row = rowindices[rowidx]
                for colidx in np.arange(neighbordistances.shape[1]):
                    curdist = neighbordistances[rowidx, colidx]
                    col = neighborcolindices[rowidx, colidx]
                    if curdist < self.maxdistances[row]:
                        replacenearest(row, col, curdist)

        def populatebatch(indices):
            rowpoints = np.empty((len(indices), datasize), dtype=datatype)

            currowbatchidx = 0
            for idx in indices:
                rowpoints[currowbatchidx] = self.getpoint(self.sourcedataset[idx][0]).flatten()
                currowbatchidx+=1
            return rowpoints

        numrowsplits = np.ceil(self.sourcelen/self.batchsize)
        rowbatch_indices = np.array_split(np.arange(self.sourcelen),numrowsplits)

        for rowbatchidx in np.arange(len(rowbatch_indices)):
            print("On row batch %i of %i" % (rowbatchidx+1, len(rowbatch_indices)), flush=True)

            rowindices = rowbatch_indices[rowbatchidx]
            rowpoints = populatebatch(rowindices)

            for colbatchidx in np.arange(len(rowbatch_indices)):

                colindices = rowbatch_indices[colbatchidx]
                colpoints = populatebatch(colindices)

                pairwisedists = self.distmetric(rowpoints, colpoints)

                sortindices = np.argsort(pairwisedists, axis=1)
                sortindices = sortindices[:, :np.amin([self.maxneighbors, colpoints.shape[0]])]
                neighborcolindices = np.empty((rowpoints.shape[0], sortindices.shape[1]), dtype=int)
                neighbordistances = np.empty((rowpoints.shape[0], sortindices.shape[1]), dtype=pairwisedists.dtype)

                for rowidx in np.arange(rowpoints.shape[0]):
                    neighborcolindices[rowidx] = colindices[sortindices[rowidx]]
                    neighbordistances[rowidx] = pairwisedists[rowidx, sortindices[rowidx]]

                mergedistances(neighbordistances, neighborcolindices, rowindices)

    def computesamples(self, store=False):
        self.sampleindices = torch.randint(low=0, high=self.sourcelen, size=(self.len,), generator=self.generator).tolist()
        rng = torch.distributions.dirichlet.Dirichlet(torch.tensor([self.alpha]*self.averageneighbors))

        self.mixcoeffs = np.zeros((self.len, self.averageneighbors))
        self.mixindices = np.zeros((self.len, self.averageneighbors), dtype=np.int)

        for idx in np.arange(self.len):
            if idx % 1000 == 0:
                print("Computing sample %i of %i" % (idx+1, self.len))
            self.mixcoeffs[idx] = rng.sample().tolist()
            #validindices = np.insert(self.indices[ np.nonzero( np.isfinite( self.distances[idx, :]))], 0, idx)
            validindices = np.insert(self.indices, 0, idx)
            self.mixindices[idx, :] = validindices[torch.randperm(len(validindices), generator=self.generator).numpy()[:self.averageneighbors]]

        if store==True:
            self.mydata = [0]*self.len
            for idx in np.arange(self.len):
                self.mydata[idx] = self.genitem(idx)

if __name__ == "__main__":
    ## Sanity check
    # To test that the nearest neighbors are being computed correctly, create an artifical dataset
    # compute the nearest neighbors, then check against the nearest neighbors computed by ConvexHullDataset

    from sklearn.neighbors import NearestNeighbors

    n = 1000
    d = 13

    bs = 51
    maxn = 23

    T = torch.Tensor(4*np.random.randn(n, d))
    Y = torch.Tensor(np.random.randn(n))

    nbrs = NearestNeighbors(n_neighbors=maxn).fit(T)
    distances, indices = nbrs.kneighbors(T)

    testDataset = TensorDataset(T, Y)
    testcvxdset = ConvexHullDataset(testDataset, batchsize=bs, maxneighbors=maxn)

    err = 0 # should remain 0
    for idx in np.arange(n):
        err = err + len(set(testcvxdset.indices[idx])-set(indices[idx]))
    print(err)

ext: py

max_stars_repo_name: gittea-rpi/neighborhoodwatch

max_issues_repo_name: gittea-rpi/neighborhoodwatch

max_forks_repo_name: gittea-rpi/neighborhoodwatch

reason: import numpy


text:
#!/usr/bin/python

import csv
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.pylab as pylab

font=13 #'x-large'
params = {'legend.fontsize': font,
         #'figure.figsize': (160, 100),
         'axes.labelsize': font,
         'axes.titlesize': font,
         'xtick.labelsize':font,
         'ytick.labelsize':font}
matplotlib.rcParams.update(params)

N = 7 # number of function
nt = 11 # number of test cases for each fucntion
n = 3 # number of test cases: no-omp, omp-thrd=2, ...
label = ['no-omp', 'omp-thrd-2', 'omp-thrd-4']
ops = ['abs', 'sin', 'erf', 'add', 'pow', 'copy', 'conv2d']
mus = [[], [], []]; stds = [[], [], []]; sizes = []
with open("openmp_cross.csv", 'rb') as csvf:
    reader = csv.reader(csvf, delimiter=',')
    for val in reader:
        sizes.append(float(val[0]))
        mus[0].append(float(val[1]))
        mus[1].append(float(val[3]))
        mus[2].append(float(val[5]))
        stds[0].append(float(val[2]))
        stds[1].append(float(val[4]))
        stds[2].append(float(val[6]))

figs = []; ax = []

fig, ax = plt.subplots(2,2)
for i in range(4):
	rects = []
	axes = ax[i / 2][i % 2]

	for j in range(n):
	    rects.append(axes.errorbar(sizes[i * nt : (i+1) * nt - 1], mus[j][i * nt : (i+1) * nt - 1],
	        yerr=stds[j][i * nt : (i + 1) * nt - 1], label=label[j]))
	axes.set_ylabel('Time(ms)')
	axes.set_xlabel('Input array size for op ' + ops[i])
	axes.set_xscale('log', nonposx='clip')
	axes.set_yscale('log')
	axes.legend()
	axes.grid(True)

fig, ax = plt.subplots(1,2)
for i in range(4,6):
	axes = ax[i - 5]
	for j in range(n):
	    rects.append(axes.errorbar(sizes[i * nt : (i+1) * nt - 1], mus[j][i * nt : (i+1) * nt - 1],
	        yerr=stds[j][i * nt : (i + 1) * nt - 1], label=label[j]))
	axes.set_ylabel('Time(ms)')
	axes.set_xlabel('Input array size for op ' + ops[i])
	axes.set_xscale('log', nonposx='clip')
	axes.set_yscale('log')
	axes.legend()
	axes.grid(True)

fig, axes = plt.subplots(1,1)
i = 6
for j in range(n):
    rects.append(axes.errorbar(sizes[i * nt : (i+1) * nt - 1], mus[j][i * nt : (i+1) * nt - 1],
        yerr=stds[j][i * nt : (i + 1) * nt - 1], label=label[j]))
axes.set_ylabel('Time(ms)')
axes.set_xlabel('Input array size for op ' + ops[i])
axes.set_xscale('log', nonposx='clip')
axes.set_yscale('log')
axes.legend()
axes.grid(True)

plt.show()

max_stars_repo_name: owlbarn/benchmark

max_issues_repo_name: owlbarn/benchmark

max_forks_repo_name: owlbarn/benchmark


text:
import logging
import os
import random
import cv2
import pickle
import numpy as np

from itertools import product

cv2.setNumThreads(0)

def load_pickled_dict(path):
    assert isinstance(path, str)
    if not os.path.exists(path):
        raise f'Path [{path}] does not exist'
    with open(path, 'rb') as f:
        return pickle.load(f)

def read_text_stat(roi, chars_patterns=None):
    '''
    Read pixel-text:
    1) split by countours
    2) match with patterns
    '''
    if not chars_patterns:
        return
    if not isinstance(chars_patterns, dict):
        return
    _, roi = cv2.threshold(roi, 127, 255, cv2.THRESH_BINARY)
    contours = cv2.findContours(roi, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[-2]
    result = ''
    for contour, (k, v) in product(reversed(contours), chars_patterns.items()):
        x, y, w, h = cv2.boundingRect(contour)
        c = get_roi(roi, x, y, h, w)
        if np.array_equal(v, c):
            result += str(k)[:1]
            continue
    return int(result or 0)

def get_roi(img, x1:int, y1:int, w:int, h:int):
    if img is None:
        return
    return img[y1:y1+h, x1:x1+w]

def get_roi3d(img, x1:int, y1:int, w:int, h:int):
    if img is None:
        return
    return img[y1:y1+h, x1:x1+w,:]

def resize_img(img, scale_factor=0.5):
    width = int(img.shape[1] * scale_factor)
    height = int(img.shape[0] * scale_factor)
    return cv2.resize(img, (width, height))


text:
# Image classifier application
# python image_classifier_project.py

#Basic usage: python train.py data_directory
#Prints out training loss, validation loss, and validation accuracy as the network trains
#Options:
# Set directory to save checkpoints: python train.py data_dir --save_dir save_directory
# Choose architecture: python train.py data_dir --arch "vgg13"
# Set hyperparameters: python train.py data_dir --learning_rate 0.01 --hidden_units 512 --epochs 20
# Use GPU for training: python train.py data_dir --gpu
# Predict flower name from an image with predict.py along with the probability of that name. That is, you'll pass in a single image /path/to/image and return the flower name and class probability.

#python train.py flowers --save_dir save --arch 'vgg13' --learning_rate 0.002 --hidden_units 512 --epochs 3 --gpu

import os
import numpy as np
import torch
import torch.nn.functional as F
import torchvision
import json
import time
import argparse

from torchvision import datasets, transforms, models
from collections import OrderedDict
from torch import nn
from torch import optim
from torch.autograd import Variable

DEFAULT_CHECKPOINT_FILENAME = 'checkpoint.pth'
DEGREES_ROTATION = 30
SIZE_CROP = 224
SIZE_RESIZE = 256
NORMALIZE_MEAN = [0.485, 0.456, 0.406]
NORMALIZE_STD = [0.229, 0.224, 0.225]
BATCH_SIZE = 64
DROPOUT_PROBABILITY = 0.5
IN_FEATURES = 25088

def parse_input_arguments():
    '''
    Parse input arguments
    Arguments:
        None
    Returns:
        data_dir (str)  : directory with the data
        save_dir (str)  : directory for saving a checkpoint
        arch (str)      : network architecture from torchvision
        learning_rate (float)   : learning rate for optimizer
        hidden_units (int)      : hidden units to use
        epochs (int)            : epochs to train
        gpu (boolean)           : Enable the use of CUDA(GPU)
    '''
    parser = argparse.ArgumentParser(description = "Train a deep neural network")
    parser.add_argument('data_dir', type = str, default = 'flowers', help = 'Dataset path')
    parser.add_argument('--save_dir', type = str, default = None, help = 'Path to save trained model checkpoint')
    parser.add_argument('--arch', type = str, default = 'vgg16', choices = ['vgg11', 'vgg13', 'vgg16', 'vgg19'], help = 'Model architecture')
    parser.add_argument('--learning_rate', type = float, default = 0.001, help = 'Learning rate for optimizator')
    parser.add_argument('--hidden_units', type = int, default = 1024, help = 'Number of hidden units')
    parser.add_argument('--epochs', type = int, default = 10, help = 'Number of epochs for training')
    parser.add_argument('--gpu', action = "store_true", default = True, help = 'Use GPU if available')

    args = parser.parse_args()
    return args.data_dir, args.save_dir, args.arch, args.learning_rate, args.hidden_units, args.epochs, args.gpu

def get_data_directories(data_directory):
    '''
    Get the directories for train, validation and test data
    Arguments:
        data_directory (str): base data directory
    Returns:
        train_directory (str): directory for training dataset
        valid_directory (str): directory for validation dataset
        test_directory (str): directory for test dataset
    '''
    train_directory = data_directory + '/train'
    valid_directory = data_directory + '/valid'
    test_directory = data_directory + '/test'

    return train_directory, valid_directory, test_directory

def get_cat_to_name(train_directory, valid_directory, test_directory):
    '''
    Load json mapping file from category label to category name
    Returns:
       category_label_to_name (dict): dictionary - category to label
    '''
    with open('cat_to_name.json', 'r') as f:
        cat_to_name = json.load(f)

    flower_types = len(cat_to_name)

    # Check if all folders have the same number of flower classes and as it's defined in JSON
    if len(os.listdir(train_directory)) == len(os.listdir(valid_directory)) == len(os.listdir(test_directory)) == flower_types:
        print(f"Fowers: {flower_types}")
    else:
        print(f"Error: Different number of flower types")
        exit()

    return cat_to_name, flower_types

def load_datasets(train_directory, valid_directory, test_directory):
    '''
    Load datasets
    Arguments:
        train_directory (str): directory for training dataset
        valid_directory (str): directory for validation dataset
        test_directory (str): directory for test dataset
    Returns:
       train_data (torchvision.datasets.ImageFolder): train dataset
       valid_data (torchvision.datasets.ImageFolder): validation dataset
       test_data (torchvision.datasets.ImageFolder): test dataset
    '''
    train_transforms = transforms.Compose([transforms.RandomRotation(DEGREES_ROTATION),
                                        transforms.RandomResizedCrop(SIZE_CROP),
                                        transforms.RandomHorizontalFlip(),
                                        transforms.ToTensor(),
                                        transforms.Normalize(NORMALIZE_MEAN, NORMALIZE_STD)
                                        ])

    valid_transforms = transforms.Compose([transforms.Resize(SIZE_RESIZE),
                                        transforms.CenterCrop(SIZE_CROP),
                                        transforms.ToTensor(),
                                        transforms.Normalize(NORMALIZE_MEAN, NORMALIZE_STD)
                                        ])

    test_transforms = transforms.Compose([transforms.Resize(SIZE_RESIZE),
                                        transforms.CenterCrop(SIZE_CROP),
                                        transforms.ToTensor(),
                                        transforms.Normalize(NORMALIZE_MEAN, NORMALIZE_STD)
                                        ])

    # Load the datasets with ImageFolder
    train_data = datasets.ImageFolder(train_directory, transform = train_transforms)
    valid_data = datasets.ImageFolder(valid_directory, transform = valid_transforms)
    test_data = datasets.ImageFolder(test_directory, transform = test_transforms)

    return train_data, valid_data, test_data

def get_data_loaders(train_data, valid_data, test_data):
    '''
    Get data loaders
    Arguments:
       train_data (torchvision.datasets.ImageFolder): train dataset
       valid_data (torchvision.datasets.ImageFolder): validation dataset
       test_data (torchvision.datasets.ImageFolder): test dataset
    Returns:
       trainloader (torch.utils.data.DataLoader): train data loader
       validloader (torch.utils.data.DataLoader): validation data loader
       testloader (torch.utils.data.DataLoader): test data loader
    '''

    # Using the image datasets and the trainforms, define the dataloaders
    trainloader = torch.utils.data.DataLoader(train_data, batch_size = BATCH_SIZE, shuffle = True)
    validloader = torch.utils.data.DataLoader(valid_data, batch_size = BATCH_SIZE)
    testloader = torch.utils.data.DataLoader(test_data, batch_size = BATCH_SIZE)

    return trainloader, validloader, testloader

def get_pretrained_model(network, number_classes, hidden_units):
    '''
    Get pretrained model and adapt it to current needs
    Arguments:
        network (str): network architecture to use
        number_classes (int): number of classes
    Returns:
       model (object): model adapted to current needs
       classifier (object): classifier adapted to current needs
    '''
    model = getattr(torchvision.models, network)(pretrained = True)
    out_features = hidden_units

    #print(model)

    # Freeze parameters so we don't backprop through them
    for param in model.parameters():
        param.requires_grad = False

    classifier = nn.Sequential(OrderedDict([('fc1', nn.Linear(IN_FEATURES, out_features)),
                                            ('drop', nn.Dropout(p = DROPOUT_PROBABILITY)),
                                            ('relu', nn.ReLU()),
                                            ('fc2', nn.Linear(out_features, number_classes)),
                                            ('output', nn.LogSoftmax(dim = 1))
                                            ]))

    model.classifier = classifier
    #print(model)
    return model, classifier

def save_model_checkpoint(model, train_data, network, number_classes, learning_rate, classifier, epochs, optimizer, save_path, checkpoint_filename):
    '''
    Save model checkpoint
    Arguments:
        network (str): network architecture used
        number_classes (int): number of classes
    Returns:
       None
    '''
    model.class_to_idx = train_data.class_to_idx

    checkpoint = {'network': network,
                'input_size': IN_FEATURES,
                'output_size': number_classes,
                'learning_rate': learning_rate,
                'batch_size': BATCH_SIZE,
                'classifier' : classifier,
                'epochs': epochs,
                'optimizer': optimizer.state_dict(),
                'state_dict': model.state_dict(),
                'class_to_idx': model.class_to_idx}

    torch.save(checkpoint, save_path)

def train(data_directory, save_directory, network, learning_rate, hidden_units, epochs, gpu):
    '''
    Train a network and save it in a checkpoint file
    Arguments:
        data_directory (str): directory were data is stored
        save_directory (str): directory to save checkpoint
        network (str): network architecture to use
        learning_rate (float): learning rate to use
        hidden_units (int): hidden units to use
        epochs (int): epochs to use
        gpu (boolean): Enable the use of GPU
    Returns:
       None
    '''
    print('Get data directories')
    train_directory, valid_directory, test_directory = get_data_directories(data_directory)

    print('Load mapping of category names to labels. Get the number of categories')
    category_label_to_name, number_classes = get_cat_to_name(train_directory, valid_directory, test_directory)

    print('Load datasets')
    train_data, valid_data, test_data = load_datasets(train_directory, valid_directory, test_directory)

    print('Get data loaders')
    trainloader, validloader, testloader = get_data_loaders(train_data, valid_data, test_data)

    print('Load pretrained model')
    model, classifier = get_pretrained_model(network, number_classes, hidden_units)

    # Train the network
    device = torch.device('cuda' if torch.cuda.is_available() and gpu == True else 'cpu')
    print('Device:', device)

    # Only train the classifier parameters, feature parameters are frozen
    optimizer = optim.Adam(model.classifier.parameters(), lr = learning_rate)

    #Loss function
    criterion = nn.NLLLoss()

    model.to(device)

    print('Start training')
    start_time = time.time()

    for epoch in range(epochs):
        running_loss = 0
        for inputs, labels in trainloader:
            inputs, labels = inputs.to(device), labels.to(device)

            optimizer.zero_grad()

            log_ps = model.forward(inputs)
            loss = criterion(log_ps, labels)
            loss.backward()
            optimizer.step()

            running_loss += loss.item()

        else:
            valid_loss = 0
            accuracy = 0
            model.eval()

            with torch.no_grad():
                for inputs, labels in validloader:
                    inputs, labels = inputs.to(device), labels.to(device)

                    log_ps = model.forward(inputs)
                    loss = criterion(log_ps, labels)

                    valid_loss += loss.item()

                    ps = torch.exp(log_ps)
                    top_p, top_class = ps.topk(1, dim = 1)
                    equals = top_class == labels.view(*top_class.shape)
                    accuracy += torch.mean(equals.type(torch.FloatTensor)).item()

            model.train()
        print(f"Epoch {epoch+1}/{epochs}.. "
              f"Train loss: {running_loss/len(trainloader):.3f}.. "
              f"Validation loss: {valid_loss/len(validloader):.3f}.. "
              f"Test accuracy: {accuracy/len(validloader):.3f}")

    end_time = time.time()

    training_time = end_time - start_time
    print(f'Training time: {training_time / 60:.0f}m {training_time % 60:.0f}s')

    # Save model checkpoint
    save_path = ''

    if save_directory == None:
        save_path = DEFAULT_CHECKPOINT_FILENAME
    else:
        save_path = save_directory + '/' + DEFAULT_CHECKPOINT_FILENAME

    save_model_checkpoint(model, train_data, network, number_classes, learning_rate,
                          classifier, epochs, optimizer, save_path, DEFAULT_CHECKPOINT_FILENAME)
    print(f'Save the checkpoint in {save_path}')

if __name__ == "__main__":
    data_directory, save_directory, network, learning_rate, hidden_units, epochs, gpu = parse_input_arguments()
    train(data_directory, save_directory, network, learning_rate, hidden_units, epochs, gpu)


text:
import numpy as np
import matplotlib.pyplot as plt

###################################################
#  Katie Chamberlain  -  February 2019            #
#  Abundance matching code relating stellar mass  #
#  to halo mass for a galaxy/DMH                  #
#  ...                                            #
#  Follows Moster, Naab, and White (2012)         #
#  https://arxiv.org/pdf/1205.5807.pdf            #
###################################################

#  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!  #
#  Note: to make plots, run plottingRoutine(z, "kind")  #
#     where "kind" can be "random" or "same"            #
#  ...                                                  #
#  Comment out the savefig line at the end of the       #
#     plotting routine if show is preferred             #
#  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!  #

f = np.random.normal

def logM1(z, kind = "median"):
    """eq. 11"""
    M10      = 11.59
    M10range = 0.236
    M11      = 1.195
    M11range = 0.353

    if kind == "median":                 # picking out only the median value
        return M10 + M11*(z/(1+z))
    else:                                # picking gaussian distributed variables
        M10Gauss, M11Gauss = f(M10, M10range,1), f(M11, M11range, 1)
        return M10Gauss + M11Gauss*(z/(1+z))

def N(z, kind="median"):
    """eq. 12"""
    N10      = 0.0351
    N10range = 0.0058
    N11      = -0.0247
    N11range = 0.0069

    if kind == "median":                 # picking out only the median value
        return N10 + N11*(z/(1+z))
    else:                                # picking gaussian distributed variables
        N10Gauss, N11Gauss = f(N10, N10range,1),f(N11, N11range, 1)
        return N10Gauss + N11Gauss*(z/(1+z))

def beta(z, kind="median"):
    """eq. 13"""
    beta10      = 1.376
    beta10range = 0.153
    beta11      = -0.826
    beta11range = 0.225

    if kind == "median":                 # picking out only the median value
        return beta10 + beta11*(z/(1+z))
    else:                                # picking gaussian distributed variables
        beta10Gauss, beta11Gauss = f(beta10, beta10range,1),f(beta11, beta11range, 1)
        return beta10Gauss + beta11Gauss*(z/(1+z))

def gamma(z, kind="median"):
    """eq. 14"""
    gamma10      = 0.608
    gamma10range = 0.059
    gamma11      = 0.329
    gamma11range = 0.173

    if kind == "median":                 # picking out only the median value
        return gamma10 + gamma11*(z/(1+z))
    else:                                # picking gaussian distributed variables
        gamma10Gauss, gamma11Gauss = f(gamma10, gamma10range,1),f( gamma11, gamma11range, 1)
        return gamma10Gauss + gamma11Gauss*(z/(1+z))

def SHMratio(M, z, kind="median"):
    """
    Inputs: masses M in solar masses (NOT in logspce)
    Outputs: Stellar mass to halo mass ratio
    """
    M1 = 10**logM1(z, kind)
    A = (M/M1)**(-beta(z,kind))
    B = (M/M1)**(gamma(z,kind))
    SHMratio = 2*N(z,kind)*(A+B)**-1
    return SHMratio

def stellarMass(M, z, kind="median"):
    """ returns the stellar mass in Msun """
    return M*SHMratio(M, z, kind)

def bounds(Ms, z, x):
    """
    finding x-sigma bounds on the stellar mass
    via error propagation
    credit: Mark Vogelsberger and Gurtina Besla
    Inputs: mass in solar masses
    Outputs: upper and lower stellar mass at x-sigma from mean (not log)
    """

    N1 = N(z)
    M1 = logM1(z)
    Beta1 = beta(z)
    Gamma1 = gamma(z)
    a = 1/(1+z)

    M = np.log10(Ms)

    M10range, M11range = 0.236, 0.353
    N10range, N11range = 0.0058, 0.0069
    B10range, B11range = 0.153, 0.225
    G10range, G11range = 0.059, 0.173

    Mstar = M + np.log10(2*N1) - np.log10( (10**(M-M1))**(-Beta1) + (10**(M-M1))**(Gamma1) )
    dmdN10 = np.log10(np.e)/N1
    dmdN11 = dmdN10*(1.0-a)

    eta = np.exp(np.log(10.0)*(M-M1))
    alpha = eta**(-Beta1) + eta**(Gamma1)
    dmdM10 = (Gamma1*eta**(Gamma1) - Beta1*eta**(-Beta1))/alpha
    dmdM11 = (Gamma1*eta**(Gamma1) - Beta1*eta**(-Beta1))/alpha*(1.0-a)
    dmdB10 = np.log10(np.e)/alpha*np.log(eta)*eta**(-Beta1)
    dmdB11 = np.log10(np.e)/alpha*np.log(eta)*eta**(-Beta1)*(1.0-a)
    dmdG10 = -np.log10(np.e)/alpha*np.log(eta)*eta**(Gamma1)
    dmdG11 = -np.log10(np.e)/alpha*np.log(eta)*eta**(Gamma1)*(1.0-a)

    sigma = np.sqrt(dmdM10**2.0*M10range**2.0 + dmdM11**2.0*M11range**2.0 + dmdN10**2.0*N10range**2.0 + dmdN11**2.0*N11range**2.0 + dmdB10**2.0*B10range**2.0 + dmdB11**2.0*B11range**2.0 + dmdG10**2.0*G10range**2.0 + dmdG11**2.0*G11range**2.0)

    Mstaru = Mstar + (sigma*x)
    Mstarl = Mstar - (sigma*x)

    return 10**Mstaru, 10**Mstarl

# ---------------------------------------#

def plottingRoutine(z=0, kind = "random"):

    if kind == "random": # to plot a random distribution of masses and their stellarMass
        Ms          = np.logspace(10,15,1000)                       # logarithmically spaced halo masses
        mstarMedian = stellarMass(Ms, z)                            # median mstar masses

        numPoints   = 100000                                        # number of random draws to make
        MsGauss     = 10**(5*np.random.rand(numPoints)+10)          # random halo mass vals
        mstarGauss  = [stellarMass(i, z, "gauss") for i in MsGauss]  # draws from gaussian dists

        numBins      = 150
        binsx, binsy = np.logspace(10,15,numBins),np.logspace(8,12, numBins) # bin definitions

    else:    # to plot a run of the stellarMass function using the same draws at each mass
        numPoints   = 10000
        Ms          = np.logspace(10,15,numPoints)                  # logarithmically spaced halo masses
        mstarMedian = stellarMass(Ms, z)                            # median mstar masses

        trials     = 10000
        MsGauss    = np.concatenate([Ms]*trials)
        mstarGauss = np.concatenate([stellarMass(Ms,0,"gauss") for i in range(trials)])

        numBins      = 150
        binsx, binsy = np.logspace(10,15,numBins),np.logspace(8,12,numBins)

    upper1, lower1 = bounds(Ms, z, 1)                           # one sigma bounds on mstar
    upper2, lower2 = bounds(Ms, z, 2)                           # two sigma bounds on mstar

    fig,ax = plt.subplots(figsize=(10,8))
    plt.hist2d(MsGauss,mstarGauss,bins=[binsx, binsy],cmap = "RdPu")
    plt.plot(Ms,mstarMedian,label = 'Median',color='black')
    plt.plot(Ms,upper1,'--',label = '1$\sigma$',lw = 1,color='black')
    plt.plot(Ms,lower1,'--',color='black',lw = 1)
    plt.plot(Ms,upper2,'--',label = '2$\sigma$',lw = .5,ms = 2,color='black')
    plt.plot(Ms,lower2,'--',color='black',lw=0.5)
    plt.title('z = '+str(z))
    plt.xlabel('M$_h$')
    plt.ylabel('m$_\star$')
    plt.legend()
    plt.yscale('log')
    plt.xscale('log')
#    plt.savefig('abundanceMatchingPlots/z'+str(z)+'.png', bbox_inches='tight',dpi=1200)
    plt.show()

#plottingRoutine()            # randomly drawn masses at z = 0
#plottingRoutine(0, "random") # gives same as above
#plottingRoutine(1, "same")   # concatenates many trials (as specified) so you get a new draw for each trial, but the same gaussian draws are made for all M_halo in each different trial; z = 1


text:
# Copyright 2016-2020 Blue Marble Analytics LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Carbon cap for each carbon_cap zone
"""

import csv
import os.path
import pandas as pd

from pyomo.environ import Set, Param, NonNegativeReals, value

def add_model_components(m, d, scenario_directory, subproblem, stage):
    """

    :param m:
    :param d:
    :return:
    """

    m.CARBON_CAP_ZONE_PERIODS_WITH_CARBON_CAP = Set(
        dimen=2, within=m.CARBON_CAP_ZONES * m.PERIODS
    )
    m.carbon_cap_target = Param(
        m.CARBON_CAP_ZONE_PERIODS_WITH_CARBON_CAP, within=NonNegativeReals
    )

def load_model_data(m, d, data_portal, scenario_directory, subproblem, stage):
    """

    :param m:
    :param d:
    :param data_portal:
    :param scenario_directory:
    :param subproblem:
    :param stage:
    :return:
    """
    data_portal.load(
        filename=os.path.join(
            scenario_directory, str(subproblem), str(stage), "inputs", "carbon_cap.tab"
        ),
        index=m.CARBON_CAP_ZONE_PERIODS_WITH_CARBON_CAP,
        param=m.carbon_cap_target,
        select=("carbon_cap_zone", "period", "carbon_cap_target"),
    )

def get_inputs_from_database(scenario_id, subscenarios, subproblem, stage, conn):
    """
    :param subscenarios: SubScenarios object with all subscenario info
    :param subproblem:
    :param stage:
    :param conn: database connection
    :return:
    """
    subproblem = 1 if subproblem == "" else subproblem
    stage = 1 if stage == "" else stage
    c = conn.cursor()
    carbon_cap_targets = c.execute(
        """SELECT carbon_cap_zone, period, carbon_cap
        FROM inputs_system_carbon_cap_targets
        JOIN
        (SELECT period
        FROM inputs_temporal_periods
        WHERE temporal_scenario_id = {}) as relevant_periods
        USING (period)
        JOIN
        (SELECT carbon_cap_zone
        FROM inputs_geography_carbon_cap_zones
        WHERE carbon_cap_zone_scenario_id = {}) as relevant_zones
        using (carbon_cap_zone)
        WHERE carbon_cap_target_scenario_id = {}
        AND subproblem_id = {}
        AND stage_id = {};
        """.format(
            subscenarios.TEMPORAL_SCENARIO_ID,
            subscenarios.CARBON_CAP_ZONE_SCENARIO_ID,
            subscenarios.CARBON_CAP_TARGET_SCENARIO_ID,
            subproblem,
            stage,
        )
    )

    return carbon_cap_targets

def validate_inputs(scenario_id, subscenarios, subproblem, stage, conn):
    """
    Get inputs from database and validate the inputs
    :param subscenarios: SubScenarios object with all subscenario info
    :param subproblem:
    :param stage:
    :param conn: database connection
    :return:
    """
    pass
    # Validation to be added
    # carbon_cap_targets = get_inputs_from_database(
    #     scenario_id, subscenarios, subproblem, stage, conn)

def write_model_inputs(
    scenario_directory, scenario_id, subscenarios, subproblem, stage, conn
):
    """
    Get inputs from database and write out the model input
    carbon_cap.tab file.
    :param scenario_directory: string, the scenario directory
    :param subscenarios: SubScenarios object with all subscenario info
    :param subproblem:
    :param stage:
    :param conn: database connection
    :return:
    """

    carbon_cap_targets = get_inputs_from_database(
        scenario_id, subscenarios, subproblem, stage, conn
    )

    with open(
        os.path.join(
            scenario_directory, str(subproblem), str(stage), "inputs", "carbon_cap.tab"
        ),
        "w",
        newline="",
    ) as carbon_cap_file:
        writer = csv.writer(carbon_cap_file, delimiter="\t", lineterminator="\n")

        # Write header
        writer.writerow(["carbon_cap_zone", "period", "carbon_cap_target"])

        for row in carbon_cap_targets:
            writer.writerow(row)

max_stars_repo_name: souissim/gridpath

max_issues_repo_name: souissim/gridpath

max_forks_repo_name: souissim/gridpath

reason: from pyomo


text:
import numpy as np
import gym
import evogym.envs
from evogym import sample_robot
from evogym.utils import get_full_connectivity

def make_env(env_name, seed=None, robot=None, **kwargs):
    if robot is None:  # Random robot
        robot, connections = sample_robot((5, 5))
    connections = get_full_connectivity(robot)
    env = gym.make(env_name, body=robot)
    if seed is not None:
        env.seed(seed)
    return env

def evaluate(agent, env, max_steps=1000, render=False):
    obs = env.reset()
    reward = 0
    steps = 0
    done = False
    while not done and steps < max_steps:
        if render:
            env.render()
        action = agent.act(obs)
        obs, r, done, _ = env.step(action)
        reward += r
        steps += 1
    return reward

def get_cfg(env_name, robot=None):
    env = make_env(env_name, robot=robot)
    cfg = {
        "n_in": env.observation_space.shape[0],
        "h_size": 32,
        "n_out": env.action_space.shape[0],
    }
    env.close()
    return cfg

def mp_eval(a, cfg):
    env = make_env(cfg["env_name"], robot=cfg["robot"])
    fit = evaluate(a, env, max_steps=cfg["max_steps"])
    env.close()
    return fit


text:
from pythran.tests import TestEnv
from pythran.typing import List

import unittest

class TestStr(TestEnv):
    def test_str_startswith0(self):
        self.run_test("def str_startswith0(s0, s1): return s0.startswith(s1)", "barbapapa", "barba", str_startswith0=[str, str])
        self.run_test("def str_startswith0a(s0, s1): return s0[0].startswith(s1)", "barbapapa", "barba", str_startswith0a=[str, str])

    def test_str_startswith1(self):
        self.run_test("def str_startswith1(s0, s1): return s0.startswith(s1)", "barbapapa", "barbi", str_startswith1=[str, str])
        self.run_test("def str_startswith1a(s0, s1): return s0[0].startswith(s1)", "barbapapa", "barbi", str_startswith1a=[str, str])

    def test_str_endswith0(self):
        self.run_test("def str_endswith0(s0, s1): return s0.endswith(s1)", "barbapapa", "papa", str_endswith0=[str, str])
        self.run_test("def str_endswith0a(s0, s1): return s0[0].endswith(s1)", "barbapapa", "papa", str_endswith0a=[str, str])

    def test_str_endswith1(self):
        self.run_test("def str_endswith1(s0, s1): return s0.endswith(s1)", "barbapapa", "papy", str_endswith1=[str, str])
        self.run_test("def str_endswith1a(s0, s1): return s0[0].endswith(s1)", "barbapapa", "papy", str_endswith1a=[str, str])

    def test_str_empty(self):
        self.run_test("def str_empty(s0): return '>o_/' if s0 else '0x0'", "", str_empty=[str])

    def test_str_failed_conversion(self):
        self.run_test("def str_failed_conversion(s):\n try: return int(s)\n except: return 42", "prout", str_failed_conversion=[str])
        self.run_test("def str_failed_conversion_a(s):\n try: return int(s[0])\n except: return 42", "prout", str_failed_conversion_a=[str])

    def test_str_replace0(self):
        self.run_test("def str_replace0(s): return s.replace('er', 'rer')", "parler", str_replace0=[str])
        self.run_test("def str_replace0a(s): return s.replace('er', 'r')", "parler", str_replace0a=[str])
        self.run_test("def str_replace0b(s): return s[0].replace('er', 'r')", "parler", str_replace0b=[str])

    def test_str_replace1(self):
        self.run_test("def str_replace1(s): return s.replace('er', 'rer', 1)", "erlang manger dessert", str_replace1=[str])
        self.run_test("def str_replace1a(s): return s.replace('er', 'r', 1)", "erlang manger dessert", str_replace1a=[str])
        self.run_test("def str_replace1b(s): return s[0].replace('er', 'r', 1)", "erlang manger dessert", str_replace1b=[str])

    def test_str_replace2(self):
        self.run_test("def str_replace2(s): return s.replace('', 'du vide surgit rien', 1)", "j aime les moulinettes a fromage", str_replace2=[str])
        self.run_test("def str_replace2a(s): return s[0].replace('', 'du vide surgit rien', 1)", "j aime les moulinettes a fromage", str_replace2a=[str])

    def test_str_ascii_letters(self):
        self.run_test("def str_ascii_letters(): import string; return string.ascii_letters", str_ascii_letters=[])

    def test_str_ascii_lowercase(self):
        self.run_test("def str_ascii_lowercase(): import string; return string.ascii_lowercase", str_ascii_lowercase=[])

    def test_str_ascii_uppercase(self):
        self.run_test("def str_ascii_uppercase(): import string; return string.ascii_uppercase", str_ascii_uppercase=[])

    def test_str_digits(self):
        self.run_test("def str_digits(): import string; return string.digits", str_digits=[])

    def test_str_with_backslash(self):
        self.run_test("def str_with_backslash(): return '''\h'''",
                      str_with_backslash=[])

    def test_str_hexdigits(self):
        self.run_test("def str_hexdigits(): import string; return string.hexdigits", str_hexdigits=[])

    def test_str_octdigits(self):
        self.run_test("def str_octdigits(): import string; return string.octdigits", str_octdigits=[])

    def test_str_lower(self):
        self.run_test("def str_lower(s): return s.lower()", "ThiS iS a TeST", str_lower=[str])
        self.run_test("def str_lower_a(s): return s[0].lower()", "ThiS iS a TeST", str_lower_a=[str])

    def test_str_upper(self):
        self.run_test("def str_upper(s): return s.upper()", "ThiS iS a TeST", str_upper=[str])
        self.run_test("def str_upper_a(s): return s[0].upper()", "ThiS iS a TeST", str_upper_a=[str])

    def test_str_capitalize(self):
        self.run_test("def str_capitalize(s): return s.capitalize()", "thiS iS a TeST", str_capitalize=[str])
        self.run_test("def str_capitalize_a(s): return s[0].capitalize()", "thiS iS a TeST", str_capitalize_a=[str])

    def test_str_strip(self):
        self.run_test("def str_strip(s): return s.strip()", "       ThiS iS a TeST        ", str_strip=[str])
        self.run_test("def str_strip_a(s): return s[0].strip()", "       ThiS iS a TeST        ", str_strip_a=[str])

    def test_str_strip2(self):
        self.run_test("def str_strip2(s): return s.strip(\"TSih\")", "ThiS iS a TeST", str_strip2=[str])

    def test_str_lstrip(self):
        self.run_test("def str_lstrip(s): return s.lstrip()", "       ThiS iS a TeST        ", str_lstrip=[str])
        self.run_test("def str_lstrip_a(s): return s[0].lstrip()", "       ThiS iS a TeST        ", str_lstrip_a=[str])

    def test_str_lstrip2(self):
        self.run_test("def str_lstrip2(s): return s.lstrip(\"TSih\")", "ThiS iS a TeST", str_lstrip2=[str])

    def test_str_rstrip(self):
        self.run_test("def str_rstrip(s): return s.rstrip()", "       ThiS iS a TeST        ", str_rstrip=[str])
        self.run_test("def str_rstrip_a(s): return s[0].rstrip()", "       ThiS iS a TeST        ", str_rstrip_a=[str])

    def test_str_rstrip2(self):
        self.run_test("def str_rstrip2(s): return s.rstrip(\"TSih\")", "ThiS iS a TeST", str_rstrip2=[str])

    def test_str_split1(self):
        self.run_test("def str_split1(s): return s.split()", "ThiS iS a TeST", str_split1=[str])
        self.run_test("def str_split1_a(s): return s[0].split()", "ThiS iS a TeST", str_split1_a=[str])

    def test_str_split2(self):
        self.run_test("def str_split2(s): return s.split('S')", "ThiS iS a TeST", str_split2=[str])

    def test_str_split3(self):
        self.run_test("def str_split3(s): return s.split()", "ThiS  iS\t a TeST", str_split3=[str])

    def test_str_split4(self):
        self.run_test("def str_split4(s): return s.split()", "", str_split4=[str])

    def test_str_format(self):
        self.run_test("def str_format(a): return '%.2f %.2f' % (a, a)", 43.23, str_format=[float])

    def test_str_join0(self):
        self.run_test("def str_join0(): a = ['1'] ; a.pop() ; return '+e+'.join(a)", str_join0=[])
        self.run_test("def str_join0a(): a = ['1'] ; a.pop() ; return 'e'.join(a)", str_join0a=[])

    def test_str_join1(self):
        self.run_test("def str_join1(): a = ['l', 'l'] ; return 'o'.join(a)", str_join1=[])

    def test_str_join2(self):
        self.run_test("def str_join2(a): return 'o'.join(filter(len, a))", ['l', 'l'], str_join2=[List[str]])

    def test_str_find0(self):
        self.run_test("def str_find0(s): return s.find('pop')", "popop", str_find0=[str])
        self.run_test("def str_find0a(s): return s.find('p')", "popop", str_find0a=[str])
        self.run_test("def str_find0b(s): return s[0].find('p')", "popop", str_find0b=[str])

    def test_str_find1(self):
        self.run_test("def str_find1(s): return s.find('pap')", "popop", str_find1=[str])

    def test_str_reversal(self):
        self.run_test("def str_reversal(s): return list(map(ord,reversed(s)))", "dear", str_reversal=[str])
        self.run_test("def str_reversal_a(s): return list(map(ord,reversed(s[0])))", "dear", str_reversal_a=[str])

    def test_str_substring_iteration(self):
        self.run_test("def str_substring_iteration(s): return list(map(ord, s[1:-1]))", "pythran", str_substring_iteration=[str])

    def test_str_isalpha(self):
        self.run_test("def str_isalpha(s, t, u): return s.isalpha(), t.isalpha(), u.isalpha()", "e", "1", "", str_isalpha=[str,str, str])
        self.run_test("def str_isalpha_a(s, t): return s[0].isalpha(), t[0].isalpha()", "e", "1", str_isalpha_a=[str,str])

    def test_str_isdigit(self):
        self.run_test("def str_isdigit(s, t, u): return s.isdigit(), t.isdigit(), u.isdigit()", "e", "1", "", str_isdigit=[str,str, str])
        self.run_test("def str_isdigit_a(s, t): return s[0].isdigit(), t[0].isdigit()", "e", "1", str_isdigit_a=[str,str])

    def test_str_count(self):
        self.run_test("def str_count(s, t, u, v): return s.count(t), s.count(u), s.count(v)",
                      "pythran is good for health", "py", "niet", "t",
                      str_count=[str, str, str, str])
        self.run_test("def str_count_a(s, t, u, v): return s[0].count(t), s[1].count(u), s.count(v[0])",
                      "pythran is good for health", "py", "niet", "t",
                      str_count_a=[str, str, str, str])

    def test_str_literal_cmp(self):
        code = '''
            def eee(a, i):
                if a == "ABCD":
                    return 2 * i
                elif a != "ZDSD":
                    return 1 * i
                return 3 * i

            def str_literal_cmp(a, i):
                if a == "EEE":
                   return eee("ZZZ", i), eee("ABCD", i)
                else:
                   return eee("YYY", i), 3'''
        self.run_test(code, "EEE", 2, str_literal_cmp=[str, int])

    def test_str_literal_cmp1(self):
        code = '''
            def eee(a, i):
                if a > "ABCD":
                    return 2 * i
                elif a <= "ZDSD":
                    return 1 * i
                return 3 * i

            def str_literal_cmp1(a, i):
                if a == "EEE":
                   return eee("ZZZ", i), eee("ABCD", i)
                else:
                   return eee("YYY", i), 3'''
        self.run_test(code, "EEE", 2, str_literal_cmp1=[str, int])

    def test_str_literal_cmp2(self):
        code = '''
            def eee(a, i):
                if a < "ABCD":
                    return 2 * i
                elif a >= "ZDSD":
                    return 1 * i
                return 3 * i

            def str_literal_cmp2(a, i):
                if a == "EEE":
                   return eee("ZZZ", i), eee("ABCD", i)
                else:
                   return eee("YYY", i), 3'''
        self.run_test(code, "EEE", 2, str_literal_cmp2=[str, int])

    def test_str_literal_add(self):
        code = '''
            def eee(a, i):
                if i > 0:
                    return a + "ABCD"
                else:
                    return a + "BCD"

            def str_literal_add(a, i):
                if a == "EEE":
                   return eee("ZZZ", i), eee("ABCD", i)
                else:
                   return eee("YYY", i), "3"'''
        self.run_test(code, "EEE", 2, str_literal_add=[str, int])

    def test_str_literal_mult(self):
        code = '''
            def eee(a, i):
                if i > 0:
                    return a * i
                else:
                    return a * 3

            def str_literal_mult(a, i):
                if a == "EEE":
                   return eee("ZZZ", i), eee("ABCD", i)
                else:
                   return eee("YYY", i), "3"'''
        self.run_test(code, "EEE", 2, str_literal_mult=[str, int])

    def test_str_float(self):
        self.run_test("def str_float(s): return float(s)", "0.000012", str_float=[str])

    def test_str_numpy_float32(self):
        self.run_test("def str_numpy_float32(s): import numpy; return numpy.float32(s)", "0.000012",
                      str_numpy_float32=[str])

    def test_str_numpy_float64(self):
        self.run_test("def str_numpy_float64(s): import numpy; return numpy.float64(s)", "0.000012",
                      str_numpy_float64=[str])

    def test_str_int(self):
        self.run_test("def str_int(s): return int(s)", "12", str_int=[str])

    def test_str_id(self):
        self.run_test("def str_id(x): return id(x) != 0", "hello", str_id=[str])

    def test_str_slice_assign(self):
        self.run_test('''
            def str_slice_assign(s, c):
                if s.startswith(c):
                    s = s[len(c):];
                return s''', "LEFT-B6", "LEFT-",
                      str_slice_assign=[str, str])

    def test_str_slice_assign2(self):
        self.run_test('''
        def sample_datatype(value):
            definitions = [
                ('LEFT-', 1),
                ('RIGHT-', 2),
                ('', 3)
            ]

            plate_number = None
            for definition in definitions:
                s, n = definition
                if value.startswith(s):
                    plate_number = n
                    value = value[len(s):]
                    break
            if plate_number is None:   # Comment for make it works
               raise ValueError("Invalid value")
            return None

        def str_slice_assign2(s1):
            sample_datatype(s1)
            return s1''', "LEFT-B6", str_slice_assign2=[str])

max_stars_repo_name: sthagen/serge-sans-paille-pythran

max_issues_repo_name: sthagen/serge-sans-paille-pythran

max_forks_repo_name: sthagen/serge-sans-paille-pythran


text:
from edgine.src.config.config_server import ConfigServer
from edgine.src.base import EdgineBase
from edgine.src.starter import EdgineStarter
from typing import Any
import cv2
import numpy as np
import random
import string
import time

class Getter(EdgineBase):

    def __init__(self,
                 **kwargs):
        EdgineBase.__init__(self,
                            name="GET",
                            **kwargs)

        self._cam_id: int = 0
        self._cap = None

    def prerun(self) -> None:
        self._cap = cv2.VideoCapture(self._cam_id)

    def blogic(self, data_in: Any = None) -> Any:
        ret, out = self._cap.read()
        return out

class Resizer(EdgineBase):

    def __init__(self,
                 config_server: ConfigServer,
                 **kwargs):
        EdgineBase.__init__(self,
                            name="RES",
                            config_server=config_server,
                            **kwargs)
        config_server.config.create_if_unknown("resize_target", (300, 300))
        config_server.save_config()

    def blogic(self, data_in: Any = None) -> Any:
        res_img = None if data_in is None else cv2.resize(data_in, tuple(self.cfg.resize_target))
        return res_img

class Canny(EdgineBase):

    def __init__(self,
                 **kwargs):
        EdgineBase.__init__(self,
                            name="CANN",
                            **kwargs)

    def blogic(self, data_in: Any = None) -> Any:
        edges = cv2.Canny(data_in, 100, 200)
        return edges

class PrintRandom(EdgineBase):

    def __init__(self,
                 **kwargs):
        EdgineBase.__init__(self,
                            name="RAND",
                            **kwargs)

    def blogic(self, data_in: Any = None) -> Any:
        letters = string.ascii_letters
        out = ''.join(random.choice(letters) for i in range(10))
        self.info(out)
        return None

if __name__ == "__main__":
    print("Canny edge detection example")
    starter = EdgineStarter(config_file="camera_canny_config.json")
    getter_id = starter.reg_service(Getter, min_runtime=0.033)
    resizer_id = starter.reg_service(Resizer, min_runtime=0.033)
    canny_id = starter.reg_service(Canny, min_runtime=0.033)
    starter.reg_service(PrintRandom, min_runtime=10)

    starter.reg_connection(getter_id, resizer_id)
    starter.reg_connection(resizer_id, canny_id)
    q3 = starter.reg_sink(canny_id)
    starter.init_services()
    starter.start()

    img = np.random.randint(255, size=(300, 300, 3), dtype=np.uint8)

    s = time.time()
    fps = 30.0

    while True:
        try:
            img = q3.get(timeout=0.5)
        except Exception:
            continue

        cv2.putText(img,
                    text=f"{fps:.1f}FPS",
                    fontFace=cv2.FONT_HERSHEY_DUPLEX,
                    color=(255, 255, 255),
                    thickness=1,
                    fontScale=0.5,
                    org=(10, 20))

        cv2.imshow('frame', img)
        if cv2.waitKey(33) & 0xFF == ord('q'):
            starter.stop()
            break

        e = time.time()
        el = e - s
        s = time.time()
        fps = 0.8 * fps + 0.2 * (1.0 / el)

    cv2.destroyAllWindows()

    print("All done!")

max_stars_repo_name: samsterckval/edgine

max_issues_repo_name: samsterckval/edgine

max_forks_repo_name: samsterckval/edgine


text:
#!/usr/bin/env python
#
# License: BSD
#   https://raw.github.com/yujinrobot/kobuki_desktop/master/kobuki_qtestsuite/LICENSE
#
##############################################################################
# Imports
##############################################################################

import operator
import numpy

#from rqt_plot.qwt_data_plot import QwtDataPlot
from rqt_plot.data_plot import DataPlot

##############################################################################
# Classes
##############################################################################

class FullSizeDataPlot(DataPlot):
    def __init__(self, parent=None):
        super(FullSizeDataPlot, self).__init__(parent)
        self.max_range = 180
        self.min_range = 0
        self.dynamic_range = False
        self._ymin = 0
        self._ymax = 0

    def reset(self):
        self._ymin = 0
        self._ymax = 0

    ######################################
    # Overrides
    ######################################
    def _update_legend(self):
        handles, labels = self._canvas.axes.get_legend_handles_labels()
        if handles:
            hl = sorted(zip(handles, labels), key=operator.itemgetter(1))
            handles, labels = zip(*hl)
        self._canvas.axes.legend(handles, labels, loc='lower right')

    def redraw(self):
        '''
          We fix the y axis and continually resize the x axis to encapsulate
          the entire domain, range of the battery profile.

          @Todo : the domain is simply the data value, we could use
        '''
        self._canvas.axes.grid(True, color='gray')
        # Set axis bounds
        xmax = 0
        for curve in self._curves.values():
            data_x, data_y, plot, min_max_y = curve
            if len(data_x) == 0:
                continue

            xmax = max(xmax, data_x[-1])
            self._ymin = min(self._ymin, min_max_y[0])
            self._ymax = max(self._ymax, min_max_y[1] + 1)
            self._canvas.axes.set_xbound(lower=0, upper=xmax)

        if self.dynamic_range:
            self._canvas.axes.set_ybound(lower=self._ymin, upper=self._ymax)
        else:
            self._canvas.axes.set_ybound(self.min_range, upper=self.max_range)

        # Set plot data on current axes
        for curve in self._curves.values():
            data_x, data_y, plot, min_max_y = curve
            plot.set_data(numpy.array(data_x), numpy.array(data_y))

        self._canvas.draw()


text:
# ------------------------------------------------------------------------------
# Copyright (c) Microsoft
# Licensed under the MIT License.
# Written by Zhipeng Zhang (zhangzhipeng2017@ia.ac.cn)
# ------------------------------------------------------------------------------
import torch
import torch.nn as nn
import numpy as np

class Ocean_(nn.Module):
    def __init__(self):
        super(Ocean_, self).__init__()
        self.features = None
        self.connect_model = None
        self.align_head = None
        self.zf = None
        self.criterion = nn.BCEWithLogitsLoss()
        self.neck = None
        self.search_size = 255
        self.score_size = 25
        self.batch = 32 if self.training else 1

        self.grids()

    def feature_extractor(self, x, online=False):
        return self.features(x, online=online)

    def extract_for_online(self, x):
        xf = self.feature_extractor(x, online=True)
        return xf

    def connector(self, template_feature, search_feature):
        pred_score = self.connect_model(template_feature, search_feature)
        return pred_score

    def _cls_loss(self, pred, label, select):
        if len(select.size()) == 0: return 0
        pred = torch.index_select(pred, 0, select)
        label = torch.index_select(label, 0, select)
        return self.criterion(pred, label)  # the same as tf version

    def _weighted_BCE(self, pred, label):
        pred = pred.view(-1)
        label = label.view(-1)
        pos = label.data.eq(1).nonzero().squeeze().cuda()
        neg = label.data.eq(0).nonzero().squeeze().cuda()

        loss_pos = self._cls_loss(pred, label, pos)
        loss_neg = self._cls_loss(pred, label, neg)
        return loss_pos * 0.5 + loss_neg * 0.5

    def _weighted_BCE_align(self, pred, label):
        pred = pred.view(-1)
        label = label.view(-1)
        pos = label.data.eq(1).nonzero().squeeze().cuda()
        neg = label.data.eq(0).nonzero().squeeze().cuda()

        loss_pos = self._cls_loss(pred, label, pos)
        loss_neg = self._cls_loss(pred, label, neg)

        return loss_pos * 0.5 + loss_neg * 0.5

    def _IOULoss(self, pred, target, weight=None):
        pred_left = pred[:, 0]
        pred_top = pred[:, 1]
        pred_right = pred[:, 2]
        pred_bottom = pred[:, 3]

        target_left = target[:, 0]
        target_top = target[:, 1]
        target_right = target[:, 2]
        target_bottom = target[:, 3]

        target_area = (target_left + target_right) * (target_top + target_bottom)
        pred_area = (pred_left + pred_right) * (pred_top + pred_bottom)

        w_intersect = torch.min(pred_left, target_left) + torch.min(pred_right, target_right)
        h_intersect = torch.min(pred_bottom, target_bottom) + torch.min(pred_top, target_top)

        area_intersect = w_intersect * h_intersect
        area_union = target_area + pred_area - area_intersect

        losses = -torch.log((area_intersect + 1.0) / (area_union + 1.0))

        if weight is not None and weight.sum() > 0:
            return (losses * weight).sum() / weight.sum()
        else:
            assert losses.numel() != 0
            return losses.mean()

    def add_iouloss(self, bbox_pred, reg_target, reg_weight):
        """

        :param bbox_pred:
        :param reg_target:
        :param reg_weight:
        :param grid_x:  used to get real target bbox
        :param grid_y:  used to get real target bbox
        :return:
        """

        bbox_pred_flatten = bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4)
        reg_target_flatten = reg_target.reshape(-1, 4)
        reg_weight_flatten = reg_weight.reshape(-1)
        pos_inds = torch.nonzero(reg_weight_flatten > 0).squeeze(1)

        bbox_pred_flatten = bbox_pred_flatten[pos_inds]
        reg_target_flatten = reg_target_flatten[pos_inds]

        loss = self._IOULoss(bbox_pred_flatten, reg_target_flatten)

        return loss

    # ---------------------
    # classification align
    # ---------------------
    def grids(self):
        """
        each element of feature map on input search image
        :return: H*W*2 (position for each element)
        """
        sz = self.score_size
        stride = 8

        sz_x = sz // 2
        sz_y = sz // 2

        x, y = np.meshgrid(np.arange(0, sz) - np.floor(float(sz_x)),
                           np.arange(0, sz) - np.floor(float(sz_y)))

        self.grid_to_search = {}
        self.grid_to_search_x = x * stride + self.search_size // 2
        self.grid_to_search_y = y * stride + self.search_size // 2

        self.grid_to_search_x = torch.Tensor(self.grid_to_search_x).unsqueeze(0).unsqueeze(0).cuda()
        self.grid_to_search_y = torch.Tensor(self.grid_to_search_y).unsqueeze(0).unsqueeze(0).cuda()

        self.grid_to_search_x = self.grid_to_search_x.repeat(self.batch, 1, 1, 1)
        self.grid_to_search_y = self.grid_to_search_y.repeat(self.batch, 1, 1, 1)

    def pred_to_image(self, bbox_pred):
        self.grid_to_search_x = self.grid_to_search_x.to(bbox_pred.device)
        self.grid_to_search_y = self.grid_to_search_y.to(bbox_pred.device)

        pred_x1 = self.grid_to_search_x - bbox_pred[:, 0, ...].unsqueeze(1)  # 17*17
        pred_y1 = self.grid_to_search_y - bbox_pred[:, 1, ...].unsqueeze(1)  # 17*17
        pred_x2 = self.grid_to_search_x + bbox_pred[:, 2, ...].unsqueeze(1)  # 17*17
        pred_y2 = self.grid_to_search_y + bbox_pred[:, 3, ...].unsqueeze(1)  # 17*17

        pred = [pred_x1, pred_y1, pred_x2, pred_y2]

        pred = torch.cat(pred, dim=1)

        return pred

    def align_label(self, pred, target, weight):
        # calc predicted box iou (treat it as aligned label)

        pred = pred.permute(0, 2, 3, 1)  # [B, 25, 25, 4]
        pred_left = pred[..., 0]
        pred_top = pred[..., 1]
        pred_right = pred[..., 2]
        pred_bottom = pred[..., 3]

        target_left = target[..., 0]
        target_top = target[..., 1]
        target_right = target[..., 2]
        target_bottom = target[..., 3]

        target_area = (target_left + target_right) * (target_top + target_bottom)
        pred_area = (pred_left + pred_right) * (pred_top + pred_bottom)

        w_intersect = torch.min(pred_left, target_left) + torch.min(pred_right, target_right)
        h_intersect = torch.min(pred_bottom, target_bottom) + torch.min(pred_top, target_top)

        area_intersect = w_intersect * h_intersect
        area_union = target_area + pred_area - area_intersect

        ious = (area_intersect + 1.0) / (area_union + 1.0)

        ious = torch.abs(weight * ious)  # delete points out of object

        ious[ious < 0] = 0
        ious[ious >= 1] = 1

        return ious

    def offset(self, boxes, featmap_sizes):
        """
        refers to Cascade RPN
        Params:
        box_list: [N, 4]   [x1, y1, x2, y2] # predicted bbox
        """

        def _shape_offset(boxes, stride):
            ks = 3
            dilation = 1
            pad = (ks - 1) // 2
            idx = torch.arange(-pad, pad + 1, dtype=dtype, device=device)
            xx, yy = torch.meshgrid(idx, idx)
            xx = xx.reshape(-1)
            yy = yy.reshape(-1)

            pad = (ks - 1) // 2
            idx = torch.arange(-pad, pad + 1, dtype=dtype, device=device)
            yy, xx = torch.meshgrid(idx, idx)  # return order matters
            xx = xx.reshape(-1)
            yy = yy.reshape(-1)
            w = (boxes[:, 2] - boxes[:, 0] + 1) / stride
            h = (boxes[:, 3] - boxes[:, 1] + 1) / stride
            w = w / (ks - 1) - dilation
            h = h / (ks - 1) - dilation
            offset_x = w[:, None] * xx  # (NA, ks**2)
            offset_y = h[:, None] * yy  # (NA, ks**2)
            return offset_x, offset_y

        def _ctr_offset(boxes, stride, featmap_size):
            feat_h, feat_w = featmap_size
            image_size = self.search_size

            assert len(boxes) == feat_h * feat_w

            x = (boxes[:, 0] + boxes[:, 2]) * 0.5
            y = (boxes[:, 1] + boxes[:, 3]) * 0.5

            # # compute centers on feature map
            # x = (x - (stride - 1) * 0.5) / stride
            # y = (y - (stride - 1) * 0.5) / stride

            # different here for Siamese
            # use center of image as coordinate origin
            x = (x - image_size * 0.5) / stride + feat_w // 2
            y = (y - image_size * 0.5) / stride + feat_h // 2

            # compute predefine centers
            # different here for Siamese
            xx = torch.arange(0, feat_w, device=boxes.device)
            yy = torch.arange(0, feat_h, device=boxes.device)
            yy, xx = torch.meshgrid(yy, xx)
            xx = xx.reshape(-1).type_as(x)
            yy = yy.reshape(-1).type_as(y)

            offset_x = x - xx  # (NA, )
            offset_y = y - yy  # (NA, )
            return offset_x, offset_y

        num_imgs = len(boxes)
        dtype = boxes[0].dtype
        device = boxes[0][0].device

        featmap_sizes = featmap_sizes[2:]

        offset_list = []
        for i in range(num_imgs):
            c_offset_x, c_offset_y = _ctr_offset(boxes[i], 8, featmap_sizes)
            s_offset_x, s_offset_y = _shape_offset(boxes[i], 8)

            # offset = ctr_offset + shape_offset
            offset_x = s_offset_x + c_offset_x[:, None]
            offset_y = s_offset_y + c_offset_y[:, None]

            # offset order (y0, x0, y1, x0, .., y9, x8, y9, x9)from torch.autograd import Variable
            offset = torch.stack([offset_y, offset_x], dim=-1)
            offset = offset.reshape(offset.size(0), -1).unsqueeze(0)  # [NA, 2*ks**2]
            offset_list.append(offset)

        offsets = torch.cat(offset_list, 0)
        return offsets

    def template(self, z):
        _, self.zf = self.feature_extractor(z)

        if self.neck is not None:
            _, self.zf = self.neck(self.zf, crop=True)

        if self.align_head is not None:
            self.update_flag = True
        else:
            pass

    def track(self, x):

        _, xf = self.feature_extractor(x)

        if self.neck is not None:
            xf = self.neck(xf)

        if self.align_head is not None:
            if self.update_flag:
                self.batch = 1
                self.search_size = x.size(-1)
                self.score_size = (self.search_size - 127) // 8 + 1 + 8
                self.grids()
                self.update_flag = False

            bbox_pred, cls_pred, cls_feature, reg_feature = self.connect_model(xf, self.zf)
            bbox_pred_to_img = self.pred_to_image(bbox_pred)
            offsets = self.offset(bbox_pred_to_img.permute(0, 2, 3, 1).reshape(bbox_pred_to_img.size(0), -1, 4), bbox_pred.size())
            cls_align = self.align_head(reg_feature, offsets)

            return cls_pred, bbox_pred, cls_align
        else:
            bbox_pred, cls_pred, _, _ = self.connect_model(xf, self.zf)

            return cls_pred, bbox_pred

    def forward(self, template, search, label=None, reg_target=None, reg_weight=None):
        """
        :return:
        """
        _, zf = self.feature_extractor(template)
        _, xf = self.feature_extractor(search)

        if self.neck is not None:
            _, zf = self.neck(zf, crop=True)
            xf = self.neck(xf, crop=False)

        # depth-wise cross correlation --> tower --> box pred
        if self.align_head is not None:
            bbox_pred, cls_pred, cls_feature, reg_feature = self.connect_model(xf, zf)

            bbox_pred_to_img = self.pred_to_image(bbox_pred)
            offsets = self.offset(bbox_pred_to_img.permute(0, 2, 3, 1).reshape(bbox_pred_to_img.size(0), -1, 4), bbox_pred.size())
            cls_align = self.align_head(reg_feature, offsets)

            # add iou loss
            reg_loss = self.add_iouloss(bbox_pred, reg_target, reg_weight)

            # add cls loss
            align_cls_label = self.align_label(bbox_pred, reg_target, reg_weight)
            cls_loss_ori = self._weighted_BCE(cls_pred, label)
            cls_loss = self.criterion(cls_align.squeeze(), align_cls_label)

            if torch.isnan(cls_loss):
                cls_loss = 0 * cls_loss_ori

            return cls_loss_ori, cls_loss, reg_loss
        else:
            bbox_pred, cls_pred, _, _ = self.connect_model(xf, zf)
            reg_loss = self.add_iouloss(bbox_pred, reg_target, reg_weight)
            cls_loss = self._weighted_BCE(cls_pred, label)
            return cls_loss, None, reg_loss

max_stars_repo_name: JackieZhai/TracKit

max_issues_repo_name: JackieZhai/TracKit

max_forks_repo_name: JackieZhai/TracKit


text:
###!/usr/bin/python
#
# front matter
#
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

#import csv
#data = open('samp_data.smp', 'r')
#datareader = csv.reader(data, delimiter=' ')

data = np.genfromtxt('data.txt', delimiter='  ')
samp = np.genfromtxt('samp.smp', delimiter='  ', skip_header=1)
#data = np.loadtxt('data_file.txt', delimiter='  ')

pandata = pd.read_csv('data.txt')
pansamp = pd.read_csv('samp.smp')

######################################################
#sampler vectorization
time = samp[:,][:,0]
conc1 = samp[:,][:,1]
var1 = samp[:,][:,2]

avg = np.mean(time)

############################################
#export to columns
time_col = time.reshape(-1,1)
conc1_col = conc1.reshape(-1,1)
var1_col = var1.reshape(-1,1)
sdata = np.concatenate((time_col, conc1_col,var1_col), axis=1)
#save vectors
np.savetxt('samp.vec',sdata, delimiter=('  ,  '))
#Space comma space ('  ,  ') is more readable!

###########################################
# plotting
#plt.figure(1)
plt.plot(time,conc1)

#plt.figure()
plt.legend(['concentration'])
plt.xlabel('Time')
plt.ylabel('kg/m3')
plt.savefig('conc1.png')
#plt.figure(1)
plt.show()

plt.figure(2)

plt.plot(time,var1,'r-')
plt.show()