text:
"""
Module colormap for creating custom color maps.  For example...
  >>> from clawpack.visclaw import colormaps
  >>> mycmap = colormaps.make_colormap({0:'r', 1.:'b'})  # red to blue
  >>> colormaps.showcolors(mycmap)   # displays resulting colormap

Note that many colormaps are also defined in matplotlib and can be set by
  >>> from matplotlib import cm
  >>> mycmap = cm.get_cmap('Greens')
for example, to get colors ranging from white to green.
See matplotlib._cm for the data defining various maps.
"""

from __future__ import absolute_import
from __future__ import print_function
import numpy
import matplotlib.colors as colors
import matplotlib.pyplot as plt
from six.moves import range

#-------------------------
def make_colormap(color_list):
#-------------------------
    """
    Define a new color map based on values specified in the dictionary
    colors, where colors[z] is the color that value z should be mapped to,
    with linear interpolation between the given values of z.

    The z values (dictionary keys) are real numbers and the values
    colors[z] can be either an RGB list, e.g. [1,0,0] for red, or an
    html hex string, e.g. "#ff0000" for red.

    Optionally, an alpha channel can be added to indicate levels of transparency.
    In this case, colors should have a fourth argument, a value between 0 and 1,
    where zero is transparent and 1 is opaque. Ex.

              blue_semi_transparent = [0.0, 0.0, 1.0, 0.5]

    """

    z = numpy.sort(list(color_list.keys()))
    n = len(z)
    z1 = min(z)
    zn = max(z)
    x0 = (z - z1) / (zn - z1)

    CC = colors.ColorConverter()
    R = []
    G = []
    B = []
    A = []
    for i in range(n):
        #i'th color at level z[i]:
        Ci = color_list[z[i]]
        if type(Ci) == str:
            # a hex string of form '#ff0000' for example (for red)
            RGBA = CC.to_rgba(Ci,1.0)
        else:
            # assume it's an RGB (or RGBA) tuple already:
            if (len(Ci) == 3):
                Ci = numpy.concatenate((Ci,[1.0]))  # Add alpha channel
            RGBA = Ci

        R.append(RGBA[0])
        G.append(RGBA[1])
        B.append(RGBA[2])
        A.append(RGBA[3])

    cmap_dict = {}
    cmap_dict['red'] = [(x0[i],R[i],R[i]) for i in range(len(R))]
    cmap_dict['green'] = [(x0[i],G[i],G[i]) for i in range(len(G))]
    cmap_dict['blue'] = [(x0[i],B[i],B[i]) for i in range(len(B))]
    cmap_dict['alpha'] = [(x0[i],A[i],A[i]) for i in range(len(A))]
    mymap = colors.LinearSegmentedColormap('mymap',cmap_dict)
    return mymap

def add_colormaps(colormaps, data_limits=[0.0,1.0], data_break=0.5,
                             colormap_name="JohnDoe"):
    r"""Concatenate colormaps in *colormaps* list and return Normalize object.

    Appends the colormaps in the list *colormaps* adjusting the mapping to the
    colormap such that it maps the data space limits *data_limits* and puts the
    break in the colormaps at *data_break* which is again in data space.  The
    argument *colormap_name* labels the colormap and is passed to the init
    method of `matplotlib.colors.LinearSegmentedColormap`.

    Originally contributed by Damon McDougall

    returns `matplotlib.colors.LinearSegmentedColormap`,
            `matplotlib.colors.Normalize`

    Example
    -------
    This example takes two colormaps whose data ranges from [-1.0, 5.0] where
    the break in the colormaps occurs at 1.0.

    >>> import matplotlib.pyplot as plt
    >>> import clawpack.visclaw.colormaps as colormaps
    >>> import numpy
    >>> cmaps = (plt.get_cmap("BuGn_r"), plt.get_cmap("Blues_r"))
    >>> new_cmap, norm = colormaps.add_colormaps(cmaps, data_limits=[-1.0, 5.0],
        data_break=1.0)
    >>> x = numpy.linspace(-1,1,100)
    >>> y = x
    >>> X, Y = numpy.meshgrid(x, y)
    >>> fig = plt.figure()
    >>> axes = fig.add_subplot(1,1,1)
    >>> plot = axes.pcolor(X, Y, 3.0 * X + 2.0, cmap=new_cmap, norm=norm)
    >>> fig.colorbar(plot)
    >>> plt.show()

    """

    lhs_dict = colormaps[0]._segmentdata
    rhs_dict = colormaps[1]._segmentdata
    new_dict = dict(red=[], green=[], blue=[], alpha=[])

    # Add first colorbar
    for key in rhs_dict:
        val_list = rhs_dict[key]
        for val in val_list:
            new_dict[key].append((val[0] * 0.5, val[1], val[2]))

    if 'alpha' not in list(rhs_dict.keys()):
        new_dict['alpha'].append((0.0,1.0,1.0))

    # Add second colorbar
    for key in lhs_dict:
        val_list = lhs_dict[key]
        for val in val_list:
            new_dict[key].append(((val[0] + 1.0) * 0.5, val[1], val[2]))

    if 'alpha' not in list(lhs_dict.keys()):
        new_dict['alpha'].append((1.0,1.0,1.0))

    N = 256
    gamma = 1.0

    cmap = colors.LinearSegmentedColormap(colormap_name, new_dict, N, gamma)

    # Compute new norm object
    bounds = numpy.empty(N)
    bounds[:int(N / 2)] = numpy.linspace(data_limits[0], data_break, int(N / 2))
    bounds[int(N / 2):] = numpy.linspace(data_break, data_limits[1],
                                                             int(N / 2) + N % 2)
    norm = colors.BoundaryNorm(boundaries=bounds, ncolors=N)

    return cmap, norm

def showcolors(cmap):
    #from scitools.easyviz.matplotlib_ import colorbar, clf, axes, linspace,\
                 #pcolor, meshgrid, show, colormap
    plt.clf()
    x = numpy.linspace(0,1,21)
    X,Y = numpy.meshgrid(x,x)
    plt.pcolor(X,Y,0.5*(X+Y), cmap=cmap, edgecolors='k')
    plt.axis('equal')
    plt.colorbar()
    plt.title('Plot of x+y using colormap')

def schlieren_colormap(color=[0,0,0]):
    """
    For Schlieren plots:
    """
    if color=='k': color = [0,0,0]
    if color=='r': color = [1,0,0]
    if color=='b': color = [0,0,1]
    if color=='g': color = [0,0.5,0]
    color = numpy.array([1,1,1]) - numpy.array(color)
    s  = numpy.linspace(0,1,20)
    colors = {}
    for key in s:
        colors[key] = numpy.array([1,1,1]) - key**0.1 * color
    schlieren_colors = make_colormap(colors)
    return schlieren_colors

# -----------------------------------------------------------------
# Some useful colormaps follow...
# There are also many colormaps in matplotlib.cm

all_white = make_colormap({0.:'w', 1.:'w'})
all_light_red = make_colormap({0.:'#ffdddd', 1.:'#ffdddd'})
all_light_blue = make_colormap({0.:'#ddddff', 1.:'#ddddff'})
all_light_green = make_colormap({0.:'#ddffdd', 1.:'#ddffdd'})
all_light_yellow = make_colormap({0.:'#ffffdd', 1.:'#ffffdd'})

red_white_blue = make_colormap({0.:'r', 0.5:'w', 1.:'b'})
blue_white_red = make_colormap({0.:'b', 0.5:'w', 1.:'r'})
red_yellow_blue = make_colormap({0.:'r', 0.5:'#ffff00', 1.:'b'})
blue_yellow_red = make_colormap({0.:'b', 0.5:'#ffff00', 1.:'r'})
yellow_red_blue = make_colormap({0.:'#ffff00', 0.5:'r', 1.:'b'})
white_red = make_colormap({0.:'w', 1.:'r'})
white_blue = make_colormap({0.:'w', 1.:'b'})

schlieren_grays = schlieren_colormap('k')
schlieren_reds = schlieren_colormap('r')
schlieren_blues = schlieren_colormap('b')
schlieren_greens = schlieren_colormap('g')

#-------------------------------
def make_amrcolors(nlevels=4):
#-------------------------------
    """
    Make lists of colors useful for distinguishing different patches when
    plotting AMR results.

    INPUT::
       nlevels: maximum number of AMR levels expected.
    OUTPUT::
       (linecolors, bgcolors)
       linecolors = list of nlevels colors for patch lines, contour lines
       bgcolors = list of nlevels pale colors for patch background
    """

    # For 4 or less levels:
    linecolors = ['k', 'b', 'r', 'g']
    # Set bgcolors to white, then light shades of blue, red, green:
    bgcolors = ['#ffffff','#ddddff','#ffdddd','#ddffdd']
    # Set bgcolors to light shades of yellow, blue, red, green:
    #bgcolors = ['#ffffdd','#ddddff','#ffdddd','#ddffdd']

    if nlevels > 4:
        linecolors = 4*linecolors  # now has length 16
        bgcolors = 4*bgcolors
    if nlevels <= 16:
        linecolors = linecolors[:nlevels]
        bgcolors = bgcolors[:nlevels]
    else:
        print("*** Warning, suggest nlevels <= 16")

    return (linecolors, bgcolors)

max_stars_repo_name: BrisaDavis/visclaw

max_issues_repo_name: BrisaDavis/visclaw

max_forks_repo_name: BrisaDavis/visclaw


text:
#!/usr/bin/env python
#
# These functions were used to evaluate the inter-rater reliability of the lesion segmentation in the study:
# Eden et al., "Spatial distribution of multiple sclerosis lesions in the cervical spinal cord", manuscript under revision (submitted to Brain, 2018).
#

import numpy as np
from scipy import ndimage
import spinalcordtoolbox.image as msct_image

def compute_lesion_wise_metrics(mask_rater, mask_consensus, threshold_overlap=0.5):
    '''
    This function compute the lesion-wise sensitivity and precision
    between the mask of a rater and the consensus reading mask.

    Input:
        - mask_rater [numpy.array]: manual segmentation mask of one rater
        - mask_rater [numpy.array]: consensus reading mask obtained considering all raters' masks
        - threshold_overlap [float]: value between 0. and 1.
            A candidate lesion was considered as correctly detected (i.e. true positive)
            when the rater segmentation connected-voxels overlapped with more than
            [threshold_overlap] of the consensus reading segmentation voxels, otherwise
            it was considered as incorrectly detected (i.e. false positive). If a lesion
            of the consensus reading had an insufficient overlap (<[threshold_overlap])
            with the rater segmentation voxels, then we defined it as not-detected
            (i.e. false negative).

    Output:
        - sensitivity [float]: value between 0. and 1., defined as the ratio of true positive lesions
            by the sum of true positive and false negative lesions.
        - precision [float]: value between 0. and 1., defined as the ratio of true positive lesions
            by the sum of true and false positive lesions.
    '''
    # label each individual lesion of both input masks
    labels, nb_les = {}, {}
    labels['rater'], nb_les['rater'] = ndimage.label(mask_rater)
    labels['consensus'], nb_les['consensus'] = ndimage.label(mask_consensus)

    # create a matrix (1xnb_les['rater']), filled with ones
    found_rater = np.ones(nb_les['rater'], np.int16)

    # initialize the number of true positive, false positive, and false negative
    tp, fp, fn = 0, 0, 0

    # loop across the lesions of the consensus mask
    for i in range(1, nb_les['consensus'] + 1):
        # measure the voxel size of the current lesion
        lesion_size = np.sum(mask_consensus[labels['consensus'] == i])

        # extract the labels of the rater lesions
        # which overlap with the current lesion from the consensus mask
        rater_lesions_list = np.unique(labels['rater'][labels['consensus'] == i])

        # measure the voxel size of the overlap between the rater segmentation
        # and the current lesion from the consensus mask
        nb_overlap = mask_rater[labels['consensus'] == i].sum()

        if nb_overlap >= threshold_overlap * lesion_size:  # if the overlap is sufficient
            tp += 1
            for rater_lesion in rater_lesions_list:
                if rater_lesion != 0:
                    found_rater[rater_lesion - 1] = 0  # this rater lesion is not a false positive
        else:  # if the overlap is unsufficient
            fn += 1

    for i in range(1, nb_les['rater'] + 1):
        if found_rater[i - 1] == 1:  # if this rater lesion did not overlap with the consensus mask
            fp += 1

    sensitivity, precision = None, None
    if tp + fn != 0:
        sensitivity = tp * 1. / (tp + fn)
    if tp + fp != 0:
        precision = tp * 1. / (tp + fp)

    return sensitivity, precision

def compute_dice_coefficient(mask_rater, mask_consensus):
    '''
    This function compute the Dice coefficient
    between the mask of a rater and the consensus reading mask.

    Input:
        - mask_rater [numpy.array]: manual segmentation mask of one rater
        - mask_rater [numpy.array]: consensus reading mask obtained considering all raters' masks

    Output:
        - dice [float]: value between 0. and 1., defined as in "Measures of the Amount of Ecologic
        Association Between Species", Lee R. Dice (1945).
    '''
    mask_rater = mask_rater.astype(np.bool)
    mask_consensus = mask_consensus.astype(np.bool)

    intersection = np.count_nonzero(mask_rater & mask_consensus)

    size_i1 = np.count_nonzero(mask_rater)
    size_i2 = np.count_nonzero(mask_consensus)

    try:
        dice = 2. * intersection / float(size_i1 + size_i2)
    except ZeroDivisionError:
        dice = None

    return dice

def generate_consensus_reading(fname_raterMasks_lst, threshold=0.5, fname_out='./consensus_reading.nii.gz'):
    '''
    This function generate a consensus reading mask from several segmentation mask of the same raw image.
    In the generated mask, 1 indicates that this voxel is part of the consensus reading, 0 otherwise.

    Input:
        - fname_raterMasks_lst [list]: list of nifti binary mask filenames.
            Note that they are suspected to all have the same size,
            since they are all the mask of the same raw image.
        - threshold [float]: value between 0. and 1.
            A voxel is labelled 1 in the output consensus mask
            if [threshold] of the raters segmented this voxel as lesion.
        - fname_out [string]: filename of the consensus reading mask generated.
    '''
    # compute the number of agreement needed to consider a voxel as part of the consensus reading mask
    threshold_n = int(threshold * len(fname_raterMasks_lst))

    # Sum all the rater segmentations
    data_sum = np.sum(np.asarray([msct_image.Image(fname_raterMask).data for fname_raterMask in fname_raterMasks_lst]), axis=0)

    # Threshold the resulting array according to the threshold
    data_thresh = np.zeros(data_sum.shape)
    data_thresh[np.where(data_sum >= threshold_n)] = 1

    # Save the resulting array as Image in the related ofolder
    im_consensus = msct_image.zeros_like(msct_image.Image(fname_raterMasks_lst[0]), dtype=np.uint8)
    im_consensus.data = data_thresh
    im_consensus.save(fname_out)

ext: py

max_stars_repo_name: neuropoly/lesion-mapping

max_issues_repo_name: neuropoly/lesion-mapping

max_forks_repo_name: neuropoly/lesion-mapping

reason: import numpy,from scipy


text:
import matplotlib.pyplot as plt
import numpy as np
from keras.layers import Conv2D, MaxPool2D, Dense, Flatten, Dropout
from keras.models import Sequential
from keras.utils import to_categorical
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split

from data_loading import signs

# Load training data from source
data, labels = signs.training_data()
classes = 43  # Total number of traffic sign classes

# Splitting data into training and test data. Does shuffle before split in order to increase randomness in the data.
# Specifying the random state allows for reproducibility.
x_train, X_test, y_train, y_test = train_test_split(data, labels, test_size=0.2, random_state=42)

# Convert labels into one hot encoding
y_train = to_categorical(y_train, classes)
y_test = to_categorical(y_test, classes)

# Function to build and compile CNN models, varying the dropout value
def build_models_diff_dropouts():
    models = []  # Array utilized for storing compiled CNN models
    dropout_percentages = [.15, .25, .5]  # Array containing various dropout values

    # Build and compile CNN models with different values for the dropout layer
    for percentage in dropout_percentages:
        # Build CNN models
        model = Sequential()
        model.add(Conv2D(filters=8, kernel_size=(3, 3), activation='relu', input_shape=x_train.shape[1:]))
        model.add(MaxPool2D(pool_size=(2, 2)))
        model.add(Dropout(percentage))
        model.add(Flatten())
        model.add(Dense(classes, activation='softmax'))
        # Compile CNN
        model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
        models.append(model)

    return models, dropout_percentages

def train_models(models, percentages):
    """
    Function to train CNN models, where each conv2d layer has varying activation functions
    :param percentages: object - Array consisting of various dropout values.
    :param models: object - Array consisting of CNN models.
    """
    epochs = 16

    # Load test data from source
    test_data, test_labels = signs.test_data()
    histories = []

    for i, model in enumerate(models):
        # Fit the CNN model using training data and labels.
        history = model.fit(x_train, y_train, batch_size=32, epochs=epochs, validation_data=(X_test, y_test))
        pred = np.argmax(model.predict(test_data), axis=-1)  # Select highest probability for all classifications
        histories.append(history)  # Append new history stats to the histories array

        # Check accuracy with test data
        print(accuracy_score(test_labels, pred))

    # Create figure 0
    plt.figure(0)
    for i, hist in enumerate(histories):
        # plotting graphs of loss from the various dropout values
        plt.plot(hist.history['accuracy'], label='training accuracy - Dropout: {}'.format(percentages[i]))

    plt.title('Accuracy')
    plt.xlabel('epochs')
    plt.ylabel('accuracy')
    plt.legend()
    plt.show()

    # Create figure 1
    plt.figure(1)
    for i, hist in enumerate(histories):
        # plotting graphs of loss from the various dropout values
        plt.plot(hist.history['loss'], label='training loss - Dropout: {}'.format(percentages[i]))
    plt.title('Loss')
    plt.xlabel('epochs')
    plt.ylabel('loss')
    plt.legend()
    plt.show()

# Build CNN models then train and display data from them.
models, drop_percentages = build_models_diff_dropouts()
train_models(models, drop_percentages)

reason: import numpy


text:
"""Calculate image height for thin lens from focal length and incident angle."""

# stdlib
import logging
import logging.config
from pathlib import Path

# external
import numpy as np

# project
from payload_designer.components import lenses
from payload_designer.libs import plotlib

# region path config
filename = Path(__file__).stem
output_path = Path(f"output/{filename}")
log_path = Path(f"logs/{filename}")
# endregion

# region logging config
log_path.mkdir(parents=True, exist_ok=True)
logging.config.fileConfig(fname="log.conf", defaults={"path": log_path})
LOG = logging.getLogger(__name__)
# endregion

# region parameter config
a_in = np.linspace(start=0, stop=10, num=100)  # [°]
f = np.linspace(start=1, stop=30, num=100)  # [mm]

# endregion

if __name__ == "__main__":
    # region component instantiation
    lens = lenses.ThinLens(f=f, a_in=a_in)
    # endregion

    # region pipeline
    h = lens.get_image_height()
    # endregion

    # region plots
    plotlib.surface(x=a_in, y=f, z=h)
    # endregion

max_stars_repo_name: spacesys-finch/payload-designer

max_issues_repo_name: spacesys-finch/payload-designer

max_forks_repo_name: spacesys-finch/payload-designer


text:
# Copyright (C) 2017 William M. Jacobs

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or (at
# your option) any later version.

# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
import sys

import math, argparse
import numpy as np
import networkx as nx
from itertools import combinations

from polymer import Polymer
from substructures import find_substructures, calc_ghost_edges, \
    calc_independent_configs, get_polymer_segments, get_polymer_from_config, get_expected_segments, \
    get_config_vertices
from fe import avg_contacts, subpolymer_epsilon, minimize_fe, segment_end_probability, find_loops

def calc_all_configs(polymer, substructures, epsilon, mu, ndim, kuhnlength, verbose=False):
    configs = calc_independent_configs(polymer, substructures)
    polymer_segments = get_polymer_segments(polymer, substructures)
    distances = polymer.distances()
    results = {}
    for L in range(polymer.number_of_residues()):
        allowed_substructures = elongation_substructures(substructures, L)
        config = tuple(sorted(c for c in allowed_substructures))
        if set(config) in configs and config not in results:
            subpolymer = get_polymer_from_config(polymer, polymer_segments, \
                                                 substructures, config, kuhnlength)
            subpolymer_segments = get_expected_segments(subpolymer, polymer_segments, \
                                                        substructures, config)
            epsilon_config = subpolymer_epsilon(epsilon, subpolymer.nodes())
            rho_config, fe_config, warning = minimize_fe(subpolymer, epsilon_config, distances, \
                                                         subpolymer_segments, \
                                                         mu=mu, d=ndim, kuhnlength=kuhnlength)
            avg_seg_end_prob = segment_end_probability(epsilon_config, rho_config, subpolymer_segments)
            results[config] = {'vertices' : list(subpolymer.nodes()),
                               'rho' : rho_config,
                               'fe' : fe_config,
                               'avg_seg_end_prob' : avg_seg_end_prob,
                               'warning' : warning}
    return results

def elongation_substructures(substructures, L):
    allowed_substructures = [i for i in range(len(substructures))
                             if all(v <= L for b in substructures[i] for v in b)]
    return allowed_substructures

def get_allowed_partial_substructures(substructures, L):
    allowed_substructures = [i for i in range(len(substructures))
                             if any(all(v <= L for v in b) for b in substructures[i])]
    return allowed_substructures

def get_subpolymer(polymer, vertices):
    return Polymer([edge for edge in polymer.edges() if all(u in vertices for u in edge)], \
                   nresidues=polymer.number_of_residues())

def calc_fe_L(polymer, distances, substructures, epsilon, mu, ndim, kuhnlength):
    polymer_segments = get_polymer_segments(polymer, substructures)
    def get_allowed_segments(vertices):
        return {c : [[u for u in seg if u in vertices] for seg in polymer_segments[c]] \
                for c in polymer_segments if all(any(u in vertices for u in seg) \
                                                 for seg in polymer_segments[c])}
    results = {}
    for L in range(min(polymer.nodes()), polymer.number_of_residues() + 1):
        allowed_substructures = get_allowed_partial_substructures(substructures, L)
        allowed_vertices = [u for u in polymer.nodes() if u <= L and any(v <= L for v in polymer[u])]
        allowed_segments = get_allowed_segments(allowed_vertices)
        polymer_L = get_subpolymer(polymer, allowed_vertices)
        epsilon_L = subpolymer_epsilon(epsilon, allowed_vertices)
        rho_L, fe_L, warning = minimize_fe(polymer_L, epsilon_L, distances, allowed_segments, \
                                           mu=mu, d=ndim, kuhnlength=kuhnlength)
        avg_seg_end_prob = segment_end_probability(epsilon_L, rho_L, allowed_segments)
        results[L] = {'vertices' : allowed_vertices,
                      'substructures' : allowed_substructures,
                      'rho' : rho_L,
                      'fe' : fe_L,
                      'loops' : find_loops(polymer_L, allowed_segments),
                      'avg_seg_end_prob' : avg_seg_end_prob,
                      'warning' : warning}
    return results

def calc_fe_Ls(polymer, distances, substructures, epsilon, mu, ndim, kuhnlength):
    polymer_segments = get_polymer_segments(polymer, substructures)
    def get_allowed_segments(vertices):
        return {c : [[u for u in seg if u in vertices] for seg in polymer_segments[c]] \
                for c in polymer_segments if all(any(u in vertices for u in seg) \
                                                 for seg in polymer_segments[c])}
    results = {}
    for L in range(min(polymer.nodes()), polymer.number_of_residues() + 1):
        allowed_substructures = get_allowed_partial_substructures(substructures, L)
        ustart = {}
        for Ls in [0] + sorted(min(seg[0] for seg in polymer_segments[c]) \
                               for c in allowed_substructures):
            allowed_vertices = tuple(sorted(u for u in polymer.nodes() if (u <= L and u >= Ls) \
                                            and any(v <= L and v >= Ls for v in polymer[u])))
            if allowed_vertices not in ustart:
                ustart[allowed_vertices] = Ls
        for allowed_vertices,Ls in ustart.items():
            allowed_segments = get_allowed_segments(allowed_vertices)
            polymer_Ls = get_subpolymer(polymer, allowed_vertices)
            epsilon_Ls = subpolymer_epsilon(epsilon, allowed_vertices)
            rho_Ls, fe_Ls, warning = minimize_fe(polymer_Ls, epsilon_Ls, distances, allowed_segments, \
                                                 mu=mu, d=ndim, kuhnlength=kuhnlength)
            avg_seg_end_prob = segment_end_probability(epsilon_Ls, rho_Ls, allowed_segments)
            results[(Ls, L)] = {'vertices' : allowed_vertices,
                                'substructures' : allowed_substructures,
                                'rho' : rho_Ls,
                                'fe' : fe_Ls,
                                'loops' : find_loops(polymer_Ls, allowed_segments),
                                'avg_seg_end_prob' : avg_seg_end_prob,
                                'warning' : warning}
    return results

def min_fe_L_Ls(results_Ls, multiregion=True):
    N = max(k[1] for k in results_Ls)
    results = {}
    stable_regions = []
    def update_stable_regions(Ls, L):
        todelete = []
        for i in range(len(stable_regions)):
            if Ls <= stable_regions[i][0] and L >= stable_regions[i][1]:
                todelete.append(i)
        for i in todelete[::-1]:
            del stable_regions[i]
        stable_regions.append((Ls, L))
        stable_regions.sort()
    def min_fe_stable_regions():
        G = nx.Graph()
        for i in range(len(stable_regions)):
            G.add_node(stable_regions[i])
            for j in range(len(stable_regions)):
                if stable_regions[j][0] >= stable_regions[i][0] \
                   and stable_regions[j][0] <= stable_regions[i][1]:
                    G.add_edge(stable_regions[i], stable_regions[j])
        ccs = list(nx.connected_components(G))
        selected = []
        for cc in ccs:
            if len(cc) == 1:
                for y in cc:
                    selected.append(y)
            else:
                ccmin = (None, 0)
                for n in range(1, len(cc)):
                    for x in combinations(cc, n):
                        if not any(x[j][0] >= x[i][0] and x[j][0] <= x[i][1] \
                                   for i in range(len(x)) for j in range(len(x)) if i != j):
                            fe = sum(results_Ls[x[i]]['fe'] for i in range(len(x)))
                            if fe < ccmin[1]:
                                ccmin = (x, fe)
                for y in ccmin[0]:
                    selected.append(y)
        return selected
    def combine_region_results(regions):
        combined_result = {}
        combined_result['vertices'] = sorted(set(u for r in regions for u in results_Ls[r]['vertices']))
        combined_result['substructures'] = sorted(set(c for r in regions \
                                                      for c in results_Ls[r]['substructures']))
        combined_result['rho'] = np.sum(results_Ls[r]['rho'] for r in regions)
        combined_result['fe'] = sum(results_Ls[r]['fe'] for r in regions)
        combined_result['loops'] = sorted(l for r in regions for l in results_Ls[r]['loops'])
        combined_result['avg_seg_end_prob'] = {k : v for r in regions \
                                               for k,v in results_Ls[r]['avg_seg_end_prob'].items()}
        combined_result['warning'] = any(results_Ls[r]['warning'] for r in regions)
        combined_result['Ls'] = max(r[0] for r in regions)
        combined_result['regions'] = sorted(regions)
        return combined_result
    for L in range(N + 1):
        keys = set(k for k in results_Ls if k[1] == L)
        if len(keys) > 0:
            minLs = min((k for k in keys), key=lambda y: results_Ls[y]['fe'])
            if multiregion:
                if results_Ls[minLs]['fe'] < 0:
                    update_stable_regions(*minLs)
                regions = min_fe_stable_regions()
                if len(regions) > 0:
                    for i in range(len(regions), 0, -1):
                        if minLs[0] <= regions[i - 1][1]:
                            del regions[i - 1]
                regions.append(minLs)
                valid_regions = [r for r in regions if results_Ls[r]['rho'].sum() >= 1. or r[1] == L]
                results[L] = combine_region_results(valid_regions)
            else:
                results[L] = results_Ls[minLs]
                results[L]['Ls'] = minLs[0]
    return results

if __name__ == '__main__':

    np.seterr(divide='ignore')

    parser = argparse.ArgumentParser()
    parser.add_argument('--gene', type=str, help="gene name")
    parser.add_argument('--polymer', type=str, help="path to input absolute-energy polymer file")
    parser.add_argument('--ndim', type=float, default=3., help="number of spatial dimensions [3]")
    parser.add_argument('--mu', type=float, default=-2., help="residue ordering penalty / kT [-2.]")
    parser.add_argument('--kuhnlength', type=float, default=2., metavar='B', help="Kuhn length [2]")
    ###
    parser.add_argument('--output', type=str, help="path to output the file")
    ###
    clargs = parser.parse_args()

###
    #print("calc_elongation...")
    #sys.exit()
###

    with open(clargs.polymer, 'r') as f:
        nresidues = int(f.readline().split()[-1])
        energies = {(int(line.split()[0]), int(line.split()[1])) : -float(line.split()[2]) \
                    for line in f}
        for edge in list(energies.keys()):
            energies[(edge[1], edge[0])] = energies[edge]
    polymer = Polymer([edge for edge in energies.keys()], nresidues=nresidues)
    distances = polymer.distances()

    print("# nresidues =", polymer.number_of_residues())

    for edge in calc_ghost_edges(polymer):
        polymer.add_edge(*edge)
    epsilon = np.zeros((polymer.number_of_residues(), polymer.number_of_residues()))
    for u,v in energies.keys():
        epsilon[u,v] = epsilon[v,u] = energies[(u,v)]

    native_L = polymer.number_of_residues()
    substructures = find_substructures(polymer)
    results_Ls = calc_fe_Ls(polymer, distances, substructures, epsilon, \
                            clargs.mu, clargs.ndim, clargs.kuhnlength)
    results = min_fe_L_Ls(results_Ls)

    print("# Writing %s_elongation_profile.dat" % clargs.gene)
    ###
    output_part1 = str(clargs.output)
    output_part2 = str('%s_elongation_profile.dat' % clargs.gene)
    output_sum = output_part1 + '/' + output_part2
    with open(output_sum, 'w') as f:
    ###
        f.write("L\tLs\tnvertices\tF_L\tF_L_constrained\trho_sum\tnloops\tregions\n")
        minfe = 0
        for L in results:
            Ls = results[L].get('Ls', 0)
            minfe = min(minfe, results[L]['fe'])
            regions = results[L].get('regions', [(Ls, L)])
            f.write("%3d\t%3d\t%3d\t%12.6g\t%12.6g\t%12.6g\t%2d\t%s\n" % \
                    (L, Ls, len(results[L]['vertices']), minfe, results[L]['fe'], \
                     results[L]['rho'].sum(), len(results[L]['loops']), str(regions)))

max_stars_repo_name: MichMourra/FariasLab

max_issues_repo_name: MichMourra/FariasLab

max_forks_repo_name: MichMourra/FariasLab

reason: import numpy,import networkx


text:
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright (c) 2014, Vispy Development Team. All Rights Reserved.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.
# -----------------------------------------------------------------------------
# Author:   Nicolas P .Rougier
# Date:     06/03/2014
# Abstract: GPU computing using the framebuffer
# Keywords: framebuffer, GPU computing, cellular automata
# -----------------------------------------------------------------------------

import numpy as np
from vispy.gloo import (Program, FrameBuffer, DepthBuffer, clear, set_viewport,
                        set_state)
from vispy import app

render_vertex = """
attribute vec2 position;
attribute vec2 texcoord;
varying vec2 v_texcoord;
void main()
{
    gl_Position = vec4(position, 0.0, 1.0);
    v_texcoord = texcoord;
}
"""

render_fragment = """
uniform int pingpong;
uniform sampler2D texture;
varying vec2 v_texcoord;
void main()
{
    float v;
    v = texture2D(texture, v_texcoord)[pingpong];
    gl_FragColor = vec4(1.0-v, 1.0-v, 1.0-v, 1.0);
}
"""

compute_vertex = """
attribute vec2 position;
attribute vec2 texcoord;
varying vec2 v_texcoord;
void main()
{
    gl_Position = vec4(position, 0.0, 1.0);
    v_texcoord = texcoord;
}
"""

compute_fragment = """
uniform int pingpong;
uniform sampler2D texture;
uniform float dx;          // horizontal distance between texels
uniform float dy;          // vertical distance between texels
varying vec2 v_texcoord;
void main(void)
{
    vec2  p = v_texcoord;
    float old_state, new_state, count;

    old_state = texture2D(texture, p)[pingpong];
    count = texture2D(texture, p + vec2(-dx,-dy))[pingpong]
            + texture2D(texture, p + vec2( dx,-dy))[pingpong]
            + texture2D(texture, p + vec2(-dx, dy))[pingpong]
            + texture2D(texture, p + vec2( dx, dy))[pingpong]
            + texture2D(texture, p + vec2(-dx, 0.0))[pingpong]
            + texture2D(texture, p + vec2( dx, 0.0))[pingpong]
            + texture2D(texture, p + vec2(0.0,-dy))[pingpong]
            + texture2D(texture, p + vec2(0.0, dy))[pingpong];

    new_state = old_state;
    if( old_state > 0.5 ) {
        // Any live cell with fewer than two live neighbours dies
        // as if caused by under-population.
        if( count  < 1.5 )
            new_state = 0.0;

        // Any live cell with two or three live neighbours
        // lives on to the next generation.

        // Any live cell with more than three live neighbours dies,
        //  as if by overcrowding.
        else if( count > 3.5 )
            new_state = 0.0;
    } else {
        // Any dead cell with exactly three live neighbours becomes
        //  a live cell, as if by reproduction.
       if( (count > 2.5) && (count < 3.5) )
           new_state = 1.0;
    }

    if( pingpong == 0) {
        gl_FragColor[1] = new_state;
        gl_FragColor[0] = old_state;
    } else {
        gl_FragColor[1] = old_state;
        gl_FragColor[0] = new_state;
    }
}
"""

class Canvas(app.Canvas):

    def __init__(self):
        app.Canvas.__init__(self, title="Conway game of life",
                            size=(512, 512), close_keys='escape')

    def on_initialize(self, event):
        # Build programs
        # --------------
        self.comp_size = (512, 512)
        size = self.comp_size + (4,)
        Z = np.zeros(size, dtype=np.float32)
        Z[...] = np.random.randint(0, 2, size)
        Z[:256, :256, :] = 0
        gun = """
        ........................O...........
        ......................O.O...........
        ............OO......OO............OO
        ...........O...O....OO............OO
        OO........O.....O...OO..............
        OO........O...O.OO....O.O...........
        ..........O.....O.......O...........
        ...........O...O....................
        ............OO......................"""
        x, y = 0, 0
        for i in range(len(gun)):
            if gun[i] == '\n':
                y += 1
                x = 0
            elif gun[i] == 'O':
                Z[y, x] = 1
            x += 1

        self.pingpong = 1
        self.compute = Program(compute_vertex, compute_fragment, 4)
        self.compute["texture"] = Z
        self.compute["position"] = [(-1, -1), (-1, +1), (+1, -1), (+1, +1)]
        self.compute["texcoord"] = [(0, 0), (0, 1), (1, 0), (1, 1)]
        self.compute['dx'] = 1.0 / size[1]
        self.compute['dy'] = 1.0 / size[0]
        self.compute['pingpong'] = self.pingpong

        self.render = Program(render_vertex, render_fragment, 4)
        self.render["position"] = [(-1, -1), (-1, +1), (+1, -1), (+1, +1)]
        self.render["texcoord"] = [(0, 0), (0, 1), (1, 0), (1, 1)]
        self.render["texture"] = self.compute["texture"]
        self.render['pingpong'] = self.pingpong

        self.fbo = FrameBuffer(self.compute["texture"],
                               DepthBuffer(self.comp_size))
        set_state(depth_test=False, clear_color='black')

    def on_draw(self, event):
        with self.fbo:
            set_viewport(0, 0, *self.comp_size)
            self.compute["texture"].interpolation = 'nearest'
            self.compute.draw('triangle_strip')
        clear()
        set_viewport(0, 0, *self.size)
        self.render["texture"].interpolation = 'linear'
        self.render.draw('triangle_strip')
        self.pingpong = 1 - self.pingpong
        self.compute["pingpong"] = self.pingpong
        self.render["pingpong"] = self.pingpong
        self.update()

    def on_reshape(self, event):
        set_viewport(0, 0, *event.size)

if __name__ == '__main__':
    canvas = Canvas()
    canvas.show()
    app.run()

max_stars_repo_name: shjoshi/vispy

max_issues_repo_name: shjoshi/vispy

max_forks_repo_name: shjoshi/vispy


text:
###############################################################################
# Raster Folder Extents
# Benjamin P. Stewart, March 2017
# Purpose: extract a zipfile of delivered imagery and extract imagery extents
###############################################################################

import os, sys, zipfile, json, shutil, logging
import rasterio, pyproj, geohash

import xml.etree.ElementTree as ET
import geopandas as gpd
import pandas as pd
import numpy as np

from datetime import datetime
from functools import partial
from shapely.geometry import Polygon, LineString, Point, box
from PIL import Image
from shapely.ops import transform
from zipfile import ZipFile

Image.MAX_IMAGE_PIXELS = 89478485000000
vendorInformation = {
    "Maxar":{'browseTag':'BROWSE.JPG'},
    'SPOT':{'browseTag':'PREVIEW'},
    'GOST':{'browseTag':''},
    'gbdx_clip':{'browseTag':''},
    'Airbus':{'browseTag':'PREVIEW_*.JPG'}
}
allFileTypes = ["TIF", "tif", 'JP2']

class deliveredImageryFolder(object):

    def __init__(self, inputFolder, outputFolder, adminBoundaries, vendor = '', metadata=''):
        ''' Generate metadata for input images in folder

        Parameters:
            inputFolder (string): folder to process
            sensor (string): sensor to process - used to define where to find important metadata
            adminBoundaries (geopandas geodataframe): file path to shapefile of global bouncaries for finding country of interest
        '''
        self.inputFolder = inputFolder
        self.outputFolder = outputFolder
        if vendor == '':
            self.vendor = self.determineVendor()
        else:
            self.vendor = vendor
        if self.vendor == "gbdx_clip":
            self.process_gbdx_xml()
        if metadata.__class__ == gpd.geodataframe.GeoDataFrame:
            self.allMetadata = metadata

        self.adminBoundaries = adminBoundaries
        self.findImages()
        self.zipFile = os.path.join(outputFolder, "%s.zip" % self.generateFilename())
        self.jsonFile = os.path.join(outputFolder, "%s.json" % self.generateFilename())
        self.thumbnail = os.path.join(outputFolder, "%s.jpg" % self.generateFilename())

    def process_gbdx_xml(self):
        ''' Images downloaded through gbdx task clipping come with an XML that should have the information we need

        RETURNS
        NULL - sets a number of object variables, such as date
        '''
        inFiles = os.listdir(self.inputFolder)
        if "gbdx_clip_log.txt" in inFiles:
            for f in inFiles:
                if f[-4:] == ".XML":
                    xml = ET.parse(os.path.join(self.inputFolder, f))
                    root_element = xml.getroot()
                    self.official_id = root_element.findall('IMD')[0].findall('IMAGE')[0].findall('CATID')[0].text
                    date = root_element.findall('IMD')[0].findall('IMAGE')[0].findall('TLCTIME')[0].text
                    dTime = datetime.strptime(date[:10], "%Y-%m-%d")
                    cDate = dTime.strftime("%Y%m%d")
                    self.Date = cDate

    def determineVendor(self):
        ''' Based on folder and file structure, determine the vendor for other calculations

        RETURNS
        [string] - vendor name matching imageryExtents.vendorInformation
        '''
        inFiles = os.listdir(self.inputFolder)
        if "gbdx_clip_log.txt" in inFiles:
            return("gbdx_clip")
        for f in inFiles:
            if f[-10:] == "BROWSE.JPG":
                return("MAXAR")
            if ("PHR1A" in f) or ("PHR1B" in f):
                return("Airbus")
            if f[:2] == "K5":
                return("SIIS")
            if "SPOT" in f:
                return("SPOT")
            if f[:3] == "BSG":
                return("Blacksky")

    def createJSON(self, pNumber, securityClassification="Official Use Only"):
        if not os.path.exists(self.jsonFile):
            m = self.allMetadata
            try:
                zippedSize = os.path.getsize(self.zipFile)
            except:
                zippedSize = 0
            self.metadataDictionary = {
                "title":"Satellite imagery for %s" % self.countryISO3,
                "iso3":self.countryISO3,
                "location":self.zipFile,
                "zippedSize":zippedSize,
                "resolution":",".join([str(round(x, 3)) for x in np.unique([m['Res'].min(), m['Res'].max()])]),
                "nBands":",".join([str(x) for x in np.unique([m['Bands'].min(), m['Bands'].max()])]),
                "vendor":self.vendor,
                "capture_date":",".join([str(x) for x in np.unique([m['Date'].min(), m['Date'].max()])]),
                "pNumber":pNumber,
                "securityClassification":securityClassification,
                "ImageExtent":self.allMetadata.unary_union.wkt,
                "originalLocation":self.inputFolder
                #"imageTileDetails":self.allMetadata
            }
            with open(self.jsonFile, 'w') as j:
                json.dump(self.metadataDictionary, j)

    def zipData(self):
        #Get a list of all files
        if not os.path.exists(self.zipFile):
            allFiles = []
            for root, dirs, files in os.walk(self.inputFolder):
                for f in files:
                    allFiles.append(os.path.join(root, f))

            with ZipFile(self.zipFile, 'w') as zip:
                for f in allFiles:
                    zip.write(f)
        return(self.zipFile)

    def getDate(self, file):
        ''' Get date of image, depending on Vendor
        '''
        try:
            return(self.Date)
        except:
            file = os.path.basename(file)
            cDate = "YYYYMMDD"
            if self.vendor == "MAXAR":
                try:
                    date = "20%s" % os.path.basename(file)[:7]
                    dTime = datetime.strptime(date, "%Y%b%d")
                    cDate = dTime.strftime("%Y%m%d")
                except:
                    logging.warning("Could not determine date for %s" % file)
            if self.vendor == "SIIS":
                cDate = file.split("_")[1][:8]
            if self.vendor == "Airbus":
                cDate = file.split("_")[3][:8]
            if self.vendor == "SPOT":
                cDate = file.split("_")[3][:8]
            if self.vendor == "Blacksky":
                cDate = file.split("-")[2][:8]
            self.Date = cDate
            return(self.Date)

    def identifyCountries(self):
        try:
            allM = self.allMetadata
        except:
            allM = self.getMetadata()
        inputExtent = allM.unary_union

        country = self.adminBoundaries[self.adminBoundaries.intersects(inputExtent)]
        if country.shape[0] > 1:
            # If more than one country is intersected, select the shape with the highest overlap
            country['OVERLAP'] = country['geometry'].apply(lambda x: x.intersection(inputExtent).area/inputExtent.area)
            country = country.sort_values('OVERLAP', ascending=False).iloc[0]
            self.countryName = country['WB_ADM0_NA']
            self.countryISO3 = country['ISO3']

        else:
            self.countryName = ";".join(country['WB_ADM0_NA'])
            self.countryISO3 = ";".join(country['ISO3'])

    def getMetadata(self):
        ''' Generate metadata dataframe for every image
        '''
        try:
            return(self.allMetadata)
        except:
            allRes = []
            for x in self.allImages:
                curRaster = rasterio.open(x)
                #Extract imagery extent
                if curRaster.crs:
                    crs = curRaster.crs
                    b = curRaster.bounds
                    bbox = box(*b)
                    project = pyproj.Transformer.from_crs(
                        pyproj.CRS(str(crs)),
                        pyproj.CRS('epsg:4326'),
                        always_xy=True).transform
                    res = curRaster.res[0]
                    bbox2 = transform(project, bbox)
                    #Need to transform resolution to a meters measure
                    if curRaster.crs.to_epsg() == 4326:
                        project = partial(
                            pyproj.transform,
                            pyproj.Proj('epsg:4326'), # source coordinate system
                            pyproj.Proj('epsg:3857')) # destination coordinate system
                        l = LineString([Point(0,0), Point(0,res)])
                        l = transform(project, l)
                        res = l.length
                    allRes.append([curRaster.count, round(res, 3),
                                    bbox2, geohash.encode(bbox2.centroid.y, bbox2.centroid.x),
                                    curRaster.shape[0], curRaster.shape[1], self.getDate(x), x])

            #create output metadata dataFrame
            if len(allRes) > 0:
                allMetadata = pd.DataFrame(allRes, columns=["Bands", "Res", "geometry", "geohash", "columns", "rows", "Date","file"])
                allMetadata = gpd.GeoDataFrame(allMetadata, geometry="geometry", crs=pyproj.CRS('epsg:4326'))
                if allMetadata.iloc[0]['Date'] == "YYYYMMDD":
                    try:
                        allMetadata['Date'] = self.Date
                    except:
                        pass
                self.allMetadata = allMetadata
                return(self.allMetadata)
            else:
                raise(ValueError("Folder does not have any valid raster datasets"))

    def generateFilename(self):
        ''' Generate a single filename based on input data
        '''
        try:
            return(self.fileName)
        except:
            try:
                allM = self.allMetadata
            except:
                allM = self.getMetadata()
            self.identifyCountries()
            u = allM.unary_union
            try:
                gHash = geohash.encode(u.centroid.y,u.centroid.x)
            except:
                gHash = "XXXXXXXXX"
            # Generate a filename
            outString = r"{ISO3}_{gHash}_{bands}_{res}_{curDate}".format(
                ISO3 = self.countryISO3,
                gHash = gHash,
                #res = ",".join([str(x) for x in np.unique([allM['Res'].min(), allM['Res'].max()])]),
                res = str(allM['Res'].min()),
                #bands = ",".join([str(x) for x in np.unique([allM['Bands'].min(), allM['Bands'].max()])]),
                bands = str(allM['Bands'].max()),
                #curDate = ",".join([str(x) for x in np.unique([allM['Date'].min(), allM['Date'].max()])])
                curDate = str(allM['Date'].max())
            )
            self.fileName = outString
        return(self.fileName)

    def findImages(self, filetype="TIF"):
        ''' Get list of imagery files in the input folder

        Parameters:
            filetype (string) (options): filetype to look for

        Returns:
            list of images
        '''
        self.allImages = []
        for root, dirs, files in os.walk(self.inputFolder):
            for f in files:
                for filetype in allFileTypes:
                    if f[-len(filetype):] == filetype:
                        # BEN TODO: Should I check if this is an actual raster?
                        self.allImages.append(os.path.join(root, f))

    def generateThumbnails(self, filetype="TIF"):
        ''' Generate thumbnails for all images in folder

        #TODO - needs more testing
        for x in self.allImages:
            outFile = os.path.join(self.thumbnailFolder, os.path.basename(x).replace(filetype, "jpg"))
            if not os.path.exists(outFile):
                try:
                    im = Image.open(x)
                    im.thumbnail((round(im.size[0]/10), round(im.size[1]/10)), Image.ANTIALIAS)
                    im.save(outFile, "JPEG")
                except:
                    logging.warning("Problem generating thumbnail for %s" % os.path.basename(x))
        '''
        try:
            if not os.path.exists(self.thumbnail):
                for root, dirs, files in os.walk(self.inputFolder):
                    for f in files:
                        if self.vendor == "Maxar":
                            #Look for a BROWSE.jpg
                            if vendorInformation[self.vendor]['browseTag'] in f:
                                shutil.copyfile(os.path.join(root, f), self.thumbnail)
                            else:
                                logging.info("could not find thumbnail for %s" % self.inputFolder)
        except:
            logging.warning("Could not find thumbnail for %s" % self.inputFolder)

    def getImageryExtents(self, inFolder):
        '''Get a list of input imagery tiles and generate extent dataframes
        RETURNS [geopandas dataframe] - Each row contains file path and shape extent for every imagery tile
        '''
        try:
            return(self.imageryExtents)
        except:
            allExtents = []
            crs = ''
            for curF in self.allImages:
                try:
                    #Try to open the file as a rasterio object
                    curRaster = rasterio.open(curF)
                    if curRaster.crs:
                        crs = curRaster.crs
                        b = curRaster.bounds
                        bbox = [[b.left, b.bottom],
                                [b.left, b.top],
                                [b.right, b.top],
                                [b.right, b.bottom],
                                [b.left, b.bottom]]
                        allExtents.append([f, Polygon(bbox), curRaster.res[0]])
                except Exception as e:
                    logging.warning("Problem generating extent for %s" % os.path.basename(x))

            if crs != '':
                curDf = gpd.GeoDataFrame(allExtents, crs=crs, columns=["fileName","geometry",'Resolution'])
                if curDf.crs != {'init': 'epsg:4326'}:
                    curDf = curDf.to_crs({'init': 'epsg:4326'})
                curDf['zipFile'] = self.zipFile
                curDf = curDf.set_geometry("geometry")
                self.imageryExtents = curDf
                return curDf
            else:
                return -1

    def valid_metadata(self, metadata):
        ''' Determine if the metadata is sufficient for cataloging

        RETURNS
        [boolean] - if TRUE, metadata should be created
        '''
        goodData = True
        #Check to see if the date field is populated
        for idx, row in metadata.iterrows():
            if row['Date'] == "YYYYMMDD":
                goodData = False
        return(goodData)

class zipFileExtents(object):
    def __init__(self, inputZip, sensor, sourceFolder):
        ''' Generate object for extracting imagery metadata
        INPUT
        inputZip [string] - path to imagery zipFile
        sensor [string] - sensor name
        '''
        self.zipFile = inputZip
        self.sensor = sensor
        self.sourceFolder = sourceFolder

        ###TODO - should this check been done here???
        ### if not os.path.exists(xx.extentsFile):
        #Check if data has been extracted
        if self.checkForSource():
            #Get imagery extents
            xy = self.fileNames
        else:
            IShould = "Raise an error here"

    def unzip(self):
        if not self.checkForSource():
            zFile = zipfile.ZipFile(self.zipFile)
            zFile.extractall(self.sourceFolder)
            return(2)
        else:
            return(1)

    def checkForSource(self):
        ''' Determine that the VENDOR_DISTRIBUTION zipfiles have been properly extracted and processed to SOURCE
                TODO: Integrate the use of an actual log as an input
                    Begin by reading in the catalog, then reading in the ...
        '''
        self.sourceExists = False
        self.getSourceNames()

        #Search the input sourceFolder for the files
        for roots, dirs, files in os.walk(self.sourceFolder):
            for name in files:
                if name in self.fileNames.keys():
                    self.fileNames[name] = True
        if len(self.fileNames.values()) > 0:
            self.sourceExists = (float(sum(self.fileNames.values())) / float(len(self.filePaths))) > 0.50
        else:
            self.sourceExists = False
        return self.sourceExists

    def getSourceNames(self):
        '''Get the filenames from the input zipFile'''
        try:
            return(self.fileNames)
        except:
            curZip = zipfile.ZipFile(self.zipFile)
            self.filePaths = curZip.namelist()
            self.fileNames = {}
            for f in self.filePaths:
                self.fileNames[os.path.basename(f)] = False
            return(self.fileNames)

    def getImageryExtents(self, inFolder):
        '''Get a list of input imagery tiles and generate extent dataframes
        RETURNS [geopandas dataframe] - Each row contains file path and shape extent for every imagery tile
        '''
        try:
            return(self.imageryExtents)
        except:
            if self.checkForSource():
                allExtents = []
                crs = ''
                for dir, subDirList, fileList in os.walk(inFolder):
                    for f in fileList:
                        curF = os.path.join(dir, f)
                        try:
                            #Try to open the file as a rasterio object
                            curRaster = rasterio.open(curF)
                            if curRaster.crs:
                                crs = curRaster.crs
                                b = curRaster.bounds
                                bbox = [[b.left, b.bottom],
                                        [b.left, b.top],
                                        [b.right, b.top],
                                        [b.right, b.bottom],
                                        [b.left, b.bottom]]
                                allExtents.append([f, Polygon(bbox), curRaster.res[0]])

                        except Exception as e:
                            FUBAR = "DO Nothing"

                if crs != '':
                    curDf = gpd.GeoDataFrame(allExtents, crs=crs, columns=["fileName","geometry",'Resolution'])
                    if curDf.crs != {'init': 'epsg:4326'}:
                        curDf = curDf.to_crs({'init': 'epsg:4326'})
                    curDf['zipFile'] = self.zipFile
                    curDf = curDf.set_geometry("geometry")
                    self.imageryExtents = curDf
                    return curDf
                else:
                    return -1
            else:
                raise ValueError("Input imagery has not been unzipped")

max_stars_repo_name: worldbank/ImageryStorage

max_issues_repo_name: worldbank/ImageryStorage

max_forks_repo_name: worldbank/ImageryStorage


text:
from __future__ import division
import cPickle as pickle
import yaml
import os
import numpy as np
import pandas as pd
import cPickle as pkl

from pnr.vis.Event import Event, EventException
from pnr.vis.Team import TeamNotFoundException
from pnr.data.extractor import ExtractorException, OneHotException
from pnr.data.utils import shuffle_2_array, make_3teams_11players
from pnr.data.constant import data_dir, game_dir
import pnr.config as CONFIG
from pnr.data.constant import game_dir

class BaseLoader:
    def __init__(self, data_config, dataset, extractor, batch_size, mode='sample', fraction_positive=.5):
        print('Running BaseLoader:init')
        self.data_config = data_config
        self.dataset = dataset
        self.extractor = extractor
        self.batch_size = batch_size
        self.fraction_positive = fraction_positive
        self.mode = mode
        self.event_index = 0
        self.games = self.dataset.games

    def next(self):
        print('Running BaseLoader:next')
        """
        """
        if self.mode == 'sample':
            return self.next_batch()
        elif self.mode == 'valid':
            return self.load_valid()
        else:
            raise Exception('unknown loader mode')

    def next_batch(self, extract=True, no_anno=False):
        print('Running BaseLoader:next_batch')
        N_pos = int(self.fraction_positive * self.batch_size)
        N_neg = self.batch_size - N_pos
        ret_val = []
        if not no_anno:
            func = [self.dataset.propose_positive_Ta,
                    self.dataset.propose_negative_Ta]
        else:
            func = [self.dataset.propose_Ta]
        Ns = [N_pos, N_neg]
        # anno = func[0]()
        for j in xrange(len(func)):
            for _ in xrange(Ns[j]):
                while True:
                    try:
                        anno = func[j]()
                        e = Event(self.dataset.games[anno['gameid']][
                                  'events'][anno['eid']], gameid=anno['gameid'])
                        e.sequence_around_t(anno, self.dataset.tfr)  # EventException
                        if extract:
                            # ExtractorException
                            ret_val.append(self.extractor.extract(e))
                        else:
                            # just to make sure event not malformed (like
                            # missing player)
                            _ = self.extractor.extract_raw(e)
                            ret_val.append(e)
                    except EventException as exc:
                        pass
                    except ExtractorException as exc:
                        pass
                    except TeamNotFoundException as exc:
                        pass
                    else:
                        break

        return (
                np.array(ret_val),
                np.vstack([np.array([[0, 1]]).repeat(N_pos, axis=0),
                np.array([[1, 0]]).repeat(N_neg, axis=0)])
            )

    def load_by_annotations(self, annotations, extract=True):
        print('Running BaseLoader:load_by_annotations')
        """
        no labels returned
        """
        ret_val = []
        ret_labels = []
        self.extractor.augment = False
        for anno in annotations:
            try:
                e = Event(self.dataset.games[anno['gameid']][
                          'events'][anno['eid']], gameid=anno['gameid'])
                e.sequence_around_t(anno, self.dataset.tfr)  # EventException
                if extract:
                    # ExtractorException
                    ret_val.append(self.extractor.extract(e))
                else:
                    # just to make sure event not malformed (like
                    # missing player)
                    _ = self.extractor.extract_raw(e)
                    ret_val.append(e)
            except EventException as exc:
                continue
            except ExtractorException as exc:
                continue
            except TeamNotFoundException as exc:
                pass
        return ret_val

    def load_split(self, split='val', extract=True, positive_only=False):
        print('Running BaseLoader:load_split')
        N_pos = 0
        ret_val = []
        ret_labels = []
        self.extractor.augment = False
        istrain = split == 'train'
        while True:
            anno = self.dataset.propose_positive_Ta(jitter=False, train=istrain, loop=True)
            if anno == None:
                break
            try:
                e = Event(self.dataset.games[anno['gameid']]['events'][anno['eid']], anno=anno, gameid=anno['gameid'])
                e.sequence_around_t(anno, self.dataset.tfr)  # EventException
                if extract:
                    # ExtractorException
                    ret_val.append(self.extractor.extract(e))
                else:
                    # just to make sure event not malformed (like
                    # missing player)
                    _ = self.extractor.extract_raw(e)
                    ret_val.append(e)
            except TeamNotFoundException as exc:
                pass
            except EventException as exc:
                continue
            except ExtractorException as exc:
                continue
            else:
                N_pos += 1
                ret_labels.append([0, 1])
        if not positive_only:
            for i in xrange(N_pos):
                while True:
                    try:
                        anno = self.dataset.propose_negative_Ta()
                        e = Event(self.dataset.games[anno['gameid']]['events'][anno['eid']], gameid=anno['gameid'], anno=anno)
                        e.sequence_around_t(anno, self.dataset.tfr)
                        if extract:
                            # ExtractorException
                            ret_val.append(self.extractor.extract(e))
                        else:
                            # just to make sure event not malformed (like
                            # missing player)
                            _ = self.extractor.extract_raw(e)
                            ret_val.append(e)
                    except EventException as exc:
                        pass
                    except ExtractorException as exc:
                        pass
                    except TeamNotFoundException as exc:
                        pass
                    else:
                        ret_labels.append([1, 0])
                        break
        self.extractor.augment = True
        return np.array(ret_val), np.array(ret_labels)

    def load_train(self, extract=True, positive_only=False):
        print('Running BaseLoader:load_train')
        return self.load_split(split='train', extract=extract, positive_only=positive_only)

    def load_valid(self, extract=True, positive_only=False):
        print('Running BaseLoader:load_valid')
        return self.load_split(split='val', extract=extract, positive_only=positive_only)

    def _load_event(self, anno, extract, every_K_frame, dont_resolve_basket=False):
        print('Running BaseLoader:_load_event')
        ret_val = []
        ret_gameclocks = []
        ret_frame_idx = []
        event_id = anno['eid']
        game_id = anno['gameid']
        try:
            e = Event(self.dataset.games[game_id]['events'][event_id], gameid=game_id, anno=anno)
        except TeamNotFoundException:
            # malformed event
            return 0
        N_moments = len(e.moments)
        for i in xrange(0, N_moments, every_K_frame):
            try:
                e = Event(self.dataset.games[game_id]['events'][event_id], gameid=game_id, anno=anno)
                game_clock = e.moments[i].game_clock
                quarter = e.moments[i].quarter
                anno = self.dataset._make_annotation(game_id, quarter, game_clock, event_id, self.dataset.tfr)
                e.sequence_around_t(anno, self.dataset.tfr)  # EventException
                # anno = {'gameclock': game_clock}
                # e.sequence_around_t(anno, self.dataset.tfr, from_beginning=False)

                # just to make sure event not malformed (like
                # missing player)
                _ = self.extractor.extract_raw(e)
                ret_val.append(e)
                ret_gameclocks.append(game_clock)
                ret_frame_idx.append(i)
            except EventException as exc:
                continue
            except ExtractorException as exc:
                continue
            except TeamNotFoundException as exc:
                pass
        if len(ret_val) == 0:  # malformed Event
            ret = 0
        else:
            if extract:
                ret_val = self.extractor.extract_batch(ret_val,dont_resolve_basket=dont_resolve_basket)
            ret = [ret_val, ret_gameclocks, ret_frame_idx]
        return ret

    def load_split_event(self, split, extract, every_K_frame=4):
        print('Running BaseLoader:load_split_event')
        if split == 'val':
            split_hash = self.dataset.val_hash
        elif split == 'train':
            split_hash = self.dataset.train_hash
        elif split == 'game':
            split_hash = self.dataset.game_hash
        else:
            raise NotImplementedError()
        if self.event_index == len(split_hash):
            self.event_index = 0
            return None
        vh = split_hash.values()[self.event_index]

        ret_labels = filter(lambda t: t != -1, [i['gameclock'] for i in vh])
        self.extractor.augment = False
        anno = vh[0]
        ret = self._load_event(anno, extract, every_K_frame)
        if ret == 0:  # malformed Event
            ret = 0
        else:
            ret_val, ret_gameclocks, ret_frame_idx = ret
            meta = [vh[0]['eid'], vh[0]['gameid']]
            ret = [ret_val, ret_labels, ret_gameclocks, meta]
        self.extractor.augment = True
        self.event_index += 1
        return ret

    def load_event(self, game_id, event_id, every_K_frame, player_id=None):
        print('Running BaseLoader:load_event')
        o = self.extractor.augment
        self.extractor.augment = False
        anno = {"gameid": game_id, "eid": event_id}
        ret = self._load_event(anno, True, every_K_frame, dont_resolve_basket=True)
        self.extractor.augment = o
        return ret

    def reset(self):
        pass

class GameSequenceLoader:
    def __init__(self, dataset, extractor, batch_size, mode='sample',
                 fraction_positive=.5, negative_fraction_hard=0):
        print('Running GameSequenceLoader:init')
        """
        """
        self.config = dataset.config
        self.negative_fraction_hard = negative_fraction_hard
        self.dataset = dataset  # not used
        self.root_dir = os.path.join(os.path.join(data_dir, self.dataset.config['preproc_dir']), str(self.dataset.fold_index))
        self.extractor = extractor
        self.batch_size = batch_size  # not used
        self.fraction_positive = fraction_positive
        self.mode = mode
        self.batch_index = 0

        games = self.dataset.config['data_config']['game_ids']
        self.detect_validation = self.dataset.config['data_config']['detect_validation']

        self.pos_x = []
        self.neg_x = []
        self.pos_t = []
        self.val_x = []
        self.val_t = []

        for game in self.detect_validation:
            if game in games:
                games.remove(game)

        for game in games:
            try:
                pos_x = np.load(os.path.join(self.root_dir, '%s_pos_x.npy' % game))
                neg_x = np.load(os.path.join(self.root_dir, '%s_neg_x.npy' % game))
                pos_t = np.load(os.path.join(self.root_dir, '%s_pos_t.npy' % game))
                val_x = np.load(os.path.join(self.root_dir, '%s_val_x.npy' % game))
                val_t = np.load(os.path.join(self.root_dir, '%s_val_t.npy' % game))
            except IOError:
                continue

            self.pos_x.extend(pos_x)
            self.neg_x.extend(neg_x)
            self.pos_t.extend(pos_t)
            self.val_x.extend(val_x)
            self.val_t.extend(val_t)

        self.val_x = np.array(self.val_x)
        self.val_t = np.array(self.val_t)
        self.pos_x = np.array(self.pos_x)
        self.neg_x = np.array(self.neg_x)
        self.pos_t = np.array(self.pos_t)
        self.neg_t = np.array([[1, 0]]).repeat(self.neg_x.shape[0], axis=0)

        self.pos_ind = 0
        self.neg_ind = 0
        self.val_ind = 0
        self.N_pos = int(batch_size * fraction_positive)
        self.N_neg = batch_size - self.N_pos

        if self.negative_fraction_hard > 0:
            self.N_hard_neg = int(self.N_neg * negative_fraction_hard)
            self.N_neg = self.N_neg - self.N_hard_neg
            self.hard_neg_ind = 0

    def _split(self, inds, fold_index=0):
        print('Running GameSequenceLoader:_split')
        if self.config['data_config']['shuffle']:
            np.random.seed(self.config['randseed'])
            np.random.shuffle(inds)
        N = len(inds)
        val_start = np.round(fold_index/self.config['data_config']['N_folds'] * N).astype('int32')
        val_end = np.round((fold_index + 1)/self.config['data_config']['N_folds'] * N).astype('int32')
        val_inds= inds[val_start:val_end]
        train_inds = inds[:val_start] + inds[val_end:]
        return train_inds, val_inds

    def next(self):
        print('Running GameSequenceLoader:next')
        """
        """
        if self.mode == 'sample':
            return self.next_batch()
        elif self.mode == 'valid':
            return self.load_valid()
        else:
            raise Exception('unknown loader mode')

    def next_batch(self, extract=True):
        print('Running GameSequenceLoader:next_batch')
        # if self.batch_index == self.dataset_size:
        #     return None

        if self.pos_ind + self.N_pos >= self.pos_x.shape[0]:
            self.pos_ind = 0
            self.pos_x, self.pos_t = shuffle_2_array(self.pos_x, self.pos_t)
        if self.neg_ind + self.N_neg >= self.neg_x.shape[0]:
            self.neg_ind = 0
            self.neg_x, self.neg_t = shuffle_2_array(self.neg_x, self.neg_t)
        if (self.negative_fraction_hard > 0 and
                self.hard_neg_ind + self.N_hard_neg >= self.hard_neg_x.shape[0]):
            self.hard_neg_ind = 0
            self.hard_neg_x, self.hard_neg_t = shuffle_2_array(self.hard_neg_x, self.hard_neg_t)

        s = list(self.pos_x.shape)
        s[0] = self.batch_size
        x = np.zeros(s)
        t = np.zeros((self.batch_size, 2))
        x[:self.N_pos] = self.pos_x[self.pos_ind:self.pos_ind + self.N_pos]
        t[:self.N_pos] = self.pos_t[self.pos_ind:self.pos_ind + self.N_pos]
        if not self.negative_fraction_hard > 0:
            x[self.N_pos:] = self.neg_x[self.neg_ind:self.neg_ind + self.N_neg]
            t[self.N_pos:] = self.neg_t[self.neg_ind:self.neg_ind + self.N_neg]
        else:
            x[self.N_pos:self.N_pos + self.N_hard_neg] = self.hard_neg_x[
                self.hard_neg_ind:self.hard_neg_ind + self.N_hard_neg]
            t[self.N_pos:self.N_pos + self.N_hard_neg] = self.hard_neg_t[
                self.hard_neg_ind:self.hard_neg_ind + self.N_hard_neg]
            x[self.N_pos + self.N_hard_neg:] = self.neg_x[self.neg_ind:self.neg_ind + self.N_neg]
            t[self.N_pos + self.N_hard_neg:] = self.neg_t[self.neg_ind:self.neg_ind + self.N_neg]
        if extract:
            x = self.extractor.extract_batch(x, input_is_sequence=True)
        self.pos_ind += self.N_pos
        self.neg_ind += self.N_neg
        if self.negative_fraction_hard > 0:
            self.hard_neg_ind += self.N_hard_neg
        return x, t

    def load_valid(self, extract=True):
        print('Running GameSequenceLoader:load_valid')
        x = self.val_x
        if extract:
            x = self.extractor.extract_batch(x, input_is_sequence=True)
        t = self.val_t
        return x, t

    def load_test(self, extract=True):
        print('Running GameSequenceLoader:load_test')
        x, t = [], []
        for game in self.detect_validation:
            pos_x = np.load(os.path.join(self.root_dir, '%s_pos_x.npy' % game))
            pos_t = np.load(os.path.join(self.root_dir, '%s_pos_t.npy' % game))
            neg_x = np.load(os.path.join(self.root_dir, '%s_neg_x.npy' % game))
            neg_t = np.load(os.path.join(self.root_dir, '%s_neg_t.npy' % game))
            val_x = np.load(os.path.join(self.root_dir, '%s_val_x.npy' % game))
            val_t = np.load(os.path.join(self.root_dir, '%s_val_t.npy' % game))

            x.extend(val_x)
            x.extend(pos_x)
            x.extend(neg_x[:len(pos_x)])
            t.extend(val_t)
            t.extend(pos_t)
            t.extend(neg_t[:len(pos_t)])

        inds = range(len(x))
        if self.config['data_config']['shuffle']:
            np.random.seed(self.config['randseed'])
            np.random.shuffle(inds)

        x = np.array(x)
        t = np.array(t)

        x = x[inds]
        t = t[inds]

        if extract:
            x = self.extractor.extract_batch(x, input_is_sequence=True)

        return x, t

    def reset(self):
        self.batch_index = 0

class TrajectoryLoader:
    def __init__(self, config, fold_index):
        print('Running TrajectoryLoader:init')
        """
        """
        pnr_dir = os.path.join(game_dir, 'pnr-annotations')

        self.config = config
        self.batch_size = self.config['batch_size']
        self.batch_index = 0
        self.fold_index = fold_index
        self.x = []
        self.annotations = []

        self.annotations = pd.read_pickle('%s/roles/annotations.pkl' % (pnr_dir))
        self.x = np.load(open('%s/roles/behaviours.npy' % (pnr_dir), 'rb'))

        self.x = np.array(self.x)
        self.ind = 0
        self.val_ind = 0
        self.set_ind = 0
        self.N = self.batch_size

        # train_inds, val_inds = self._split(list(range(len(self.x))))
        # self.val_x = self.x[val_inds]
        # self.train_x = self.x[train_inds]
        self.train_x = self.x

    def _split(self, inds, fold_index=0):
        print('Running TrajectoryLoader:_split')
        if self.config['data_config']['shuffle']:
            np.random.seed(self.config['randseed'])
            np.random.shuffle(inds)
        N = len(inds)
        val_start = np.round(fold_index / self.config['data_config']['N_folds'] * N).astype('int32')
        val_end = np.round((fold_index + 1) / self.config['data_config']['N_folds'] * N).astype('int32')
        val_inds = inds[val_start:val_end]
        train_inds = inds[:val_start] + inds[val_end:]
        return train_inds, val_inds

    def next(self):
        print('Running TrajectoryLoader:next')
        return self.next_batch()

    def next_batch(self):
        print('Running TrajectoryLoader:next_batch')
        if self.ind + self.N >= self.train_x.shape[0]:
            self.ind = 0
            np.random.shuffle(self.train_x)

        try:
            s = list(self.train_x.shape)
        except Exception as err:
            print(None)
        s[0] = self.batch_size
        x = np.zeros(s)
        x[:self.N] = self.train_x[self.ind:self.ind + self.N]
        self.ind += self.N
        return x

    def load_valid(self):
        print('Running TrajectoryLoader:load_valid')
        if self.val_ind + self.batch_size > len(self.val_x):
            self.val_ind = 0
            return None
        x = self.val_x[self.val_ind:self.val_ind + self.batch_size]
        self.val_ind += self.batch_size
        return x

    def load_set(self):
        print('Running TrajectoryLoader:load_set')
        if self.set_ind + self.batch_size > len(self.x):
            self.set_ind = 0
            return None
        x = self.x[self.set_ind:self.set_ind + self.batch_size]
        annotations = self.annotations[self.set_ind:self.set_ind + self.batch_size]
        self.set_ind += self.batch_size
        return x, annotations

    def reset(self):
        print('Running TrajectoryLoader:reset')
        self.batch_index = 0

max_stars_repo_name: dkStephanos/pnrRebuild

max_issues_repo_name: dkStephanos/pnrRebuild

max_forks_repo_name: dkStephanos/pnrRebuild


text:
import json
import os
from os.path import join
import subprocess

import numpy as np
from remimi.utils.binary_image import create_roi_from_u8_mask

from matplotlib import pyplot as plt
from remimi.utils.image import show_image_ui
from remimi.segmentation.u2net_wrapper import U2MaskModel, U2MaskModelOption
import click
import glob
import cv2
import streamlit as st
import tqdm
import shutil
import tempfile
import yaml

@click.command()
@click.option("--model-name")
@click.option("--dataset-definition-file")
@click.option("--output-path")
def main(model_name, dataset_definition_file, output_path):
    u2_mask_model = U2MaskModel(U2MaskModelOption(model_name=model_name))
    datasets = yaml.load(open(dataset_definition_file, "r"))

    categories = [
        {
            "id": 1,
            "name": "alchol sheet"
        },
        {
            "id": 2,
            "name": "ipad"
        },
    ]
    annotations_train = []
    annotations_val = []
    images_train = []
    images_val = []

    image_output_path = os.path.join(output_path, "images")
    os.makedirs(image_output_path, exist_ok=True)

    next_image_id = 0
    for dataset in datasets:
        with tempfile.TemporaryDirectory() as image_path:
            video_path = dataset['video_path']
            file_template = os.path.join(image_path, "$filename%06d.png")
            command = f"ffmpeg -i {video_path} -r 60/1 {file_template}"
            subprocess.run(command, shell=True, check=True)

            image_paths = sorted(list(glob.glob(join(image_path, "*.png"))) + list(glob.glob(join(image_path, "*.jpg"))))
            areas = []
            masks = []
            annotations = []
            images = []
            for image_path in tqdm.tqdm(image_paths):
                image_bgr = cv2.imread(image_path)
                mask = u2_mask_model.predict_mask(image_bgr)
                mask_numpy = mask.squeeze(0).cpu().detach().numpy()
                mask_u8 = u2_mask_model.convert_probability_mask_to_binary_mask(mask_numpy)
                roi = create_roi_from_u8_mask(mask_u8)

                contour_points = roi.contours[0].flatten().tolist()
                area = cv2.contourArea(np.array(roi.contours[0])) if len(roi.contours) != 0 else 0

                masks.append(mask_u8)

                if area == 0:
                    continue

                areas.append(area)

                # import IPython; IPython.embed()
                annotation = {
                    "segmentation": [contour_points],
                    # "area": float(max_x - min_x) * float(max_y - min_y),
                    "area": area,
                    "iscrowd": 0,
                    "image_id": next_image_id,
                    "bbox": [
                        float(roi.min_x), float(roi.min_y),
                        float(roi.max_x - roi.min_x), float(roi.max_y - roi.min_y)],
                    "category_id": dataset["category_id"],
                    "id": next_image_id
                }
                image_name = os.path.basename(image_path)
                video_name = os.path.basename(video_path)
                new_image_name = f"{next_image_id}_{video_name}_{image_name}"
                image = {
                    "file_name": new_image_name,
                    "width": image_bgr.shape[1],
                    "height": image_bgr.shape[0],
                    "id": next_image_id,
                    "image_id": next_image_id
                }
                annotations.append(annotation)
                images.append(image)

                shutil.copy2(image_path, os.path.join(image_output_path, new_image_name))
                next_image_id += 1

            import scipy
            WINDOW_LENGTH = 5
            # area_moving_median = np.convolve(areas, np.ones(WINDOW_LENGTH), 'valid') / WINDOW_LENGTH
            area_moving_median = scipy.signal.medfilt(areas, WINDOW_LENGTH)
            st.write(area_moving_median)
            near_median_indices = []
            for i, area in enumerate(areas):
                if area_moving_median[i] * 0.7 < area and area < area_moving_median[i] * 1.3:
                    near_median_indices.append(i)

            # fig, ax = plt.subplots(1, 1)
            # ax.plot(list(range(0, len(areas))), areas)
            # st.pyplot(fig)

            # import IPython; IPython.embed()

            NUM_VAL_PER_NUM = 3
            num_added = 0

            for i in near_median_indices:
                if num_added % NUM_VAL_PER_NUM == 0:
                    annotations_val.append(annotations[i])
                    images_val.append(images[i])
                else:
                    annotations_train.append(annotations[i])
                    images_train.append(images[i])

                num_added += 1

    json.dump({
        "images": images_train,
        "annotations": annotations_train,
        "categories": categories,
        "info": {
            "description": "COCO 2017 Dataset"
        }
    }, open(os.path.join(output_path, "annotations.json"), "w"), indent=4)

    json.dump({
        "images": images_val,
        "annotations": annotations_val,
        "categories": categories,
        "info": {
            "description": "COCO 2017 Dataset"
        }
    }, open(os.path.join(output_path, "annotations_val.json"), "w"), indent=4)

if __name__ == "__main__":
    main()

max_stars_repo_name: xiong-jie-y/remimi

max_issues_repo_name: xiong-jie-y/remimi

max_forks_repo_name: xiong-jie-y/remimi

reason: import numpy,import scipy


text:
import numpy as np
import torch

import neuron.ops as ops
from neuron.config import registry

__all__ = ['ReID_Transforms']

class Compose(object):

    def __init__(self, transforms):
        self.transforms = transforms

    def __call__(self, img, target):
        for t in self.transforms:
            img, target = t(img, target)
        return img, target

    def __getitem__(self, index):
        if isinstance(index, int):
            return self.transforms[index]
        elif isinstance(index, slice):
            return Compose(self.transforms[index])
        else:
            raise TypeError('Invalid type of index.')

    def __add__(self, other):
        if isinstance(other, Compose):
            return Compose(self.transforms + other.transforms)
        else:
            raise TypeError('Invalid type of other.')

class CropAndResize(object):

    def __init__(self,
                 train=True,
                 context=0.5,
                 out_size=(128, 128),
                 random_shift=10):
        self.train = train
        self.context = context
        self.out_size = ops.make_pair(out_size)
        self.random_shift = random_shift

    def __call__(self, img, target):
        bbox = target['bbox']
        f_out_size = np.array(self.out_size, dtype=np.float32)

        # calculate the cropping size
        bbox_size = bbox[2:] - bbox[:2] + 1
        context = self.context * np.sum(bbox_size)
        crop_size = np.sqrt(np.prod(bbox_size + context))

        # calculate the cropping center
        bbox_center = (bbox[:2] + bbox[2:]) / 2.
        if self.train:
            # randomly shift in the cropped image
            shift = np.random.uniform(
                -self.random_shift, self.random_shift, 2)
            shift *= crop_size / f_out_size
        else:
            shift = np.zeros(2)
        crop_center = bbox_center + shift

        # crop and resize
        avg_color = np.mean(img, axis=(0, 1))
        crop_img = ops.crop_and_resize(
            img, crop_center, crop_size,
            out_size=self.out_size, border_value=avg_color)

        # bounding box in the cropped image
        scale = f_out_size / crop_size
        bbox_center = (f_out_size - 1) / 2. - shift * scale
        bbox_size *= scale
        crop_bbox = np.concatenate([
            bbox_center - (bbox_size - 1) / 2.,
            bbox_center + (bbox_size - 1) / 2.])
        target.update({'bbox': crop_bbox.astype(np.float32)})

        return crop_img, target

class RandomHorizontalFlip(object):

    def __init__(self, p=0.5):
        self.p = p

    def __call__(self, img, target):
        if np.random.rand() < self.p:
            img, target['bbox'] = ops.flip_img(img, target['bbox'])
        return img, target

class Normalize(object):

    def __init__(self,
                 rgb_mean=[0.485, 0.456, 0.406],
                 rgb_std=[0.229, 0.224, 0.225]):
        self.rgb_mean = rgb_mean
        self.rgb_std = rgb_std

    def __call__(self, img, target):
        img = ops.normalize_img(
            img / 255., self.rgb_mean, self.rgb_std)
        return img, target

class RandomErasing(object):

    def __init__(self,
                 p=0.5,
                 min_area=0.02,
                 max_area=0.4,
                 min_aspect_ratio=0.3,
                 rgb_mean=[0.485, 0.456, 0.406]):
        self.p = p
        self.min_area = min_area
        self.max_area = max_area
        self.min_aspect_ratio = min_aspect_ratio
        self.rgb_mean = rgb_mean

    def __call__(self, img, target):
        if np.random.rand() > self.p:
            return img, target

        h, w, c = img.shape
        for _ in range(100):
            area = np.prod(img.shape[:2])

            erase_area = area * np.random.uniform(
                self.min_area, self.max_area)
            erase_aspect_ratio = np.random.uniform(
                self.min_aspect_ratio, 1. / self.min_aspect_ratio)
            erase_h = int(round(np.sqrt(
                erase_area * erase_aspect_ratio)))
            erase_w = int(round(np.sqrt(
                erase_area / erase_aspect_ratio)))

            if erase_w < w and erase_h < h:
                x1 = np.random.randint(0, w - erase_w)
                y1 = np.random.randint(0, h - erase_h)
                img[y1:y1 + erase_h, x1:x1 + erase_w] = self.rgb_mean
                return img, target

        return img, target

class ToTensor(object):

    def __call__(self, img, target):
        img = torch.from_numpy(img).permute(2, 0, 1).float()
        bbox = torch.from_numpy(target['bbox']).float()
        label = torch.LongTensor([target['ins_id']])[0] - 1
        target = {'bbox': bbox, 'label': label}
        return img, target

@registry.register_module
class ReID_Transforms(Compose):

    def __init__(self,
                 train=True,
                 context=0.5,
                 out_size=(128, 128),
                 random_shift=10,
                 flip_p=0.5,
                 rgb_mean=[0.485, 0.456, 0.406],
                 rgb_std=[0.229, 0.224, 0.225],
                 erase_p=0.5,
                 min_area=0.02,
                 max_area=0.4,
                 min_aspect_ratio=0.3):
        if train:
            super(ReID_Transforms, self).__init__(transforms=[
                CropAndResize(train, context, out_size, random_shift),
                RandomHorizontalFlip(flip_p),
                Normalize(rgb_mean, rgb_std),
                RandomErasing(erase_p, min_area, max_area,
                              min_aspect_ratio, rgb_mean),
                ToTensor()])
        else:
            super(ReID_Transforms, self).__init__(transforms=[
                CropAndResize(train, context, out_size, random_shift),
                Normalize(rgb_mean, rgb_std),
                ToTensor()])

max_stars_repo_name: tsingqguo/ABA

max_issues_repo_name: tsingqguo/ABA

max_forks_repo_name: tsingqguo/ABA


text:
from abc import ABC

import numpy as np

from .implicit_method import ImplicitMethod as IM

class Model(ABC):

    name = 'Model'

    parameters = {}
    inputs = {}
    outputs = {}
    states = {}

    def __init__(self, x_0=0, u_0=0, u=0, method=None):

        assert isinstance(x_0, np.ndarray), "Initial states must be given in a numpy array"
        assert isinstance(u_0, np.ndarray), "Initial input must be given in a numpy array"
        assert isinstance(u, np.ndarray), "Input must be given in a numpy array"

        assert isinstance(method, IM), "Method must be of class IM"

        self.p = np.array([])

        self.x_0 = x_0
        self.u_0 = u_0
        self.u = u
        self.y = np.array([])

        self.method = method

    def __str__(self, title: bool = True, as_html: bool = False):
        string = 'Parameters: \n' if title else ''
        for name, value in list(zip(self.parameters.keys(), self.p)):
            if title:
                string += '\t'
            string += '{}: {:.4f}\n'.format(name, value)

        if as_html:
            string = string.replace('\n', '<br>')

        return string

    def f(self, x=None, u=None, factor=0):

        if x is None:
            x = self.x_0
        if u is None:
            u = self.u

        assert isinstance(x, np.ndarray), "States vector must be given in a numpy array"
        assert isinstance(u, np.ndarray), "Input vector must be given in a numpy array"

        return x, u

    def g(self, x=None, u=None):

        if x is None:
            x = self.x_0
        if u is None:
            u = self.u

        assert isinstance(x, np.ndarray), "States vector must be given in a numpy array"
        assert isinstance(u, np.ndarray), "Input vector must be given in a numpy array"

        return x, u

    def update_parameters(self, p):
        assert isinstance(p, np.ndarray), "Parameters must be given in a numpy array"
        assert len(p) == len(self.parameters), "Parameter vector must be of length %d" % len(self.parameters)

        self.p = p

        self.initial_conditions()

    def update_output(self, p=None):
        if p is not None:
            self.update_parameters(p)

        self.method(self)

    def save_output(self, filename='example.csv'):

        assert isinstance(self.y, np.ndarray), "Output must be a numpy array in order to save"

        np.savetxt('Sample_Data/' + filename, self.y, delimiter=',')

    def initial_conditions(self):
        pass

max_stars_repo_name: gnegrelli/identPy

max_issues_repo_name: gnegrelli/identPy

max_forks_repo_name: gnegrelli/identPy


text:
"""
Scenario-based stochastic optimal binning testing.
"""

# Guillermo Navas-Palencia <g.navas.palencia@gmail.com>
# Copyright (C) 2020

import numpy as np
import pandas as pd

from pytest import approx, raises

from optbinning.binning.binning_statistics import BinningTable
from optbinning.binning.uncertainty import SBOptimalBinning
from sklearn.datasets import load_breast_cancer
from sklearn.exceptions import NotFittedError
from sklearn.model_selection import train_test_split

data = load_breast_cancer()
df = pd.DataFrame(data.data, columns=data.feature_names)

variable = "mean radius"
x = df[variable].values
y = data.target

x1, x2, y1, y2 = train_test_split(x, y, test_size=0.5, random_state=42)
x3, x4, y3, y4 = train_test_split(x1, y1, test_size=0.2, random_state=42)
x_s = [x1, x3, x4]
y_s = [y1, y3, y4]

def test_params():
    with raises(TypeError):
        sboptb = SBOptimalBinning(name=1)
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(prebinning_method="new_method")
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(max_n_prebins=-2)
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(min_prebin_size=0.6)
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(min_n_bins=-2)
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(max_n_bins=-2.2)
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(min_n_bins=3, max_n_bins=2)
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(min_bin_size=0.6)
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(max_bin_size=-0.6)
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(min_bin_size=0.5, max_bin_size=0.3)
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(monotonic_trend="new_trend")
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(min_event_rate_diff=1.1)
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(max_pvalue=1.1)
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(max_pvalue_policy="new_policy")
        sboptb.fit(x_s, y_s)

    with raises(TypeError):
        sboptb = SBOptimalBinning(class_weight=[0, 1])
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(class_weight="unbalanced")
        sboptb.fit(x_s, y_s)

    with raises(TypeError):
        sboptb = SBOptimalBinning(user_splits={"a": [1, 2]})
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(user_splits_fixed=[True, False])
        sboptb.fit(x_s, y_s)

    with raises(TypeError):
        sboptb = SBOptimalBinning(user_splits=[2], user_splits_fixed={0: True})
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(user_splits=[2, 3],
                                  user_splits_fixed=[True, 0])
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(user_splits=[2, 3],
                                  user_splits_fixed=[True])
        sboptb.fit(x_s, y_s)

    with raises(TypeError):
        sboptb = SBOptimalBinning(special_codes={1, 2, 3})
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(split_digits=9)
        sboptb.fit(x_s, y_s)

    with raises(ValueError):
        sboptb = SBOptimalBinning(time_limit=-2)
        sboptb.fit(x_s, y_s)

    with raises(TypeError):
        sboptb = SBOptimalBinning(verbose=1)
        sboptb.fit(x_s, y_s)

def test_input_scenarios():
    with raises(TypeError):
        sboptb = SBOptimalBinning()
        sboptb.fit(x1, y_s)

    with raises(TypeError):
        sboptb = SBOptimalBinning()
        sboptb.fit(x_s, y1)

    with raises(ValueError):
        sboptb = SBOptimalBinning()
        sboptb.fit([x1, x3], [y1])

    with raises(ValueError):
        sboptb = SBOptimalBinning()
        sboptb.fit(x_s, y_s, [0.2, 0.8])

def test_default():
    sboptb = SBOptimalBinning(monotonic_trend="descending")
    sboptb.fit(x_s, y_s)

    assert sboptb.status == "OPTIMAL"
    assert sboptb.splits == approx([13.09499979, 14.14999962, 15.24499989],
                                   rel=1e-6)

def test_default_transform():
    sboptb = SBOptimalBinning(monotonic_trend="descending")
    sboptb.fit(x_s, y_s)

    x_transform = sboptb.transform([12, 14, 15, 21], metric="woe")
    assert x_transform == approx([-2.46667422, -0.55591146, 0.24544931,
                                  2.8963411], rel=1e-6)

def test_default_fit_transform():
    sboptb = SBOptimalBinning(monotonic_trend="descending")

    x_transform = sboptb.fit_transform([12, 14, 15, 21], x_s, y_s)
    assert x_transform == approx([-2.46667422, -0.55591146, 0.24544931,
                                  2.8963411], rel=1e-6)

def test_user_splits():
    user_splits = [11, 12, 13, 14, 15, 16, 17]

    sboptb = SBOptimalBinning(monotonic_trend="descending",
                              user_splits=user_splits)
    sboptb.fit(x_s, y_s)

    assert sboptb.status == "OPTIMAL"
    assert sboptb.splits == approx([12, 14, 15], rel=1e-6)

def test_user_splits_non_unique():
    user_splits = [11, 12, 13, 13, 15, 16, 17]

    sboptb = SBOptimalBinning(monotonic_trend="descending",
                              user_splits=user_splits)

    with raises(ValueError):
        sboptb.fit(x_s, y_s)

def test_user_splits_fixed():
    user_splits = [11, 12, 13, 14, 14.7, 15.5, 17]
    user_splits_fixed = [False, False, False, False, False, True, False]

    sboptb = SBOptimalBinning(monotonic_trend="descending",
                              user_splits=user_splits,
                              user_splits_fixed=user_splits_fixed)
    sboptb.fit(x_s, y_s)

    assert sboptb.status == "OPTIMAL"
    assert 15.5 in sboptb.splits

def test_min_bin_size():
    sboptb = SBOptimalBinning(monotonic_trend="descending", min_bin_size=0.1)
    sboptb.fit(x_s, y_s)

    n_scenarios = len(x_s)

    for s in range(n_scenarios):
        bt_s = sboptb.binning_table_scenario(s)
        bt = bt_s.build()

        count = bt["Count (%)"].values[:-3]
        assert np.all(count >= 0.1)

def test_max_bin_size():
    sboptb = SBOptimalBinning(monotonic_trend="descending", max_bin_size=0.6)
    sboptb.fit(x_s, y_s)

    assert sboptb.status == "OPTIMAL"

    n_scenarios = len(x_s)

    for s in range(n_scenarios):
        bt_s = sboptb.binning_table_scenario(s)
        bt = bt_s.build()

        count = bt["Count (%)"].values[:-3]
        assert np.all(count <= 0.6)

def test_binning_table_scenario():
    sboptb = SBOptimalBinning(monotonic_trend="descending")

    with raises(NotFittedError):
        sboptb.binning_table_scenario(scenario_id=2)

    sboptb.fit(x_s, y_s)

    with raises(ValueError):
        bt = sboptb.binning_table_scenario(scenario_id=4)

    bt = sboptb.binning_table_scenario(scenario_id=2)

    assert isinstance(bt, BinningTable)

def test_verbose():
    sboptb = SBOptimalBinning(monotonic_trend="descending", verbose=True)
    sboptb.fit(x_s, y_s)

    assert sboptb.status == "OPTIMAL"

max_stars_repo_name: jensgk/optbinning

max_issues_repo_name: jensgk/optbinning

max_forks_repo_name: jensgk/optbinning