text:
# -*- coding: utf-8 -*-
'''Module to define the ``Triangle``, ``Circle`` and ``Polygon`` classes. Some
properties related to the geometry of these classes are determined.
These classes are the basic inputs to pack circular particles in a
closed polygon in :math:`\\mathbb{R}^2`.
'''

import numpy as np
import matplotlib.pyplot as plt
from scipy.spatial.distance import euclidean

# %%
class Circle:
    '''Creates an instance of an object that defines a Circle once the
    cartesian coordinates of its center and the radius are given.

    Attributes
    ----------
        center : `tuple` or `list`
            (x, y)-cartesian coordinates of circle center.
        radius : `float` or `int`
            Length of the segment that joins the center with any point of\
            the circumference.

    Examples
    --------

        >>> center, radius = (0, 0), 1
        >>> circle = Circle(center, radius)
        >>> circle.__dict__
        {'area': 3.141592653589793,
         'center': array([0, 0]),
         'curvature': 1.0,
         'diameter': 2,
         'perimeter': 6.283185307179586,
         'radius': 1}

        >>> center, radius = (2, 5), 2.5
        >>> circle = Circle(center, radius)
        >>> circle.__dict__
        {'area': 19.634954084936208,
        'center': array([2, 5]),
        'curvature': 0.4,
        'diameter': 5.0,
        'perimeter': 15.707963267948966,
        'radius': 2.5}
    '''

    def __init__(self, center, radius):
        '''Method for initializing the attributes of the class.'''
        self.center = np.array(center)
        self.radius = radius
        self.curvature = 1 / radius
        self.diameter = 2 * radius
        self.area = np.pi * radius**2
        self.perimeter = 2 * radius * np.pi

# %%
class Triangle:
    '''Creates an instance of an object that defines a Triangle once the
    coordinates of its three vertices in cartesian :math:`\\mathbb{R}^2` space
    are given.

    It considers the usual notation for the triangle ``ABC`` in which `A`, `B`
    and `C` represent the vertices and ``a``, ``b``, ``c`` are the
    lengths of `the segments BC`, `CA` and `AB` respectively.

    Attributes
    ----------
        coordinates : (3, 2) `numpy.ndarray`
            Coordinates of three vertices of the triangle.

    Examples
    --------
        >>> from numpy import array
        >>> from pc4bims.basegeom import Triangle
        >>> coords = array([(2, 1.5), (4.5, 4), (6, 2)])
        >>> triangle = Triangle(coords)
        >>> triangle.__dict__.keys()
        dict_keys(['vertices', 'area', 'sides', 'perimeter', 'distToIncenter',
                   'incircle'])

        >>> from numpy import array
        >>> from pc4bims.basegeom import Triangle
        >>> coords = array([[2, 1], [6, 1], [4, 5.5]])
        >>> triangle = Triangle(coords)
        >>> triangle.__dict__.keys()
        dict_keys(['vertices', 'area', 'sides', 'perimeter', 'distToIncenter',
                   'incircle'])
    '''

    def __init__(self, coordinates):
        '''Method for initializing the attributes of the class.'''
        self.vertices = dict(zip('ABC', coordinates))
        # same geometric properties of the triangle
        self.getGeomProperties()

    def getGeomProperties(self):
        '''Method to set the attributes to the instanced object

        The established geometric attributes are the following:
            * Area.
            * Lenght of its three sides.
            * Perimeter.
            * Incircle (``Circle`` Object)
            * Distance of each vertice to incenter.
        '''

        vertsArray = np.array([*self.vertices.values()])
        v = np.vstack((vertsArray, vertsArray[0]))
        # Area (Gauss Equation)
        area = 0.5*abs(sum(v[:-1, 0] * v[1:, 1] - v[:-1, 1] * v[1:, 0]))
        # length three sides triangle.
        sides = {'a': euclidean(self.vertices['B'], self.vertices['C']),
                 'b': euclidean(self.vertices['A'], self.vertices['C']),
                 'c': euclidean(self.vertices['A'], self.vertices['B'])}
        perimeter = sides['a'] + sides['b'] + sides['c']
        # Inscribed circle radius
        radius = 2*area / perimeter
        # Inscribed circle center
        center = (sides['a'] * self.vertices['A'] +
                  sides['b'] * self.vertices['B'] +
                  sides['c'] * self.vertices['C']) / perimeter
        # Distances of each vertice to incenter
        distToIncenter = [euclidean(center, v) for v in self.vertices.values()]
        # Set the attribute to the instanced object.
        setattr(self, 'area', area)
        setattr(self, 'sides', sides)
        setattr(self, 'perimeter', perimeter)
        setattr(self, 'distToIncenter', distToIncenter)
        setattr(self, 'incircle', Circle(center, radius))
        return

    def packCircles(self, depth=None, want2plot=False):
        '''Method to pack circular particles within of a triangle. It apply
        the Descartes theorem (special and general case) to generate mutually
        tangent circles in a fractal way in the triangle.

        Parameters
        ----------
            depth :`int`
                Fractal depth. Number that indicate how many circles are\
                fractally generated from the `incirle` to each vertice of the\
                triangle. If this number is not given, then, the fractal\
                generation of circles is done up to a circle reachs a radius\
                to lower than the five percent of the incircle radius.
            want2plot : `bool`
                Variable to check if a plot is wanted. Default is ``False``.

        Returns
        -------
            listCircles : `list`
                `list` that contains all the circular particles packed in the\
                triangle.

        Note
        ----
            Large values of `depth` might produce internal variables that tend
            to infinte, then a ``ValueError`` is produced with a warning
            message ``array must not contain infs or NaNs``.

        Examples
        --------
            >>> from numpy import array
            >>> from pc4bims.basegeom import Triangle
            >>> coords = array([(2, 1.5), (4.5, 4), (6, 2)])
            >>> triangle = Triangle(coords)
            >>> cirsInTri = triangle.packCircles(depth=2, want2plot=True)

            >>> from numpy import array
            >>> from pc4bims.basegeom import Triangle
            >>> coords = array([[2, 1], [6, 1], [4, 5.5]])
            >>> triangle = Triangle(coords)
            >>> cirsInTri = triangle.packCircles(depth=5, want2plot=True)

            .. plot::

                from numpy import array
                from pc4bims.basegeom import Triangle
                coords = array([(2, 1), (2, 8), (7, 1)])
                triangle = Triangle(coords)
                triangle.packCircles(want2plot=True)
        '''

        listCircles = [self.incircle]
        for vert, distance in zip(self.vertices.values(), self.distToIncenter):
            c_0 = self.incircle
            auxDist = distance
            if depth is None:
                while True:
                    radius = (c_0.radius*auxDist -
                              c_0.radius**2) / (c_0.radius + auxDist)
                    # Checking condition stop
                    # (circle with center on the bisectrix)
                    if radius < 0.050*self.incircle.radius:
                        break
                    dist = c_0.radius + radius  # dist between centers
                    center = np.array(c_0.center) + dist/auxDist * (
                            np.array(vert) - np.array(c_0.center))
                    c_1 = Circle(center, radius)
                    listCircles.append(c_1)
                    # Updating variables
                    auxDist = euclidean(vert, c_1.center)
                    c_0 = c_1
            else:
                for _ in range(1, depth+1):
                    radius = (c_0.radius*auxDist -
                              c_0.radius**2) / (c_0.radius + auxDist)
                    dist = c_0.radius + radius  # dist between centers
                    center = np.array(c_0.center) + dist/auxDist * (
                            np.array(vert) - np.array(c_0.center))
                    c_1 = Circle(center, radius)
                    listCircles.append(c_1)
                    # Updating variables
                    auxDist = euclidean(vert, c_1.center)
                    c_0 = c_1
        # plotting
        if want2plot:
            vert = np.array([*self.vertices.values()])
            fig = plt.figure()
            ax = fig.add_subplot(111)
        # plotting
            ax.plot(np.hstack((vert[:, 0], vert[0, 0])),
                    np.hstack((vert[:, 1], vert[0, 1])),
                    'k-', lw=2, label='Triangle')
            ax.axis('equal')
            for circle in listCircles:
                ax.add_patch(plt.Circle(circle.center, circle.radius,
                                        fill=True, ec='k', lw=1, fc='k'))
        else:
            return listCircles

# %%
class Polygon:
    '''Creates an instance of an object that defines a Polygon once the
    cartesian coordinates of its vertices are given.

    Attributes
    ----------
        coordinates : (n, 2) `numpy.ndarray`
            Coordinates of the vertices of the polygon.
    '''

    def __init__(self, coordinates):
        '''Method for initializing the attributes of the class.'''
        self.coords = coordinates
        self.boundCoords = np.vstack((coordinates, coordinates[0]))
        self.area()

    def area(self):
        '''Method for determine the area of the polygon.

        Returns
        -------
            area : `float`
                Area of the polygon surface.

        Examples
        --------
            >>> from numpy import array
            >>> from pc4bims.basegeom import Polygon
            >>> coords = array([(1, 1), (4, 8), (8, 5)])
            >>> polygon = Polygon(coords)
            >>> polygon.area
            18.5

            >>> from numpy import array
            >>> from pc4bims.basegeom import Polygon
            >>> coords = array([[1, 1], [2, 5], [4.5, 6], [8, 3],
                                [7, 1], [4, 0]])
            >>> polygon = Polygon(coords)
            >>> polygon.area
            27.5
        '''

        # polygon area by applying the gauss equation
        area = 0.5*abs(sum(self.coords[:-1, 0] * self.coords[1:, 1] -
                           self.coords[:-1, 1] * self.coords[1:, 0]))
        setattr(self, 'area', area)
        return area

    def plot(self):
        '''Method for show the graph of the polygon.

        Examples
        --------

            .. plot::

                from numpy import array
                from pc4bims.basegeom import Polygon
                coords = array([[1, 1], [2, 5], [4.5, 6], [8, 3],
                                [7, 1], [4, 0]])
                polygon = Polygon(coords)
                polygon.plot()
        '''

        fig = plt.figure()
        ax = fig.add_subplot(111)
        ax.plot(*self.boundCoords.T, '-k', lw=2)
        ax.plot(*self.coords.T, 'ok', ms=5)
        ax.grid(ls='--', lw=0.5)
        ax.axis('equal')
        return

# %%
'''
BSD 2 license.

Copyright (c) 2018, Universidad Nacional de Colombia, Andres Ariza-Triana
and Ludger O. Suarez-Burgoa.
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''

reason: import numpy,from numpy,from scipy


text:
#!/usr/bin/env python
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import cv2 as cv

#np.set_printoptions(threshold='nan')
def loadData(directory,inputdatatype="rgb",labeldatatype="cmd_vel",resize=None,datarange=None):
	f=open(directory+'/data.txt','r')
	alldata=f.readlines()
	Ndata=len(alldata)//10
	imagedata=[]
	labeldata=[]

	for i in range(Ndata):
		label=int(alldata[i*10]);

		if datarange:
			if not (label in datarange):
				continue;

		if inputdatatype=='rgb':
			rgbimg=cv.imread(directory+'/image'+str(label)+'_rgb.png')
			rgbimg=rgbimg/255.0
			if resize:
				rgbimg=cv.resize(rgbimg,resize)
			imagedata.append(rgbimg);
		if inputdatatype=='gray':
			rgbimg=cv.imread(directory+'/image'+str(label)+'_rgb.png')
			rgbimg=cv.cvtColor(rgbimg,cv.COLOR_RGB2BGR)
			grayimg=cv.cvtColor(rgbimg,cv.COLOR_BGR2GRAY)/255.0
			if resize:
				grayimg=cv.resize(grayimg,resize)
			grayimg=grayimg.reshape(resize[1],resize[0],1)
			imagedata.append(grayimg)
		elif inputdatatype=='d':
			dimg=cv.imread(directory+'/image'+str(label)+'_depth.png')
			dimg=dimg/32.0
			if resize:
				dimg=cv.resize(dimg,resize)
			imagedata.append(dimg);
		elif inputdatatype=='rgbd':
			rgbimg=cv.imread(directory+'/image'+str(label)+'_rgb.png')
			dimg=cv.imread(directory+'/image'+str(label)+'_depth.png')[:,:,0:1]
			rgbimg=rgbimg/255.0
			dimg=dimg/32.0
			if resize:
				rgbimg=cv.resize(rgbimg,resize)
				dimg=cv.resize(dimg,resize)
			imagedata.append(np.concatenate((rgbimg,dimg),2));

		if labeldatatype=='cmd_vel':
			labeldata.append([float(alldata[10*i+2]),float(alldata[10*i+3])])
		elif labeldatatype=='vel':
			labeldata.append([float(alldata[10*i+7]),float(alldata[10*i+8])])
		elif labeldatatype=='pose':
			labeldata.append([float(alldata[10*i+4]),float(alldata[10*i+5]),float(alldata[10*i+6])])

	return imagedata, labeldata

def imageenhance(images,labels,intensitybiases,applyNoise=True,maxnoise=0.01,numnoise=3,truncate=True):
	enhanceimages=[]
	enhancelabels=[]
	imgshape=images[0].shape
	for i,image in enumerate(images):
		for intensitybias in intensitybiases:
			newimage=image+intensitybias
			if truncate:
				newimage[newimage>1.0]=1.0
				newimage[newimage<0.0]=0.0
			enhanceimages.append(newimage)
			enhancelabels.append(labels[i])
			if applyNoise==True:
				for n in range(numnoise):
					noiseimage=newimage+(np.random.rand(imgshape[0],imgshape[1],imgshape[2])*maxnoise*2-maxnoise)
					if truncate:
						noiseimage[noiseimage>1.0]=1.0
						noiseimage[noiseimage<0.0]=0.0
					enhanceimages.append(noiseimage)
					enhancelabels.append(labels[i])
	return enhanceimages,enhancelabels;
#TODO: delete the point with speed (0.0,0.0)?

if __name__=="__main__":
	images,labels=loadData("./data/data518",inputdatatype='gray',resize=(160,120));
	images,labels=imageenhance(images,labels,[-0.1,0.0,0.1])

	# print(images[0].shape)
	# print(images[0])
	print(len(images))
	sp=images[0].shape

	for i in range(48,72):
		cv.imshow("i",images[i])
		cv.waitKey(2000)
		print(labels[i])

ext: py

max_stars_repo_name: DrarPan/vDriving

max_issues_repo_name: DrarPan/vDriving

max_forks_repo_name: DrarPan/vDriving

reason: import numpy


text:
import sys
import os
# lucas_path = os.environ['LUCAS_PATH']
# sys.path.insert(1, lucas_path)

from sys_simulator.parameters.parameters import LearningParameters
import numpy as np
import torch

class QTable:
    def __init__(self, num_states: int, num_actions: int, params: LearningParameters):
        self.table = np.zeros((num_states, num_actions))
        self.gamma = params.gamma
        self.alpha = params.alpha

    # calculates Q-table values
    def learn(self, obs, action_index, reward, next_obs):
        deltaQ = reward + self.gamma*np.max(self.table[next_obs]) - self.table[obs, action_index]
        self.table[obs, action_index] = self.table[obs,action_index] + self.alpha*deltaQ

class QTensor:
    def __init__(self, tensor: np.array, params: LearningParameters):
        self.tensor = tensor
        self.gamma = params.gamma
        self.alpha = params.alpha

    # calculates Q-table values
    def learn(self, obs, action, reward, next_obs):
        deltaQ = reward + self.gamma*self.tensor[next_obs].max() - self.tensor[(*obs, action)]
        self.tensor[obs] += self.alpha*deltaQ

class QTableTorch:
    def __init__(self, num_states: int, num_actions: int, params: LearningParameters):
        self.device = torch.device('cuda')
        self.table = torch.zeros((num_states, num_actions), device=self.device)
        self.gamma = params.gamma
        self.alpha = params.alpha

    # calculates Q-table values
    def learn(self, obs, action_index, reward, next_obs):
        deltaQ = reward + self.gamma*torch.max(self.table[next_obs]) - self.table[obs, action_index]
        self.table[obs, action_index] = self.table[obs,action_index] + self.alpha*deltaQ

class DistributedQTable(QTable):
    def __init__(self, num_states: int, num_actions: int, params: LearningParameters):
        super(DistributedQTable, self).__init__(num_states, num_actions, params)

    # calculates Q-table values
    def learn(self, obs, action_index, reward, next_obs):
        deltaQ = reward + self.gamma*np.max(self.table[next_obs]) - self.table[obs, action_index]
        q_value = self.table[obs,action_index] + self.alpha*deltaQ
        if q_value > self.table[obs, action_index]:
            self.table[obs, action_index] = q_value

class ModDistributedQTable(QTable):
    def __init__(self, num_states: int, num_actions: int, params: LearningParameters):
        super(ModDistributedQTable, self).__init__(num_states, num_actions, params)

    # calculates Q-table values
    def learn(self, obs, action_index, reward, next_obs):
        deltaQ = reward + self.gamma*np.max(self.table[next_obs]) - self.table[obs, action_index]
        q_value = self.table[obs,action_index] + self.alpha*deltaQ
        self.table[obs, action_index] = q_value

class DistributedQTensor:
    """
    table is now a tensor. obs is a now a two-element-tuple
    """
    def __init__(self, num_actions: int, num_mue_states: int, params: LearningParameters):
        self.tensor = np.zeros((num_actions, num_mue_states, num_actions))
        self.gamma = params.gamma
        self.alpha = params.alpha

    # calculates Q-table values
    def learn(self, obs, action_index, reward, next_obs):
        deltaQ = reward + self.gamma*np.max(self.tensor[next_obs]) - self.tensor[(obs[0], obs[1], action_index)]
        q_value = self.tensor[(obs[0], obs[1], action_index)] + self.alpha*deltaQ
        if q_value > self.tensor[(obs[0], obs[1], action_index)]:
            self.tensor[(obs[0], obs[1], action_index)] = q_value


text:
from flask import Flask, request
from seldon_core.utils import (
    json_to_seldon_message,
    extract_request_parts,
    array_to_grpc_datadef,
    seldon_message_to_json,
)
from seldon_core.proto import prediction_pb2
import numpy as np
import json
import logging
import sys
import log_helper
from collections.abc import Iterable

MAX_PAYLOAD_BYTES = 300000
app = Flask(__name__)
print("starting logger")
sys.stdout.flush()

log = logging.getLogger("werkzeug")
log.setLevel(logging.ERROR)

es = log_helper.connect_elasticsearch()

@app.route("/", methods=["GET", "POST"])
def index():

    request_id = log_helper.extract_request_id(request.headers)
    type_header = request.headers.get(log_helper.TYPE_HEADER_NAME)
    message_type = log_helper.parse_message_type(type_header)
    index_name = log_helper.build_index_name(request.headers)

    body = request.get_json(force=True)

    # max size is configurable with env var or defaults to constant
    max_payload_bytes = log_helper.get_max_payload_bytes(MAX_PAYLOAD_BYTES)

    body_length = request.headers.get(log_helper.LENGTH_HEADER_NAME)
    if body_length and int(body_length) > int(max_payload_bytes):
        too_large_message = (
            "body too large for "
            + index_name
            + "/"
            + log_helper.DOC_TYPE_NAME
            + "/"
            + request_id
            + " adding "
            + message_type
        )
        print(too_large_message)
        sys.stdout.flush()
        return too_large_message

    if not type(body) is dict:
        body = json.loads(body)

    # print('RECEIVED MESSAGE.')
    # print(str(request.headers))
    # print(str(body))
    # print('----')
    # sys.stdout.flush()

    try:

        # now process and update the doc
        doc = process_and_update_elastic_doc(
            es, message_type, body, request_id, request.headers, index_name
        )

        return str(doc)
    except Exception as ex:
        print(ex)
    sys.stdout.flush()
    return "problem logging request"

def process_and_update_elastic_doc(
    elastic_object, message_type, message_body, request_id, headers, index_name
):

    if message_type == "unknown":
        print("UNKNOWN REQUEST TYPE FOR " + request_id + " - NOT PROCESSING")
        sys.stdout.flush()

    # first do any needed transformations
    new_content_part = process_content(message_type, message_body)

    # set metadata to go just in this part (request or response) and not top-level
    log_helper.field_from_header(
        content=new_content_part, header_name="ce-time", headers=headers
    )
    log_helper.field_from_header(
        content=new_content_part, header_name="ce-source", headers=headers
    )

    doc_body = {message_type: new_content_part}

    log_helper.set_metadata(doc_body, headers, message_type, request_id)

    # req or res might be batches of instances so split out into individual docs
    if "instance" in new_content_part:

        if type(new_content_part["instance"]) == type([]):
            # if we've a list then this is batch
            # we assume first dimension is always batch

            no_items_in_batch = len(new_content_part["instance"])
            index = 0
            for item in new_content_part["instance"]:

                item_body = doc_body.copy()

                item_body[message_type]["instance"] = item
                item_request_id = build_request_id_batched(
                    request_id, no_items_in_batch, index
                )
                upsert_doc_to_elastic(
                    elastic_object, message_type, item_body, item_request_id, index_name
                )
                index = index + 1
        else:
            item_request_id = build_request_id_batched(request_id, 1, 0)
            upsert_doc_to_elastic(
                elastic_object, message_type, doc_body, item_request_id, index_name
            )
    elif "data" in new_content_part and message_type == "outlier":
        no_items_in_batch = len(doc_body[message_type]["data"]["is_outlier"])
        index = 0
        for item in doc_body[message_type]["data"]["is_outlier"]:
            item_body = doc_body.copy()
            item_body[message_type]["data"]["is_outlier"] = item
            if (
                "feature_score" in item_body[message_type]["data"]
                and item_body[message_type]["data"]["feature_score"] is not None
                and len(item_body[message_type]["data"]["feature_score"])
                == no_items_in_batch
            ):
                item_body[message_type]["data"]["feature_score"] = item_body[
                    message_type
                ]["data"]["feature_score"][index]
            if (
                "instance_score" in item_body[message_type]["data"]
                and item_body[message_type]["data"]["instance_score"] is not None
                and len(item_body[message_type]["data"]["instance_score"])
                == no_items_in_batch
            ):
                item_body[message_type]["data"]["instance_score"] = item_body[
                    message_type
                ]["data"]["instance_score"][index]
            item_request_id = build_request_id_batched(
                request_id, no_items_in_batch, index
            )
            upsert_doc_to_elastic(
                elastic_object, message_type, item_body, item_request_id, index_name
            )
            index = index + 1
    else:
        print("unexpected data format")
        print(new_content_part)
    return

def build_request_id_batched(request_id, no_items_in_batch, item_index):
    item_request_id = request_id
    if no_items_in_batch > 1:
        item_request_id = item_request_id + "-item-" + str(item_index)
    return item_request_id

def upsert_doc_to_elastic(
    elastic_object, message_type, upsert_body, request_id, index_name
):
    upsert_doc = {
        "doc_as_upsert": True,
        "doc": upsert_body,
    }
    new_content = elastic_object.update(
        index=index_name,
        doc_type=log_helper.DOC_TYPE_NAME,
        id=request_id,
        body=upsert_doc,
        retry_on_conflict=3,
        refresh=True,
        timeout="60s",
    )
    print(
        "upserted to doc "
        + index_name
        + "/"
        + log_helper.DOC_TYPE_NAME
        + "/"
        + request_id
        + " adding "
        + message_type
    )
    sys.stdout.flush()
    return str(new_content)

# take request or response part and process it by deriving metadata
def process_content(message_type, content):

    if content is None:
        print("content is empty")
        sys.stdout.flush()
        return content

    # if we have dataType then have already parsed before
    if "dataType" in content:
        print("dataType already in content")
        sys.stdout.flush()
        return content

    requestCopy = content.copy()

    # extract data part out and process for req or resp - handle differently later for outlier
    if message_type == "request" or message_type == "response":
        requestCopy = extract_data_part(content)

    return requestCopy

def extract_data_part(content):
    copy = content.copy()

    # if 'instances' in body then tensorflow request protocol
    # if 'predictions' then tensorflow response
    # otherwise can use seldon logic for parsing and inferring type (won't be in here if outlier)

    if "instances" in copy:

        copy["instance"] = copy["instances"]
        content_np = np.array(copy["instance"])

        copy["dataType"] = "tabular"
        first_element = content_np.item(0)

        set_datatype_from_numpy(content_np, copy, first_element)
        del copy["instances"]
    elif "predictions" in copy:
        copy["instance"] = copy["predictions"]
        content_np = np.array(copy["predictions"])

        copy["dataType"] = "tabular"
        first_element = content_np.item(0)

        set_datatype_from_numpy(content_np, copy, first_element)
        del copy["predictions"]
    else:
        requestMsg = json_to_seldon_message(copy)

        (req_features, _, req_datadef, req_datatype) = extract_request_parts(requestMsg)

        # set sensible defaults for non-tabular dataTypes
        # tabular should be iterable and get inferred through later block
        if req_datatype == "strData":
            copy["dataType"] = "text"
        if req_datatype == "binData":
            copy["dataType"] = "image"

        if isinstance(req_features, Iterable):

            elements = createElelmentsArray(req_features, list(req_datadef.names))

            if isinstance(elements, Iterable):

                for i, e in enumerate(elements):
                    reqJson = extractRow(
                        i, requestMsg, req_datatype, req_features, req_datadef
                    )
                    reqJson["elements"] = e
                    copy = reqJson

        copy["instance"] = json.loads(
            json.dumps(req_features, cls=log_helper.NumpyEncoder)
        )

        if isinstance(req_features, np.ndarray):
            set_datatype_from_numpy(req_features, copy, req_features.item(0))

    # copy names into its own section of request
    if "data" in content:
        if "names" in content["data"]:
            copy["names"] = content["data"]["names"]

    # should now have processed content of seldon message so don't want its various constructs on top-level anymore
    if "data" in copy:
        del copy["data"]
    if "strData" in copy:
        del copy["strData"]
    if "jsonData" in copy:
        del copy["jsonData"]
    if "binData" in copy:
        del copy["binData"]

    copy["payload"] = content

    return copy

def set_datatype_from_numpy(content_np, copy, first_element):

    if first_element is not None and not isinstance(first_element, (int, float)):
        copy["dataType"] = "text"
    if first_element is not None and isinstance(first_element, (int, float)):
        copy["dataType"] = "number"
    if content_np.shape is not None and len(content_np.shape) > 1:
        # first dim is batch so second reveals whether instance is array
        if content_np.shape[1] > 1:
            copy["dataType"] = "tabular"
    if len(content_np.shape) > 2:
        copy["dataType"] = "tabular"
    if len(content_np.shape) > 3:
        copy["dataType"] = "image"

def extractRow(
    i: int,
    requestMsg: prediction_pb2.SeldonMessage,
    req_datatype: str,
    req_features: np.ndarray,
    req_datadef: "prediction_pb2.SeldonMessage.data",
):
    datatyReq = "ndarray"
    dataType = "tabular"
    if len(req_features.shape) == 2:
        dataReq = array_to_grpc_datadef(
            datatyReq, np.expand_dims(req_features[i], axis=0), req_datadef.names
        )
    else:
        if len(req_features.shape) > 2:
            dataType = "image"
        else:
            dataType = "text"
            req_features = np.char.decode(req_features.astype("S"), "utf-8")
        dataReq = array_to_grpc_datadef(
            datatyReq, np.expand_dims(req_features[i], axis=0), req_datadef.names
        )
    requestMsg2 = prediction_pb2.SeldonMessage(data=dataReq, meta=requestMsg.meta)
    reqJson = {}
    reqJson["payload"] = seldon_message_to_json(requestMsg2)
    # setting dataType here temporarily so calling method will be able to access it
    # don't want to set it at the payload level
    reqJson["dataType"] = dataType
    return reqJson

def createElelmentsArray(X: np.ndarray, names: list):
    results = None
    if isinstance(X, np.ndarray):
        if len(X.shape) == 1:
            results = []
            for i in range(X.shape[0]):
                d = {}
                for num, name in enumerate(names, start=0):
                    if isinstance(X[i], bytes):
                        d[name] = X[i].decode("utf-8")
                    else:
                        d[name] = X[i]
                results.append(d)
        elif len(X.shape) >= 2:
            results = []
            for i in range(X.shape[0]):
                d = {}
                for num, name in enumerate(names, start=0):
                    d[name] = np.expand_dims(X[i, num], axis=0).tolist()
                results.append(d)
    return results

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=8080)

max_stars_repo_name: glindsell/seldon-core

max_issues_repo_name: glindsell/seldon-core

max_forks_repo_name: glindsell/seldon-core


text:
from data_loader.data_loaders import *
from data_loader.TR_data_loaders import *
from data_loader.CT_Wiki_data_loaders import *
from data_loader.hybrid_data_loaders import *
import pdb
import torch
import numpy as np
from utils.util import *

if __name__ == "__main__":
    torch.manual_seed(0)
    np.random.seed(0)
    mode = 0
    data_dir = "./data/wikitables_v2"
    entity_vocab = load_entity_vocab(data_dir, ignore_bad_title=True, min_ent_count=2)
    dev_dataset = WikiHybridTableDataset(data_dir,\
        entity_vocab,max_cell=100, max_input_tok=350, max_input_ent=150, src="dev", max_length = [50, 10, 10], force_new=True, tokenizer = None, mode = mode)
    train_dataset = WikiHybridTableDataset(data_dir,\
        entity_vocab,max_cell=100, max_input_tok=350, max_input_ent=150, src="train", max_length = [50, 10, 10], force_new=True, tokenizer = None, mode = mode)
    pdb.set_trace()
    dev_data_generator = HybridTableLoader(dev_dataset,10,num_workers=0,mlm_probability=0.5,ent_mlm_probability=0.5,is_train=False,use_cand=True, mode = mode)
    train_data_generator = HybridTableLoader(train_dataset,10,num_workers=0,mlm_probability=0.5,ent_mlm_probability=0.5,is_train=True,use_cand=False, mode = mode)
    for x in train_data_generator:
        pass
        pdb.set_trace()
        break
    for x in dev_data_generator:
        pass
        pdb.set_trace()
        break

    # data_dir = "data/WebQueryTable_Dataset"
    # train_dataset = WebQueryTableDataset(data_dir, src="train", force_new=True)
    # train_dataloader = TRLoader(train_dataset, 10, is_train=True)
    # for x in train_dataloader:
    #     pass
        # pdb.set_trace()
        # break

    # data_dir = "data/T2D_IO"
    # train_dataset = SemColDataset(data_dir,entity_vocab, src="train", max_cell_length=10, force_new=True)
    # train_dataloader = CTLoader(train_dataset, 10, is_train=True)
    # for x in train_dataloader:
    #     pass
    #     pdb.set_trace()
    #     break

max_stars_repo_name: kimlindner/TURL

max_issues_repo_name: kimlindner/TURL

max_forks_repo_name: kimlindner/TURL


text:
import numpy as np
import matplotlib.pyplot as plt
import os
A = np.zeros((10,3))
for i in range(10):
    A[i, :] = [i, i+1, i+2]
print(A/6)

'''

def plott(N):
    A = np.array(list(range(N)))

    z2 = 0.4
    m = 0
    for i in range(N):
        m += 0.7 * pow(1 / (i + 1), z2)

    K_2 = np.zeros(N)
    for i in range(N):
        ind = i / 10
        dice = np.random.rand(1)
        if dice < 0.05:
            penalty = -100
            K_2[i] = 10 * (-1000 + 1000 * (0.11 - (1 / m) * 0.7 * pow(1 / (ind + 1), z2)) + np.random.normal(0, 1,
                                                                                1)) + penalty
        else:
            K_2[i] = 10 * (-1000 + 1000 * (0.11 - (1 / m) * 0.7 * pow(1 / (ind + 1), z2)) + np.random.normal(0, 1, 1))

    for obs in range(N):
        i = 0
        while i < 1e6:
            i += 1
        else:
            K = K_2[obs]
            print('\rNumber of Timestep: {}\tCumulative Reward: {:.4f}'.format(obs, K))

    plt.plot(A, K_2)
    plt.xlabel('Time Step (s)')
    plt.ylabel('Cumulative Reward')
    plt.title('Test Cumulative Reward vs Time Step in DDPG method')
    plt.show()
    return K_2

def compute(N):
    for i in range(200):
        i += 1
        J = i + N
    K_2 = plott(N)
    os._exit(0)

'''


text:
import tensorflow as tf
import numpy as np
from model import GPModel
from gpr import GPR
from param import Param
from mean_functions import Zero
import likelihoods
from tf_hacks import eye
import kernel_expectations as ke
import transforms

def PCA_reduce(X, Q):
    """
    A helpful function for linearly reducing the dimensionality of the data X
    to Q.

    X is a N x D numpy array.
    Q is an integer, Q < D

    returns a numpy array, N x Q
    """
    evecs, evals = np.linalg.eigh(np.cov(X.T))
    i = np.argsort(evecs)[::-1]
    W = evals[:, i]
    W = W[:, :Q]
    return (X-X.mean(0)).dot(W)

class GPLVM(GPR):
    """
    Standard GPLVM where the likelihood can be optimised with respect to the  latent X.
    """
    def __init__(self, Y, X_mean, kern, mean_function=Zero()):
        """
        Y is a data matrix, size N x R
        Z is a matrix of pseudo inputs, size M x D
        X_mean is a matrix, size N x Q, for the initialisation of the latent space.
        kern, mean_function are appropriate GPflow objects

        This method only works with a Gaussian likelihood.

        """
        self.num_latent = X_mean.shape[1]
        assert Y.shape[1] >= self.num_latent, 'More latent dimensions than observed.'
        GPR. __init__(self, X_mean, Y, kern, mean_function=mean_function)
        del self.X  # in GPLVM this is a Param
        self.X = Param(X_mean)

class BayesianGPLVM(GPModel):

    def __init__(self, X_mean, X_var, Y, kern, Z, X_prior_mean=None, X_prior_var=None):
        """
        X_mean is a data matrix, size N x D
        X_var is a data matrix, size N x D (X_var > 0)
        Y is a data matrix, size N x R
        Z is a matrix of pseudo inputs, size M x D
        kern, mean_function are appropriate GPflow objects

        This method only works with a Gaussian likelihood.

        """
        GPModel.__init__(self, X_mean, Y, kern, likelihood=likelihoods.Gaussian(), mean_function=Zero())
        del self.X
        self.X_mean = Param(X_mean)
        self.X_var = Param(X_var, transforms.positive)
        self.Z = Param(Z)
        self.num_data = X_mean.shape[0]
        self.num_latent = Z.shape[1]
        self.output_dim = Y.shape[1]

        assert np.all(X_mean.shape == X_var.shape)
        assert X_mean.shape[1] == self.num_latent
        assert X_mean.shape[0] == Y.shape[0], 'X mean and Y must be same size.'
        assert X_var.shape[0] == Y.shape[0], 'X var and Y must be same size.'

        # deal with parameters for the prior mean variance of X
        if X_prior_mean is None:
            X_prior_mean = np.zeros((self.num_data, self.num_latent))
        self.X_prior_mean = X_prior_mean
        if X_prior_var is None:
            X_prior_var = np.ones((self.num_data, self.num_latent))
        self.X_prior_var = X_prior_var

        assert X_prior_mean.shape[0] == self.num_data
        assert X_prior_mean.shape[1] == self.num_latent
        assert X_prior_var.shape[0] == self.num_data
        assert X_prior_var.shape[1] == self.num_latent

    def build_likelihood(self):
        """
        Construct a tensorflow function to compute the bound on the marginal
        likelihood.
        """
        num_inducing = tf.shape(self.Z)[0]

        psi0, psi1, psi2 = ke.build_psi_stats(self.Z, self.kern, self.X_mean, self.X_var)
        Kuu = self.kern.K(self.Z) + eye(num_inducing) * 1e-6
        L = tf.cholesky(Kuu)
        sigma2 = self.likelihood.variance
        sigma = tf.sqrt(sigma2)

        # Compute intermediate matrices
        A = tf.matrix_triangular_solve(L, tf.transpose(psi1), lower=True) / sigma
        tmp = tf.matrix_triangular_solve(L, psi2, lower=True)
        AAT = tf.matrix_triangular_solve(L, tf.transpose(tmp), lower=True) / sigma2
        B = AAT + eye(num_inducing)
        LB = tf.cholesky(B)
        log_det_B = 2. * tf.reduce_sum(tf.log(tf.diag_part(LB)))
        c = tf.matrix_triangular_solve(LB, tf.matmul(A, self.Y), lower=True) / sigma

        # KL[q(x) || p(x)]
        NQ = tf.cast(tf.size(self.X_mean), tf.float64)
        D = tf.cast(tf.shape(self.Y)[1], tf.float64)
        KL = -0.5*tf.reduce_sum(tf.log(self.X_var)) \
            + 0.5*tf.reduce_sum(tf.log(self.X_prior_var))\
            - 0.5 * NQ\
            + 0.5 * tf.reduce_sum((tf.square(self.X_mean - self.X_prior_mean) + self.X_var) / self.X_prior_var)

        # compute log marginal bound
        ND = tf.cast(tf.size(self.Y), tf.float64)
        bound = -0.5 * ND * tf.log(2 * np.pi * sigma2)
        bound += -0.5 * D * log_det_B
        bound += -0.5 * tf.reduce_sum(tf.square(self.Y)) / sigma2
        bound += 0.5 * tf.reduce_sum(tf.square(c))
        bound += -0.5 * D * (tf.reduce_sum(psi0) / sigma2 -
                             tf.reduce_sum(tf.diag_part(AAT)))
        bound -= KL

        return bound

    def build_predict(self, Xnew, full_cov=False):
        """
        Compute the mean and variance of the latent function at some new points
        Xnew. Note that this is very similar to the SGPR prediction, for whcih
        there are notes in the SGPR notebook.
        """
        num_inducing = tf.shape(self.Z)[0]
        psi0, psi1, psi2 = ke.build_psi_stats(self.Z, self.kern, self.X_mean, self.X_var)
        Kuu = self.kern.K(self.Z) + eye(num_inducing) * 1e-6
        Kus = self.kern.K(self.Z, Xnew)
        sigma2 = self.likelihood.variance
        sigma = tf.sqrt(sigma2)
        L = tf.cholesky(Kuu)

        A = tf.matrix_triangular_solve(L, tf.transpose(psi1), lower=True) / sigma
        tmp = tf.matrix_triangular_solve(L, psi2, lower=True)
        AAT = tf.matrix_triangular_solve(L, tf.transpose(tmp), lower=True) / sigma2
        B = AAT + eye(num_inducing)
        LB = tf.cholesky(B)
        c = tf.matrix_triangular_solve(LB, tf.matmul(A, self.Y), lower=True) / sigma
        tmp1 = tf.matrix_triangular_solve(L, Kus, lower=True)
        tmp2 = tf.matrix_triangular_solve(LB, tmp1, lower=True)
        mean = tf.matmul(tf.transpose(tmp2), c)
        if full_cov:
            var = self.kern.K(Xnew) + tf.matmul(tf.transpose(tmp2), tmp2)\
                - tf.matmul(tf.transpose(tmp1), tmp1)
            shape = tf.pack([1, 1, tf.shape(self.Y)[1]])
            var = tf.tile(tf.expand_dims(var, 2), shape)
        else:
            var = self.kern.Kdiag(Xnew) + tf.reduce_sum(tf.square(tmp2), 0)\
                - tf.reduce_sum(tf.square(tmp1), 0)
            shape = tf.pack([1, tf.shape(self.Y)[1]])
            var = tf.tile(tf.expand_dims(var, 1), shape)
        return mean + self.mean_function(Xnew), var

max_stars_repo_name: blutooth/dgp

max_issues_repo_name: blutooth/dgp

max_forks_repo_name: blutooth/dgp


text:
# First import the library
import pyrealsense2 as rs
# Import Numpy for easy array manipulation
import numpy as np
# Import OpenCV for easy image rendering
import cv2
from PIL import Image
import os

class Depth():

    def __init__(self, config):
        self.config = config

        self.pipeline = rs.pipeline()

        # Create a config and configure the pipeline to stream
        #  different resolutions of color and depth streams
        config = rs.config()
        config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30)
        # config.enable_stream(rs.stream.depth, 1280, 720, rs.format.z16, 6)
        config.enable_stream(rs.stream.color, 1280, 720, rs.format.bgr8, 6)
        # Start streaming
        self.profile = self.pipeline.start(config)

    def __del__(self):
        self.pipeline.stop()

    def get_depth_image(self):
        # Create a pipeline
        # pipeline = rs.pipeline()
        # Create a config and configure the pipeline to stream
        #  different resolutions of color and depth streams
        # config = rs.config()
        # config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30)
        # if not os.path.exists('./frame'):
        #    os.mkdir('./frame')
        # count = len(os.walk('C:/Users/vclab/PycharmProjects/nlos/data_generation/real_data/depth').__next__()[1])
        # filename = './frame/depth_{}'.format(count)
        # os.mkdir(filename)
        # config.enable_stream(rs.stream.depth, 1280, 720, rs.format.z16, 6)
        # config.enable_stream(rs.stream.color, 1280, 720, rs.format.bgr8, 6)

        # Start streaming
        # profile = pipeline.start(config)
        # Getting the depth sensor's depth scale (see rs-align example for explanation)
        # depth_sensor = profile.get_device().first_depth_sensor()
        # depth_scale = depth_sensor.get_depth_scale()
        # print("Depth Scale is: ", depth_scale)

        # We will be removing the background of objects more than
        #  clipping_distance_in_meters meters away
        # clipping_distance_in_meters = 5  # 1 meter
        # clipping_distance = clipping_distance_in_meters / depth_scale
        # Create an align object
        # rs.align allows us to perform alignment of depth frames to others frames
        # The "align_to" is the stream type to which we plan to align depth frames.
        # Streaming loop
        try:
            # Get frameset of color and depthprint('4')
            # for x in range(5):
            #     self.pipeline.wait_for_frames()

            frames = self.pipeline.wait_for_frames()
            # frames.get_depth_frame() is a 640x360 depth image

            # Align the depth frame to color frame

            # Get aligned frames
            depth_frame = frames.get_depth_frame()  # aligned_depth_frame is a 640x480 depth image
            # Validate that both frames are valid
            if not depth_frame:
                print("ERROR : NO FRAME")
                return

            colorizer = rs.colorizer()
            depth_image = np.asanyarray(colorizer.colorize(depth_frame).get_data())

            # Remove background - Set pixels further than clipping_distance to grey
            # grey_color = 153
            # depth_image_3d = np.dstack((depth_image, depth_image, depth_image))  # depth image is 1 channel, color is 3 channels
            # bg_removed = np.where((depth_image_3d > clipping_distance) | (depth_image_3d <= 0), grey_color, color_image)

            # Render images
            # depth_colormap = cv2.applyColorMap(cv2.convertScaleAbs(depth_image, alpha=0.03), cv2.COLORMAP_JET)

            # images = np.hstack((bg_removed, depth_colormap))

            ### DEPTH ARRAY TO IMAGE ###

            # filename = "C:/Users/vclab/PycharmProjects/nlos/data_generation/real_data/depth"
            # cv2.imwrite('{}/DEPTH.png'.format(filename), depth_image)
            # print("success")
            # cv2.imwrite('{}/DEPTH.png'p.format(filename), images)
            # cv2.imwrite('{}/DEPTH.png'.format(filename), depth_colormap)

            return depth_image
        except:
            print('Exception Error')

        # finally:
        #     pipeline.stop()

    def get_rgb(self):
        # Create a pipeline
        # pipeline = rs.pipeline()

        # Create a config and configure the pipeline to stream
        #  different resolutions of color and depth streams
        # config = rs.config()
        # config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30)
        # config.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 30)

        # Start streaming
        # profile = pipeline.start(config)

        # Create an align object
        # rs.align allows us to perform alignment of depth frames to others frames
        # The "align_to" is the stream type to which we plan to align depth frames.
        # Streaming loop
        try:
            # Get frameset of color and depth
            frames = self.pipeline.wait_for_frames()
            # frames.get_depth_frame() is a 640x360 depth image

            # Align the depth frame to color frame

            # Get aligned frames
            color_frame = frames.get_color_frame()

            # Validate that both frames are valid
            if not color_frame:
                print("ERROR : NO FRAME")
                return
            # colorizer = rs.colorizer()
            color_image = np.asanyarray(color_frame.get_data())
            # print(color_image)

            # filename = "C:/Users/vclab/PycharmProjects/nlos/data_generation/real_data/depth"
            # cv2.imwrite('{}/rgb_image.png'.format(filename), color_image)

            return color_image
        except:
            print('Exception Error')

        # finally:
        #     pipeline.stop()

if __name__ == "__main__":
    depth = Depth(config=None)
    a = depth.get_depth_image()
    b = depth.get_rgb()

max_stars_repo_name: vclabskku/nlos

max_issues_repo_name: vclabskku/nlos

max_forks_repo_name: vclabskku/nlos


text:
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
from lxml import etree
import csv
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import statistics
from scipy import stats

patterns = []

stopset = set(stopwords.words('english'))

tokenizer = RegexpTokenizer(r'\w+')

glossary = etree.parse("glossary.xml")

tags = {}

tfidf_vectorizer = TfidfVectorizer()

# outFile = open("timeseries.csv", "wb")
# wr = csv.writer(outFile, quoting=csv.QUOTE_ALL)

# outFile = open("timeseries.txt", "wb")

def nucleous(s):
	docs = [s]
	instr = ""
	conList = []

	words = tokenizer.tokenize(s)

	for w in words:
		instr += (' ' + w)
		docs.append(instr)

	tfidf_matrix = tfidf_vectorizer.fit_transform(docs)
	matrix = cosine_similarity(tfidf_matrix[0:1], tfidf_matrix)

 	for row in matrix:
 		for x in row[1:]:
			conList.append(x)

	mean = statistics.mean(conList)
  	conf = stats.norm.interval(0.95 ,loc=mean,scale=statistics.pstdev(conList))

  	thld = 1 - statistics.pstdev(conList)

  	return thld

for w in stopset:
	print w

# for entry_obj in glossary.findall("Entry"):

# 	for concept_obj in entry_obj.findall("Concept"):

# 		concept_text = concept_obj.text.lower()

# 		words = tokenizer.tokenize(concept_text)

# 		pattern = ""

# 		for w in words:
# 			if w in stopset:
# 				pattern += "0"
# 			else:
# 				pattern += "1"

# 		# if pattern == "11011":
# 		# 	outFile.write(concept_text + "\n")
# 		# 	outFile.write(pattern + "\n")
# 		# 	outFile.write('%.7f' % nucleous(concept_text))
# 		# 	outFile.write("\n")

# 		# print pattern , " - ", nucleous(concept_text), "\n"
# 		if pattern == "11011":
# 		# if pattern not in tags:
# 		# 	tags[pattern] = 1
# 		# else:
# 		# 	newvalue = tags[pattern]
# 		# 	tags[pattern] = (newvalue + 1)
# 			print concept_text, " - ", pattern
# 			print nucleous(concept_text)

# for keys, values in tags.items():
# 	print keys, " - ", values

		# 	print concept_text, "\n"
		# patterns.append(pattern)

# outFile.close()
# print len(set(patterns))

max_stars_repo_path: Similarity Nucleus/timeSeries.py

max_issues_repo_path: Similarity Nucleus/timeSeries.py

max_forks_repo_path: Similarity Nucleus/timeSeries.py

reason: from scipy


text:
"""
Code based on https://github.com/carpedm20/DCGAN-tensorflow
and https://github.com/igul222/improved_wgan_training
"""

import os, random, time, math, pickle, csv, re
from glob import glob
import numpy as np
from six.moves import xrange
import heapq as hq
import copy

from ops import *
from utils import *

import tensorflow as tf

def conv_out_size_same(size, stride):
  return int(math.ceil(float(size) / float(stride)))

class PacGANTaskWrapper:
    def __init__(self, _work_dir, _config):
        self._work_dir = _work_dir
        self._config = _config

    def main(self):
        seed = random.randint(1, 100000)
        path = os.path.join(self._work_dir, ".numpyseed")
        f = open(path, "w")
        f.write("{}\n".format(seed))
        f.close()
        np.random.seed(seed)

        run_config = tf.ConfigProto()
        run_config.gpu_options.allow_growth = True
        with tf.Session(config=run_config) as sess:
            self.packing_num = self._config["packing_num"]

            self.sess = sess
            self.batch_size = 64
            self.sample_num = 64

            self.input_height=108
            self.input_width=108
            self.output_height = 64
            self.output_width = 64

            self.z_dim = 100

            self.gf_dim = 64
            self.df_dim = 64

            self.d_bn1 = batch_norm(name="d_bn1")
            self.d_bn2 = batch_norm(name="d_bn2")
            self.d_bn3 = batch_norm(name="d_bn3")

            self.g_bn0 = batch_norm(name='g_bn0')
            self.g_bn1 = batch_norm(name='g_bn1')
            self.g_bn2 = batch_norm(name='g_bn2')
            self.g_bn3 = batch_norm(name='g_bn3')

            self.checkpoint_dir = os.path.join(self._work_dir, "checkpoint")
            if not os.path.exists(self.checkpoint_dir):
                os.makedirs(self.checkpoint_dir)
            self.sample_dir = os.path.join(self._work_dir, "samples")
            if not os.path.exists(self.sample_dir):
                os.makedirs(self.sample_dir)
            self.data_dir = "celebA" #"cifar-10-batches-py"
            self.metric_path = os.path.join(self._work_dir, "metrics.csv")

            self.learning_rate = 0.0002
            self.epoch = 25
            self.beta1 = 0.5

            self.c_dim = 3
            self.data = glob(os.path.join("celebA/img_align_celeba", "*.jpg"))

            self.inception_score_sample_num = 50000

            self.build_model()
            self.train()
            nbatch, topK, runs = 16, 20, 1
            self.get_collisions(nbatch, topK, runs)

    def build_model(self):
        image_dims = [self.output_height, self.output_width, self.c_dim * self.packing_num]
        self.inputs = tf.placeholder(tf.float32, [self.batch_size] + image_dims, name='real_images')

        self.z = []
        for i in range(self.packing_num):
            self.z.append(tf.placeholder(
                tf.float32, [None, self.z_dim], name='z{}'.format(i)))

        self.G_sep = []
        for i in range(self.packing_num):
            self.G_sep.append(self.generator(self.z[i]))
        self.G                  = tf.concat(self.G_sep, 3)

        self.D, self.D_logits = self.discriminator(self.inputs, reuse=False)
        self.sampler = self.sampler(self.z[0])
        self.D_, self.D_logits_ = self.discriminator(self.G, reuse=True)

        def sigmoid_cross_entropy_with_logits(x, y):
            try:
                return tf.nn.sigmoid_cross_entropy_with_logits(logits=x, labels=y)
            except:
                return tf.nn.sigmoid_cross_entropy_with_logits(logits=x, targets=y)
        self.d_loss_real = tf.reduce_mean(
            sigmoid_cross_entropy_with_logits(self.D_logits, tf.ones_like(self.D)))
        self.d_loss_fake = tf.reduce_mean(
            sigmoid_cross_entropy_with_logits(self.D_logits_, tf.zeros_like(self.D_)))
        self.g_loss = tf.reduce_mean(
            sigmoid_cross_entropy_with_logits(self.D_logits_, tf.ones_like(self.D_)))

        self.d_loss = self.d_loss_real + self.d_loss_fake

        t_vars = tf.trainable_variables()

        self.d_vars = [var for var in t_vars if 'd_' in var.name]
        self.g_vars = [var for var in t_vars if 'g_' in var.name]

        print("Number of parameters in discriminator: {}".format(np.sum([np.prod(v.get_shape().as_list()) for v in self.d_vars])))
        print("Number of parameters in generator: {}".format(np.sum([np.prod(v.get_shape().as_list()) for v in self.g_vars])))

        self.saver = tf.train.Saver()

    def generator(self, z):
        with tf.variable_scope("generator", reuse = tf.AUTO_REUSE) as scope:
            s_h, s_w = self.output_height, self.output_width
            s_h2, s_w2 = conv_out_size_same(s_h, 2), conv_out_size_same(s_w, 2)
            s_h4, s_w4 = conv_out_size_same(s_h2, 2), conv_out_size_same(s_w2, 2)
            s_h8, s_w8 = conv_out_size_same(s_h4, 2), conv_out_size_same(s_w4, 2)
            s_h16, s_w16 = conv_out_size_same(s_h8, 2), conv_out_size_same(s_w8, 2)

            # project `z` and reshape
            self.z_, self.h0_w, self.h0_b = linear(
                z, self.gf_dim*8*s_h16*s_w16, 'g_h0_lin', with_w=True)

            self.h0 = tf.reshape(
                self.z_, [-1, s_h16, s_w16, self.gf_dim * 8])
            h0 = tf.nn.relu(self.g_bn0(self.h0))

            self.h1, self.h1_w, self.h1_b = deconv2d(
                h0, [self.batch_size, s_h8, s_w8, self.gf_dim*4], name='g_h1', with_w=True)
            h1 = tf.nn.relu(self.g_bn1(self.h1))

            h2, self.h2_w, self.h2_b = deconv2d(
                h1, [self.batch_size, s_h4, s_w4, self.gf_dim*2], name='g_h2', with_w=True)
            h2 = tf.nn.relu(self.g_bn2(h2))

            h3, self.h3_w, self.h3_b = deconv2d(
                h2, [self.batch_size, s_h2, s_w2, self.gf_dim*1], name='g_h3', with_w=True)
            h3 = tf.nn.relu(self.g_bn3(h3))

            h4, self.h4_w, self.h4_b = deconv2d(
                h3, [self.batch_size, s_h, s_w, self.c_dim], name='g_h4', with_w=True)

            return tf.nn.tanh(h4)

    def sampler(self, z):
        with tf.variable_scope("generator") as scope:
            scope.reuse_variables()

            s_h, s_w = self.output_height, self.output_width
            s_h2, s_w2 = conv_out_size_same(s_h, 2), conv_out_size_same(s_w, 2)
            s_h4, s_w4 = conv_out_size_same(s_h2, 2), conv_out_size_same(s_w2, 2)
            s_h8, s_w8 = conv_out_size_same(s_h4, 2), conv_out_size_same(s_w4, 2)
            s_h16, s_w16 = conv_out_size_same(s_h8, 2), conv_out_size_same(s_w8, 2)

            # project `z` and reshape
            self.z_, self.h0_w, self.h0_b = linear(
                z, self.gf_dim*8*s_h16*s_w16, 'g_h0_lin', with_w=True)

            self.h0 = tf.reshape(
                self.z_, [-1, s_h16, s_w16, self.gf_dim * 8])
            h0 = tf.nn.relu(self.g_bn0(self.h0))

            self.h1, self.h1_w, self.h1_b = deconv2d(
                h0, [self.batch_size, s_h8, s_w8, self.gf_dim*4], name='g_h1', with_w=True)
            h1 = tf.nn.relu(self.g_bn1(self.h1))

            h2, self.h2_w, self.h2_b = deconv2d(
                h1, [self.batch_size, s_h4, s_w4, self.gf_dim*2], name='g_h2', with_w=True)
            h2 = tf.nn.relu(self.g_bn2(h2))

            h3, self.h3_w, self.h3_b = deconv2d(
                h2, [self.batch_size, s_h2, s_w2, self.gf_dim*1], name='g_h3', with_w=True)
            h3 = tf.nn.relu(self.g_bn3(h3))

            h4, self.h4_w, self.h4_b = deconv2d(
                h3, [self.batch_size, s_h, s_w, self.c_dim], name='g_h4', with_w=True)

            return tf.nn.tanh(h4)

    def discriminator(self, image, reuse=False):
        with tf.variable_scope("discriminator") as scope:
            if reuse:
                scope.reuse_variables()
            h0 = lrelu(conv2d(image, self.df_dim, name='d_h0_conv'))
            h1 = lrelu(self.d_bn1(conv2d(h0, self.df_dim*2, name='d_h1_conv')))
            h2 = lrelu(self.d_bn2(conv2d(h1, self.df_dim*4, name='d_h2_conv')))
            h3 = lrelu(self.d_bn3(conv2d(h2, self.df_dim*8, name='d_h3_conv')))
            h4 = linear(tf.reshape(h3, [self.batch_size, -1]), 1, 'd_h4_lin')

            return tf.nn.sigmoid(h4), h4

    def train(self):
        self.field_names = ["epoch", "idx", "inception score", "inception score std"]
        with open(self.metric_path, "wb") as csv_file:
            writer = csv.DictWriter(csv_file, fieldnames = self.field_names)
            writer.writeheader()

        d_optim = tf.train.AdamOptimizer(self.learning_rate, beta1=self.beta1) \
            .minimize(self.d_loss, var_list=self.d_vars)
        g_optim = tf.train.AdamOptimizer(self.learning_rate, beta1=self.beta1) \
            .minimize(self.g_loss, var_list=self.g_vars)

        try:
            tf.global_variables_initializer().run()
        except:
            tf.initialize_all_variables().run()

        sample_feed_dict_z = {}
        for i in range(self.packing_num):
            sample_feed_dict_z[self.z[i]] = np.random.normal(size = (self.batch_size, self.z_dim)).astype(np.float32)
        #sample_inputs = self.data[0 : self.batch_size]

        counter = 0
        start_time = time.time()

        for epoch in xrange(self.epoch):
            ## random packing and random ordering of real data
            ids = np.reshape(np.arange(0,len(self.data)),(len(self.data),1))
            np.random.shuffle(ids)
            print ids.shape
            for temp in range(self.packing_num-1):
                ids_pack = np.reshape(np.arange(0,len(self.data)),(len(self.data),1))
                np.random.shuffle(ids_pack)
                ids = np.concatenate((ids,ids_pack), axis=1)
            batch_idxs = len(self.data) // self.batch_size
            for idx in xrange(0, batch_idxs):
                batch_files = []
                for index in xrange(idx * self.batch_size, (idx + 1) * self.batch_size):
                    batch_files.append([self.data[pack_index] for pack_index in ids[index]])
                batch = []
                for pack_files in batch_files:
                    batch.append(np.concatenate([get_image(batch_file,
                            input_height=self.input_height,
                            input_width=self.input_width,
                            resize_height=self.output_height,
                            resize_width=self.output_width,
                            crop=True,
                            grayscale=False) for batch_file in pack_files], axis=2))
                batch_images = np.array(batch).astype(np.float32)
                feed_dict_z = {}
                for i in range(self.packing_num):
                    feed_dict_z[self.z[i]] = np.random.normal(size = (self.batch_size, self.z_dim)).astype(np.float32)

                # Update D network
                _ = self.sess.run([d_optim],
                    feed_dict = dict({
                        self.inputs: batch_images
                    }, **feed_dict_z))

                # Update G network
                _ = self.sess.run([g_optim],
                    feed_dict = dict({
                    }, **feed_dict_z))

                # Update G network
                _ = self.sess.run([g_optim],
                    feed_dict = dict({
                    }, **feed_dict_z))

                errD_fake = self.d_loss_fake.eval(dict({}, **feed_dict_z))
                errD_real = self.d_loss_real.eval({self.inputs: batch_images})
                errG = self.g_loss.eval(dict({}, **feed_dict_z))

                counter += 1
                print("Epoch: [%2d] [%8d/%8d] time: %4.4f, d_loss: %.8f, g_loss: %.8f" \
                    % (epoch, idx, batch_idxs,
                    time.time() - start_time, errD_fake + errD_real, errG))

                if np.mod(counter, 3165) == 0:
                    sample_inputs = batch_images
                    samples, d_loss, g_loss = self.sess.run(
                        [self.sampler, self.d_loss, self.g_loss],
                        feed_dict=dict({
                            self.inputs: sample_inputs,
                        }, **sample_feed_dict_z))
                    save_images(samples[0 : 64], image_manifold_size(samples[0 : 64].shape[0]),
                        '{}/train_{:02d}_{:08d}.png'.format(self.sample_dir, epoch, idx))
                    print("[Sample] d_loss: %.8f, g_loss: %.8f" % (d_loss, g_loss))

                if np.mod(counter, 3165) == 0:
                    self.save(counter)

    def save(self, step):
        model_name = "DCGAN.model"
        self.saver.save(self.sess,
            os.path.join(self.checkpoint_dir, model_name),
            global_step=step)

    def load(self):
        ckpt = tf.train.get_checkpoint_state(self.checkpoint_dir)
        if ckpt and ckpt.model_checkpoint_path:
            ckpt_name = os.path.basename(ckpt.model_checkpoint_path)
            self.saver.restore(self.sess, os.path.join(self.checkpoint_dir, ckpt_name))
            counter = int(next(re.finditer("(\d+)(?!.*\d)",ckpt_name)).group(0))
            print(" [*] Success to read {}".format(ckpt_name))
            return True, counter
        else:
            print(" [*] Failed to find a checkpoint")
            return False, 0

    def get_collisions(self, nbatch, topK, runs):
        could_load, checkpoint_counter = self.load()
        print(could_load, checkpoint_counter)
        for ii in range(runs):
            samples = []
            z_space = []
            for i in range(nbatch):
                feed_dict_z = {}
                feed_dict_z[self.z[0]] = np.random.normal(size = (self.batch_size, self.z_dim)).astype(np.float32)
                z_space.append(feed_dict_z[self.z[0]])
                samples.append(self.sess.run(self.sampler,
                    feed_dict=dict({
                    }, **feed_dict_z)
                    ))
            images = np.concatenate(samples)
            z_space1 = np.concatenate(np.array(z_space).astype(np.float32))
            queue = []
            n_image = images.shape[0]

            for i in range(n_image):
                for j in range(i+1, n_image):
                    dist = np.sum((images[i] - images[j])**2)
                    dist_z = np.sum((z_space1[i] - z_space1[j])**2)
                    if len(queue) == 0 or -1*dist > queue[0][0]:
                        hq.heappush(queue, (-1*copy.deepcopy(dist), copy.deepcopy(dist_z), copy.deepcopy(images[i]), copy.deepcopy(images[j])))
                        if len(queue) > topK:
                            hq.heappop(queue)
            f_images = []
            for idx in range(topK):
                neg_dist, dist_z, img1, img2 = hq.heappop(queue)
                f_images.append(img1)
                f_images.append(img2)
                print -neg_dist, dist_z
                #scipy.misc.imsave(self.sample_dir + '/pair#%d_%f_%d.png'%(idx, -1*neg_dist, 1), (img1+1.)/2)
                #scipy.misc.imsave(self.sample_dir + '/pair#%d_%f_%d.png'%(idx, -1*neg_dist, 2), (img2+1.)/2)
            f_images = np.asarray(f_images)
            save_images(f_images[0 : 2*topK], image_manifold_size_pairs(f_images[0 : 2*topK].shape[0]),
                    '{}/top_{}_pairs_from_{:01d}_run_{:01d}.png'.format(self.sample_dir,topK,nbatch*64,ii))
            print "\n"

max_stars_repo_name: RyanHonwad/Two-Samples-in-GANs

max_issues_repo_name: RyanHonwad/Two-Samples-in-GANs

max_forks_repo_name: RyanHonwad/Two-Samples-in-GANs


text:
# Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
This module defines text regions in both pixel and sky coordinates.
"""

from astropy.wcs.utils import pixel_to_skycoord, skycoord_to_pixel

from .point import PointPixelRegion, PointSkyRegion
from ..core import PixCoord

__all__ = ['TextSkyRegion', 'TextPixelRegion']

class TextPixelRegion(PointPixelRegion):
    """
    A text string in pixel coordinates.

    Parameters
    ----------
    center : `~regions.PixCoord`
        The position of the leftmost point of the text string.
    text : str
        The text string.
    meta : `~regions.RegionMeta`, optional
        A dictionary that stores the meta attributes of this region.
    visual : `~regions.RegionVisual`, optional
        A dictionary that stores the visual meta attributes of this
        region.

    Examples
    --------
    .. plot::
        :include-source:

        from regions import PixCoord, TextPixelRegion, RegionVisual
        import matplotlib.pyplot as plt

        x, y = 15, 10
        textangle = '30'

        fig, ax = plt.subplots(1, 1)

        center = PixCoord(x=x, y=y)
        reg = TextPixelRegion(center=center, text="Hello World!",
                              visual=RegionVisual(textangle=textangle))
        reg.plot(ax=ax)

        plt.xlim(10, 30)
        plt.ylim(2.5, 20)
        ax.set_aspect('equal')
    """

    _params = ('center', 'text')
    mpl_artist = 'Text'

    def __init__(self, center, text, meta=None, visual=None):
        super().__init__(center, meta, visual)
        self.text = text

    def to_sky(self, wcs):
        center = pixel_to_skycoord(self.center.x, self.center.y, wcs=wcs)
        return TextSkyRegion(center, self.text)

    def as_artist(self, origin=(0, 0), **kwargs):
        """
        Return a matplotlib Text object for this region
        (`matplotlib.text.Text`).

        Parameters
        ----------
        origin : array_like, optional
            The ``(x, y)`` pixel position of the origin of the displayed
            image.

        **kwargs : dict
            Any keyword arguments accepted by `~matplotlib.text.Text`.
            These keywords will override any visual meta attributes of
            this region.

        Returns
        -------
        artist : `~matplotlib.text.Text`
            A matplotlib Text object.
        """
        from matplotlib.text import Text

        mpl_kwargs = self.visual.define_mpl_kwargs(self.mpl_artist)
        mpl_kwargs.update(kwargs)

        return Text(self.center.x - origin[0], self.center.y - origin[1],
                    self.text, **mpl_kwargs)

class TextSkyRegion(PointSkyRegion):
    """
    A text string in sky coordinates.

    Parameters
    ----------
    center : `~astropy.coordinates.SkyCoord`
        The position of the leftmost point of the text string.
    text : str
        The text string.
    meta : `~regions.RegionMeta`, optional
        A dictionary that stores the meta attributes of this region.
    visual : `~regions.RegionVisual`, optional
        A dictionary that stores the visual meta attributes of this
        region.
    """

    _params = ('center', 'text')

    def __init__(self, center, text, meta=None, visual=None):
        super().__init__(center, meta, visual)
        self.text = text

    def to_pixel(self, wcs):
        center_x, center_y = skycoord_to_pixel(self.center, wcs=wcs)
        center = PixCoord(center_x, center_y)
        return TextPixelRegion(center, self.text, self.meta, self.visual)

max_stars_repo_name: astropy/regions

max_issues_repo_name: pllim/regions

max_forks_repo_name: pllim/regions

reason: from astropy


text:
import numpy
import os
import numpy as np
import logging
from theano.tensor.signal import pool
from theano.tensor.nnet.abstract_conv import bilinear_upsampling
import joblib
from theano.tensor.nnet import conv2d
from theano.tensor.nnet import relu,softmax
import theano
import theano.tensor as T

from theano.tensor.signal.pool import pool_2d
cwd =  os.path.dirname(os.path.realpath(__file__))

'''
['convolution2d_4_weights',
 'dense_1_weights',
 'dense_2_weights',
 'convolution2d_8_weights',
 'convolution2d_5_weights',
 'convolution2d_13_weights',
 'convolution2d_7_weights',
 'convolution2d_15_weights',
 'convolution2d_59_weights',
 'convolution2d_14_weights',
 'convolution2d_16_weights',
 'dense_1_weights',
 'convolution2d_6_weights',
 'convolution2d_3_weights',
 'convolution2d_10_weights',
 'convolution2d_1_weights',
 'convolution2d_10_weights',
 'convolution2d_60_weights',
 'convolution2d_2_weights']
'''

# Read in layer weights and save to a dictionary
# Read in layer weights and save to a dictionary

cwd = os.getcwd()
direct = os.path.join(cwd,'theano_model','weights')
# weights_layer_paths = joblib.load(os.path.join(cwd,'weight_names','layer_names_weights'))
layer_weights = {}
for layer_weight_path in range(16):
#     head,layer_name = os.path.split(layer_weight_path)
    layer_weights[str(layer_weight_path) + '_w'] = joblib.load(os.path.join(direct,str(layer_weight_path) + '_w'))

# Read in bias weights and save to a dictionary
for bias_layer_path in range(16):
    layer_weights[str(bias_layer_path) + '_b'] = joblib.load(os.path.join(direct,str(bias_layer_path) + '_b'))

def drop(input, p=0.5):
    """
    :type input: numpy.array
    :param input: layer or weight matrix on which dropout is applied

    :type p: float or double between 0. and 1.
    :param p: p probability of NOT dropping out a unit, therefore (1.-p) is the drop rate.

    """
    rng = numpy.random.RandomState(1234)
    srng = T.shared_randomstreams.RandomStreams(rng.randint(999999))
    mask = srng.binomial(n=1, p=p, size=input.shape, dtype=theano.config.floatX)
    return input * mask

class DropoutHiddenLayer(object):
    def __init__(self, is_train, input, W=None, b=None,
                 activation=relu, p=0.5):
        self.input = input

        self.W = W
        self.b = b

        lin_output = T.dot(input, self.W) + self.b

        self.output = activation(lin_output)
        # train_output = drop(output, p)
        # self.output = T.switch(T.neq(is_train, 0), train_output, p * output)
        self.params = [self.W, self.b]

class VGG19_conv2d_layer(object):
    """Pool Layer of a convolutional network """

    def __init__(self, input, layer_name,image_shape,
                 activation=relu, border_mode=(2,2)):
        self.activation = activation
        self.input = input

        self.W = theano.shared(value=np.array(layer_weights[layer_name + '_w'],
                                              dtype=theano.config.floatX),
                               borrow=True)
        self.b = theano.shared(value=np.array(layer_weights[layer_name + '_b'],
                                              dtype=theano.config.floatX
                                              )
                               , borrow=True)

        self.conv_out = conv2d(
            input=input,
            input_shape=image_shape,
            filters=self.W,
            filter_shape=layer_weights[layer_name + '_w'].shape,
            border_mode=border_mode
        )

        self.output = activation(self.conv_out + self.b.dimshuffle('x', 0, 'x', 'x'))
        self.params = [self.W, self.b]
        self.input = input

class VGG19(object):

    def __init__(self,input_image_shape,pool_method = 'average_exc_pad'):
        IMAGE_H = input_image_shape[2]
        IMAGE_W = input_image_shape[3]
        self.input = theano.tensor.tensor4('input')
        self.conv1_1 = VGG19_conv2d_layer(input=self.input,
                                    layer_name='0',
                                    image_shape=input_image_shape,
                                    border_mode=(1,1))

        self.conv1_2 = VGG19_conv2d_layer(input=self.conv1_1.output,
                                              layer_name='1',
                                              image_shape=(None, 64, IMAGE_H, IMAGE_W),
                                              border_mode=(1,1))
        self.pool1 = pool_2d(
            input = self.conv1_2.output,
            ds = (2,2),
            mode = pool_method,
            ignore_border=True,
            st = (2,2))

        self.conv2_1 = VGG19_conv2d_layer(input=self.pool1,
                                                   layer_name='2',
                                                   image_shape=(None, 64, IMAGE_H/2, IMAGE_W/2),
                                                   border_mode=(1,1))

        self.conv2_2 = VGG19_conv2d_layer(input=self.conv2_1.output,
                                                   layer_name='3',
                                                   image_shape=(None, 128, IMAGE_H/2, IMAGE_W/2),
                                                   border_mode=(1,1))

        self.pool2 = pool_2d(
            input=self.conv2_2.output,
            ds=(2, 2),
            mode=pool_method,
            ignore_border=True,
            st=(2, 2))

        self.conv3_1 = VGG19_conv2d_layer(input=self.pool2,
                                                   layer_name='4',
                                                   image_shape=(None, 128, IMAGE_H/4, IMAGE_W/4),
                                                   border_mode=(1, 1))
        self.conv3_2= VGG19_conv2d_layer(input=self.conv3_1.output,
                                                   layer_name='5',
                                                   image_shape=(None, 128, IMAGE_H/4, IMAGE_W/4),
                                                   border_mode=(1, 1))
        self.conv3_3 = VGG19_conv2d_layer(input=self.conv3_2.output,
                                                   layer_name='6',
                                                   image_shape=(None, 128, IMAGE_H/4, IMAGE_W/4),
                                                   border_mode=(1, 1))

        self.conv3_4 = VGG19_conv2d_layer(input=self.conv3_3.output,
                                                   layer_name='7',
                                                   image_shape=(None, 128, IMAGE_H/4, IMAGE_W/4),
                                                   border_mode=(1, 1))

        self.pool3 = pool_2d(
            input=self.conv3_4.output,
            ds=(2, 2),
            mode=pool_method,
            ignore_border=True,
            st=(2, 2))

        self.conv4_1 = VGG19_conv2d_layer(input=self.pool3,
                                                   layer_name='8',
                                                   image_shape=(None, 512, IMAGE_H/8, IMAGE_W/8),
                                                   border_mode=(1, 1))
        self.conv4_2 = VGG19_conv2d_layer(input=self.conv4_1.output,
                                                   layer_name='9',
                                                   image_shape=(None, 512, IMAGE_H/8, IMAGE_W/8),
                                                   border_mode=(1, 1))
        self.conv4_3 = VGG19_conv2d_layer(input=self.conv4_2.output,
                                                   layer_name='10',
                                                   image_shape=(None, 512, IMAGE_H/8, IMAGE_W/8),
                                                   border_mode=(1, 1))

        self.conv4_4 = VGG19_conv2d_layer(input=self.conv4_3.output,
                                                   layer_name='11',
                                                   image_shape=(None, 512, IMAGE_H/8, IMAGE_W/8),
                                                   border_mode=(1, 1))

        self.pool4 = pool_2d(
            input=self.conv4_4.output,
            ds=(2, 2),
            mode=pool_method,
            ignore_border=True,
            st=(2, 2))

        self.conv5_1 = VGG19_conv2d_layer(input=self.pool4,
                                                   layer_name='12',
                                                   image_shape=(None, 512, IMAGE_H/16, IMAGE_W/16),
                                                   border_mode=(1, 1))
        self.conv5_2 = VGG19_conv2d_layer(input=self.conv5_1.output,
                                                   layer_name='13',
                                                   image_shape=(None, 512, IMAGE_H/16, IMAGE_W/16),
                                                   border_mode=(1, 1))
        self.conv5_3 = VGG19_conv2d_layer(input=self.conv5_2.output,
                                                   layer_name='14',
                                                   image_shape=(None, 512, IMAGE_H/16, IMAGE_W/16),
                                                   border_mode=(1, 1))

        self.conv5_4 = VGG19_conv2d_layer(input=self.conv5_3.output,
                                                   layer_name='15',
                                                   image_shape=(None, 512, IMAGE_H/16, IMAGE_W/16),
                                                   border_mode=(1, 1))

        self.pool5 = pool_2d(
            input=self.conv5_4.output,
            ds=(2, 2),
            mode=pool_method,
            ignore_border=True,
            st=(2, 2))

        # self.dense_1_input= self.pool5.flatten(2)
        #
        # self.dense_1 = DropoutHiddenLayer(is_train = numpy.cast['int32'](0),
        #                    input = self.dense_1_input,
        #                    W=layer_weights['dense_1_weights'],
        #                    b=layer_weights['dense_1_bias'],)
        #
        # self.dense_2 = DropoutHiddenLayer(is_train=numpy.cast['int32'](0),
        #                                input=self.dense_1.output,
        #                                W=layer_weights['dense_2_weights'],
        #                                b=layer_weights['dense_2_bias'], )
        #
        # self.dense_3 = DropoutHiddenLayer(is_train=numpy.cast['int32'](0),
        #                                input=self.dense_2.output,
        #                                W=layer_weights['dense_3_weights'],
        #                                b=layer_weights['dense_3_bias'],
        #                                    activation=softmax )

# model.add(Flatten())
# model.add(Dense(4096, activation='relu'))
# model.add(Dropout(0.5))
# model.add(Dense(4096, activation='relu'))
# model.add(Dropout(0.5))
# model.add(Dense(1000, activation='softmax'))

class VGG16(object):

    def __init__(self,input_image_shape,pool_method = 'max'):
        self.input = theano.tensor.tensor4('input')
        self.conv1_1 = VGG19_conv2d_layer(input=self.input,
                                    layer_name='convolution2d_1',
                                    image_shape=input_image_shape,
                                    border_mode=(1,1))

        self.conv1_2 = VGG19_conv2d_layer(input=self.conv1_1.output,
                                              layer_name='convolution2d_2',
                                              image_shape=(None, 64, 224, 224),
                                              border_mode=(1,1))
        self.pool1 = pool_2d(
            input = self.conv1_2.output,
            ds = (2,2),
            mode = pool_method,
            ignore_border=True,
            st = (2,2))

        self.conv2_1 = VGG19_conv2d_layer(input=self.pool1,
                                                   layer_name='convolution2d_3',
                                                   image_shape=(None, 64, 112, 112),
                                                   border_mode=(1,1))

        self.conv2_2 = VGG19_conv2d_layer(input=self.conv2_1.output,
                                                   layer_name='convolution2d_4',
                                                   image_shape=(None, 128, 112, 112),
                                                   border_mode=(1,1))

        self.pool2 = pool_2d(
            input=self.conv2_2.output,
            ds=(2, 2),
            mode=pool_method,
            ignore_border=True,
            st=(2, 2))

        self.conv3_1 = VGG19_conv2d_layer(input=self.pool2,
                                                   layer_name='convolution2d_5',
                                                   image_shape=(None, 128, 56, 56),
                                                   border_mode=(1, 1))
        self.conv3_2= VGG19_conv2d_layer(input=self.conv3_1.output,
                                                   layer_name='convolution2d_6',
                                                   image_shape=(None, 128, 56, 56),
                                                   border_mode=(1, 1))
        self.conv3_3 = VGG19_conv2d_layer(input=self.conv3_2.output,
                                                   layer_name='convolution2d_7',
                                                   image_shape=(None, 128, 56, 56),
                                                   border_mode=(1, 1))

        self.conv3_4 = VGG19_conv2d_layer(input=self.conv3_3.output,
                                                   layer_name='convolution2d_8',
                                                   image_shape=(None, 128, 56, 56),
                                                   border_mode=(1, 1))

        self.pool3 = pool_2d(
            input=self.conv3_4.output,
            ds=(2, 2),
            mode=pool_method,
            ignore_border=True,
            st=(2, 2))

        self.conv4_1 = VGG19_conv2d_layer(input=self.pool3,
                                                   layer_name='convolution2d_9',
                                                   image_shape=(None, 512, 28, 28),
                                                   border_mode=(1, 1))
        self.conv4_2 = VGG19_conv2d_layer(input=self.conv4_1.output,
                                                   layer_name='convolution2d_10',
                                                   image_shape=(None, 512, 28, 28),
                                                   border_mode=(1, 1))
        self.conv4_3 = VGG19_conv2d_layer(input=self.conv4_2.output,
                                                   layer_name='convolution2d_11',
                                                   image_shape=(None, 512, 28, 28),
                                                   border_mode=(1, 1))

        self.conv4_4 = VGG19_conv2d_layer(input=self.conv4_3.output,
                                                   layer_name='convolution2d_12',
                                                   image_shape=(None, 512, 28, 28),
                                                   border_mode=(1, 1))

        self.pool4 = pool_2d(
            input=self.conv4_4.output,
            ds=(2, 2),
            mode=pool_method,
            ignore_border=True,
            st=(2, 2))

        self.conv5_1 = VGG19_conv2d_layer(input=self.pool4,
                                                   layer_name='convolution2d_13',
                                                   image_shape=(None, 512, 14, 14),
                                                   border_mode=(1, 1))
        self.conv5_2 = VGG19_conv2d_layer(input=self.conv5_1.output,
                                                   layer_name='convolution2d_14',
                                                   image_shape=(None, 512, 14, 14),
                                                   border_mode=(1, 1))
        self.conv5_3 = VGG19_conv2d_layer(input=self.conv5_2.output,
                                                   layer_name='convolution2d_15',
                                                   image_shape=(None, 512, 14, 14),
                                                   border_mode=(1, 1))

        self.conv5_4 = VGG19_conv2d_layer(input=self.conv5_3.output,
                                                   layer_name='convolution2d_16',
                                                   image_shape=(None, 512, 14, 14),
                                                   border_mode=(1, 1))

max_stars_repo_name: dkdanielkost/Theano-Style-Transfer

max_issues_repo_name: dkdanielkost/Theano-Style-Transfer

max_forks_repo_name: dkdanielkost/Theano-Style-Transfer

reason: import numpy,import theano,from theano


text:
import cv2
import os
import numpy as np

def mean_yaxis_error_mat(pred, label):
    # pred and label should all in the shape of img_shape
    assert pred.shape == label.shape

    img_shape = pred.shape
    y_errors = np.zeros(img_shape[1], dtype=np.float32)
    y_ranges = [i for i in range(img_shape[0])]
    y_ranges = np.array(y_ranges)

    for i in range(img_shape[1]):
        if (label[:, i] == 0).all() or (pred[:, i] == 0).all():
            continue
        label_y = np.sum(label[:, i] * y_ranges) / np.sum(label[:, i])
        pred_y = np.sum(pred[:, i] * y_ranges) / np.sum(pred[:, i])
        y_errors[i] = np.absolute(label_y - pred_y)

    return np.mean(y_errors)

def eval_score(pred_path, label_path, pred_suffix, label_suffix, output=False, output_path=None, img_path=None):
    pred_file_names = sorted(os.listdir(pred_path))
    label_file_names = sorted(os.listdir(label_path))

    assert len(pred_file_names) == len(label_file_names)

    nums = len(pred_file_names)
    all = 0

    for i in range(nums):
        # 将文件名去掉后缀，以便匹配
        pred_name = os.path.splitext(pred_file_names[i])
        pred_name = pred_name[0].replace(pred_suffix, '')

        label_name = os.path.splitext(label_file_names[i])
        label_name = label_name[0].replace(label_suffix, '')

        if pred_name != label_name:
            print('Wrong match:', pred_file_names[i], label_file_names[i])
            exit()

        print(pred_name)

        pred_file = os.path.join(pred_path + pred_file_names[i])
        label_file = os.path.join(label_path + label_file_names[i])
        img_file = os.path.join(img_path + pred_name + '.tif')

        pred = cv2.imread(pred_file, 0)
        label = cv2.imread(label_file, 0)
        img = cv2.imread(img_file)

        # 计算误差
        all += mean_yaxis_error_mat(pred, label)

        # 将预测结果叠加在原图上
        if output:
            if output_path == None or img_path == None:
                print('No output path or image path!')
                return
            else:
                if not os.path.exists(output_path):
                    os.mkdir(output_path)

                for h_x in range(pred.shape[0]):
                    for w_x in range(pred.shape[1]):
                        if pred[h_x][w_x] >= 250:
                            img[h_x][w_x] = [0, 0, 255]  # 预测结果以红色表示

                cv2.imwrite(output_path + pred_name + '.jpg', img)

    print('Mean error: %.4f' % (all / nums))

if __name__ == "__main__":
    pred_path = './output3_p/'  # 预测出来的标签文件夹地址
    pred_suffix = '_OUT'  # 预测结果的文件名后缀
    label_path = './data/train_label2/'  # 真实标签文件夹地址
    label_suffix = '_mask2'  # 真实标签文件名后缀
    img_path = './data/train_img/'  # 原始图片文件夹地址
    output_path = './res2/'  # 叠加回原图后输出文件夹地址
    eval_score(pred_path, label_path, pred_suffix, label_suffix, True, output_path, img_path)


text:
from ingest_data import *
import pickle
import numpy as np

sequences_input_fn = 'new_filtered_bt_intersect_ML_input.csv'

with open(sequences_input_fn, 'r') as f:

    sequences = f.readlines()

subseq_length = 100
subseq_window_stride = 20

ks = [2,3]

vec_length = [4 ** x for x in ks]
vec_length = sum(vec_length)
print(vec_length)

train_lRNA_collector = []
test_lRNA_collector = []

for s in sequences[:]:

    ind = sequences.index(s)

    if ind % 200 == 0:

        print(ind)

    row = s.split(',')

    for i in range(0,len(row[1]), subseq_window_stride):

        try:
            subseq = row[1][i : i + subseq_length]

            #print(i, k_mer_frequencies(subseq, ks, include_missing=True, vector=True))

            z = list(k_mer_frequencies(subseq, ks, include_missing=True, vector=True))
            z.append(int(row[-1]))

            if len(z) == vec_length + 1:

                if ind % 5 == 0:

                    test_lRNA_collector.append(z)

                else:

                    train_lRNA_collector.append(z)

        except:

            pass

#print(lRNA_collector)
train_a = np.array(train_lRNA_collector)
test_a = np.array(test_lRNA_collector)
print(train_a.shape)
print(test_a.shape)

with open('train_featurized_5.p', 'wb') as f:

    pickle.dump(train_a, f)

with open('test_featurized_5.p', 'wb') as f:

    pickle.dump(test_a, f)

max_stars_repo_name: NCBI-Hackathons/LNCPEP

max_issues_repo_name: NCBI-Hackathons/LNCPEP

max_forks_repo_name: NCBI-Hackathons/LNCPEP


text:
import sys

# Setting the Qt bindings for QtPy
import os
os.environ["QT_API"] = "pyqt5"

from PyQt5 import QtWidgets
from PyQt5.QtWidgets import QMainWindow
from PyQt5.QtWidgets import QRadioButton

import numpy as np

import pyvista as pv
from pyvistaqt import QtInteractor

import Style as styling
class Render3D(QMainWindow):

    def __init__(self, data, parent=None, show=True):
        QtWidgets.QMainWindow.__init__(self, parent)

        # create the frame
        self.frame = QtWidgets.QFrame()
        vlayout = QtWidgets.QVBoxLayout()

        # add the pyvista interactor object
        self.plotter = QtInteractor(self.frame)
        self.plotter.set_background("royalblue", top="aliceblue")
        vlayout.addWidget(self.plotter.interactor)

        self.frame.setLayout(vlayout)
        self.setCentralWidget(self.frame)

        self.update3D(data)

    def update3D(self, data):
        self.plotter.clear()
        self.plotter.add_bounding_box()
        self.plotter.add_volume(data, cmap="gray")
        self.show()

    def updateSlice(self, data):
        self.plotter.clear()
        self.plotter.add_bounding_box()
        opacity = [0, 0.1, 0.1, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
        self.plotter.add_volume(data, cmap="gray", opacity=opacity)
        self.show()

max_stars_repo_name: jolyDev/orto-ray

max_issues_repo_name: jolyDev/orto-ray

max_forks_repo_name: jolyDev/orto-ray


text:
import numpy as np
import random

from .shuttering_utils import checkShardingValidity

def _random_sharding(nShards, rDocsByTopic, rDocsList, nrDocsList, emptyShards):
    allowed_empty = (emptyShards == 'E')

    nRelDocs = len(rDocsList)
    nNRelDocs = len(nrDocsList)
    shardsList = np.arange(nShards)

    rDocsByShard = {}

    # TODO - a very bad sharding apporach: we should do a stratified sampling. This is very slow... but works
    found = False
    while not found:
        rShards = random.choices(shardsList, k=nRelDocs)
        rDocsByShard = {i: set([d for e, d in enumerate(rDocsList) if rShards[e] == i]) for i in shardsList}
        found = (allowed_empty or checkShardingValidity(rDocsByShard, rDocsByTopic))

    nrShards = random.choices(shardsList, k=nNRelDocs)
    nrDocsByShard = {i: [d for e, d in enumerate(nrDocsList) if nrShards[e] == i] for i in shardsList}

    shards = {i: set(list(rDocsByShard[i]) + nrDocsByShard[i]) for i in shardsList}

    return shards

max_stars_repo_name: guglielmof/pyRe

max_issues_repo_name: guglielmof/pyRe

max_forks_repo_name: guglielmof/pyRe