text:
# Copyright (c) 2018 Ultimaker B.V.
# Uranium is released under the terms of the LGPLv3 or higher.
from typing import Optional, Tuple, List, Union

import numpy
import scipy.spatial

from UM.Logger import Logger
from UM.Math import NumPyUtil
from UM.Math import ShapelyUtil

##  A class representing an immutable arbitrary 2-dimensional polygon.
class Polygon:
    ##  Return vertices from an approximate circle.
    #
    #   An octagon is returned, which comes close enough to a circle.
    #
    #   \param radius The radius of the circle.
    #   \return A polygon that approximates a circle.
    @staticmethod
    def approximatedCircle(radius):
        return Polygon(points = numpy.array([
            [-radius, 0],
            [-radius * 0.707, radius * 0.707],
            [0, radius],
            [radius * 0.707, radius * 0.707],
            [radius, 0],
            [radius * 0.707, -radius * 0.707],
            [0, -radius],
            [-radius * 0.707, -radius * 0.707]
        ], numpy.float32))

    def __init__(self, points: Optional[Union[numpy.ndarray, List]] = None):
        self._points = NumPyUtil.immutableNDArray(points)

    def __eq__(self, other):
        if self is other:
            return True
        if type(other) is not Polygon:
            return False

        point_count = len(self._points) if self._points is not None else 0
        point_count2 = len(other.getPoints()) if other.getPoints() is not None else 0
        if point_count != point_count2:
            return False
        return numpy.array_equal(self._points, other.getPoints())

    ##  Gives a debugging representation of the polygon.
    #
    #   This lists the polygon's coordinates, like so::
    #     [[0,0], [1,3], [3,0]]
    #
    #   \return A representation of the polygon that is useful for debugging.
    def __repr__(self):
        coordinates = (("[" + str(point[0]) + "," + str(point[1]) + "]") for point in self._points)
        return "[" + ", ".join(coordinates) + "]"

    def isValid(self) -> bool:
        return bool(self._points is not None and len(self._points) >= 3)

    def getPoints(self) -> numpy.array:
        return self._points

    ##  Project this polygon on a line described by a normal.
    #
    #   \param normal The normal to project on.
    #   \return A tuple describing the line segment of this Polygon projected on to the infinite line described by normal.
    #           The first element is the minimum value, the second the maximum.
    def project(self, normal) -> Tuple[float, float]:
        projection_min = numpy.dot(normal, self._points[0])

        projection_max = projection_min
        for point in self._points:
            projection = numpy.dot(normal, point)
            projection_min = min(projection_min, projection)
            projection_max = max(projection_max, projection)

        return projection_min, projection_max

    ##  Moves the polygon by a fixed offset.
    #
    #   \param x The distance to move along the X-axis.
    #   \param y The distance to move along the Y-axis.
    def translate(self, x: float = 0, y: float = 0) -> "Polygon":
        if self.isValid():
            return Polygon(numpy.add(self._points, numpy.array([[x, y]])))
        else:
            return self

    ##  Mirrors this polygon across the specified axis.
    #
    #   \param point_on_axis A point on the axis to mirror across.
    #   \param axis_direction The direction vector of the axis to mirror across.
    def mirror(self, point_on_axis: List[float], axis_direction: List[float]) -> "Polygon":
        # Input checking.
        if axis_direction == [0, 0]:
            Logger.log("w", "Tried to mirror a polygon over an axis with direction [0, 0].")
            return self  # Axis has no direction. Can't expect us to mirror anything!
        axis_direction /= numpy.linalg.norm(axis_direction)  # Normalise the direction.
        if not self.isValid():  # Not a valid polygon, so don't do anything.
            return self

        # In order to be able to mirror points around an arbitrary axis, we have to normalize the axis and all points
        # such that the axis goes through the origin.
        point_matrix = numpy.matrix(self._points)
        point_matrix -= point_on_axis  # Moves all points such that the axis origin is at [0,0].

        # To mirror a coordinate, we have to add the projection of the point to the axis twice
        # (where v is the vector to reflect):
        #  reflection(v) = 2 * projection(v) - v
        # Writing out the projection, this becomes (where l is the normalised direction of the line):
        #  reflection(v) = 2 * (l . v) l - v
        # With Snell's law this can be simplified to the Householder transformation matrix:
        #  reflection(v) = R v
        #  R = 2 l l^T - I
        # This simplifies the entire reflection to one big matrix transformation.
        axis_matrix = numpy.matrix(axis_direction)
        reflection = 2 * numpy.transpose(axis_matrix) * axis_matrix - numpy.identity(2)
        point_matrix = point_matrix * reflection  # Apply the actual transformation.

        # Shift the points back to the original coordinate space before the axis was normalised to the origin.
        point_matrix += point_on_axis
        return Polygon(point_matrix.getA()[::-1])

    ##  Computes the intersection of the convex hulls of this and another
    #   polygon.
    #
    #   \param other The other polygon to intersect convex hulls with.
    #   \return The intersection of the two polygons' convex hulls.
    def intersectionConvexHulls(self, other: "Polygon") -> "Polygon":
        me = self.getConvexHull()
        him = other.getConvexHull()

        # If either polygon has no surface area, then the intersection is empty.
        if len(me._points) <= 2 or len(him._points) <= 2:
            return Polygon()

        polygen_me = ShapelyUtil.polygon2ShapelyPolygon(me)
        polygon_him = ShapelyUtil.polygon2ShapelyPolygon(him)

        polygon_intersection = polygen_me.intersection(polygon_him)
        if polygon_intersection.area == 0:
            return Polygon()

        return Polygon(points = [list(p) for p in polygon_intersection.exterior.coords[:4]])

    #  Computes the convex hull of the union of the convex hulls of this and another polygon.
    #
    #   \param other The other polygon to combine convex hulls with.
    #   \return The convex hull of the union of the two polygons' convex hulls.
    def unionConvexHulls(self, other: "Polygon") -> "Polygon":
        my_hull = self.getConvexHull()
        other_hull = other.getConvexHull()

        if not my_hull.isValid():
            return other_hull
        if not other_hull.isValid():
            return my_hull

        my_polygon = ShapelyUtil.polygon2ShapelyPolygon(my_hull)
        other_polygon = ShapelyUtil.polygon2ShapelyPolygon(other_hull)

        polygon_union = my_polygon.union(other_polygon).convex_hull
        if polygon_union.area == 0:
            return Polygon()

        return Polygon(points = [list(p) for p in polygon_union.exterior.coords[:-1]])

    ##  Check to see whether this polygon intersects with another polygon.
    #
    #   \param other \type{Polygon} The polygon to check for intersection.
    #   \return A tuple of the x and y distance of intersection, or None if no intersection occured.
    def intersectsPolygon(self, other: "Polygon") -> Optional[Tuple[float, float]]:
        if not self.isValid() or not other.isValid():
            return None

        polygon_me = ShapelyUtil.polygon2ShapelyPolygon(self)
        polygon_other = ShapelyUtil.polygon2ShapelyPolygon(other)
        if polygon_other.is_empty or polygon_me.is_empty:
            return None
        if not (polygon_me.is_valid and polygon_other.is_valid):  # If not valid
            return None

        polygon_intersection = polygon_me.intersection(polygon_other)
        ret_size = None
        if polygon_intersection:
            ret_size = (polygon_intersection.bounds[2] - polygon_intersection.bounds[0],
                        polygon_intersection.bounds[3] - polygon_intersection.bounds[1],
                        )
        return ret_size

    ##  Calculate the convex hull around the set of points of this polygon.
    #   \return The convex hull around the points of this polygon.
    def getConvexHull(self) -> "Polygon":
        points = self._points

        if len(points) < 1:
            return Polygon(numpy.zeros((0, 2), numpy.float64))
        if len(points) <= 2:
            return Polygon(numpy.array(points, numpy.float64))

        try:
            hull = scipy.spatial.ConvexHull(points)
        except scipy.spatial.qhull.QhullError:
            return Polygon(numpy.zeros((0, 2), numpy.float64))

        return Polygon(numpy.flipud(hull.points[hull.vertices]))

    ##  Perform a Minkowski sum of this polygon with another polygon.
    #
    #   \param other The polygon to perform a Minkowski sum with.
    #   \return \type{Polygon} The Minkowski sum of this polygon with other.
    def getMinkowskiSum(self, other: "Polygon") -> "Polygon":
        points = numpy.zeros((len(self._points) * len(other._points), 2))
        for n in range(0, len(self._points)):
            for m in range(0, len(other._points)):
                points[n * len(other._points) + m] = self._points[n] + other._points[m]

        return Polygon(points)

    ##  Create a Minkowski hull from this polygon and another polygon.
    #
    #   The Minkowski hull is the convex hull around the Minkowski sum of this
    #   polygon with other.
    #
    #   \param other \type{Polygon} The Polygon to do a Minkowski addition with.
    #   \return The convex hull around the Minkowski sum of this Polygon with other
    def getMinkowskiHull(self, other: "Polygon") -> "Polygon":
        sum = self.getMinkowskiSum(other)
        return sum.getConvexHull()

    ##  Whether the specified point is inside this polygon.
    #
    #   If the point is exactly on the border or on a vector, it does not count
    #   as being inside the polygon.
    #
    #   \param point The point to check of whether it is inside.
    #   \return True if it is inside, or False otherwise.
    def isInside(self, point) -> bool:
        for i in range(0, len(self._points)):
            if self._isRightTurn(self._points[i], self._points[(i + 1) % len(self._points)], point) == -1: #Outside this halfplane!
                return False
        return True

    def _isRightTurn(self, p: numpy.ndarray, q: numpy.ndarray, r: numpy.ndarray) -> float:
        sum1 = q[0] * r[1] + p[0] * q[1] + r[0] * p[1]
        sum2 = q[0] * p[1] + r[0] * q[1] + p[0] * r[1]

        if sum1 - sum2 < 0:
            return 1
        elif sum1 == sum2:
            return 0
        else:
            return -1

__all__ = ["Polygon"]

reason: import numpy,import scipy


text:
import sys
import numpy as np
#import cv2
#sys.path.append('/home/ibu6429/shared_file_system/code/math_functions/')
try:
    from algebra import Algebra
except ImportError:
    from localseg.data_generators.algebra import Algebra

def extractEquirectangular(wrap,source_image,result_image,euler_angles):
	extractEquirectangular_quick(wrap,source_image,result_image,Algebra.rotation_matrix(euler_angles))

def extractEquirectangular_quick(wrap, source_image, result_image,R):

        # First dimension: Yaw
        # First dimension: Roll
        # Third dimenstion: Pitch

        Algebra.test_polar_to_polar()
        min_theta = np.pi * 4
        max_theta = -np.pi * 4;
        min_psi = np.pi * 4;
        max_psi = -np.pi * 4;

        result_image[:,:,0]=255
        width=result_image.shape[1]
        height=result_image.shape[0]

        row,col = np.mgrid[0:height,0:width]
        polar_point=np.zeros((height,width,3))
        polar_point [:,:,0]=1.0
        polar_point[:,:,1] = (row / (1.0 * height))*np.pi
        polar_point[:,:,2]= ((col - width // 2) / (0.5 * width)) *np.pi

        max_1=np.max(np.max(polar_point[:,:,1]))
        min_1=np.min(np.min(polar_point[:,:,1]))
        abs_max_2=np.max(np.max(np.abs(polar_point[:,:,2])))

        #print('max 1 min 1 absmax2 ' +str(max_1)+ ' '+str(min_1)+' ' +str(abs_max_2))
        #assert(max_1 <= np.pi) and(min_1>=0)and(abs_max_2<=np.pi) #disabled for speed

        plane_point = Algebra.polar_to_cartesian_array(polar_point)

        # assert( np.max(np.max(np.abs(Algebra.magnitude_array(plane_point)-polar_point[:,:,0]))) < 0.0001) #disabled for speed

        plane_rot=Algebra.rotate_array(R,plane_point)

        #assert(np.max(np.max(np.abs(Algebra.magnitude_array(plane_point) - Algebra.magnitude_array(plane_rot)))) < 0.0001) #disbled for speed

        eq_row,eq_col=Algebra.cartesian_to_polar_quantised_array(wrap,plane_rot,width,height)

        #assert ((np.min(np.min(eq_row))>=0) and (np.max(np.max(eq_row))<height)and (np.min(np.min(eq_col))>=0)  and (np.max(np.max(eq_row))<width)) #disabled for speed

        result_image[row, col, :] = source_image[ eq_row ,eq_col, :]

ext: py

max_stars_repo_name: MarvinTeichmann/TorchLab

max_issues_repo_name: MarvinTeichmann/TorchLab

max_forks_repo_name: MarvinTeichmann/TorchLab

reason: import numpy


text:
from __future__ import print_function
import numpy as np
import torch
import torch.utils.data as data
import torchvision
import torchvision.transforms as transforms

from PIL import Image
import os
import os.path
import tarfile

import random
import boto3
import time

from data_loader.cifar10_dataset import CIFAR10_subset

s3 = boto3.resource('s3')
base_folder = 'cifar-10-batches-py'
processed_folder = 'processed'

# Download files from S3 and unzip them
def preprocess_cifar10(bucket_name='cifar10dataset', key="cifar-10-python.tar.gz",
                       data_path='/tmp/data', num_workers=8):
    # download zipped file from S3
    print('==> Downloading data from S3..')
    if not os.path.exists(data_path):
        os.makedirs(data_path)
    s3.Bucket(bucket_name).download_file(key, os.path.join(data_path, key))

    # extract file
    cwd = os.getcwd()
    tar = tarfile.open(os.path.join(data_path, key), "r:gz")
    os.chdir(data_path)
    tar.extractall()
    tar.close()
    os.chdir(cwd)
    # delete the zipped file
    os.remove(os.path.join(data_path, key))

    # Data
    print('==> Preprocessing data..')
    transform_train = transforms.Compose([
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])

    transform_test = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])

    # training set
    trainset = torchvision.datasets.CIFAR10(root=data_path, train=True, download=False, transform=transform_train)
    # test set
    testset = torchvision.datasets.CIFAR10(root=data_path, train=False, download=False, transform=transform_test)

    # save test dataset back to S3
    with open(os.path.join(data_path, "test.pt"), 'wb') as f:
        torch.save(testset, f)
    s3.Bucket(bucket_name).upload_file(os.path.join(data_path, "test.pt"), 'test.pt')

    #remove the file after uploading it to S3
    os.remove(os.path.join(data_path, "test.pt"))

    if num_workers == 1:
        with open(os.path.join(data_path, "training_0.pt"), 'wb') as f:
            torch.save(trainset, f)
        s3.Bucket(bucket_name).upload_file(os.path.join(data_path, "training_0.pt"), 'training_0.pt')
    else:
        # shuffle training set
        print('==> Shuffling and partitioning training data..')
        num_examples = len(trainset)
        indices = list(range(num_examples))
        random.shuffle(indices)

        num_examples_per_worker = num_examples // num_workers
        residue = num_examples % num_workers

        for i in range(num_workers):
            start = (num_examples_per_worker * i) + min(residue, i)
            num_examples_real = num_examples_per_worker + (1 if i < residue else 0)

            # print("first 10 labels of original[{}]:{}".format(i, [trainset.train_labels[i] for i in indices[start:start+num_examples_real]][0:10]))
            training_subset = CIFAR10_subset(train=True,
                                             train_data=[trainset.train_data[i] for i in indices[start:start+num_examples_real]],
                                             train_labels=[trainset.train_labels[i] for i in indices[start:start+num_examples_real]],
                                             test_data=None, test_labels=None, transform=transform_train)
            # print("first 10 labels of subset[{}]:{}".format(i, training_subset.train_labels[0:10]))

            # save training subset back to S3
            with open(os.path.join(data_path, 'training_{}.pt'.format(i)), 'wb') as f:
                torch.save(training_subset, f)
            # subset_load = torch.load(os.path.join(data_path, 'training_{}.pt'.format(i)))
            # print("first 10 labels of subset_load[{}]:{}".format(i, subset_load.train_labels[0:10]))

            s3.Bucket(bucket_name).upload_file(os.path.join(data_path, 'training_{}.pt'.format(i)), 'training_{}.pt'.format(i))
            os.remove(os.path.join(data_path, 'training_{}.pt'.format(i)))

# if __name__ == "__main__":
#     preprocess_cifar10(bucket_name = ' ', key = "cifar-10-python.tar.gz", data_path = '/home/pytorch-cifar/data/', num_workers = 4)


text:
import numpy as np
import numpy.random as random

# 设置随机数种子
random.seed(42)

# 产生一个1x3，[0,1)之间的浮点型随机数
# array([[ 0.37454012,  0.95071431,  0.73199394]])
# 后面的例子就不在注释中给出具体结果了
random.rand(1, 3)

# 产生一个[0,1)之间的浮点型随机数
random.random()

# 下边4个没有区别，都是按照指定大小产生[0,1)之间的浮点型随机数array，不Pythonic…
random.random((3, 3))
random.sample((3, 3))
random.random_sample((3, 3))
random.ranf((3, 3))

# 产生10个[1,6)之间的浮点型随机数
5 * random.random(10) + 1
random.uniform(1, 6, 10)

# 产生10个[1,6]之间的整型随机数
random.randint(1, 6, 10)

# 产生2x5的标准正态分布样本
random.normal(size=(5, 2))

# 产生5个，n=5，p=0.5的二项分布样本
random.binomial(n=5, p=0.5, size=5)

a = np.arange(10)

# 从a中有回放的随机采样7个
random.choice(a, 7)

# 从a中无回放的随机采样7个
random.choice(a, 7, replace=False)

# 对a进行乱序并返回一个新的array
b = random.permutation(a)

# 对a进行in-place乱序
random.shuffle(a)

# 生成一个长度为9的随机bytes序列并作为str返回
# '\x96\x9d\xd1?\xe6\x18\xbb\x9a\xec'
random.bytes(9)


text:
#!/usr/bin/env python

"""This is the causal matrix module.

"""

__all__ = [
    'CausalMatrix'
]

import numpy as np
import pandas as pd
import logging
from podspy.log import constants as cnst

logger = logging.getLogger(__file__)

class CausalMatrix:
    def __init__(self, activity_list=list(), matrix=None):
        self.activity_list = activity_list
        self.matrix = matrix

    def __repr__(self):
        return '{}({}, {})'.format(self.__class__.__name__,
                                   self.activity_list,
                                   self.matrix.shape)

    def __str__(self):
        return '{}'.format(self.matrix)

    @staticmethod
    def build_from_logtable(logtable, sort=True):
        """Factory method to build a causal matrix from a log table.

        :param logtable: log table
        :param sorted: whether to sort the activities
        :return: built causal matrix
        """
        activity_list = logtable.event_df[cnst.ACTIVITY].unique()

        if sort:
            activity_list = sorted(activity_list)

        def get_causal_mat(df):
            to_join = ([np.nan], df[cnst.ACTIVITY].values)
            shifted_down = np.concatenate(to_join)
            shifted_df = pd.DataFrame({'shifted_down': shifted_down})
            # need to reset the index of df so that concatenation align
            df = pd.concat([df.reset_index(drop=True), shifted_df], axis=1)
            df = df[[cnst.CASEID, 'shifted_down', cnst.ACTIVITY]]
            return df

        grouped = logtable.event_df.groupby(cnst.CASEID, as_index=False)
        grouped = grouped.apply(get_causal_mat)
        # exclude any NaN rows
        grouped = grouped.dropna(axis=0, subset=[cnst.ACTIVITY, 'shifted_down'])

        grouped = grouped.groupby(['shifted_down', cnst.ACTIVITY], as_index=False)
        counts = grouped.agg('count')
        # give the counts column a good column name
        counts = counts.rename(columns={cnst.CASEID: 'count'}, index=str)
        counts = counts[['shifted_down', cnst.ACTIVITY, 'count']]

        zeros = np.zeros((len(activity_list), len(activity_list)))
        mat = pd.DataFrame(zeros, columns=activity_list, index=activity_list, dtype=np.int)

        for activity in logtable.event_df[cnst.ACTIVITY].unique():
            cols = counts.loc[counts['shifted_down'] == activity, cnst.ACTIVITY].values
            vals = counts.loc[counts['shifted_down'] == activity, 'count'].values
            logger.debug('Setting columns {} related to "{}" row with values {}'.format(cols, activity, vals))
            mat.loc[activity, cols] = vals

        logger.debug('\n{}'.format(counts))
        logger.debug('\n{}'.format(mat))

        mat.reset_index(drop=True, inplace=True)
        mat.columns = list(range(len(activity_list)))

        return CausalMatrix(activity_list, mat)

max_stars_repo_name: jwllee/podspy

max_issues_repo_name: wailamjonathanlee/podspy

max_forks_repo_name: wailamjonathanlee/podspy


text:
import os
import __main__

import numpy as np

from desicos.logger import log, warn
from desicos.constants import FLOAT

DOC_COMMON = '''
    scaling_factor     - scales the original imperfection (default = 1.)
    r_TOL              - defines a tolerance to ignore measured points
                         (default = 1. percent)
    num_closest_points - number of measured points to be interpolated for each
                         node (default = 5)
    power_parameter    - power of inverse weighted interpolation function
                         (default = 2.)
    num_sec_z          - number of cross-sections in Z to classify the measured
                         points (default = 25)
'''
def read_file(file_name,
               frequency             = 1,
               forced_average_radius = None,
               H_measured            = None,
               R_best_fit            = None,
               stretch_H             = False,
               z_offset_bot            = None,
               r_TOL                 = 1.):
    log('Reading imperfection file: {0} ...'.format(file_name))
    # user warnings
    if stretch_H:
        if z_offset_bot:
            warn('Because of the stretch_H option, '+
                 'consider setting z_offset_bot to None')
    # reading the imperfection file
    ignore = False
    mps = np.loadtxt(file_name, dtype=FLOAT)
    r = np.sqrt(mps[:, 0]**2 + mps[:, 1]**2)
    # measuring model dimensions
    if R_best_fit is None:
        R_best_fit   = np.average(r)
        warn('The cylinder average radius of the measured points ' +
             'assumed to be {0:1.2f}'.format(R_best_fit))
    z_min = mps[:, 2].min()
    z_max = mps[:, 2].max()
    z_center  = (z_max + z_min)/2.
    H_points  = (z_max - z_min)
    log('R_best_fit     : {0}'.format(R_best_fit))
    # applying user inputs
    R = R_best_fit
    if forced_average_radius:
        R = forced_average_radius
        log('Forced measured radius: {0}'.format(forced_average_radius))
    H_model = H_points
    if not H_measured:
        H_measured = H_points
        warn('The cylinder height of the measured points assumed ' +
             'to be {0:1.2f}'.format(H_measured))
    # calculating default z_offset_bot
    if not z_offset_bot:
        if stretch_H:
            z_offset_bot = 0.
        else:
            z_offset_bot = (H_measured - H_points) / 2.
    offset_z = z_offset_bot - z_min
    log('H_points       : {0}'.format(H_points))
    log('H_measured     : {0}'.format(H_measured))
    log('z_min          : {0}'.format(z_min))
    log('z_max          : {0}'.format(z_max))
    log('offset_z       : {0}'.format(offset_z))
    r_TOL_min = (R * (1-r_TOL/100.))
    r_TOL_max = (R * (1+r_TOL/100.))
    cond = np.all(np.array((r > r_TOL_max,
                            r < r_TOL_min)), axis=0)
    skept = mps[cond]
    log('Skipping {0} points'.format(len(skept)))
    mps = mps[np.logical_not(cond)]
    offset_mps = mps.copy()
    offset_mps[:, 2] += offset_z
    norm_mps = offset_mps.copy()
    norm_mps[:, 0] /= R
    norm_mps[:, 1] /= R
    if stretch_H:
        norm_mps[:, 2] /= H_points
    else:
        norm_mps[:, 2] /= H_measured
    return mps, offset_mps, norm_mps

def plot_file(file_name,
               R_model,
               H_model,
               semi_angle,
               H_measured,
               R_best_fit=None,
               stretch_H        = False,
               z_offset_bot       = None,
               r_TOL            = 1.,
               frequency        = 100):
    # reading imperfection file
    m, o, mps = read_file(file_name             = file_name,
                          H_measured            = H_measured,
                          forced_average_radius = R_best_fit,
                          stretch_H              = stretch_H,
                          z_offset_bot            = z_offset_bot,
                          r_TOL                 = r_TOL)
    R_top = R_model - np.tan(np.deg2rad(semi_angle)) * H_model
    semi_angle = abs(semi_angle)
    def local_radius(z):
        r = R_model + (R_top - R_model) * z / H_model
        return r
    #
    model_name = os.path.basename(file_name).split('.')[0]
    mod = __main__.mdb.Model(model_name)
    part = mod.Part(model_name)
    # applying scalings when necessary
    mps[:, 2] *= H_model
    if semi_angle < 1.e-6:
        R_local = R_model
    else:
        R_local = local_radius(mps[:, 2])
    x = mps[:, 0] * R_local
    y = mps[:, 1] * R_local
    for i, xi in enumerate(x):
        part.DatumPointByCoordinate(coords=(xi, y[i], z[i]))
    #
    #
    from abaqusConstants import PARALLEL
    vpname = __main__.session.currentViewportName
    session = __main__.session
    session.viewports[vpname].setValues(displayedObject=part)
    session.viewports[vpname].view.setProjection(projection=PARALLEL)
    session.viewports[vpname].view.setValues(nearPlane=1232.67,
        farPlane=2195.51, cameraPosition=(-528.899, -505.13, 1304.85),
        cameraUpVector=(-0.00801664, 0.951535, 0.307437),
        cameraTarget=(-0.331696, 0.209839, -245.425))
    session.viewports[vpname].view.fitView()

def calc_nodal_translations(imperfection_file_name,
                            nodes,
                            H_model,
                            H_measured,
                            R_model,
                            R_best_fit,
                            semi_angle,
                            stretch_H,
                            z_offset_bot,
                            rotatedeg,
                            r_TOL,
                            num_closest_points,
                            power_parameter,
                            num_sec_z=100,
                            sample_size=None):
    # reading imperfection file
    m, o, mps = read_file(file_name = imperfection_file_name,
                          H_measured = H_measured,
                          R_best_fit = R_best_fit,
                          forced_average_radius = R_best_fit,
                          stretch_H = stretch_H,
                          z_offset_bot = z_offset_bot,
                          r_TOL = r_TOL)

    log('Calculating nodal translations!')
    if sample_size:
        num = mps.shape[0]
        if sample_size < num:
            log('Using sample_size={0}'.format(sample_size), level=1)
            mps = mps[sample(range(num), int(sample_size)), :]
    num_nodes = nodes.shape[0]
    R_top = R_model - np.tan(np.deg2rad(semi_angle)) * H_model
    semi_angle = abs(semi_angle)
    def local_radius(z):
        return R_model + (R_top - R_model) * z / H_model
    mps[:, 2] *= H_model
    if semi_angle < 1.e-6:
        R_local = R_model
    else:
        R_local = local_radius(mps[:, 2])
    thetarads = np.arctan2(mps[:, 1], mps[:, 0])
    if rotatedeg:
        thetarads += np.deg2rad(rotatedeg)
    mps[:, 0] = R_local*np.cos(thetarads)
    mps[:, 1] = R_local*np.sin(thetarads)
    num_sec_z = int(num_sec_z)
    mem_limit = 1024*1024*1024*8*2    # 2 GB
    mem_entries = int(mem_limit / 64) # if float64 is used
    sec_size = int(num_nodes/num_sec_z)
    #TODO better memory control...
    if sec_size**2*10 > mem_entries:
        while True:
            num_sec_z +=1
            sec_size = int(num_nodes/num_sec_z)
            if sec_size**2*10 <= mem_entries:
                warn('New sec_size {0}'.format(sec_size))
                break
    ncp = num_closest_points
    nodes = nodes[np.argsort(nodes[:, 2])]
    mps = mps[np.argsort(mps[:, 2])]
    nodal_t = np.zeros(nodes.shape, dtype=nodes.dtype)
    limit = int(num_sec_z/5)
    for i in xrange(num_sec_z+1):
        i_inf = sec_size*i
        i_sup = sec_size*(i+1)
        if i % limit == 0:
            log('processed {0:7d} out of {1:7d} entries'.format(
                min(i_sup, num_nodes), num_nodes), level=1)
        sub_nodes = nodes[i_inf : i_sup]
        if not np.any(sub_nodes):
            continue
        inf_z = sub_nodes[:, 2].min()
        sup_z = sub_nodes[:, 2].max()
        tol = 0.01
        if i == 0 or i == num_sec_z:
            tol = 0.05
        while True:
            cond1 = mps[:, 2] >= inf_z - tol*H_model
            cond2 = mps[:, 2] <= sup_z + tol*H_model
            cond = np.all(np.array((cond1, cond2)), axis=0)
            sub_mps = mps[cond]
            if not np.any(sub_mps):
                tol += 0.01
            else:
                break
        dist  = np.subtract.outer(sub_nodes[:, 0], sub_mps[:, 0])**2
        dist += np.subtract.outer(sub_nodes[:, 1], sub_mps[:, 1])**2
        dist += np.subtract.outer(sub_nodes[:, 2], sub_mps[:, 2])**2
        asort = np.argsort(dist, axis=1)
        lenn = sub_nodes.shape[0]
        lenp = sub_mps.shape[0]
        asort_mesh = asort + np.meshgrid(np.arange(lenn)*lenp,
                                         np.arange(lenp))[0].transpose()
        # getting the z coordinate of the closest points
        sub_mps_z = np.take(sub_mps[:, 2], asort[:, :ncp])
        # getting the distance of the closest points
        dist_ncp = np.take(dist, asort_mesh[:, :ncp])
        # avoiding division by zero
        dist_ncp[(dist_ncp==0)] == 1.e-12
        # calculating the radius of the sub-group of measured points
        radius = np.sqrt(sub_mps[:, 0]**2 + sub_mps[:, 1]**2)
        # taking only the radius of the closest points
        radius_ncp = np.take(radius, asort[:, :ncp])
        # weight calculation
        total_weight = np.sum(1./(dist_ncp**power_parameter), axis=1)
        weight = 1./(dist_ncp**power_parameter)
        # calculating the local radius for the closest points
        r_local_ncp = local_radius(sub_mps_z)
        # computing the new radius
        r_new = np.sum(radius_ncp*weight/r_local_ncp, axis=1)/total_weight
        r_new *= local_radius(sub_nodes[:, 2])
        #NOTE modified after Regina, Mariano and Saullo decided to use
        #     the imperfection amplitude constant along the whole cone
        #     surface, which represents better the real manufacturing
        #     conditions. In that case the amplitude will be re-scaled
        #     using only  the bottom radius
        # calculating the local radius for the nodes for the new assumption
        r_local_nodes = np.sqrt(sub_nodes[:,0]**2 + sub_nodes[:, 1]**2)
        # calculating the scaling factor required for the new assumption
        sf = R_model/r_local_nodes
        theta = np.arctan2(sub_nodes[:, 1], sub_nodes[:, 0])
        nodal_t[i_inf : i_sup][:, 0] = \
                (r_new*np.cos(theta) - sub_nodes[:, 0])*sf
        nodal_t[i_inf : i_sup][:, 1] = \
                (r_new*np.sin(theta) - sub_nodes[:, 1])*sf
        nodal_t[i_inf : i_sup][:, 3] = sub_nodes[:, 3]
    nodal_t = nodal_t[np.argsort(nodal_t[:, 3])]
    log('Nodal translations calculated!')

    return nodal_t

def get_nodes_from_txt_file(nodes_file_name):
    '''The file name must be: x y z node_id
    '''
    nodes = np.loadtxt(nodes_file_name, dtype=FLOAT)
    nodes = nodes[np.argsort(nodes[:, 3])]
    return

def disturb_my_model(imperfection_file_name,
                     nodes_file_name,
                     output_file_name,
                     H_model,
                     H_measured,
                     R_model,
                     R_best_fit=None,
                     semi_angle=0.,
                     stretch_H=False,
                     z_offset_bot=None,
                     rotatedeg=0.,
                     scaling_factor=1.,
                     r_TOL=1.,
                     num_closest_points=5,
                     power_parameter=2,
                     num_sec_z=25,
                     sample_size=None):
    # reading nodes data
    log('Reading nodes data from {0} ...'.format(nodes_file_name))
    nodes = get_nodes_from_txt_file(nodes_file_name)
    # calling translate_nodes function
    nodal_translations = calc_nodal_translations(
                                     imperfection_file_name,
                                     nodes = nodes,
                                     H_model = H_model,
                                     H_measured = H_measured,
                                     R_model = R_model,
                                     R_best_fit = R_best_fit,
                                     semi_angle = semi_angle,
                                     stretch_H = stretch_H,
                                     z_offset_bot = z_offset_bot,
                                     rotatedeg = rotatedeg,
                                     r_TOL = r_TOL,
                                     num_closest_points = num_closest_points,
                                     power_parameter = power_parameter,
                                     num_sec_z = num_sec_z,
                                     sample_size=sample_size)
    # writing output file
    log('Writing output file "{0}" ...'.format(output_file_name))
    outfile = open(output_file_name, 'w')
    for i, node in enumerate(nodal_translations):
        original_coords = node[0:3]
        translations = nodal_translations[k]
        new_coords = original_coords + translations * scaling_factor
        outfile.write('{0:d} {1:f} {2:f} {3:f}\n'.format(k, coords[0],
                      coords[1], coords[2]))
    outfile.close()
    return True

disturb_my_model.__doc__ = DOC_COMMON

if __name__ == '__main__':
    from  gui.gui_defaults import imps
    disturb_my_model(
             imperfection_file_name = imps['z18'],
             nodes_file_name    = r'C:\Temp\z15_nodes.txt',
             output_file_name   = r'C:\Temp\z15_nodes_new.txt',
             H_model            = 500,
             H_measured         = 510,
             R_model            = 250,
             R_best_fit         = None,
             stretch_H          = False,
             z_offset_bot         = None,
             scaling_factor     = 10.,
             r_TOL              = 10.,
             num_closest_points = 5,
             power_parameter    = 2,
             num_sec_z          = 100)

max_stars_repo_name: saullocastro/desicos

max_issues_repo_name: saullocastro/desicos

max_forks_repo_name: saullocastro/desicos


text:
from gensim.models.keyedvectors import KeyedVectors
import numpy as np
import numpy.linalg as la
from konlpy.tag import Okt
import scipy.stats as st
import argparse

testset = "test_dataset/kor_ws353.csv"
okt = Okt()

def normalize(array):
    norm = la.norm(array)
    return array / norm

def create_word_vector(word, pos_vectors):
    pos_list = okt.pos(word, norm=True)
    word_vector = np.sum([pos_vectors.word_vec(str(pos).replace(" ", "")) for pos in pos_list], axis=0)
    return normalize(word_vector)

def word_sim_test(filename, pos_vectors):
    delim = ','
    actual_sim_list, pred_sim_list = [], []
    missed = 0

    with open(filename, 'r',encoding='utf-8') as pairs:
        for pair in pairs:
            w1, w2, actual_sim = pair.strip().split(delim)

            try:
                w1_vec = create_word_vector(w1, pos_vectors)
                w2_vec = create_word_vector(w2, pos_vectors)
                pred = float(np.inner(w1_vec, w2_vec))
                actual_sim_list.append(float(actual_sim))
                pred_sim_list.append(pred)

            except KeyError:
                missed += 1

    spearman, _ = st.spearmanr(actual_sim_list, pred_sim_list)
    pearson, _ = st.pearsonr(actual_sim_list, pred_sim_list)

    return spearman, pearson, missed

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("pos_file", type=str, help="trained morpheme vectors file")
    args = parser.parse_args()

    pos_vectors = KeyedVectors.load_word2vec_format(args.pos_file, binary=False)

    spearman, pearson, missed = word_sim_test(testset, pos_vectors)
    print("Missing words :", missed)
    print("Spearman coefficient :", spearman)
    print("Pearson coefficient :", pearson)


text:
import cv2
import numpy as np

import multiprocessing as mp

import time

def image_capture_proc(video, out_q):
    # while True:
    check, frame = video.read()
    out_q.put(frame)

def preprocess_proc(inp_q, out_q):
    # while not inp_q.empty():
    img = inp_q.get()

    res_img = cv2.fastNlMeansDenoisingColored(img, None, 5, 5, 10, 8)

    out_q.put(res_img)

def feature_ext_proc(inp_q, out_q):
    # while not inp_q.empty():
    img = inp_q.get()

    # hu moments
    bw_image = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    hu_moments = cv2.HuMoments(cv2.moments(bw_image)).flatten()

    # means
    img_mean = np.mean(img)
    mean_b = np.mean(img[0, :, :])
    mean_g = np.mean(img[1, :, :])
    mean_r = np.mean(img[2, :, :])
    means_vec = [img_mean, mean_r, mean_g, mean_b]

    # orb # debug it
    orb = cv2.ORB_create()
    kp, orb_descriptors = orb.detectAndCompute(bw_image, None)

    out_q.put({"hu": hu_moments, "orb": orb_descriptors, "means": means_vec})

if __name__ == "__main__":
    img_capt_queue = mp.Queue(3)
    prep_queue = mp.Queue(3)
    feat_ext_queue = mp.Queue(3)

    video_inp = cv2.VideoCapture(0)

    p1 = mp.Process(target=image_capture_proc, args=(video_inp, img_capt_queue))
    p2 = mp.Process(target=preprocess_proc, args=(img_capt_queue, prep_queue))
    p3 = mp.Process(target=feature_ext_proc, args=(prep_queue, feat_ext_queue))

    [p.start() for p in (p1, p2, p3)]

    while True:
        # image_capture_proc(video_inp, img_capt_queue)
        print(feat_ext_queue.get())

        key = cv2.waitKey(1)
        if key == ord("q"):
            break

    [p.join() for p in (p1, p2, p3)]


text:
# Copyright 2021-present, Nike, Inc.
# All rights reserved.
#
# This source code is licensed under the Apache-2.0 license found in
# the LICENSE file in the root directory of this source tree.

import numpy as np
from faker import Faker

class ColumnFactory(object):
    """
    ColumnFactory is a callable that wraps another callable
    so that it can return a dict with the column as the key
    for the value returned by the callable.
    """
    def __init__(self, column, callable_, depends_on=None):
        """
        :param column: str
            Column name that will be used as the
            key for the return
        :param callable_: function
        :param depends_on: list[str], default None
            If this is not None, the strings provided here will
            be used by the KnockoffTable as keys in the kwargs
            when making a call to this instance. The values provided
            for those kwargs are looked up from a dict populated with
            previously returned key-values from calls to preceding factories.
        """
        self.column = column
        self.callable = callable_
        self.depends_on = depends_on

    def __call__(self, *args, **kwargs):
        return {self.column: self.callable(*args,**kwargs)}

class ChoiceFactory(object):
    def __init__(self, choices, p=None, replace=True):
        self.choices = choices
        self.p = p
        self.replace = replace

    def __call__(self, size=None, p=None, replace=None):
        if replace is None:
            replace = self.replace
        return np.random.choice(self.choices,
                                size=size,
                                replace=replace,
                                p=p or self.p)

class FakerFactory(object):
    def __init__(self, method, faker=None, **kwargs):
        self.method = method
        self.faker = faker or Faker()
        self.kwargs = kwargs

    def __call__(self):
        return getattr(self.faker, self.method)(**self.kwargs)

max_stars_repo_name: Nike-Inc/knockoff-factory

max_issues_repo_name: Nike-Inc/knockoff-factory

max_forks_repo_name: Nike-Inc/knockoff-factory


text:
import numpy as np

from scipy.ndimage.morphology import distance_transform_edt

from .morphology import find_perimeter

def bottom_top_area(X, Y, Z, ignore_ground=False):
    """computes top quad and bottom quad areas for distmap
    and SOR algorithms"""
    """ignore_ground is an adjustment used in distmap
    but not in SOR"""
    h, w = Z.shape

    i2 = slice(0, h - 1)
    i1 = slice(1, h)
    ia2 = slice(0, w - 1)
    ia1 = slice(1, w)

    # create linesegs AB for all quadrilaterals
    AB1, AB2, AB3 = [xyz[i2, ia2] - xyz[i1, ia2] for xyz in [X, Y, Z]]
    # create linesegs AD for all quadrilaterals
    AD1, AD2, AD3 = [xyz[i2, ia2] - xyz[i1, ia1] for xyz in [X, Y, Z]]
    # create linesegs AD for all quadrilaterals
    CD1, CD2, CD3 = [xyz[i2, ia1] - xyz[i1, ia1] for xyz in [X, Y, Z]]

    # triangle formed by AB and AD for all quadrilaterals
    leg1 = ((AB2 * AD3) - (AB3 * AD2)) ** 2
    leg2 = ((AB3 * AD1) - (AB1 * AD3)) ** 2
    leg3 = ((AB1 * AD2) - (AB2 * AD1)) ** 2
    # bottom area
    area_bot = 0.5 * np.sqrt(leg1 + leg2 + leg3)

    # triangle formed by CD and AD for all quadrilaterals
    leg1 = ((CD2 * AD3) - (CD3 * AD2)) ** 2
    leg2 = ((CD3 * AD1) - (CD1 * AD3)) ** 2
    leg3 = ((CD1 * AD2) - (CD2 * AD1)) ** 2
    # top area
    area_top = 0.5 * np.sqrt(leg1 + leg2 + leg3)

    if ignore_ground:
        ind = np.abs(AB3) + np.abs(AD3) + np.abs(CD3) + Z[i2, ia2]
        area_bot[ind == 0] = 0
        area_top[ind == 0] = 0

    return area_bot, area_top

def distmap_volume_surface_area(B, perimeter_image=None):
    """Moberg & Sosik biovolume algorithm
    returns volume and representative transect"""
    if perimeter_image is None:
        perimeter_image = find_perimeter(B)
    # elementwise distance to perimeter + 1
    D = distance_transform_edt(1 - perimeter_image) + 1
    # mask distances outside blob
    D = D * (B > 0)
    Dm = np.ma.array(D, mask=1 - B)
    # representative transect
    x = 4 * np.ma.mean(Dm) - 2
    # diamond correction
    c1 = x ** 2 / (x ** 2 + 2 * x + 0.5)
    # circle correction
    # c2 = np.pi / 2
    # volume = c1 * c2 * 2 * np.sum(D)
    volume = c1 * np.pi * np.sum(D)
    # surface area
    h, w = D.shape
    Y, X = np.mgrid[0:h, 0:w]
    area_bot, area_top = bottom_top_area(X, Y, D, ignore_ground=True)
    # final correction of the diamond cross-section
    # inherent in the distance map to be circular instead
    c = (np.pi * x / 2) / (2 * np.sqrt(2) * x / 2 + (1 + np.sqrt(2)) / 2)
    sa = 2 * c * (np.sum(area_bot) + np.sum(area_top))
    # return volume, representative transect, and surface area
    return volume, x, sa

def sor_volume_surface_area(B):
    """pass in rotated blob"""
    """Sosik and Kilfoyle surface area / volume algorithm"""
    # find the bottom point of the circle for each column
    m = np.argmax(B, axis=0)
    # compute the radius of each slice
    d = np.sum(B, axis=0)
    # exclude 0s
    r = (d / 2.0)[d > 0]
    m = m[d > 0]
    n_slices = r.size
    # compute 721 angles between 0 and 180 degrees inclusive, in radians
    n_angles = 721
    angR = np.linspace(0, 180, n_angles) * (np.pi / 180)

    # make everything the same shape: (nslices, nangles)
    angR = np.vstack([angR] * m.size)
    m = np.vstack([m] * n_angles).T
    r = np.vstack([r] * n_angles).T

    # compute the center of each slice
    center = m + r
    # correct for edge effects
    center[0, :] = center[1, :]
    center[-1, :] = center[-2, :]

    # y coordinates of all angles on all slices
    Y = center + np.cos(angR) * r
    # z coordinates of all angles on all slices
    Z = np.sin(angR) * r

    # compute index of slice in y matrix
    x = np.array(range(r.shape[0])) + 1.0
    # half-pixel adjustment of edges
    x[0] -= 0.5
    x[-1] += 0.5
    X = np.vstack([x] * n_angles).T

    # compute bottom and top area
    area_bot, area_top = bottom_top_area(X, Y, Z)

    # surface area
    # multiply sum of areas of quadrilaterals by 2 to account for angles 180-360
    sa = 2 * (np.sum(area_bot) + np.sum(area_top))
    # add flat end caps
    sa += np.sum(np.pi * r[[0, -1], 0] ** 2)

    # compute height of cone slices
    b1 = np.pi * r[1:n_slices, 0] ** 2
    b2 = np.pi * r[0 : n_slices - 1, 0] ** 2
    h = np.diff(x)
    # volume
    v = np.sum((h / 3) * (b1 + b2 + np.sqrt(b1 * b2)))

    # representative width
    xr = np.mean(r[:, 0] * 2)

    # return volume, representative width, and surface area
    return v, xr, sa

max_stars_repo_name: veot/ifcb-features

max_issues_repo_name: veot/ifcb-features

max_forks_repo_name: veot/ifcb-features

reason: import numpy,from scipy


text:
from utility import TOOLBOX
from Model import SNeL_RFS_ANNC

import argparse
import os
import configparser
import numpy as np
import json

import time

import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)

API_Description = """
***** Radiomics Analysis Platform  *****
API Name: TRAIN Model
Version:    1.0
Developer: Pei-Yan Li
Email:     d05548014@ntu.edu.tw
****************************************

"""

_pwd_ = os.getcwd()

parser = argparse.ArgumentParser(prog = 'Train.py',
                                 formatter_class = argparse.RawDescriptionHelpFormatter,
                                 description = API_Description)

parser.add_argument('-f', action = 'store', type = str, help='the path to the dataset stored in npz file format')
parser.add_argument('-hyp', action = 'store', type = str, help='the path to the hyper-parameter file')
parser.add_argument('-cfg', action = 'store', type = str, help='the architecture of neural netowrk designed for model')
parser.add_argument('-k', action = 'store_true', default=False, help='True/False to use 5-Fold cross validation for model training with 7:1:2 train-validation-test ratio.')
parser.add_argument('-s', action = 'store_true', default=False, help='True/False to standardize train data')
parser.add_argument('-p', action = 'store_true', default=False, help='True/False to use imputation algorithm for missing data')
parser.add_argument('-d', action = 'store', type = str, help='the folder for the storage of model checkpoints.')
parser.add_argument('-gpu', action = 'store', type = int, default = 0, help='the gpu id for training')
args = parser.parse_args()

if __name__ == '__main__':

    assert ( os.path.isfile(args.f) and ".npz" == os.path.splitext(args.f)[1] ), "Wrong! Given dataset cannot be found (*.npz)."
    assert ( os.path.isfile(args.hyp) and ".ini" == os.path.splitext(args.hyp)[1] ), "Wrong! Given hyper-parameter file cannot be found (*.ini)."
    assert ( os.path.isfile(args.cfg) and ".cfg" == os.path.splitext(args.cfg)[1] ), "Wrong! Given NN architecture file cannot be found (*.cfg)."

    if ( not os.path.isdir(args.d) ):
        os.mkdir(args.d)

    hyperparameters = configparser.ConfigParser()
    hyperparameters.read(args.hyp)

    nn_cfg = configparser.ConfigParser()
    nn_cfg.read(args.cfg)

    toolbox = TOOLBOX(kfold=args.k, normalize=args.s, file=args.f, hype=hyperparameters)

    if (args.k):
        print("Applying 5-fold cross validation for model training...")
        NNs = []
        for i in range(5):
            save_folder = os.path.join( args.d, f"Fold_{i+1}" )
            if ( not os.path.isdir(save_folder) ):
                os.mkdir(save_folder)
            NNs.append(
                SNeL_RFS_ANNC(cfg=nn_cfg,
                        store=save_folder, gpu=int(args.gpu),
                        batchsize=toolbox.batch_size,
                        learning_rate=toolbox.learning_rate,
                        ID=i)
            )

        save_folder = os.path.join( args.d, f"Fold_Best" )
        if ( not os.path.isdir(save_folder) ):
            os.mkdir(save_folder)
        NNs.append(
                SNeL_RFS_ANNC(cfg=nn_cfg,
                        store=save_folder, gpu=int(args.gpu),
                        batchsize=toolbox.batch_size,
                        learning_rate=toolbox.learning_rate,
                        ID=5)
            )

    Best_Set = {"Train":[], "Val":[], "Test":[]}
    Best_F1  = 0.0

    Fold_Count = 0
    CV_table = {}
    for TRAIN, VAL, TEST in toolbox.data_splitter():

        Train_X, Train_y = TRAIN
        Val_X, Val_y = VAL
        Test_X, Test_y = TEST

        if ( args.p ):
            Train_X, Val_X, Test_X = toolbox.Imputer_KNN()
        else:
            print("Replacing nans to zeros")
            Train_X, Val_X, Test_X = toolbox.fillzeros()

        if ( args.s ):
            print("Standardizing ...")
            Train_X, Val_X, Test_X = toolbox.Standard_Scaler()

        if (  0 < np.count_nonzero(np.isnan(Train_X))
           or 0 < np.count_nonzero(np.isnan(Val_X))
           or 0 < np.count_nonzero(np.isnan(Test_X))  ):
            print("NaN!!!")
            print(f"Train_X {np.count_nonzero(np.isnan(Train_X))}")
            print(f"Val_X {np.count_nonzero(np.isnan(Val_X))}")
            print(f"Test_X {np.count_nonzero(np.isnan(Test_X))}")
            exit()

        NNs[0].initModel()
        NNs[0].init_variables()

        EPOCHs = toolbox.get_final_epochs() - 1
        epoch_count = 0
        best_val_f1 = 0.0
        for lambda_s, lambda_a in toolbox.KKTmultipliers():

            print(f"CV-iteration: {Fold_Count+1}, Epoch: {epoch_count}")

            NNs[0].set_KKTmultipliers(lambda_s=lambda_s, lambda_a=lambda_a)

            start_time = time.time()

            loss = 0
            for i in range( Train_y.size ):
                x = Train_X[i, :]
                y = Train_y[i, :]

                try:
                    loss_,loss_softmax_ = NNs[0].run_train( x=np.reshape(x, (toolbox.batch_size, int(x.size / toolbox.batch_size)) ),
                                        label=y+1,
                                        step = epoch_count )
                    loss += loss_
                    print(f"epoch {epoch_count}, iter {i}, loss = {loss_}, loss_softmax={loss_softmax_}")
                except Exception as err:
                    print(f"Input x: {x}")
                    print(f"Input y: {y}")
                    print(err)

            train_loss = loss / Train_y.size

            best_val_f1, report = NNs[0].run_eval(X=Val_X, Y=Val_y, best_val_f1=best_val_f1, step=epoch_count)

            epoch_time = time.time() - start_time
            print("Epoch " + str(epoch_count) + " of " + str(EPOCHs) + " took " + str(epoch_time) + "s")
            print(f"Avergae training loss: {train_loss}")
            print(f"Best Validation Performance (F1): {best_val_f1}")

            epoch_count+=1

        CV_table[Fold_Count] = {'report':report, 'best f1':best_val_f1}

        if ( best_val_f1 > Best_F1 ):
            Best_F1 = best_val_f1
            Best_Set['Train'] = [Train_X, Train_y]
            Best_Set['Val']   = [Val_X, Val_y]
            Best_Set['Test']  = [Test_X, Test_y]

        del NNs[0]
        Fold_Count+=1

    json_path = os.path.join( args.d, 'CV_table.json' )
    json_file = open(json_path, 'w')
    json_content = json.dump( CV_table, indent=4 )
    json_file.writelines(json_content)
    json_file.close()

    NNs[0].initModel()
    NNs[0].init_variables()

    EPOCHs = toolbox.get_final_epochs()
    epoch_count = 0
    best_val_f1 = 0.0

    Train_X, Train_y = Best_Set['Train']
    Val_X, Val_y     = Best_Set['Val']
    Test_X, Test_y  = Best_Set['Test']

    Train_X = np.concatenate((Train_X, Val_X), axis=0)
    Train_y = np.concatenate((Train_y, Val_y), axis=0)

    best_report = None
    Final_test_table = {}

    for lambda_s, lambda_a in toolbox.KKTmultipliers():

        NNs[0].set_KKTmultipliers(lambda_s=lambda_s, lambda_a=lambda_a)

        start_time = time.time()

        loss = 0
        for i in range( Train_y.size ):
            x = Train_X[i, :]
            y = Train_y[i, :]

            loss_,loss_softmax_ = NNs[0].run_train( x=np.reshape(x, (toolbox.batch_size, int(x.size / toolbox.batch_size)) ),
                                    label=y+1,
                                    step = epoch_count )
            loss += loss_
            print(f"epoch {epoch_count}, iter {i}, loss = {loss_}, loss_softmax={loss_softmax_}, KKT_omega_s={KKT_omega_s_}, KKT_omega_a={KKT_omega_a_}")
        train_loss = loss / Train_y.size

        best_val_f1, best_report = NNs[0].run_eval(X=Test_X, Y=Test_y,
                                            best_val_f1=best_val_f1, best_report=best_report,
                                            step=epoch_count)

        epoch_time = time.time() - start_time
        print("Epoch " + str(epoch_count) + " of " + str(EPOCHs) + " took " + str(epoch_time) + "s")

        Final_test_table['report'] = report
        Final_test_table['best f1'] = best_val_f1

        epoch_count+=1

    json_path = os.path.join( args.d, 'Final_test_table.json' )
    json_file = open(json_path, 'w')
    json_content = json.dump( Final_test_table, indent=4 )
    json_file.writelines(json_content)
    json_file.close()

    print("\n")
    print(f"Best Test Performance (F1): {best_val_f1}")
    print(best_report)


text:
from typing import (
    Tuple,
    List
)

import onnxruntime
import numpy as np

from espnet_onnx.asr.frontend.frontend import Frontend
from espnet_onnx.asr.frontend.global_mvn import GlobalMVN
from espnet_onnx.asr.frontend.utterance_mvn import UtteranceMVN
from espnet_onnx.asr.scorer.interface import BatchScorerInterface
from espnet_onnx.utils.function import (
    subsequent_mask,
    make_pad_mask,
    mask_fill
)
from espnet_onnx.utils.config import Config

class Encoder:
    def __init__(
        self,
        encoder_config: Config,
        providers: List[str],
        use_quantized: bool = False,
    ):
        self.config = encoder_config
        if use_quantized:
            self.encoder = onnxruntime.InferenceSession(
                self.config.quantized_model_path,
                providers=providers
            )
        else:
            self.encoder = onnxruntime.InferenceSession(
                self.config.model_path,
                providers=providers
            )

        self.frontend = Frontend(self.config.frontend)
        if self.config.do_normalize:
            if self.config.normalize.type == 'gmvn':
                self.normalize = GlobalMVN(self.config.normalize)
            elif self.config.normalize.type == 'utterance_mvn':
                self.normalize = UtteranceMVN(self.config.normalize)

        # if self.config.do_preencoder:
        #     self.preencoder = Preencoder(self.config.preencoder)

        # if self.config.do_postencoder:
        #     self.postencoder = Postencoder(self.config.postencoder)

    def __call__(
        self, speech: np.ndarray, speech_length: np.ndarray
    ) -> Tuple[np.ndarray, np.ndarray]:
        """Frontend + Encoder. Note that this method is used by asr_inference.py
        Args:
            speech: (Batch, Length, ...)
            speech_lengths: (Batch, )
        """
        # 1. Extract feature
        feats, feat_length = self.frontend(speech, speech_length)

        # 2. normalize with global MVN
        if self.config.do_normalize:
            feats, feat_length = self.normalize(feats, feat_length)

        # if self.config.do_preencoder:
        #     feats, feats_lengths = self.preencoder(feats, feats_lengths)

        # 3. forward encoder
        encoder_out, encoder_out_lens = \
            self.forward_encoder(feats, feat_length)
        encoder_out = self.mask_output(encoder_out, encoder_out_lens)

        # if self.config.do_postencoder:
        #     encoder_out, encoder_out_lens = self.postencoder(
        #         encoder_out, encoder_out_lens
        #     )

        return encoder_out, encoder_out_lens

    def mask_output(self, feats, feat_length):
        if self.config.is_vggrnn:
            feats = mask_fill(feats, make_pad_mask(feat_length, feats, 1), 0.0)
        return feats, feat_length

    def forward_encoder(self, feats, feat_length):
        if self.config.enc_type == 'RNNEncoder':
            encoder_out, encoder_out_lens = \
                self.encoder.run(["encoder_out", "encoder_out_lens"], {
                    "feats": feats,
                    "feats_length": feat_length
                })
            encoder_out = mask_fill(encoder_out, make_pad_mask(
                feat_length, encoder_out, 1), 0.0)

        elif self.config.enc_type == 'XformerEncoder':
            mask = (make_pad_mask(feat_length)[
                :, None, :] == False).astype(np.float64)
            # 3. forward encoder
            encoder_out, encoder_out_lens = \
                self.encoder.run(["encoder_out", "encoder_out_lens"], {
                    "feats": feats,
                    "mask": mask
                })
        return encoder_out, encoder_out_lens


text:
import numpy as np
import scipy.stats as stats
import matplotlib.pyplot as plt
import scipy.optimize as opt
import emcee
import triangle, walkers
import priors

np.random.seed(666)

def randomData(a, b, sig, npts = 100):
    x    = np.random.random(npts)
    mean = a + b * x
    y = stats.norm(loc=mean, scale=sig).rvs(size=mean.shape)
    return x, y

if __name__ == "__main__":
    aTrue    = 7.1
    bTrue    = 10.0
    sigTrue  = np.sqrt(2.0)
    aGuess   = 6.9
    bGuess   = 10.5
    sigGuess = np.sqrt(1.8)

    x, y = randomData(aTrue, bTrue, sigTrue)
    yerrTrue = sigTrue*np.ones_like(x)

    sigPrior = priors.HalfCauchyPrior(0, 0.1)
    # Step 5: MCMC modeling, unknown errors, prior on sigma
    def chi2(params, *args):
        a, b, sigma = params
        xf, yf = args
        prediction = a + b * xf
        chi2  = np.sum( (prediction - yf)**2 / sigma**2 )
        chi2 += np.log(2 * np.pi * sigma**2)
        return chi2
    def lnprior(params):
        a, b, sigma = params
        #if sigma < 1e-10 or sigma > 10:
        #    return -np.inf
        #return 0.0
        return sigPrior.lnlike(sigma)
    def lnlike(params, *args):
        lp = lnprior(params)
        if not np.isfinite(lp):
            return -np.inf
        return -0.5 * chi2(params, *args) + lp

    result = opt.minimize(chi2, [aGuess, bGuess, sigGuess], args=(x, y), method="BFGS")
    print "Nonlinear fit, no errors"
    print " a'   = %.3f +/- %.3f" % (result.x[0], np.sqrt(result.hess_inv[0][0]))
    print " b'   = %.3f +/- %.3f" % (result.x[1], np.sqrt(result.hess_inv[1][1]))
    print " sig' = %.3f +/- %.3f" % (result.x[2], np.sqrt(result.hess_inv[2][2]))

    ndim, nwalkers, nburn, nstep = 3, 100, 1000, 10000
    pos = [np.array((aGuess, bGuess, sigGuess)) + 1e-4*np.random.randn(ndim) for i in range(nwalkers)]
    sampler = emcee.EnsembleSampler(nwalkers, ndim, lnlike, args=(x, y))
    pos, prob, state = sampler.run_mcmc(pos, nburn)
    sampler.reset()
    pos, prob, state = sampler.run_mcmc(pos, nstep, rstate0=state)
    fig = plt.figure()
    sp = fig.add_subplot(111)
    sp.errorbar(x, y, yerr=yerrTrue, fmt="ro")
    sp.plot(x, aTrue + bTrue * x, "r-")
    flatchain = sampler.flatchain
    print "MCMC analysis"
    flata   = flatchain[:,0]
    flatb   = flatchain[:,1]
    flatsig = flatchain[:,2]
    print " a = %.3f +%.3f -%.3f" % (np.percentile(flata, 50),
                                     np.percentile(flata, 50)-np.percentile(flata, 50-68.27/2),
                                     np.percentile(flata, 50+68.27/2)-np.percentile(flata, 50),
                                     )
    print " b = %.3f +%.3f -%.3f" % (np.percentile(flatb, 50),
                                     np.percentile(flatb, 50)-np.percentile(flatb, 50-68.27/2),
                                     np.percentile(flatb, 50+68.27/2)-np.percentile(flatb, 50),
                                     )
    print " sig = %.3f +%.3f -%.3f" % (np.percentile(flatsig, 50),
                                       np.percentile(flatsig, 50)-np.percentile(flatsig, 50-68.27/2),
                                       np.percentile(flatsig, 50+68.27/2)-np.percentile(flatsig, 50),
                                       )

    for a, b in flatchain[np.random.randint(len(flatchain), size=100)][:,:2]:
        sp.plot(x, a + b*x, color="k", alpha=0.05)
    triangle.triangle(flatchain, ("a", "b", "sig"))
    walkers.walkers(sampler.chain, ("a", "b", "sig"))
    plt.show()

max_stars_repo_name: acbecker/pyhm

max_issues_repo_name: acbecker/pyhm

max_forks_repo_name: acbecker/pyhm


text:
from distutils.core import setup
import numpy as np
import os
import sys
from setuptools import setup, find_packages, Extension
from setuptools.command.build_ext import build_ext

def list_cpp_files(package_dir='wikipedia2vec'):
    if sys.platform.startswith("win"):
        compile_args = []
        link_args = []
    else:
        compile_args = ['-Wno-unused-function', '-std=c++11']
        link_args = ['-std=c++11']

    ret = []
    for (dir_name, _, files) in os.walk(package_dir):
        for file_name in files:
            (module_name, ext) = os.path.splitext(file_name)
            if ext == '.cpp':
                module_name = '.'.join(dir_name.split(os.sep) + [module_name])
                path = os.path.join(dir_name, file_name)
                ret.append((module_name, dict(
                    sources=[path], language='c++', extra_compile_args=compile_args,
                    extra_link_args=link_args
                )))

    return ret

class custom_build_ext(build_ext):
    def finalize_options(self):
        build_ext.finalize_options(self)

        # Prevent numpy from thinking it is still in its setup process:
        # https://docs.python.org/2/library/__builtin__.html#module-__builtin__
        if isinstance(__builtins__, dict):
            __builtins__["__NUMPY_SETUP__"] = False
        else:
            __builtins__.__NUMPY_SETUP__ = False

        import numpy
        self.include_dirs.append(numpy.get_include())

kwargs = dict(
    language='c++',
    extra_compile_args=['-Wno-unused-function', '-std=c++11'],
    extra_link_args=['-std=c++11']
)
setup(
    cmdclass=dict(build_ext=custom_build_ext),
    ext_modules = [
        Extension('word2vec.data', ['word2vec/data.pyx'], **kwargs),
        Extension('word2vec.random', ['word2vec/random.pyx'], **kwargs),
        Extension('word2vec.train', ['word2vec/train.pyx'], **kwargs)
    ]
)


text:
"""Actors add exploration to model predictions and convert them to the appropriate format for acting.

Actors become associated with Agents in a 1-to-1 relationship and collect information
about the Model and Environment that they're working with through the Agent.
They use this information to convert predictions to the appropriate format,
such as converting a policy vector from a Policy model to an integer for a Discrete action space.

Often, an Actor will require at least one Schedule to govern a particular value, such as
epsilon for the epsilon-greedy approach.

Currently implemented exploration methods are:
- Epsilon greedy.
"""

import numpy as np
from custom_inherit import DocInheritMeta
from ..util import ActionModelMismatchError

class Actor(metaclass=DocInheritMeta(style="numpy")):
    """Component responsible both for exploration and converting model predictions to actions.

    Returns 2 actions at a time: the first to be consumed by the model,
                                 the second to be consumed by the environment.

    Attributes
    -------
    explore_and_convert_fns : dict of {str : dict of {str : function}}
        Functions corresponding to every combination of action space and model type.
        Action space is the first key, model type is the second key.
        The functions add exploration and convert the chosen action to the correct format.
    action_space : gym.Space
        The action space object of the associated environment.
    action_type : str
        The class name of the action space as a lowercased string.
    prediction_type : {'value', 'policy'}
        Indicates whether the model outputs action-values or a policy vector for the state.
    """
    def __init__(self):
        # a dictionary of functions makes choosing the method of conversion easier
        self.explore_and_convert_fns = {
            'discrete': {'policy': self._discrete_policy,
                         'value': self._discrete_value},
            'multidiscrete': {'policy': self._multidiscrete_policy,
                              'value': self._multidiscrete_value},
            'box': {'policy': self._box_policy, 'value': self._box_value},
            'multibinary': {'policy': self._multibinary_policy,
                            'value': self._multibinary_value}}
        self.action_space = None
        self.action_type = None
        self.prediction_type = None

    def configure(self, agent):
        """Associate actor with agent, picking up information about its action space and model."""
        self.action_space = agent.env.action_space
        self.action_type = self.action_space.__class__.__name__.lower()
        self.prediction_type = agent.model.prediction_type

    def convert_pred(self, pred):
        """Look up explore_and_convert function and apply it to model prediction."""
        return self.explore_and_convert_fns[self.action_type][self.prediction_type](pred)

    def warming_action(self):
        """Choose a random action without involving the model.

        Returns
        -------
        action_for_model : numpy.ndarray
            The action that will be consumed by the model for learning.
        action_for_env : numpy.ndarray or int or float
            The action that will be consumed by the environment to step.
        """
        action_for_env = self.action_space.sample()
        if self.action_type == 'discrete':
            action_for_model = np.eye(self.action_space.n)[action_for_env]
        elif self.action_type == 'multidiscrete':
            action_for_model = [np.eye(n)[action_for_env[i]]
                           for i, n in enumerate(self.action_space.nvec)]
        elif self.action_type == 'box':
            action_for_model = action_for_env
        elif self.action_type == 'multibinary':
            action_for_model = [np.eye(2)[env_a] for env_a in action_for_env]
        return action_for_model, action_for_env

    def step(self, new_episode):
        """Move one timestep ahead. Main purpose is to advance value schedules.

        Parameters
        ----------
        new_episode : bool
            A flag indicating whether this step is one that resets the environment.
        """
        pass

    def _discrete_policy(self, pred):
        """Exploration and conversion function for a Discrete action space + Policy model.

        Parameters
        ----------
        pred : numpy.ndarray
            The prediction output by the model, to be converted.

        Raises
        ------
        NotImplementedError.
        """
        raise NotImplementedError

    def _discrete_value(self, pred):
        """Exploration and conversion function for a Discrete action space + Value model.

        Parameters
        ----------
        pred : numpy.ndarray
            The prediction output by the model, to be converted.

        Raises
        ------
        NotImplementedError.
        """
        raise NotImplementedError

    def _multidiscrete_policy(self, pred):
        """Exploration and conversion function for a Multi-Discrete action space + Policy model.

        Parameters
        ----------
        pred : numpy.ndarray
            The prediction output by the model, to be converted.

        Raises
        ------
        NotImplementedError.
        """
        raise NotImplementedError

    def _multidiscrete_value(self, pred):
        """Exploration and conversion function for a Multi-Discrete action space + Value model.

        Parameters
        ----------
        pred : numpy.ndarray
            The prediction output by the model, to be converted.

        Raises
        ------
        NotImplementedError.
        """
        raise NotImplementedError

    def _box_policy(self, pred):
        """Exploration and conversion function for a Box action space + Policy model.

        Parameters
        ----------
        pred : numpy.ndarray
            The prediction output by the model, to be converted.

        Raises
        ------
        NotImplementedError.
        """
        raise NotImplementedError

    def _box_value(self, pred):
        """Exploration and conversion function for a Box action space + Value model.

        Parameters
        ----------
        pred : numpy.ndarray
            The prediction output by the model, to be converted.

        Raises
        ------
        NotImplementedError.
        """
        raise NotImplementedError

    def _multibinary_policy(self, pred):
        """Exploration and conversion function for a Multi-Binary action space + Policy model.

        Parameters
        ----------
        pred : numpy.ndarray
            The prediction output by the model, to be converted.

        Raises
        ------
        NotImplementedError.
        """
        raise NotImplementedError

    def _multibinary_value(self, pred):
        """Exploration and conversion function for a Multi-Binary action space + Value model.

        Parameters
        ----------
        pred : numpy.ndarray
            The prediction output by the model, to be converted.

        Raises
        ------
        NotImplementedError.
        """
        pass

class EpsilonGreedyActor(Actor):
    """Takes a random action with a varying probability, otherwise does what it considers optimal.

    Parameters
    ----------
    epsilon_schedule : pavlov.auxiliary.Schedule
        The schedule that governs the value of epsilon.

    Attributes
    ----------
    epsilon_schedule : pavlov.auxiliary.Schedule
        The schedule that governs the value of epsilon.
    epsilon : float
        The current value of epsilon to be applied.
    """
    def __init__(self, epsilon_schedule):
        super().__init__()
        self.epsilon_schedule = epsilon_schedule
        self.epsilon = self.epsilon_schedule.get()

    def step(self, new_episode):
        """Iterate the epsilon value according to its schedule."""
        self.epsilon_schedule.step(new_episode)
        self.epsilon = self.epsilon_schedule.get()

    def _discrete_policy(self, pred):
        """Apply epsilon-greedy to discrete action space with a policy-based model.

        Returns
        -------
        action_for_model : numpy.ndarray
            The action that will be consumed by the model for learning.
        action_for_env : numpy.ndarray or int or float
            The action that will be consumed by the environment to step.
        """
        if np.random.random() < self.epsilon:
            action_for_env = self.action_space.sample()
        else:
            action_for_env = np.argmax(pred)
        action_for_model = np.eye(self.action_space.n)[action_for_env]
        return action_for_model, action_for_env

    def _discrete_value(self, pred):
        """Apply epsilon-greedy to discrete action space with a value-based model.

        Returns
        -------
        action_for_model : numpy.ndarray
            The action that will be consumed by the model for learning.
        action_for_env : numpy.ndarray or int or float
            The action that will be consumed by the environment to step.
        """
        return self._discrete_policy(pred)

    def _multidiscrete_policy(self, pred):
        """Apply epsilon-greedy to multidiscrete action space with a policy-based model.

        Returns
        -------
        action_for_model : numpy.ndarray
            The action that will be consumed by the model for learning.
        action_for_env : numpy.ndarray or int or float
            The action that will be consumed by the environment to step.
        """
        if np.random.random() < self.epsilon:
            action_for_env = self.action_space.sample()
        else:
            action_for_env = np.array(list(map(np.argmax, pred)))
        action_for_model = [np.eye(n)[action_for_env[i]]
                       for i, n in enumerate(self.action_space.nvec)]
        return action_for_model, action_for_env

    def _multidiscrete_value(self, pred):
        """Apply epsilon-greedy to multidiscrete action space with a value-based model.

        Returns
        -------
        action_for_model : numpy.ndarray
            The action that will be consumed by the model for learning.
        action_for_env : numpy.ndarray or int or float
            The action that will be consumed by the environment to step.
        """
        return self._multidiscrete_policy(pred)

    def _box_policy(self, pred):
        """Apply epsilon-greedy to box action space with a policy-based model.

        Returns
        -------
        action_for_model : numpy.ndarray
            The action that will be consumed by the model for learning.
        action_for_env : numpy.ndarray or int or float
            The action that will be consumed by the environment to step.
        """
        if np.random.random() < self.epsilon:
            both_actions = self.action_space.sample()
            return both_actions, both_actions
        else:
            return pred, pred

    def _box_value(self, pred):
        """Apply epsilon-greedy to box action space with a value-based model.

        Returns
        -------
        action_for_model : numpy.ndarray
            The action that will be consumed by the model for learning.
        action_for_env : numpy.ndarray or int or float
            The action that will be consumed by the environment to step.
        """
        raise ActionModelMismatchError('box', 'value')

    def _multibinary_policy(self, pred):
        """Apply epsilon-greedy to multibinary action space with a policy-based model.

        Returns
        -------
        action_for_model : numpy.ndarray
            The action that will be consumed by the model for learning.
        action_for_env : numpy.ndarray or int or float
            The action that will be consumed by the environment to step.
        """
        return self._discrete_policy(pred)

    def _multibinary_value(self, pred):
        """Apply epsilon-greedy to multibinary action space with a value-based model.

        Returns
        -------
        action_for_model : numpy.ndarray
            The action that will be consumed by the model for learning.
        action_for_env : numpy.ndarray or int or float
            The action that will be consumed by the environment to step.
        """
        return self._discrete_policy(pred)

'''
class EpsilonNoisyActor(Actor):
    def __init__(self, eps_schedule, mu_schedule, sigma_schedule):
        super.__init__()
        self.eps_schedule = eps_schedule
        self.mu_schedule = mu_schedule
        self.sigma_schedule = sigma_schedule
        self.epsilon = self.eps_schedule.get()
        self.mu = self.mu_schedule.get()
        self.sigma = self.sigma_schedule.get()

    # TODO: this whole thing
'''

max_stars_repo_name: ntaylorwss/pavlov

max_issues_repo_name: ntaylorwss/kerasgym

max_forks_repo_name: ntaylorwss/pavlov


text:
import numpy as np
import generic
import torch

def evaluate(env, agent, valid_test="valid", verbose=False):
    with torch.no_grad():
        env.seed(42)
        env.split_reset(valid_test)
        agent.eval()
        print_qa_acc, print_ig_acc, print_steps = [], [], []
        to_print, counter = [], 0

        while(True):
            obs, infos = env.reset(random=False)
            batch_size = len(obs)
            agent.init(obs, infos)
            quest_list = agent.get_game_quest_info(infos)
            agent.kg.push_batch_question(quest_list, [item["q_srl"] for item in infos])
            previous_belief = None

            input_quest, input_quest_mask, quest_id_list = agent.get_agent_inputs(quest_list)
            previous_dynamics = None

            tmp_replay_buffer = []
            prev_commands = ["restart" for _ in range(batch_size)]
            for _ in range(agent.max_nb_steps_per_episode):
                commands, replay_info, current_dynamics, current_belief = agent.act_greedy(obs, infos, input_quest, input_quest_mask, quest_id_list, prev_commands, previous_dynamics, previous_belief)
                tmp_replay_buffer.append(replay_info)
                obs, infos = env.step(commands)
                prev_commands = commands
                previous_dynamics = current_dynamics
                previous_belief = current_belief

                still_running = generic.to_np(replay_info[-1])
                if np.sum(still_running) == 0:
                    break

            # The agent has exhausted all steps, now answer question.
            chosen_head_tails = agent.answer_question_act(agent.naozi.get(), quest_list, current_belief)  # batch
            chosen_head_tails_np = generic.to_np(chosen_head_tails)
            chosen_answer_strings = generic.get_answer_strings(agent.naozi.get(), chosen_head_tails_np, agent.bert_tokenizer, agent.special_token_ids)
            answer_strings = [item["a_string"] for item in infos]
            answer_token_ids = [item["a"] for item in infos]
            masks_np = [generic.to_np(item[-1]) for item in tmp_replay_buffer]

            qa_reward_np = generic.get_qa_reward(chosen_answer_strings, answer_strings)
            ig_reward_np = generic.get_sufficient_info_reward(agent.naozi.get(), answer_token_ids)

            if verbose:
                for i in range(batch_size):
                    to_print.append(str(counter) + " -------------------------------------------- ig: " + str(ig_reward_np[i]) + ", qa: " + str(qa_reward_np[i]))
                    to_print.append("Q: %s " % (agent.bert_tokenizer.decode(infos[0]["q"])))
                    to_print.append("OBS: %s " % (agent.bert_tokenizer.decode(agent.naozi.get(i))))
                    to_print.append("PRED: %s " % (chosen_answer_strings[i]))
                    to_print.append("GT: %s " % (" | ".join(answer_strings[i])))
                    to_print.append("")
                    counter += 1

            step_masks_np = np.sum(np.array(masks_np), 0)
            for i in range(len(qa_reward_np)):
                # if the answer is totally wrong, we assume it used all steps
                if qa_reward_np[i] == 0.0:
                    step_masks_np[i] = agent.max_nb_steps_per_episode
            print_qa_acc += qa_reward_np.tolist()
            print_ig_acc += ig_reward_np.tolist()
            print_steps += step_masks_np.tolist()
            if env.batch_pointer == 0:
                break

        if verbose:
            with open(agent.experiment_tag + "_output.txt", "w") as f:
                f.write("\n".join(to_print))
        print("===== Eval =====: qa acc: {:2.3f} | correct state: {:2.3f} | used steps: {:2.3f}".format(np.mean(np.array(print_qa_acc)), np.mean(np.array(print_ig_acc)), np.mean(np.array(print_steps))))
        return np.mean(np.array(print_qa_acc)), np.mean(np.array(print_ig_acc)), np.mean(np.array(print_steps))

def evaluate_observation_infomax(env, agent, valid_test="valid"):
    env.split_reset(valid_test)
    agent.eval()
    total_valid_loss = []
    total_valid_accuracy = []

    while(True):

        positive_samples, negative_samples = env.get_batch()
        with torch.no_grad():
            valid_loss, acc = agent.get_observation_infomax_loss(positive_samples, negative_samples, evaluate=True)

        total_valid_loss = total_valid_loss + valid_loss
        total_valid_accuracy = total_valid_accuracy + acc
        if env.batch_pointer == 0:
            break
    report_loss = np.mean(np.array(total_valid_loss))
    report_accuracy = np.mean(np.array(total_valid_accuracy))
    print("evaluating " + valid_test + " set, loss: " + str(report_loss) + ", acc: " + str(report_accuracy))
    return report_loss, report_accuracy

def ensemble_evaluate(env, agents, valid_test="valid"):
    with torch.no_grad():
        env.seed(42)
        env.split_reset(valid_test)
        num_agents = len(agents)
        for i in range(num_agents):
            agents[i].eval()
        print_qa_acc, print_ig_acc, print_steps = [], [], []

        while(True):
            obs, infos = env.reset(random=False)
            batch_size = len(obs)
            quest_list = agents[0].get_game_quest_info(infos)
            input_quest, input_quest_mask, quest_id_list = agents[0].get_agent_inputs(quest_list)

            tmp_replay_buffer = []
            previous_dynamics = []
            previous_belief = []
            for i in range(num_agents):
                agents[i].init(obs, infos)
                agents[i].kg.push_batch_question(quest_list, [item["q_srl"] for item in infos])
                previous_belief.append(None)
                previous_dynamics.append(None)

            prev_commands = ["restart" for _ in range(batch_size)]
            for _ in range(agents[0].max_nb_steps_per_episode):

                overall_a_rank, overall_c_rank = None, None
                overall_ctrlf_word_mask = None
                for i in range(num_agents):
                    action_rank, ctrlf_rank, ctrlf_word_mask, current_dynamics, current_belief, replay_info = agents[i].get_ranks_greedy(obs, infos, input_quest, input_quest_mask, quest_id_list, prev_commands, previous_dynamics[i], previous_belief[i])
                    if i == 0:
                        tmp_replay_buffer.append(replay_info)
                    if overall_a_rank is None:
                        overall_a_rank = action_rank
                        overall_c_rank = ctrlf_rank
                        overall_ctrlf_word_mask = ctrlf_word_mask
                    else:
                        overall_a_rank = overall_a_rank + action_rank
                        overall_c_rank = overall_c_rank + ctrlf_rank
                    previous_dynamics[i] = current_dynamics
                    previous_belief[i] = current_belief

                action_indices = agents[0].choose_maxQ_command(overall_a_rank)
                ctrlf_indices = agents[0].choose_maxQ_command(overall_c_rank, overall_ctrlf_word_mask)
                commands = agents[0].generate_commands(action_indices, ctrlf_indices)

                for i in range(num_agents):
                    for j in range(batch_size):
                        if commands[j] == "stop":
                            agents[i].not_finished_yet[j] = 0.0
                    agents[i].prev_actions.append(commands)

                obs, infos = env.step(commands)
                prev_commands = commands
                still_running = generic.to_np(replay_info[-1])
                if np.sum(still_running) == 0:
                    break

            # The agent has exhausted all steps, now answer question.
            overall_qa_rank, overall_qa_mask = None, None
            for i in range(num_agents):
                qa_rank, qa_mask = agents[i].get_qa_ranks_greedy(agents[0].naozi.get(), quest_list, previous_belief[i])  # batch
                if overall_qa_rank is None:
                    overall_qa_rank = qa_rank
                    overall_qa_mask = qa_mask
                else:
                    overall_qa_rank = overall_qa_rank + qa_rank

            chosen_head_tails = agents[0].point_maxq_position(overall_qa_rank, overall_qa_mask)  # batch x 2
            chosen_head_tails_np = generic.to_np(chosen_head_tails)
            chosen_answer_strings = generic.get_answer_strings(agents[0].naozi.get(), chosen_head_tails_np, agents[0].bert_tokenizer, agents[0].special_token_ids)

            answer_strings = [item["a_string"] for item in infos]
            answer_token_ids = [item["a"] for item in infos]
            masks_np = [generic.to_np(item[-1]) for item in tmp_replay_buffer]

            qa_reward_np = generic.get_qa_reward(chosen_answer_strings, answer_strings)
            ig_reward_np = generic.get_sufficient_info_reward(agents[0].naozi.get(), answer_token_ids)

            step_masks_np = np.sum(np.array(masks_np), 0)
            for i in range(len(qa_reward_np)):
                # if the answer is totally wrong, we assume it used all steps
                if qa_reward_np[i] == 0.0:
                    step_masks_np[i] = agents[0].max_nb_steps_per_episode
            print_qa_acc += qa_reward_np.tolist()
            print_ig_acc += ig_reward_np.tolist()
            print_steps += step_masks_np.tolist()
            if env.batch_pointer == 0:
                break

        print("===== Eval =====: qa acc: {:2.3f} | correct state: {:2.3f} | used steps: {:2.3f}".format(np.mean(np.array(print_qa_acc)), np.mean(np.array(print_ig_acc)), np.mean(np.array(print_steps))))
        return np.mean(np.array(print_qa_acc)), np.mean(np.array(print_ig_acc)), np.mean(np.array(print_steps))


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Oct 16 16:51:00 2017

@author: r.dewinter
"""
import numpy as np

def regpoly0(S):
    m,n = S.shape
    f = np.ones((m,1))
    df = np.zeros(n)
    return [f, df]

max_stars_repo_name: napa-jmm/CEGO

max_issues_repo_name: napa-jmm/CEGO

max_forks_repo_name: napa-jmm/CEGO


text:
#!/usr/bin/env python

'''
This module defines the `Worker` class and related helper functions.

Workers in Spartan manage array data and computation; methods are
available for creating, updating, reading and deleting *tiles* of
arrays.  Workers can also run a user-specified function on a set
of tiles.

Workers periodically send a heartbeat message to the master; if the
master cannot be contacted for a sufficiently long interval, workers
shut themselves down.
'''

import multiprocessing
from multiprocessing.pool import ThreadPool
import os
import pstats
import socket
import sys
import threading
import time

from . import config, util, rpc, core, blob_ctx
from .config import FLAGS, StrFlag, IntFlag, BoolFlag
from .rpc import zeromq, TimeoutException, rlock
from .util import Assert
import psutil
import weakref
import os
import numpy as np

#timeout for hearbeat messsage
HEARTBEAT_TIMEOUT=1000000
_init_lock = rlock.FastRLock()

class Worker(object):
  '''
  Spartan workers generally correspond to one core of a machine.

  Workers manage the storage of array data and running of kernel
  functions.

  Attributes:
      id (int): The unique identifier for this worker
      _peers (dict): Mapping from worker id to RPC client
      _blobs (dict): Mapping from tile id to tile.
  '''
  def __init__(self, master):
    # Reseed the Numpy random number state.
    #
    # The default initialization results in all worker processes on the machine having the
    # same random seed.
    np.random.seed(seed=os.getpid())

    self.id = -1
    self._initialized = False
    self._peers = {}
    self._blobs = {}
    self._master = master
    self._running = True
    self._ctx = None
    self.worker_status = core.WorkerStatus(psutil.virtual_memory().total, psutil.cpu_count(),
                                           psutil.virtual_memory().percent,
                                           psutil.cpu_percent(),
                                           time.time(),
                                           [], [])

    self._lock = rlock.FastRLock()
    #self._lock = threading.Lock()

    #Patch to fix buggy assumption by multiprocessing library
    if not hasattr(threading.current_thread(), "_children"):
      threading.current_thread()._children = weakref.WeakKeyDictionary()

    self._kernel_threads = ThreadPool(processes=1)
    self._kernel_remain_tiles = []

    if FLAGS.profile_worker:
      import yappi
      yappi.start()

    hostname = socket.gethostname()
    self._server = rpc.listen_on_random_port(hostname)
    self._server.register_object(self)

    if FLAGS.profile_worker:
      self._server._socket._event_loop.enable_profiling()

    self._server.serve_nonblock()

    req = core.RegisterReq()
    req.host = hostname
    req.port = self._server.addr[1]
    req.worker_status = self.worker_status

    with _init_lock:
      # There is a race-condition in the initialization code for zeromq; this causes
      # sporadic crashes when running in multi-thread mode.  We lock the first
      # client RPC to workaround this issue.
      master.register(req)

  def initialize(self, req, handle):
    '''
    Initialize worker.

    Assigns this worker a unique identifier and sets up connections to all other workers in the process.

    Args:
        req (InitializeReq): foo
        handle (PendingRequest): bar

    '''
    util.log_debug('Worker %d initializing...', req.id)
    for id, (host, port) in req.peers.iteritems():
      self._peers[id] = rpc.connect(host, port)
    self._peers[blob_ctx.MASTER_ID] = self._master

    self.id = req.id
    self._ctx = blob_ctx.BlobCtx(self.id, self._peers, self)
    blob_ctx.set(self._ctx)
    self._initialized = True
    handle.done()

  def create(self, req, handle):
    '''
    Create a new tile.

    :param req: `CreateTileReq`
    :param handle: `PendingRequest`

    '''
    with self._lock:
      assert self._initialized
      #util.log_info('Creating: %s', req.tile_id)
      Assert.eq(req.tile_id.worker, self.id)

      if req.tile_id.id == -1:
        id = self._ctx.new_tile_id()
      else:
        id = req.tile_id
      self._blobs[id] = req.data

    resp = core.TileIdMessage(tile_id=id)
    handle.done(resp)

  def tile_op(self, req, handle):
    resp = core.RunKernelResp(result=req.fn(self._blobs[req.tile_id]))
    handle.done(resp)

  def destroy(self, req, handle):
    '''
    Delete zero or more blobs.

    :param req: `DestroyReq`
    :param handle: `PendingRequest`

    '''
    with self._lock:
      for id in req.ids:
        if id in self._blobs:
          blob = self._blobs[id]
          blob.refcnt -= 1
          if blob.refcnt == 0:
            del self._blobs[id]
          #util.log_info('Destroyed blob %s', id)

    #util.log_info('Destroy...')
    handle.done()

  def update(self, req, handle):
    '''
    Apply an update to a tile.

    :param req: `UpdateReq`
    :param handle: `PendingRequest`

    '''
    #util.log_info('W%d Update: %s', self.id, req.id)
    with self._lock:
      blob =  self._blobs[req.id]
      self._blobs[req.id] = blob.update(req.region, req.data, req.reducer)

    handle.done()

  def get(self, req, handle):
    '''
    Fetch a portion of a tile.

    :param req: `GetReq`
    :param handle: `PendingRequest`

    '''
    if req.subslice is None:
      #util.log_info('GET: %s', type(self._blobs[req.id]))
      resp = core.GetResp(data=self._blobs[req.id])
      handle.done(resp)
    else:
      resp = core.GetResp(data=self._blobs[req.id].get(req.subslice))
      handle.done(resp)

  def get_flatten(self, req, handle):
    '''
    Fetch a flatten portion of the flatten format of a tile.

    :param req: `GetReq`
    :param handle: `PendingRequest`

    '''
    if req.subslice is None:
      #util.log_info('GET: %s', type(self._blobs[req.id]))
      resp = core.GetResp(data=self._blobs[req.id].data.flatten())
      handle.done(resp)
    else:
      resp = core.GetResp(data=self._blobs[req.id].data.flatten()[req.subslice])
      handle.done(resp)

  def cancel_tile(self, req, handle):
    '''
    Cancel the tile from the kernel remain tile list. The tile will not be executed in this worker.

    :param req: `TileIdMessage`
    :param handle: `PendingRequest`
    '''
    try:
      self._kernel_remain_tiles.remove(req.tile_id)
      handle.done(True)
    except:
      handle.done(False)

  def _run_kernel(self, req, handle):
    '''
    Run a kernel over the tiles resident on this worker.

    (This operation is run in a separate thread from the main worker loop).

    :param req: `KernelReq`
    :param handle: `PendingRequest`

    '''
    start_time = time.time()
    futures = []
    original_tile_id_set = set(self._blobs.iterkeys())
    try:
      blob_ctx.set(self._ctx)
      results = {}
      for tile_id in req.blobs:
        if tile_id.worker == self.id:
          self._kernel_remain_tiles.append(tile_id)

      # sort all tiles
      self._kernel_remain_tiles.sort(key=lambda x: np.size(self._blobs[x].data))

      while len(self._kernel_remain_tiles) > 0:
        tile_id = self._kernel_remain_tiles.pop()

        blob = self._blobs[tile_id]
        map_result = req.mapper_fn(tile_id, blob, **req.kw)
        results[tile_id] = map_result.result

        if map_result.futures is not None:
          futures.append(map_result.futures)

      # wait for all kernel update operations to finish
      rpc.wait_for_all(futures)

      # We've finished processing our local set of tiles.
      # If we are load balancing, check with the master if it's possible to steal
      # a tile from another worker.
      if FLAGS.load_balance:
        tile_id = self._ctx.maybe_steal_tile(None, None).tile_id
        while tile_id is not None:
          blob = self._ctx.get(tile_id, None)
          map_result = req.mapper_fn(tile_id, blob, **req.kw)
          with self._lock:
            id = self._ctx.new_tile_id()
            self._blobs[id] = blob
          results[id] = map_result.result
          if map_result.futures is not None:
            rpc.wait_for_all(map_result.futures)

          tile_id = self._ctx.maybe_steal_tile(tile_id, id).tile_id

      # Some expression reuse tiles from previous distarray. In such cases,
      # the results contain tile_id this worker already has.
      result_tile_id_set = set()
      for result in results.iteritems():
        if isinstance(result[1], list):
          for _blob in result[1]:
            result_tile_id_set.add(_blob[1])

      with self._lock:
        for tile_id in result_tile_id_set.intersection(original_tile_id_set):
          self._blobs[tile_id].refcnt += 1

      finish_time = time.time()
      handle.done(results)
    except:
      util.log_warn('Exception occurred during kernel call', exc_info=1)
      self.worker_status.add_task_failure(req)
      handle.exception()

    util.log_debug('worker(%s) kernel run time:%s', self.id, finish_time - start_time)

  def run_kernel(self, req, handle):
    '''
    Run a kernel on tiles local to this worker.

    :param req: `KernelReq`
    :param handle: `PendingRequest`

    '''
    #threading.Thread(target=self._run_kernel, args=(req, handle)).start()
    self._kernel_threads.apply_async(self._run_kernel, args=(req, handle))

  def shutdown(self, req, handle):
    '''
    Shutdown this worker.

    Shutdown is deferred to another thread to ensure the RPC reply
    is sent before the poll loop is killed.

    :param req: `EmptyMessage`
    :param handle: `PendingRequest`

    '''
    if FLAGS.profile_worker:
        util.log_info('Working shutting down... writing profile.', self.id, FLAGS.profile_worker)

    if FLAGS.profile_worker:
      try:
        os.system('mkdir -p ./_worker_profiles/')
        import yappi
        yappi.get_func_stats().save('_worker_profiles/%d' % self.id, type='pstat')
      except Exception, ex:
        print 'Failed to write profile.', ex
    handle.done()
    threading.Thread(target=self._shutdown).start()

  def _shutdown(self):
    util.log_debug('Closing server...')
    time.sleep(0.1)
    self._running = False
    self._server.shutdown()

  def wait_for_shutdown(self):
    '''
    Wait for the worker to shutdown.

    Periodically send heartbeat updates to the master.
    '''
    last_heartbeat = time.time()
    while self._running:
      now = time.time()
      if now - last_heartbeat < FLAGS.heartbeat_interval or not self._initialized:
        time.sleep(0.1)
        continue

      self.worker_status.update_status(psutil.virtual_memory().percent, psutil.cpu_percent(), now, self._kernel_remain_tiles)
      future = self._ctx.heartbeat(self.worker_status, HEARTBEAT_TIMEOUT)
      try:
        future.wait()
      except TimeoutException:
        util.log_error("Exit due to heartbeat message timeout.")
        sys.exit(0)

      last_heartbeat = time.time()

    util.log_debug('Worker shutdown.  Exiting.')

def _start_worker(master, local_id):
  '''
  Start a worker, register it with the master process, and wait
  until the worker is shutdown.

  Runs in a subprocess.

  :param master: (host, port)
  :param local_id: index (from 0..#num_workers) of this worker on the local machine.
  '''
  util.log_info('Worker starting up... Master: %s Profile: %s', master, FLAGS.profile_worker)
  rpc.set_default_timeout(FLAGS.default_rpc_timeout)
  if FLAGS.use_single_core:
    pid = os.getpid()
    os.system('taskset -pc %d %d > /dev/null' % (local_id * 2, pid))

  master = rpc.connect(*master)
  worker = Worker(master)
  worker.wait_for_shutdown()

  if FLAGS.dump_timers:
    util.TIMER.dump()

if __name__ == '__main__':
  sys.path.append('./tests')

  FLAGS.add(StrFlag('master'))
  FLAGS.add(IntFlag('count'))
  FLAGS.add(IntFlag('heartbeat_interval'))

  #resource.setrlimit(resource.RLIMIT_AS, (8 * 1000 * 1000 * 1000,
  #                                        8 * 1000 * 1000 * 1000))

  config.parse(sys.argv)
  assert FLAGS.master is not None
  assert FLAGS.count > 0
  assert FLAGS.heartbeat_interval > 0

  util.log_info('Starting %d workers on %s', FLAGS.count, socket.gethostname())

  m_host, m_port = FLAGS.master.split(':')
  master = (m_host, int(m_port))

  workers = []
  for i in range(FLAGS.count):
    p = multiprocessing.Process(target=_start_worker, args=(master, i))
    p.start()
    workers.append(p)

  for w in workers:
    w.join()

  print >>sys.stderr, 'Worker: all worker processes exited!'

max_stars_repo_name: spartan-array/spartan

max_issues_repo_name: spartan-array/spartan

max_forks_repo_name: spartan-array/spartan