text:
from models.basic_model import BasicModel
from data_providers.base_provider import DataProvider
from models.layers import ConvLayer, PoolLayer, FCLayer
from models.layer_cascade import LayerCascade
import tensorflow as tf
import numpy as np

class SimpleConvnetConfig:
	def __init__(self):
		self.net_config = {
			'weight_decay': None,
			'bn_epsilon': None,
			'bn_decay': None,
			'drop_scheme': None,
		}
		self.layer_cascade = None

	@property
	def weight_decay(self): return self.net_config['weight_decay']

	@property
	def bn_epsilon(self): return self.net_config['bn_epsilon']

	@property
	def bn_decay(self): return self.net_config['bn_decay']

	@property
	def drop_scheme(self): return self.net_config['drop_scheme']

	@property
	def depth(self): return self.layer_cascade.depth

	def get_config(self):
		return {
			'name': 'SimpleConvnet',
			**self.net_config,
			'layer_cascade': self.layer_cascade.get_config()
		}

	def copy(self):
		net_config = SimpleConvnetConfig()
		net_config.set_net_from_config(self.get_config(), self.renew_init(None), print_info=False)
		return net_config

	def renew_init(self, convnet):
		return {
			'layer_cascade': self.layer_cascade.renew_init(convnet)
		}

	def set_standard_convnet(self, data_provider: DataProvider, conv_blocks_config, fc_block_config, weight_decay,
							 drop_scheme, bn_epsilon, bn_decay, print_info=True, **kwargs):
		self.net_config = {
			'weight_decay': weight_decay,
			'bn_epsilon': bn_epsilon,
			'bn_decay': bn_decay,
			'drop_scheme': drop_scheme,
		}

		image_size = data_provider.data_shape[0]

		layers = []
		conv_id = 0
		for _i, block_config in enumerate(conv_blocks_config):
			num_layers, kernel_size, filter_num = block_config
			for _j in range(num_layers):
				keep_prob = 1.0
				if 'conv' in drop_scheme['type']:
					keep_prob = 1.0 if _i + _j == 0 else drop_scheme.get('conv_drop', 1.0)
				conv_layer = ConvLayer('conv_%d' % conv_id, filter_num, kernel_size=kernel_size, keep_prob=keep_prob,
									   pre_activation=False)
				conv_id += 1
				layers.append(conv_layer)
			if _i < len(conv_blocks_config) - 1:
				keep_prob = 1.0
				if 'pool' in drop_scheme['type']:
					keep_prob = drop_scheme.get('pool_drop', 1.0)
				pool_layer = PoolLayer('pool_%d' % _i, 'max', keep_prob=keep_prob, pre_activation=False)
				layers.append(pool_layer)
				image_size = image_size // 2
		global_avg_pool = PoolLayer('pool_%d' % len(conv_blocks_config), 'avg',
									kernel_size=image_size, strides=image_size, pre_activation=False)
		layers.append(global_avg_pool)
		for _i, units in enumerate(fc_block_config):
			keep_prob = 1.0
			if 'fc' in drop_scheme['type']:
				keep_prob = drop_scheme.get('fc_drop', 1.0)
			fc_layer = FCLayer('fc_%d' % _i, units, keep_prob=keep_prob)
			layers.append(fc_layer)
		final_fc_layer = FCLayer('fc_%d' % len(fc_block_config), data_provider.n_classes, use_bn=False, use_bias=True,
								 activation=None)
		layers.append(final_fc_layer)
		self.layer_cascade = LayerCascade('SimpleConvNet', layers)

		if print_info:
			pass
		return self

	def set_net_from_config(self, net_config_json, init=None, print_info=True):
		for key in self.net_config.keys():
			self.net_config[key] = net_config_json[key]
		init = init['layer_cascade'] if init is not None else None
		self.layer_cascade = LayerCascade.set_from_config(net_config_json['layer_cascade'], init)
		if print_info:
			pass
		return self

	def widen(self, layer_idx, new_width, widen_type='output_dim', noise=None):
		change_out_dim, _, _ = self.layer_cascade.widen(layer_idx, new_width, widen_type, noise)
		if change_out_dim:
			raise ValueError('Can not change the final logits number')

	def deepen(self, layer_idx, new_layer_config):
		return self.layer_cascade.deepen(layer_idx, new_layer_config, None)

	def set_identity4deepen(self, to_set_layers, data_provider, batch_size, batch_num=1, strict=True, noise=None):
		"""
		to_set_layers = [(new_layer, prev_layer), ...]
		"""
		task_list = {}
		for new_layer, prev_layer in to_set_layers:
			if new_layer.ready: continue
			if new_layer.use_bn and strict:
				task_id = id(prev_layer)
				if task_id in task_list:
					task_list[task_id][1].append(new_layer)
				else:
					task_list[task_id] = (prev_layer, [new_layer])
			else:
				new_layer.set_identity_layer(strict=strict, noise=noise)
		if len(task_list) > 0:
			model = SimpleConvnet(None, data_provider, None, net_config=self, only_forward=True)
			task_list = list(task_list.values())
			fetches = [prev_layer.output_op for prev_layer, _ in task_list]
			statistics = [[0, 0] for _ in task_list]
			for _i in range(batch_num):
				input_images, _ = data_provider.train.next_batch(batch_size)
				outputs = model.sess.run(fetches, feed_dict={model.images: input_images, model.is_training: False})
				for _j, out in enumerate(outputs):
					out = out.astype('float32')
					axis = tuple(range(len(out.shape) - 1))
					mean = np.mean(out, axis=axis, keepdims=True)
					variance = np.mean(np.square(out - mean), axis=axis, keepdims=True)
					mean, variance = np.squeeze(mean), np.squeeze(variance)
					statistics[_j][0] += mean
					statistics[_j][1] += variance
			for _j, (prev_layer, new_layers) in enumerate(task_list):
				mean, variance = statistics[_j][0] / batch_num, statistics[_j][1] / batch_num
				for new_layer in new_layers:
					if new_layer.ready: continue
					param = {
						'moving_mean': mean,
						'moving_variance': variance,
						'epsilon': self.bn_epsilon,
					}
					new_layer.set_identity_layer(strict=strict, param=param, noise=noise)

class SimpleConvnet(BasicModel):
	def _build_graph(self, only_forward=False):
		_input = self.images
		output = _input

		output = self.net_config.layer_cascade.build(output, self, store_output_op=only_forward)

		if not only_forward:
			logits = output
			with tf.variable_scope('L2_Loss'):
				l2_loss = tf.add_n([tf.nn.l2_loss(var) for var in tf.trainable_variables()])

			prediction = tf.nn.softmax(logits)

			# losses
			cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
				logits=logits, labels=self.labels))
			self.cross_entropy = cross_entropy

			# optimizer and train step
			optimizer = self.build_optimizer(self.learning_rate,
											 self.run_config.opt_config[0], self.run_config.opt_config[1])
			self.train_step = optimizer.minimize(
				cross_entropy + l2_loss * self.net_config.weight_decay)
			correct_prediction = tf.equal(
				tf.argmax(prediction, 1),
				tf.argmax(self.labels, 1))
			self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

max_stars_repo_name: tchittesh/EAS-

max_issues_repo_name: tchittesh/EAS-

max_forks_repo_name: tchittesh/EAS-


text:
"""Machinery for exps, cs, varlocs data -- common to nomials and programs"""
from collections import defaultdict
import numpy as np
from ..small_classes import HashVector
from ..keydict import KeySet
from .map import NomialMap
from ..repr_conventions import _repr
from ..varkey import VarKey

class NomialData(object):
    """Object for holding cs, exps, and other basic 'nomial' properties.

    cs: array (coefficient of each monomial term)
    exps: tuple of {VarKey: float} (exponents of each monomial term)
    varlocs: {VarKey: list} (terms each variable appears in)
    units: pint.UnitsContainer
    """
    # pylint: disable=too-many-instance-attributes
    _hashvalue = _varlocs = _exps = _cs = _varkeys = None

    def _reset(self):
        self._hashvalue = \
            self._varlocs = \
            self._exps = \
            self._cs = \
            self._varkeys = \
            self._values = None

    def __init__(self, hmap):
        self.hmap = hmap

        self.vks = set()
        for exp in self.hmap:
            self.vks.update(exp)
        self.units = self.hmap.units
        self.any_nonpositive_cs = any(c <= 0 for c in self.hmap.values())
        self._reset()

    @property
    def varlocs(self):
        "Create varlocs or return cached varlocs"
        if self._varlocs is None:
            self._varlocs = defaultdict(list)
            for i, exp in enumerate(self.exps):
                for var in exp:
                    self._varlocs[var].append(i)
        return self._varlocs

    @property
    def exps(self):
        "Create exps or return cached exps"
        if self._exps is None:
            self._exps = tuple(self.hmap.keys())
        return self._exps

    @property
    def cs(self):
        "Create cs or return cached cs"
        if self._cs is None:
            self._cs = np.array(self.hmap.values())
            if self.hmap.units:
                self._cs = self._cs*self.hmap.units
        return self._cs

    __repr__ = _repr

    def __hash__(self):
        if self._hashvalue is None:
            self._hashvalue = hash(hash(self.hmap) + hash(str(self.hmap.units)))
        return self._hashvalue

    @property
    def varkeys(self):
        "The NomialData's varkeys, created when necessary for a substitution."
        if self._varkeys is None:
            self._varkeys = KeySet(self.vks)
        return self._varkeys

    @property
    def values(self):  # TODO: if it's none presume it stays that way?
        "The NomialData's values, created when necessary."
        return {k: k.descr["value"] for k in self.vks
                if "value" in k.descr}

    def diff(self, var):
        """Derivative of this with respect to a Variable

        Arguments
        ---------
        var (Variable):
            Variable to take derivative with respect to

        Returns
        -------
        NomialData
        """
        # pylint:disable=len-as-condition
        varset = self.varkeys[var]
        if len(varset) > 1:
            raise ValueError("multiple variables %s found for key %s"
                             % (list(varset), var))
        elif len(varset) == 0:
            hmap = NomialMap({HashVector(): 0})
            hmap.units = None
        else:
            var, = varset
            hmap = self.hmap.diff(var)
        return NomialData(hmap)

    def __eq__(self, other):
        "Equality test"
        if not hasattr(other, "hmap"):
            return NotImplemented
        if isinstance(other, VarKey):
            return False
        if self.hmap != other.hmap:
            return False
        if self.units != other.units:
            return False
        return True

    def __ne__(self, other):
        return not self.__eq__(other)

ext: py

max_stars_repo_name: JamesVorder/gpkit

max_issues_repo_name: JamesVorder/gpkit

max_forks_repo_name: UCLA-StarAI/LearnFairNB

reason: import numpy


text:
import os
import random
import math

import cv2
import torch
import torch.utils.data as data
from pycocotools.coco import COCO
import numpy as np
from PIL import Image
from scipy import ndimage
from lib.utils.linemod import linemod_config
from lib.utils.pvnet import (pvnet_data_utils, pvnet_pose_utils,
                             visualize_utils)
from lib.config import cfg

from transforms3d.euler import mat2euler, euler2mat

class Dataset(data.Dataset):
    def __init__(self, ann_file, data_root, split, transforms=None):
        super(Dataset, self).__init__()

        self.data_root = data_root
        self.split = split

        cgmodel_dir = os.path.join(cfg['cgmodel_dir'], cfg.model,
                                   f'{cfg.model}.ply')
        self.model = pvnet_data_utils.get_ply_model(cgmodel_dir)

        self.coco = COCO(ann_file)
        self.img_ids = np.array(sorted(self.coco.getImgIds()))
        self._transforms = transforms
        self.cfg = cfg

    def read_data(self, img_id):
        ann_ids = self.coco.getAnnIds(imgIds=img_id)
        anno = self.coco.loadAnns(ann_ids)[0]
        path = self.coco.loadImgs(int(img_id))[0]['file_name']
        path = path.replace('benchwise', 'benchvise')
        mask_path = anno['mask_path'].replace('benchwise', 'benchvise')
        inp = Image.open(path)
        w, h = inp.size
        K = np.array(anno['K']).astype(np.float32)
        pose = np.array(anno['pose']).astype(np.float32)
        R = pose[:, :3]
        R = cv2.Rodrigues(R)[0].reshape(3)
        t = np.expand_dims(pose[:, 3], axis=1)
        cls = anno['cls']
        if cls == 'benchwise':
            cls = 'benchvise'
        cls_idx = linemod_config.linemod_cls_names.index(cls) + 1
        mask = pvnet_data_utils.read_linemod_mask(mask_path, anno['type'],
                                                  cls_idx)

        if 'occlusion' in self.data_root:
            dataset = 'LinemodOccTest'
        else:
            dataset = cfg.train.dataset if self.split == 'train' else cfg.test.dataset
        directory = f'cache/{dataset}/{cfg.model}'

        filename = f'{directory}/{img_id}.npy'
        result = np.load(filename, allow_pickle=True).item()
        bbox = result['bbox']
        x_ini = result['x_ini']

        filename = f'{directory}/{img_id}_features.npz'
        features = np.load(filename, allow_pickle=True)

        inp = np.asarray(inp).astype(np.uint8)
        mask = np.asarray(mask).astype(np.uint8)

        return inp, K, R, t, x_ini, bbox, features

    def __getitem__(self, index_tuple):
        index, height, width = index_tuple
        img_id = self.img_ids[index]

        img, K, R, t, x_ini, bbox, features = self.read_data(img_id)

        if self._transforms is not None:
            img = self._transforms(img)

        ret = {
            'inp': img,
            'K': K,
            'x_ini': x_ini,
            'bbox': bbox,
            'R': R,
            't': t,
            'img_id': img_id,
            'x2s': features['x2s'][0],
            'x4s': features['x4s'][0],
            'x8s': features['x8s'][0],
            'xfc': features['xfc'][0],
        }

        return ret

    def __len__(self):
        return len(self.img_ids)

max_stars_repo_name: hiyyg/RePOSE

max_issues_repo_name: hiyyg/RePOSE

max_forks_repo_name: hiyyg/RePOSE

reason: import numpy,from scipy


text:
# -*- coding: utf-8 -*-
import numpy as np
import sklearn.metrics
import sklearn.neighbors

from .utils_complexity_embedding import complexity_embedding

# =============================================================================
# Phi
# =============================================================================

def _phi(
    signal,
    delay=1,
    dimension=2,
    tolerance=0.1,
    distance="chebyshev",
    approximate=True,
    fuzzy=False,
    **kwargs,
):
    """Common internal for `entropy_approximate`, `entropy_sample` and `entropy_range`."""
    # Initialize phi
    phi = np.zeros(2)

    embedded1, count1 = _get_embedded(
        signal, delay, dimension, tolerance, distance=distance, approximate=approximate, fuzzy=fuzzy
    )

    embedded2, count2 = _get_embedded(
        signal, delay, dimension + 1, tolerance, distance=distance, approximate=True, fuzzy=fuzzy
    )

    if approximate is True:
        phi[0] = np.mean(np.log(count1 / embedded1.shape[0]))
        phi[1] = np.mean(np.log(count2 / embedded2.shape[0]))
    else:
        phi[0] = np.mean((count1 - 1) / (embedded1.shape[0] - 1))
        phi[1] = np.mean((count2 - 1) / (embedded2.shape[0] - 1))

    return phi

def _phi_divide(phi):
    if phi[0] == 0:
        return -np.inf
    division = np.divide(phi[1], phi[0])
    if division == 0:
        return np.inf
    return -np.log(division)

# =============================================================================
# Get Embedded
# =============================================================================

def _get_embedded(
    signal,
    delay=1,
    dimension=2,
    tolerance=0.1,
    distance="chebyshev",
    approximate=True,
    fuzzy=False,
):
    """Examples
    -----------
    .. ipython:: python

      import neurokit2 as nk
      import sklearn
      complexity_embedding = nk.complexity_embedding

      signal = nk.signal_simulate(duration=2, frequency=5)

      embeded, count = _get_embedded(signal, delay=8, dimension=2, tolerance=0.07,
                                      distance='chebyshev', approximate=False)
    """
    # Sanity checks
    if distance not in sklearn.neighbors.KDTree.valid_metrics + ["range"]:
        raise ValueError(
            "NeuroKit error: _get_embedded(): The given metric (%s) is not valid."
            "The valid metric names are: %s"
            % (distance, sklearn.neighbors.KDTree.valid_metrics + ["range"])
        )

    # Get embedded
    embedded = complexity_embedding(signal, delay=delay, dimension=dimension)
    if approximate is False:
        embedded = embedded[:-1]  # Removes the last line

    if fuzzy is False:
        # Get neighbors count
        count = _get_count(embedded, tolerance=tolerance, distance=distance)
    else:
        # FuzzyEn: Remove the local baselines of vectors
        embedded -= np.mean(embedded, axis=1, keepdims=True)
        count = _get_count_fuzzy(embedded, tolerance=tolerance, distance=distance, n=1)

    return embedded, count

# =============================================================================
# Get Count
# =============================================================================
def _get_count(embedded, tolerance, distance="chebyshev"):
    """Examples
    -----------
    .. ipython:: python

      import neurokit2 as nk
      signal = nk.signal_simulate(duration=1, frequency=[5, 6])
      embedded = nk.complexity_embedding(signal, delay=8, dimension=3)
      tolerance = 0.07
      distance = "range"
      x = embedded
      y = embedded[0]

    """

    if distance == "range":
        # internal function for distrange
        def distrange(x, y):
            numerator = np.max(np.abs(x - y), axis=1) - np.min(np.abs(x - y), axis=1)
            denominator = np.max(np.abs(x - y), axis=1) + np.min(np.abs(x - y), axis=1)
            valid = np.where(denominator != 0)  # To prevent division by 0
            return np.divide(numerator[valid], denominator[valid])

        # Count for each row
        count = np.array(
            [np.sum(distrange(embedded, embedded[i]) < tolerance) for i in range(len(embedded))]
        )

    else:  # chebyshev and other sklearn methods
        # Perhaps scipy.spatial.KDTree would be faster? Especially since its query() method
        # has a `workers` argument to use multiple cores? Benchmark or opinion required!
        kdtree = sklearn.neighbors.KDTree(embedded, metric=distance)
        count = kdtree.query_radius(embedded, tolerance, count_only=True).astype(np.float64)

    return count

def _get_count_fuzzy(embedded, tolerance, distance="chebyshev", n=1):
    # TODO: it would be good to implement 'distrange' here to have fuzzy RangeEn
    dist = sklearn.metrics.DistanceMetric.get_metric(distance)
    dist = dist.pairwise(embedded)

    if n > 1:
        sim = np.exp(-(dist ** n) / tolerance)
    else:
        sim = np.exp(-dist / tolerance)
    # Return the count
    return np.sum(sim, axis=0)

max_stars_repo_name: danibene/NeuroKit

max_issues_repo_name: danibene/NeuroKit

max_forks_repo_name: danibene/NeuroKit


text:
import numpy as np
import torch
from data.load_datasets import SynthSpectra

class AutofitterSpectra(SynthSpectra):
    def __init__(self, redshifts, ivar=None, npca=10, SN=10, datapath=None, test=False):
        '''Needs clean splitter method.'''

        super().__init__(regridded=False, npca=npca, noise=True, SN=SN, datapath=datapath, boss=True,
                         test=test)

        self.n_qso = self.flux.shape[0]
        self.n_pix = self.flux.shape[-1]

        if isinstance(redshifts, np.ndarray) and (redshifts.shape == (self.n_qso,)):
            self.redshifts = redshifts
        elif isinstance(redshifts, float):
            self.redshifts = redshifts * np.ones(self.n_qso)
        else:
            raise ValueError("Parameter 'redshifts' should be a 1D array of length n_qso.")

        self.wave_rest = np.full((self.n_qso, self.n_pix), self.wave_grid)
        redshifts2d = np.full((self.n_pix, self.n_qso), self.redshifts).T
        self.wave_obs = self.wave_rest * (1 + redshifts2d)

        if ivar is None:
            # assume homoscedastic noise described by the given signal-to-noise ratio
            self.ivar = np.full((self.n_qso, self.n_pix), SN**2)
        elif self.ivar is not None:
            pass
        else:
            self.ivar = ivar


text:
from itertools import product
import numpy as np
import pandas as pd
from openpyxl import load_workbook

def parse_data(data_file, nbd_residues, activators, datatypes, reps, col_types,
               FIRST_COL_INDEX, FIRST_ROW_INDEX, LAST_ROW_INDEX,
               sheet_name='Dataset'):
    """Parse kinetic data from Excel spreadsheets."""
    wb = load_workbook(data_file, data_only=True)

    # Load the worksheet with the data
    sheet = wb[sheet_name]

    # Initialize some values before iterating
    col_index = FIRST_COL_INDEX
    col_tuples = []
    data = []

    # For all residues...
    for nbd_ix, nbd_residue in enumerate(nbd_residues):
        # ...and replicates...
        for rep_ix, rep in enumerate(reps):
            # ...and activators...
            for act_ix, activator in enumerate(activators):
                # Set the time vector to None so we can make sure that it is set
                # correctly
                time_vector = None
                # Now iterate over the Time, Release, FRET, NBD columns
                for dtype_ix, dtype in enumerate(datatypes):
                    # Skip the FRET and NBD datatypes if we're currently on the
                    # WT Bax section
                    if nbd_residue == 'WT' and \
                       (dtype == 'FRET' or dtype == 'NBD'):
                        col_index += 1
                        continue
                    # Iterate and fill the array of column values
                    value_vector = []
                    data_col = list(sheet.columns)[col_index]\
                                             [FIRST_ROW_INDEX:LAST_ROW_INDEX]
                    for cell in data_col:
                        if (cell.value == None):
                            value_vector.append(np.nan)
                        else:
                            value_vector.append(float(cell.value))
                    # If this is the time column, get and save the time vector
                    if dtype_ix == 0:
                        time_vector = value_vector
                    # Otherwise, get the data and add it along with a time
                    # column
                    else:
                        assert time_vector is not None
                        time_tuple = (activator, dtype, nbd_residue, rep,
                                      'TIME')
                        value_tuple = (activator, dtype, nbd_residue, rep,
                                       'VALUE')
                        col_tuples.append(time_tuple)
                        col_tuples.append(value_tuple)
                        data.append(time_vector)
                        data.append(value_vector)
                    # Increment the index of the current column
                    col_index += 1
    # Create the Pandas dataframe
    data_matrix = np.array(data)
    col_multi_index = pd.MultiIndex.from_tuples(col_tuples,
                        names=('Activator', 'Datatype', 'NBD Site', 'Replicate',
                               'Column'))
    df = pd.DataFrame(data_matrix.T,
                      index=range(LAST_ROW_INDEX - FIRST_ROW_INDEX),
                      columns=col_multi_index)
    return df


text:
#!/usr/bin/python
import rospy
import serial
import struct
import numpy as np

from sensor_msgs.msg import Joy

class SimpleArm:
    def __init__(self):
        self.velocities = {
            'grip': 0,
            'wrist_roll': 0,
            'wrist_pitch': 0,
            'upper_elbow': 0,
            'lower_elbow': 0,
            'base_yaw': 0
        }
        self.positions = {
            'camera': 0
        }
        baudrate = rospy.get_param('~baudrate', 9600)
        self.serialDev = serial.Serial(baudrate=baudrate)
        self.serialDev.port = rospy.get_param("~serial_device")
        self.serialDev.open()
        self.arm_sub = rospy.Subscriber("/joy_arm", Joy, self.arm_joy_callback)

    def write_serial(self):
      # Execute arm position
        rospy.loginfo('velocities:%s\n' % self.velocities)
        rospy.loginfo('positions:%s\n' % self.positions)
        encoded_position = struct.pack("<fffffff",
                                        self.velocities['grip'],
                                        self.velocities['wrist_roll'],
                                        self.velocities['wrist_pitch'],
                                        self.velocities['upper_elbow'],
                                        self.velocities['lower_elbow'],
                                        self.velocities['base_yaw'],
                                        self.positions['camera']
                                        )
        self.serialDev.write(encoded_position)

    def arm_joy_callback(self, data):
        self.velocities['grip'] = 0
        self.velocities['wrist_roll'] = 0
        self.velocities['wrist_pitch'] = 0
        self.velocities['upper_elbow'] = 0
        self.velocities['lower_elbow'] = 0
        self.velocities['base_yaw'] = 0

        # button 12 = hold button to double motor speed
        SPEED_MULTIPLIER = 1
        if data.buttons[3]:
            SPEED_MULTIPLIER = 2
        # button 10 = hold button to half motor speed
        if data.buttons[9]:
            SPEED_MULTIPLIER = 0.5

        # big stick twist = base clockwise
        # big stick twist = base counter-clockwise
        self.velocities['base_yaw'] = data.buttons[1] / 5.0 * SPEED_MULTIPLIER

        # big stick forward = lower elbow up
        # big stick back = lower elbow down
        self.velocities['lower_elbow'] = data.axes[1] / 3.0 * SPEED_MULTIPLIER

        # thumb stick forward = upper elbow down
        # thumb stick back = upper elbow up
        self.velocities['upper_elbow'] = data.buttons[0] / 5.0 * SPEED_MULTIPLIER

        # paddle forward = arm camera up
        # paddle back = arm camera down
        self.positions['camera'] = np.interp(data.axes[3],[-1,1],[7,77]) # this value is angular postion in degrees

        # button 9 = wrist clockwise
        # button 11 = wrist counter-clockwise
        if data.buttons[8]:
            self.velocities['wrist_roll'] = 0.35 * SPEED_MULTIPLIER
        if data.buttons[10]:
            self.velocities['wrist_roll'] = -0.35 * SPEED_MULTIPLIER

        # button 6 = wrist down
        # button 4 = wrist up
        if data.buttons[4]:
            self.velocities['wrist_pitch'] = 0.15 * SPEED_MULTIPLIER
        if data.buttons[2]:
            self.velocities['wrist_pitch'] = -0.15 * SPEED_MULTIPLIER

        # trigger = close gripper
        # thumb button = open gripper
        if data.axes[5] == -1:
            self.velocities['grip'] = 1
        if data.axes[2] == -1:
            self.velocities['grip'] = -1

        # MOVE ARM
        self.write_serial()

def main():
    rospy.init_node("simple_arm")
    controller = SimpleArm()
    rospy.spin()


text:
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""dataset related classes and methods."""

import ctypes
import sys
import time

import numpy as np
import tensorflow as tf

MEAN_RGB = [123.68, 116.78, 103.94]
IMAGE_SIZE = 224

class Item(object):

  def __init__(self, label, img, idx):
    self.label = label
    self.img = img
    self.idx = idx
    self.start = time.time()

def usleep(sec):
  """A cross-plattform sleep utility function."""
  if sys.platform == 'win32':
    # on windows time.sleep() doesn't work to well
    kernel32 = ctypes.windll.kernel32
    timer = kernel32.CreateWaitableTimerA(ctypes.c_void_p(), True,
                                          ctypes.c_void_p())
    delay = ctypes.c_longlong(int(-1 * (10 * 1000000 * sec)))
    kernel32.SetWaitableTimer(timer, ctypes.byref(delay), 0, ctypes.c_void_p(),
                              ctypes.c_void_p(), False)
    kernel32.WaitForSingleObject(timer, 0xffffffff)
  else:
    time.sleep(sec)

class Dataset(object):
  """Dataset class definition."""

  def __init__(self):
    self.arrival = None
    self.image_list = []
    self.label_list = []
    self.image_list_inmemory = []
    self.image_map = {}
    self.last_loaded = -1

  def preprocess(self, use_cache=True):
    raise NotImplementedError('Dataset:preprocess')

  def get_item_count(self):
    return len(self.image_list)

  def get_list(self):
    raise NotImplementedError('Dataset:get_list')

  def load_query_samples(self, sample_list):
    self.image_list_inmemory = []
    self.image_map = {}
    for i, sample in enumerate(sample_list):
      self.image_map[sample] = i
      img, _ = self.get_item(sample)
      self.image_list_inmemory.append(img)
    self.last_loaded = time.time()

  def unload_query_samples(self, sample_list):
    if sample_list:
      for sample in sample_list:
        del self.image_map[sample]
    else:
      self.image_map = {}
    self.image_list_inmemory = []

  def get_image_list_inmemory(self):
    return np.array(self.image_list_inmemory)

  def get_indices(self, item_list):
    data = [self.image_map[item] for item in item_list]
    data = np.array(data)
    return data, self.label_list[item_list]

  def get_item_loc(self, item):
    raise NotImplementedError('Dataset:get_item_loc')

#
# Post processing
#
class PostProcessCommon(object):
  """Class definition for common post-processing."""

  def __init__(self, offset=0):
    self.offset = offset
    self.good = 0
    self.total = 0

  def __call__(self, results, ids, expected=None, result_dict=None):
    n = len(results[0])
    for idx in range(0, n):
      if results[0][idx] + self.offset == expected[idx]:
        self.good += 1
    self.total += n
    return results

  def start(self):
    self.good = 0
    self.total = 0

  def finalize(self, results, unused_ds=False, unused_output_dir=None):
    results['good'] = self.good
    results['total'] = self.total

class PostProcessArgMax(object):
  """Class definition for argmax post-processing."""

  def __init__(self, offset=0):
    self.offset = offset
    self.good = 0
    self.total = 0

  def __call__(self, results, ids, expected=None, result_dict=None):
    result = np.argmax(results[0], axis=1)
    n = result.shape[0]
    for idx in range(0, n):
      if result[idx] + self.offset == expected[idx]:
        self.good += 1
    self.total += n
    return results

  def start(self):
    self.good = 0
    self.total = 0

  def finalize(self, results, unused_ds=False, unused_output_dir=None):
    results['good'] = self.good
    results['total'] = self.total

#
# pre-processing
#
def _resize_with_aspectratio(image_bytes,
                             out_height=IMAGE_SIZE,
                             out_width=IMAGE_SIZE,
                             scale=87.5):
  """Resize with aspect ratio."""

  # Reference: https://github.com/mlperf/inference/blob/master/v0.5/classification_and_detection/python/dataset.py#L154  # pylint: disable=line-too-long
  # Take width/height
  shape = tf.shape(image_bytes)
  height = shape[0]
  width = shape[1]

  new_height = 100.0 * tf.to_float(out_height) / scale
  new_width = 100.0 * tf.to_float(out_width) / scale
  h_greater_than_w = tf.greater(height, width)
  # Reference code:
  # if height > width:
  #      w = new_width
  #      h = int(height * new_width / width)
  #  else:
  #      h = new_height
  #      w = int(width * new_height / height)
  w = tf.to_int32(
      tf.cond(h_greater_than_w, lambda: new_width,
              lambda: tf.to_float(width) * new_height / tf.to_float(height)))
  h = tf.to_int32(
      tf.cond(h_greater_than_w,
              lambda: tf.to_float(height) * new_width / tf.to_float(width),
              lambda: new_height))

  image_bytes = tf.reshape(image_bytes, [1, height, width, 3])
  image_bytes = tf.image.resize_bilinear(image_bytes, (h, w))
  image_bytes = tf.reshape(image_bytes, [h, w, 3])
  return image_bytes

def _center_crop(image, size):
  """Crops to center of image with specified `size`."""

  # Reference: https://github.com/mlperf/inference/blob/master/v0.5/classification_and_detection/python/dataset.py#L144  # pylint: disable=line-too-long

  height = tf.shape(image)[0]
  width = tf.shape(image)[1]
  out_height = size
  out_width = size
  # Reference code:
  # left = (width - out_width) / 2
  # right = (width + out_width) / 2
  # top = (height - out_height) / 2
  # bottom = (height + out_height) / 2
  # img = img.crop((left, top, right, bottom))

  offset_height = tf.to_int32((height - out_height) / 2)
  offset_width = tf.to_int32((width - out_width) / 2)
  image = tf.image.crop_to_bounding_box(
      image,
      offset_height,
      offset_width,
      target_height=out_height,
      target_width=out_width,
  )
  return image

def _normalize(image):
  """Normalize the image."""
  offset = tf.constant(MEAN_RGB, shape=[1, 1, 3])
  image -= offset
  return image

def pre_process_vgg(image_bytes, label, image_name):
  """Preprocesses the given image for evaluation.

  Note that image normalization is moved to TPU. see backend_tf.py.

  Args:
    image_bytes: `Tensor` representing an image binary of arbitrary size.
    label: label for that image.
    image_name: unique image identifier

  Returns:
    A preprocessed image `Tensor`.
  """
  image_size = IMAGE_SIZE
  image = _resize_with_aspectratio(image_bytes)
  image = _center_crop(image, image_size)
  image = tf.reshape(image, [image_size, image_size, 3])
  return image, label, image_name


text:
import unittest

import torch
import numpy as np
from nose.plugins.attrib import attr

from embryovision.tvmaskrcnn.objectsegmentation import ObjectSegmentationModel

@attr('slow')
class TestOutputObjectSegmentation(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        # Create model
        cls.base_model = ObjectSegmentationModel(input_size=(500, 500),
                                                 num_classes=2)
        cls.base_model.eval()
        # Generate input image
        nsamples = 2
        input_shape = (nsamples,) + cls.base_model.input_shape
        valid_input = torch.rand(input_shape).type(torch.float32)
        valid_input = [img for img in valid_input]
        # Predict output
        cls.pred_out = cls.base_model.predict(valid_input)

    def test_output_is_list(self):
        self.assertEqual(type(self.pred_out), type(list()))

    def test_output_confidence_is_float(self):
        for pred_inst in self.pred_out:
            self.assertEqual(pred_inst.confidence.dtype, np.float32)

    def test_output_mask_is_float(self):
        for pred_inst in self.pred_out:
            self.assertEqual(pred_inst.mask.dtype, np.float32)

    def test_output_mask_has_correct_shape(self):
        for pred_inst in self.pred_out:
            self.assertEqual(pred_inst.mask.shape,
                             (self.base_model.input_shape[1],
                              self.base_model.input_shape[2]))

    def test_output_box_is_float(self):
        for pred_inst in self.pred_out:
            self.assertEqual(pred_inst.box.dtype, np.float32)

    def test_output_box_has_correct_shape(self):
        for pred_inst in self.pred_out:
            self.assertEqual(pred_inst.box.shape, (4,))

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: briandleahy/embryovision

max_issues_repo_name: briandleahy/embryovision

max_forks_repo_name: briandleahy/embryovision


text:
import os

import numpy as np
import torch
from torch import nn, optim
from torch.utils.data import DataLoader, TensorDataset, random_split

import config
import cv2
import damage_detector
import data_handler
import nms

def make_xml(vfilename, vname, vvbox):
    # FUNCTION: output results as xml format
    import xml.etree.ElementTree as ET
    root = ET.Element('annotations')
    for index in range(len(vfilename)):
        annotation = ET.SubElement(root, 'annotation')
        filename = ET.SubElement(annotation, 'filename')
        filename.text = os.path.basename(vfilename[index])
        for index_object in range(len(vname[index])):
            for index_class in range(len(vname[index][index_object])):
                object_tag = ET.SubElement(annotation, 'object')
                name = ET.SubElement(object_tag, 'name')
                name.text = str(vname[index][index_object][index_class])
                pose = ET.SubElement(object_tag, 'pose')
                pose.text = "Unspecified"
                truncated = ET.SubElement(object_tag, 'truncated')
                truncated.text = "0"
                difficult = ET.SubElement(object_tag, 'difficult')
                difficult.text = "0"
                bndbox = ET.SubElement(object_tag, 'bndbox')
                xmin = ET.SubElement(bndbox, 'xmin')
                ymin = ET.SubElement(bndbox, 'ymin')
                xmax = ET.SubElement(bndbox, 'xmax')
                ymax = ET.SubElement(bndbox, 'ymax')
                # xmin, xmax, ymin, ymax range [0, config.width-1] to [1, config]
                xmin.text = str(vvbox[index][index_object][index_class][0]+1)
                ymin.text = str(vvbox[index][index_object][index_class][1]+1)
                xmax.text = str(vvbox[index][index_object][index_class][2]+1)
                ymax.text = str(vvbox[index][index_object][index_class][3]+1)

    from xml.dom import minidom
    xmlstr = minidom.parseString(ET.tostring(root)).toprettyxml(indent="   ")
    with open('results/answer.xml', 'w') as f:
        f.write(xmlstr)

color_set = [(255, 0, 0), (255, 128, 0), (255, 255, 0), (128, 255, 0), (0, 255, 255), (0, 255, 255), (255, 0, 255), (128, 128, 128)]
device = torch.device("cuda" if config.cuda else "cpu")
detector = damage_detector.Detector()
detector.load_state_dict(torch.load(config.fn_model))
if config.multi_gpu:
    n_gpu = torch.cuda.device_count()
    print('Multi GPU mode Use {} GPU'.format(n_gpu))
    detector = nn.DataParallel(detector)
detector.to(device)
detector.eval()

vfilename = [os.path.join(config.dirname_testimage, filename) for filename in os.listdir(config.dirname_testimage)]
vimage = [data_handler.load_image(filename) for filename in vfilename]
# transform for input to net
vimage_transformed = np.array([data_handler.transform_image(image) for image in vimage])

vvbox = []
vname = []
with torch.no_grad():
    for index, image_transformed in enumerate(vimage_transformed):
        vvbox.append([])
        vname.append([])
        image_transformed = torch.from_numpy(image_transformed).float()
        image_transformed = image_transformed.view(1, image_transformed.size(0), image_transformed.size(1), image_transformed.size(2))
        boxmatrix_confidence = detector(image_transformed)
        boxmatrix_confidence = boxmatrix_confidence.cpu()
        boxmatrix = boxmatrix_confidence[0, :, :, 0:4]
        print(f'predicted w h : {boxmatrix[:, :, 2]} {boxmatrix[:, :, 3]}')
        # (config.n_cell, config.n_cell)
        confidence = boxmatrix_confidence[0, :, :, 4]
        class_predict = boxmatrix_confidence[0, :, :, config.n_offset_class:]
        # (config.n_cell, config.n_cell)
        class_predict = torch.argmax(class_predict, dim=2)
        mask_matrix = (confidence > 0.5).float()
        # get class predict box
        image = vimage[index]
        for class_number in range(config.n_class):
            class_mask = (class_predict == class_number).float()
            # w to 0 if confidence <= 0.5
            boxmatrix_temp = boxmatrix.clone()
            boxmatrix_temp[:, :, 2] = boxmatrix[:, :, 2] * mask_matrix * class_mask
            boxmatrix_temp = boxmatrix_temp.numpy()
            vbox, confidence_temp = data_handler.boxmatrix2box(boxmatrix_temp, confidence)
            vbox = np.array(vbox)
            vbox = nms.non_max_suppression_slow(vbox, confidence_temp, config.threshold_intersection_of_union)
            vvbox[index].append(vbox)
            vname[index].append([class_number+1 for _ in range(len(vbox))])
            for xmin, ymin, xmax, ymax in vbox:
                image = cv2.rectangle(image, (xmin, ymin), (xmax, ymax), color=color_set[class_number], thickness=4)
        filename_result = os.path.join(config.dirname_testimage_predict, os.path.basename(vfilename[index]))
        cv2.imwrite(filename_result, image)
        print(f'saved {filename_result}')
# make results xml file
make_xml(vfilename, vname, vvbox)

max_stars_repo_name: Soyukke/RoadDamageDetector

max_issues_repo_name: Soyukke/RoadDamageDetector

max_forks_repo_name: Soyukke/RoadDamageDetector


text:
# using trained checkpoin to identfiy image
''' print out the top 5 possibilities species of one image

Options:
- Load the trained modle form directory, python predict.py --save_dir './assests'
- Choose image data from directory, python train.py --image './flowers/test/1/'
- Show top 5 possibility for image reconginization --topk 5

Example usage:
 python predict  --save_dir --image
'''
# libraries

import argparse
import numpy as np

from torchvision import datasets, transforms, models
import torch
import torch.nn as nn
from torch import optim

import time
import copy
from PIL import Image
import pandas as pd
import os
from workspace_utils import active_session
import matplotlib.pyplot as plt

from load_model_and_recoginze_image import load_checkpoint,predict_topk

# Get the command line inputs
parser = argparse.ArgumentParser()

# Basic usage: python train.py data_directory
parser.add_argument('--image_path', action='store',
                    default = 'flowers/test/10/image_07090.jpg',
                    help='choose data directory to load training data, e.g., "./flowers/image_name"')

# Set directory to save checkpoints: python train.py data_dir --save_dir save_directory
parser.add_argument('--checkpoint_dir', action='store',
                    default = 'resnet101_eopch_.pth',
                    dest='checkpoint_dir',
                    help='Select  saving checkpoints to load the model, e.g., "./resnet101_eopch_.pth"')

# Use GPU for training: python train.py data_dir --gpu
parser.add_argument('--gpu', action='store_true',
                    default=False,
                    dest='gpu',
                    help='Use GPU for training, set a switch to true')
# Choose pretrained architecture: python train.py data_dir --arch "resnet101"
parser.add_argument('--topk', action='store',
                    default = 5,
                    dest='topk',
                    help='Choose architecture, e.g., 5')
# clas to name jason file
parser.add_argument('--mapping_json', action='store',
                    default = 'cat_to_name.json',
                    dest='mapping_json',
                    help='label to name json file, e.g., "cat_to_name"')

parse_results = parser.parse_args()
# parse
image_path = parse_results.image_path
checkpoint_dir = parse_results.checkpoint_dir
topk = int(parse_results.topk)
mapping_json = parse_results.mapping_json
gpu = parse_results.gpu

#import the model from checkpoint
model = load_checkpoint(checkpoint_dir)
#predict the image class and show the top5 of those
probs,top_classes= predict_topk(image_path, model, gpu,topk,mapping_json)


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Nov 19 12:53:16 2019

Contains all results of one specific model. Performs the statistics on the set of results.

@author: Dr. Dr. Danny E. P. Vanpoucke
@web   : https://dannyvanpoucke.be
"""
from TDataPoint import TDataPoint

class TModelResults:
    """
    This class contains the following information:
        - Name  : string with the model name (unique)
        - NameModel : string with the model name (general class)
        - NData : integer number indicating the number of data-points
        - Average : average value of a dataset (array)
        - StDev   : standard deviation of a dataset (array)
        - RMSEfulltrain : RMSE on the full training set (absolute values, array)
        - RMSEfulltest : RMSE on the full test set (absolute values, array)
        - MAEfulltrain : Mean-Absolute-Error on the full training set (array)
        - MAEfulltest  : Mean-Absolute-Error on the full test set (array)
        - RMSEtrainLoO_mean : the mean RMSE on the training data using Leave-One-Out
        - RMSEtrainLoO_2sig : the 95% (2sigma) range
        - RMSEtrainCV5_mean : the mean RMSE on the training data using 5-fold Cross-Validation
        - RMSEtrainCV5_2sig : the 95% (2sigma) range
        - modelCoef : Array of dictionaries, containing the (hyper-)parameters for each model instance
    """
    def __init__(self, Name: str, NameModel: str, initialNDataRuns: int=0):
        """
        Initialise all arrays as zero-arrays of length initialNDataRuns.

        parameters:
            - Name :  name of the model (preferably unique)
            - initialNDataRuns : length of the arrays upon initialisation. Default=0
        """
        self.Name=Name
        self.NameModel=NameModel
        self.NData=initialNDataRuns
        self.Average=[0.0]*initialNDataRuns
        self.StDev=[0.0]*initialNDataRuns
        self.RMSEfulltrain=[0.0]*initialNDataRuns
        self.RMSEfulltest=[0.0]*initialNDataRuns
        self.MAEfulltrain=[0.0]*initialNDataRuns
        self.MAEfulltest=[0.0]*initialNDataRuns
        self.RMSEtrainLoO_mean=[0.0]*initialNDataRuns
        self.RMSEtrainLoO_2sig=[0.0]*initialNDataRuns
        self.RMSEtrainCV5_mean=[0.0]*initialNDataRuns
        self.RMSEtrainCV5_2sig=[0.0]*initialNDataRuns
        self.modelCoef=[0]*initialNDataRuns  #this will become an array of dictionaries

    def addDataPoint(self, dp: TDataPoint):
                     #Avg, StD, RMSEtrain,  MAEtrain, RMSEtest, MAEtest: float,
                     #trainLoO_mean, trainLoO_2sig,
                     #trainCV5_mean, trainCV5_2sig:float,
                     #modelCoef: dict):
        """
        Adding the results of a single data-point to arrays of values.

        parameters:
            - self
            - dp : TDataPoint object of which the values need to be appended to the arrays
        """
        self.NData+=1
        self.Average.append(dp.mean)
        self.StDev.append(dp.std)
        self.RMSEfulltrain.append(dp.RMSEtrain)
        self.RMSEfulltest.append(dp.RMSEtest)
        self.MAEfulltrain.append(dp.MAEtrain)
        self.MAEfulltest.append(dp.MAEtest)
        self.RMSEtrainLoO_mean.append(dp.trainLoO_mean)
        self.RMSEtrainLoO_2sig.append(dp.trainLoO_2sig)
        self.RMSEtrainCV5_mean.append(dp.trainCV5_mean)
        self.RMSEtrainCV5_2sig.append(dp.trainCV5_2sig)
        self.modelCoef.append(dp.modelCoef)

    def setDataPoint(self, dp: TDataPoint):
                     #index:int, Avg, StD, RMSEtrain, MAEtrain, RMSEtest, MAEtest: float,
                     #trainLoO_mean, trainLoO_2sig,
                     #trainCV5_mean, trainCV5_2sig:float,
                     #modelCoef: dict):
        """
        Setting the results of a single data-point.

        parameters:
            - index : position of the datapoint to modify.
            --> if the index is larger than NData, then this data is appended at the end of the list
        """
        index=dp.index
        if (index<self.NData):
            self.Average[index]=dp.mean
            self.StDev[index]=dp.std
            self.RMSEfulltrain[index]=dp.RMSEtrain
            self.RMSEfulltest[index]=dp.RMSEtest
            self.MAEfulltrain[index]=dp.MAEtrain
            self.MAEfulltest[index]=dp.MAEtest
            self.RMSEtrainLoO_mean[index]=dp.trainLoO_mean
            self.RMSEtrainLoO_2sig[index]=dp.trainLoO_2sig
            self.RMSEtrainCV5_mean[index]=dp.trainCV5_mean
            self.RMSEtrainCV5_2sig[index]=dp.trainCV5_2sig
            self.modelCoef[index]=dp.modelCoef
        else:
            self.addDataPoint(dp)
            #Avg, StD, RMSEtrain, MAEtrain, RMSEtest, MAEtest,
            #         trainLoO_mean, trainLoO_2sig,
            #         trainCV5_mean, trainCV5_2sig, modelCoef)

    def printQualityDataPoint(self, index: int, printCoef: bool=False):
        """
        Prints a quality-control block of a model.

        parameters:
            - index: index of the data-point to print
            - printCoef : should the coefficients and hyper-parameters be printed as well?

        """

        if (index<0)or(index>=self.NData): #out of range: print warning and do nothing
            print("WARNING: index %i is out of range (>%i , or <0). Cannot printQualityDataPoint::TModelResults.")
            return
        #If we get here, we just print
        print("============ Quality measures Datablock ==================")
        print("Quality measures for model: ",self.Name," which is a ",self.NameModel," .")
        print("Real target properties: ")
        print("  - Average       = %0.3f" % self.Average[index])
        print("  - Standard Dev. = %0.3f" % self.StDev[index])
        print("")
        print("1. Measure on whole set:")
        pct=(self.RMSEfulltrain[index]/self.Average[index])*100.0
        print("RMSE of the prediction by the model= %0.3f  --> %0.2f %%" % (self.RMSEfulltrain[index], pct ))
        pct=(self.MAEfulltrain[index]/self.Average[index])*100.0
        print("MAE of the prediction by the model= %0.3f  --> %0.2f %%" % (self.MAEfulltrain[index], pct ))
        print("2. Measure of cross-validation  (95% interval):")
        pct=(self.RMSEtrainLoO_mean[index]/self.Average[index])*100.0
        print("Leave-One-Out: RMSE: %0.3f (+/- %0.3f)      --> %0.2f %%" % (self.RMSEtrainLoO_mean[index], self.RMSEtrainLoO_2sig[index], pct))
        pct=(self.RMSEtrainCV5_mean[index]/self.Average[index])*100.0
        print("5-fold CV    : RMSE: %0.3f (+/- %0.3f)      --> %0.2f %%" % (self.RMSEtrainCV5_mean[index], self.RMSEtrainCV5_2sig[index], pct))
        if printCoef:
            print("")
            textBlock=[""]*len(self.modelCoef[index])
            for key in self.modelCoef[index]:
                value=self.modelCoef[index][key] #this should be an array
                textBlock[abs(value[0])]=''.join(str(element) for element in value[1:])
            for i in range(len(self.modelCoef[index])):
                print(textBlock[i])
        print("=============================================================")

    def PrintResults(self):
        """
        Prints the averaged results for 1 model
        """
        import numpy as np

        print(self.Name,"  AVG= %0.4f  STDEV= %0.4f " % (np.mean(self.Average),np.mean(self.StDev)),
              "RMSE train: %0.4f RMSE test: %0.4f " % (np.mean(self.RMSEfulltrain),np.mean(self.RMSEfulltest)),
              "MAE train: %0.4f MAE test: %0.4f " % (np.mean(self.MAEfulltrain),np.mean(self.MAEfulltest)),
              "LoO_tr= %0.4f (+- %0.4f )" % (np.mean(self.RMSEtrainLoO_mean),np.mean(self.RMSEtrainLoO_2sig)),
              "CV5_tr= %0.4f (+- %0.4f )" % (np.mean(self.RMSEtrainCV5_mean),np.mean(self.RMSEtrainCV5_2sig))
              )

    def PrintStatistics(self, file: str=None):
        """
        Prints the list of results for all runs to a textfile

        parameters:
            file: filename of the file to append with the data. (default= None, i.e. model name)

        columns:
        - 1 : run index
        - 2 :

        """
        import numpy as np

        if (file==None):
            filename=self.Name + ".dat"
            f=open(filename,"w+")
        else:
            f=open(file,"a+")

        #For LS-SVM we need to sort the coeff and add zeros for missing vectors
        if (self.NameModel=='LS-SVM'):
            mfc=[0]*self.NData
            FullSet=set()
            for i in range(self.NData):
                FullSet.update(set(self.modelCoef[i]['data_pt_index'][1].flatten())) #flatten because nd-array
            SizeSet=max(FullSet)+1 #if for some reason some support vectors are missing altogether
            for i in range(self.NData):
                coef=np.zeros(2*SizeSet+1)#+1 ofset in modelcoef
                coefar=np.asarray(self.modelCoef[i]['coef_'][1]).flatten()
                idxar=self.modelCoef[i]['data_pt_index'][1].flatten()
                for j in range(self.modelCoef[i]['coef_'][1].shape[1]):
                    coef[idxar[j]+1]=coefar[j]
                    coef[SizeSet+idxar[j]+1]=idxar[j]
                locdict=dict()
                locdict['coef_']=coef
                mfc[i]=locdict
        else:#all non-LS-SVM
            mfc=self.modelCoef

        for i in range(self.NData):
            line=str(i+1)+"  "+str(self.RMSEfulltrain[i]
                 )+"  "+str(self.RMSEfulltest[i]
                 )+"  "+str(self.MAEfulltrain[i]
                 )+"  "+str(self.MAEfulltest[i]
                 )+"  "+str(self.RMSEtrainLoO_mean[i]
                 )+"  "+str(self.RMSEtrainLoO_2sig[i]
                 )+"  "+str(self.RMSEtrainCV5_mean[i]
                 )+"  "+str(self.RMSEtrainCV5_2sig[i])
            #the replace line-breaks are because of numpy's need to help out by inserting linebreaks...
            #this resolves the issue here, without breaking the numpy behaviour globally
            a=str(self.modelCoef[i]["intercept_"][1]).replace('[','').replace(']','').replace('\n','').replace('\r','')
            b=" ".join(str(n) for n in mfc[i]["coef_"][1:]).replace('[','').replace(']','').replace('\n','').replace('\r','')
            line=line+"  "+a+"  "+b
            print(line)
            f.write(line+" \n")
        f.close()

    def PrintStatisticsHyperparameters(self, file: str=None):
        """
        Prints the list of hyperparameter-values for all runs to a textfile

        If no hyper-parameters are present, nothing is written.

        parameters:
            file: filename of the file to append with the data. (default= None, i.e. model name)

        columns:
        - 1 : run index
        - 2 :

        """
        if (file==None):
            filename="HyperParam_"+self.Name + ".dat"
            f=open(filename,"w+")
        else:
            f=open(file,"a+")

        for i in range(self.NData):
            line=str(i+1)+"  "
            cnt=0
            for key, value in self.modelCoef[i].items():
                if (int(value[0])<0):
                    cnt+=1
                    line=line+value[1]+" "
            if (cnt>0): #there are hyperparameters to print
                print(line)
                f.write(line+" \n")

        f.close()

max_stars_repo_name: DannyVanpoucke/Amadeus

max_issues_repo_name: DannyVanpoucke/Amadeus

max_forks_repo_name: DannyVanpoucke/Amadeus


text:
from .. import arr
import numpy as np

cfg = {
    'name': 'Baseline - Polynomial',
    'group': 'Signal processing',
    'desc': "Remove baseline wander using polynomial curve",
    'reference': "Chouhan, V.S. Computing: Theory and Applications, 2007. ICCTA '07. International Conference on",
    'interval': 30,
    'inputs': [{'name': 'ecg', 'type': 'wav'}],
    'outputs': [
        {'type': 'wav', 'name': 'ecg_filtered'},
        {'type': 'wav', 'name': 'baseline'}
    ]
}

def run(inp, opt, cfg):
    data = arr.interp_undefined(inp['ecg']['vals'])
    srate = inp['ecg']['srate']

    span = int(srate * 10)  # Cut every 10 second regardless of interval
    baseline = [0] * len(data)
    x = np.arange(0, span)
    for spos in range(0, len(data), span):  # start position of this segment
        if spos + span > len(data):
            span = len(data) - spos
            x = x[0:span]

        med = np.median(data[spos:spos + span])  # compute overall median for the entire waveform
        for j in range(span):
            data[spos + j] -= med  # shift each sample of the entire waveform by this median value
        y = data[spos:spos + span]
        p = np.polyfit(x, y, 4)
        y = np.polyval(p, x)

        for j in range(span):
            baseline[spos + j] = y[j]
            data[spos + j] -= y[j]

    r_list = arr.detect_qrs(data, srate) # detect r-peak

    for i in range(len(r_list)-1):  # for each rr interval
        idx1 = r_list[i]
        idx2 = r_list[i+1]
        if idx1 + 1 > idx2:
            continue
        med = np.median(data[idx1 + 1:idx2])
        for j in range(idx1 + 1, idx2):
            data[j] -= med

    return [
        {'srate': srate, 'vals': data},
        {'srate': srate, 'vals': baseline}
    ]

max_stars_repo_name: vitaldb/pyvital

max_issues_repo_name: vitaldb/pyvital

max_forks_repo_name: vitaldb/pyvital


text:
import numpy as np
import pylib as py
import tensorflow as tf
import tflib as tl

def make_dataset(img_dir,
                 batch_size,
                 load_size=286,
                 crop_size=256,
                 n_channels=3,
                 training=True,
                 drop_remainder=True,
                 shuffle=True,
                 repeat=1):
    img_paths = sorted(py.glob(img_dir, '*'))

    if shuffle:
        img_paths = np.random.permutation(img_paths)

    if training:
        def _map_fn(img):
            if n_channels == 1:
                img = tf.image.rgb_to_grayscale(img)
            img = tf.image.resize(img, [load_size, load_size])
            img = tf.image.random_flip_left_right(img)
            img = tl.center_crop(img, size=crop_size)
            # img = tf.image.random_crop(img, [crop_size, crop_size, n_channels])
            img = tf.clip_by_value(img, 0, 255) / 127.5 - 1
            return img
    else:
        def _map_fn(img):
            if n_channels == 1:
                img = tf.image.rgb_to_grayscale(img)
            img = tf.image.resize(img, [load_size, load_size])
            img = tl.center_crop(img, size=crop_size)
            img = tf.clip_by_value(img, 0, 255) / 127.5 - 1
            return img

    dataset = tl.disk_image_batch_dataset(img_paths,
                                          batch_size,
                                          drop_remainder=drop_remainder,
                                          map_fn=_map_fn,
                                          shuffle=shuffle,
                                          repeat=repeat)

    if drop_remainder:
        len_dataset = len(img_paths) // batch_size
    else:
        len_dataset = int(np.ceil(len(img_paths) / batch_size))

    return dataset, len_dataset

max_issues_repo_name: gokulsg/EigenGAN-Tensorflow

max_forks_repo_name: gokulsg/EigenGAN-Tensorflow


text:
"""
AMRVAC-specific fields

"""

import functools

import numpy as np

from yt import mylog
from yt.fields.field_info_container import FieldInfoContainer
from yt.fields.magnetic_field import setup_magnetic_field_aliases
from yt.units import dimensions

# We need to specify which fields we might have in our dataset.  The field info
# container subclass here will define which fields it knows about.  There are
# optionally methods on it that get called which can be subclassed.

direction_aliases = {
    "cartesian": ("x", "y", "z"),
    "polar": ("r", "theta", "z"),
    "cylindrical": ("r", "z", "theta"),
    "spherical": ("r", "theta", "phi"),
}

def _velocity(field, data, idir, prefix=None):
    """Velocity = linear momentum / density"""
    # This is meant to be used with functools.partial to produce
    # functions with only 2 arguments (field, data)
    # idir : int
    #    the direction index (1, 2 or 3)
    # prefix : str
    #    used to generalize to dust fields
    if prefix is None:
        prefix = ""
    moment = data["gas", "%smoment_%d" % (prefix, idir)]
    rho = data["gas", "%sdensity" % prefix]

    mask1 = rho == 0
    if mask1.any():
        mylog.info(
            "zeros found in %sdensity, patching them to compute corresponding velocity field."
            % prefix
        )
        mask2 = moment == 0
        if not ((mask1 & mask2) == mask1).all():
            raise RuntimeError
        rho[mask1] = 1
    return moment / rho

code_density = "code_mass / code_length**3"
code_moment = "code_mass / code_length**2 / code_time"
code_pressure = "code_mass / code_length / code_time**2"

# for now, define a finite family of dust fields (up to 100 species, should be enough)
MAXN_DUST_SPECIES = 100
known_dust_fields = [
    ("rhod%d" % idust, (code_density, ["dust%d_density" % idust], None))
    for idust in range(1, MAXN_DUST_SPECIES + 1)
]
for idir in (1, 2, 3):
    known_dust_fields += [
        (
            "m%dd%d" % (idir, idust),
            (code_moment, ["dust%d_moment_%d" % (idust, idir)], None),
        )
        for idust in range(1, MAXN_DUST_SPECIES + 1)
    ]

class AMRVACFieldInfo(FieldInfoContainer):

    # format: (native(?) field, (units, [aliases], display_name))
    # note: aliases will correspond to "gas" typed fields, whereas the native ones are "amrvac" typed
    known_other_fields = tuple(
        list(
            (
                ("rho", (code_density, ["density"], None)),
                ("m1", (code_moment, ["moment_1"], None)),
                ("m2", (code_moment, ["moment_2"], None)),
                ("m3", (code_moment, ["moment_3"], None)),
                ("e", (code_pressure, ["energy_density"], None)),
                ("b1", ("code_magnetic", ["magnetic_1"], None)),
                ("b2", ("code_magnetic", ["magnetic_2"], None)),
                ("b3", ("code_magnetic", ["magnetic_3"], None)),
            )
        )
        + known_dust_fields
        # in python3, there is no need for this tuple+list conversion, it suffices to write
        # known_other_fields = (..., *known_dust_fields)
    )

    known_particle_fields = ()

    def _setup_velocity_fields(self, idust=None):
        if idust is None:
            dust_flag = dust_label = ""
        else:
            dust_flag = "d%d" % idust
            dust_label = "dust%d_" % idust

        us = self.ds.unit_system
        for idir, alias in enumerate(direction_aliases[self.ds.geometry], start=1):
            if not ("amrvac", "m%d%s" % (idir, dust_flag)) in self.field_list:
                break
            velocity_fn = functools.partial(_velocity, idir=idir, prefix=dust_label)
            functools.update_wrapper(velocity_fn, _velocity)
            self.add_field(
                ("gas", "%svelocity_%s" % (dust_label, alias)),
                function=velocity_fn,
                units=us["velocity"],
                dimensions=dimensions.velocity,
                sampling_type="cell",
            )
            self.alias(
                ("gas", "%svelocity_%d" % (dust_label, idir)),
                ("gas", "%svelocity_%s" % (dust_label, alias)),
                units=us["velocity"],
            )
            self.alias(
                ("gas", "%smoment_%s" % (dust_label, alias)),
                ("gas", "%smoment_%d" % (dust_label, idir)),
                units=us["density"] * us["velocity"],
            )

    def _setup_dust_fields(self):
        idust = 1
        while ("amrvac", "rhod%d" % idust) in self.field_list:
            if idust > MAXN_DUST_SPECIES:
                mylog.error(
                    "Only the first %d dust species are currently read by yt. "
                    "If you read this, please consider issuing a ticket. "
                    % MAXN_DUST_SPECIES
                )
                break
            self._setup_velocity_fields(idust)
            idust += 1
        n_dust_found = idust - 1

        us = self.ds.unit_system
        if n_dust_found > 0:

            def _total_dust_density(field, data):
                tot = np.zeros_like(data["density"])
                for idust in range(1, n_dust_found + 1):
                    tot += data["dust%d_density" % idust]
                return tot

            self.add_field(
                ("gas", "total_dust_density"),
                function=_total_dust_density,
                dimensions=dimensions.density,
                units=us["density"],
                sampling_type="cell",
            )

            def dust_to_gas_ratio(field, data):
                return data["total_dust_density"] / data["density"]

            self.add_field(
                ("gas", "dust_to_gas_ratio"),
                function=dust_to_gas_ratio,
                dimensions=dimensions.dimensionless,
                sampling_type="cell",
            )

    def setup_fluid_fields(self):

        setup_magnetic_field_aliases(self, "amrvac", ["mag%s" % ax for ax in "xyz"])
        self._setup_velocity_fields()  # gas velocities
        self._setup_dust_fields()  # dust derived fields (including velocities)

        # fields with nested dependencies are defined thereafter by increasing level of complexity
        us = self.ds.unit_system

        def _kinetic_energy_density(field, data):
            # devnote : have a look at issue 1301
            return 0.5 * data["gas", "density"] * data["gas", "velocity_magnitude"] ** 2

        self.add_field(
            ("gas", "kinetic_energy_density"),
            function=_kinetic_energy_density,
            units=us["density"] * us["velocity"] ** 2,
            dimensions=dimensions.density * dimensions.velocity ** 2,
            sampling_type="cell",
        )

        # magnetic energy density
        if ("amrvac", "b1") in self.field_list:

            def _magnetic_energy_density(field, data):
                emag = 0.5 * data["gas", "magnetic_1"] ** 2
                for idim in "23":
                    if not ("amrvac", "b%s" % idim) in self.field_list:
                        break
                    emag += 0.5 * data["gas", "magnetic_%s" % idim] ** 2
                # important note: in AMRVAC the magnetic field is defined in units where mu0 = 1,
                # such that
                # Emag = 0.5*B**2 instead of Emag = 0.5*B**2 / mu0
                # To correctly transform the dimensionality from gauss**2 -> rho*v**2, we have to
                # take mu0 into account. If we divide here, units when adding the field should be
                # us["density"]*us["velocity"]**2. If not, they should be us["magnetic_field"]**2
                # and division should happen elsewhere.
                emag /= 4 * np.pi
                # divided by mu0 = 4pi in cgs, yt handles 'mks' and 'code' unit systems internally.
                return emag

            self.add_field(
                ("gas", "magnetic_energy_density"),
                function=_magnetic_energy_density,
                units=us["density"] * us["velocity"] ** 2,
                dimensions=dimensions.density * dimensions.velocity ** 2,
                sampling_type="cell",
            )

        # Adding the thermal pressure field.
        # In AMRVAC we have multiple physics possibilities:
        # - if HD/MHD + energy equation, pressure is (gamma-1)*(e - ekin (- emag)) for (M)HD
        # - if HD/MHD but solve_internal_e is true in parfile, pressure is (gamma-1)*e for both
        # - if (m)hd_energy is false in parfile (isothermal), pressure is c_adiab * rho**gamma

        def _full_thermal_pressure_HD(field, data):
            # important note : energy density and pressure are actually expressed in the same unit
            pthermal = (data.ds.gamma - 1) * (
                data["gas", "energy_density"] - data["gas", "kinetic_energy_density"]
            )
            return pthermal

        def _full_thermal_pressure_MHD(field, data):
            pthermal = (
                _full_thermal_pressure_HD(field, data)
                - (data.ds.gamma - 1) * data["gas", "magnetic_energy_density"]
            )
            return pthermal

        def _polytropic_thermal_pressure(field, data):
            return (data.ds.gamma - 1) * data["gas", "energy_density"]

        def _adiabatic_thermal_pressure(field, data):
            return data.ds._c_adiab * data["gas", "density"] ** data.ds.gamma

        pressure_recipe = None
        if ("amrvac", "e") in self.field_list:
            if self.ds._e_is_internal:
                pressure_recipe = _polytropic_thermal_pressure
                mylog.info("Using polytropic EoS for thermal pressure.")
            elif ("amrvac", "b1") in self.field_list:
                pressure_recipe = _full_thermal_pressure_MHD
                mylog.info("Using full MHD energy for thermal pressure.")
            else:
                pressure_recipe = _full_thermal_pressure_HD
                mylog.info("Using full HD energy for thermal pressure.")
        elif self.ds._c_adiab is not None:
            pressure_recipe = _adiabatic_thermal_pressure
            mylog.info("Using adiabatic EoS for thermal pressure (isothermal).")
            mylog.warning(
                "If you used usr_set_pthermal you should redefine the thermal_pressure field."
            )

        if pressure_recipe is not None:
            self.add_field(
                ("gas", "thermal_pressure"),
                function=pressure_recipe,
                units=us["density"] * us["velocity"] ** 2,
                dimensions=dimensions.density * dimensions.velocity ** 2,
                sampling_type="cell",
            )

            # sound speed and temperature depend on thermal pressure
            def _sound_speed(field, data):
                return np.sqrt(
                    data.ds.gamma
                    * data["gas", "thermal_pressure"]
                    / data["gas", "density"]
                )

            self.add_field(
                ("gas", "sound_speed"),
                function=_sound_speed,
                units=us["velocity"],
                dimensions=dimensions.velocity,
                sampling_type="cell",
            )
        else:
            mylog.warning(
                "e not found and no parfile passed, can not set thermal_pressure."
            )


text:
import matplotlib.pyplot as plt
import datetime as dt
import numpy as np

from sentinelhub import (SHConfig, CRS, BBox, DataSource, SentinelHubRequest,
                         MimeType, bbox_to_dimensions)

PRECISION_SCORES = 4
PRECISION_THRESHOLD = None

BANDS = ['B02', 'B03', 'B04', 'B08', 'B11']
BANDS_STR = ','.join(BANDS)
MODEL_INPUTS = ['B02', 'B03', 'B04', 'NDWI', 'NDMI']
MODEL_INPUTS_STR = ', '.join(MODEL_INPUTS)

def parse_subtree(node, brackets=True):
    if 'leaf_index' in node:
        score = float(node["leaf_value"])
        if PRECISION_SCORES is not None:
            score = round(score, PRECISION_SCORES)
        return f'{score}'

    feature = MODEL_INPUTS[int(node["split_feature"])]

    threshold = float(node["threshold"])
    if PRECISION_THRESHOLD is not None:
        threshold = round(threshold, PRECISION_THRESHOLD)

    condition = f'{feature}{node["decision_type"]}{threshold}'

    left = parse_subtree(node['left_child'])
    right = parse_subtree(node['right_child'])

    result = f'({condition})?{left}:{right}'
    if brackets:
        return f'({result})'
    return result

def parse_one_tree(root, index):
    return \
f"""
function pt{index}({MODEL_INPUTS_STR}) {{
   return {parse_subtree(root, brackets=False)};
}}
"""

def parse_trees(trees):

    tree_functions = '\n'.join([parse_one_tree(tree['tree_structure'], idx)
                                  for idx, tree in enumerate(trees)])
    function_sum = '+'.join([f'pt{i}({MODEL_INPUTS_STR})' for i in range(len(trees))])

    return f"""
//VERSION=3

function setup() {{
    return {{
        input: [{{
            bands: [{','.join(f'"{band}"' for band in BANDS)}],
            units: "reflectance"
        }}],
        output: {{
            id:"default",
            bands: 1,
            sampleType: "FLOAT32"
        }}
    }}
}}

function evaluatePixel(sample) {{
    let NDWI = index(sample.B03, sample.B08);
    let NDMI = index(sample.B08, sample.B11);

    return [predict(sample.B02, sample.B03, sample.B04, NDWI, NDMI)]
}}

{tree_functions}

function predict({MODEL_INPUTS_STR}) {{
    return [1/(1+Math.exp(-1*({function_sum})))];
}}
"""

def parse_model(model, js_output_filename=None):
    model_json = model.booster_.dump_model()

    model_javascript = parse_trees(model_json['tree_info'])

    if js_output_filename:
        with open(js_output_filename, 'w') as f:
            f.write(model_javascript)

    return model_javascript

def visualize(patch, factor=3.5):
    fig, ax = plt.subplots(ncols=3, figsize=(22,7))
    ax[0].imshow(factor*patch.data['BANDS-S2-L1C'][0][...,[3,2,1]].squeeze())
    ax[0].set_title(f'True color, {patch.timestamp[0]}')

    ax[1].imshow(patch.data_timeless['DEM'].squeeze())
    ax[1].set_title('DEM')

    ax[2].imshow(patch.mask_timeless['water_label'].squeeze(), vmin=0, vmax=1)
    ax[2].set_title('water mask')

def predict_on_sh(model_script, bbox, size, timestamp, config):
    request = SentinelHubRequest(
        evalscript=model_script,
        input_data=[
            SentinelHubRequest.input_data(
                data_source=DataSource.SENTINEL2_L1C,
                time_interval=(timestamp-dt.timedelta(minutes=5),timestamp+dt.timedelta(minutes=5)),
                maxcc=1,
            )
        ],
        responses=[SentinelHubRequest.output_response('default',MimeType.TIFF)],
        bbox=bbox,
        size=size,
        config=config
    )
    return request.get_data()[0]

def get_predictions(patch, model, config):
    model_script = parse_model(model, None)
    sh_prediction = predict_on_sh(model_script, patch.bbox, (64, 64), patch.timestamp[0], config)

    features = patch.data['FEATURES'][0]
    f_s = features.shape
    model_prediction = model.predict_proba(features.reshape(f_s[0]*f_s[1],f_s[2]))[...,-1].reshape(f_s[0],f_s[1])

    return features[..., [2, 1, 000]], sh_prediction, model_prediction

def plot_comparison(rgb_array, sh_prediction, model_prediction, threshold=0.5):
    fig, ax = plt.subplots(nrows=2, ncols=4, figsize=(24, 12), sharex=True, sharey=True)

    for axx in ax.flatten():
        axx.set_xticks([])
        axx.set_yticks([])

    ax[0][0].imshow(3.5*rgb_array)
    ax[0][0].set_title('RGB image')

    ax[0][1].imshow(sh_prediction, vmin=0, vmax=1)
    ax[0][1].set_title('[a] water prediction probabilities with evalscript on SH', fontsize=14)

    ax[0][2].imshow(model_prediction, vmin=0, vmax=1)
    ax[0][2].set_title('[b] water prediction probabilities with model, locally', fontsize=14)

    ax[0][3].imshow(sh_prediction - model_prediction, vmin=-0.2, vmax=0.2, cmap="RdBu")
    ax[0][3].set_title('differences between [a] and [b]', fontsize=14)

    sh_thr = np.where(sh_prediction>threshold, 1, 0)
    ax[1][0].imshow(3.5*rgb_array)
    ax[1][0].set_title('RGB image')

    ax[1][1].imshow(sh_thr, vmin=0, vmax=1)
    ax[1][1].set_title('[c] water prediction with evalscript on SH', fontsize=14)

    model_thr = np.where(model_prediction>threshold, 1, 0)
    ax[1][2].imshow(model_thr, vmin=0, vmax=1)
    ax[1][2].set_title('[d] water prediction with model, locally', fontsize=14)

    ax[1][3].imshow(sh_thr - model_thr, vmin=-1, vmax=1, cmap="RdBu")
    ax[1][3].set_title('differences between [d] and [d]', fontsize=14)

    plt.tight_layout(pad=0.05)

max_stars_repo_name: rpitonak/eo-learn

max_issues_repo_name: rpitonak/eo-learn

max_forks_repo_name: rpitonak/eo-learn