text:
"""The Fast Gradient Method attack."""

import numpy as np
import tensorflow as tf

def fast_gradient_method(model_fn, x, eps, ord, clip_min=None, clip_max=None, y=None,
                         targeted=False, sanity_checks=False):
  """
  Tensorflow 2.0 implementation of the Fast Gradient Method.
  :param model_fn: a callable that takes an input tensor and returns the model logits.
  :param x: input tensor.
  :param eps: epsilon (input variation parameter); see https://arxiv.org/abs/1412.6572.
  :param ord: Order of the norm (mimics NumPy). Possible values: np.inf, 1 or 2.
  :param clip_min: (optional) float. Minimum float value for adversarial example components.
  :param clip_max: (optional) float. Maximum float value for adversarial example components.
  :param y: (optional) Tensor with true labels. If targeted is true, then provide the
            target label. Otherwise, only provide this parameter if you'd like to use true
            labels when crafting adversarial samples. Otherwise, model predictions are used
            as labels to avoid the "label leaking" effect (explained in this paper:
            https://arxiv.org/abs/1611.01236). Default is None.
  :param targeted: (optional) bool. Is the attack targeted or untargeted?
            Untargeted, the default, will try to make the label incorrect.
            Targeted will instead try to move in the direction of being more like y.
  :param sanity_checks: bool, if True, include asserts (Turn them off to use less runtime /
            memory or for unit tests that intentionally pass strange input)
  :return: a tensor for the adversarial example
  """
  if ord not in [np.inf, 1, 2]:
    raise ValueError("Norm order must be either np.inf, 1, or 2.")

  asserts = []

  # If a data range was specified, check that the input was in that range
  if clip_min is not None:
    asserts.append(tf.math.greater_equal(x, clip_min))

  if clip_max is not None:
    asserts.append(tf.math.less_equal(x, clip_max))

  if y is None:
    # Using model predictions as ground truth to avoid label leaking
    y = tf.argmax(model_fn(x), 1)

  grad = compute_gradient(model_fn, x, y, targeted)

  optimal_perturbation = optimize_linear(grad, eps, ord)
  # Add perturbation to original example to obtain adversarial example
  adv_x = x + optimal_perturbation

  # If clipping is needed, reset all values outside of [clip_min, clip_max]
  if (clip_min is not None) or (clip_max is not None):
    # We don't currently support one-sided clipping
    assert clip_min is not None and clip_max is not None
    adv_x = tf.clip_by_value(adv_x, clip_min, clip_max)

  if sanity_checks:
    assert np.all(asserts)
  return adv_x

# Due to performance reasons, this function is wrapped inside of tf.function decorator.
# Not using the decorator here, or letting the user wrap the attack in tf.function is way
# slower on Tensorflow 2.0.0-alpha0.
@tf.function
def compute_gradient(model_fn, x, y, targeted):
  """
  Computes the gradient of the loss with respect to the input tensor.
  :param model_fn: a callable that takes an input tensor and returns the model logits.
  :param x: input tensor
  :param y: Tensor with true labels. If targeted is true, then provide the target label.
  :param targeted:  bool. Is the attack targeted or untargeted? Untargeted, the default, will
                    try to make the label incorrect. Targeted will instead try to move in the
                    direction of being more like y.
  :return: A tensor containing the gradient of the loss with respect to the input tensor.
  """
  loss_fn = tf.nn.sparse_softmax_cross_entropy_with_logits
  with tf.GradientTape() as g:
    g.watch(x)
    # Compute loss
    loss = loss_fn(labels=y, logits=model_fn(x))
    if targeted:  # attack is targeted, minimize loss of target label rather than maximize loss of correct label
      loss = -loss

  # Define gradient of loss wrt input
  grad = g.gradient(loss, x)
  return grad

def optimize_linear(grad, eps, ord=np.inf):
  """
  Solves for the optimal input to a linear function under a norm constraint.

  Optimal_perturbation = argmax_{eta, ||eta||_{ord} < eps} dot(eta, grad)

  :param grad: tf tensor containing a batch of gradients
  :param eps: float scalar specifying size of constraint region
  :param ord: int specifying order of norm
  :returns:
    tf tensor containing optimal perturbation
  """

  # Convert the iterator returned by `range` into a list.
  axis = list(range(1, len(grad.get_shape())))
  avoid_zero_div = 1e-12
  if ord == np.inf:
    # Take sign of gradient
    optimal_perturbation = tf.sign(grad)
    # The following line should not change the numerical results. It applies only because
    # `optimal_perturbation` is the output of a `sign` op, which has zero derivative anyway.
    # It should not be applied for the other norms, where the perturbation has a non-zero derivative.
    optimal_perturbation = tf.stop_gradient(optimal_perturbation)
  elif ord == 1:
    abs_grad = tf.abs(grad)
    sign = tf.sign(grad)
    max_abs_grad = tf.reduce_max(abs_grad, axis, keepdims=True)
    tied_for_max = tf.dtypes.cast(tf.equal(abs_grad, max_abs_grad), dtype=tf.float32)
    num_ties = tf.reduce_sum(tied_for_max, axis, keepdims=True)
    optimal_perturbation = sign * tied_for_max / num_ties
  elif ord == 2:
    square = tf.maximum(avoid_zero_div, tf.reduce_sum(tf.square(grad), axis, keepdims=True))
    optimal_perturbation = grad / tf.sqrt(square)
  else:
    raise NotImplementedError("Only L-inf, L1 and L2 norms are currently implemented.")

  # Scale perturbation to be the solution for the norm=eps rather than norm=1 problem
  scaled_perturbation = tf.multiply(eps, optimal_perturbation)
  return scaled_perturbation

max_stars_repo_name: HaojieYuan/cleverhans

max_issues_repo_name: HaojieYuan/cleverhans

max_forks_repo_name: HaojieYuan/cleverhans


text:
"""
This module contains the base objects for use in base xml/serializable functionality.
"""

__classification__ = "UNCLASSIFIED"
__author__ = "Thomas McCullough"

import logging
from xml.etree import ElementTree
from weakref import WeakKeyDictionary

import numpy
from numpy.linalg import norm

from sarpy.compliance import integer_types, int_func, string_types
from sarpy.io.xml.base import DEFAULT_STRICT, get_node_value, find_children, \
    Arrayable, ParametersCollection, SerializableArray, \
    parse_str, parse_bool, parse_int, parse_float, parse_complex, parse_datetime, \
    parse_serializable, parse_serializable_list

logger = logging.getLogger(__name__)

class BasicDescriptor(object):
    """A descriptor object for reusable properties. Note that is is required that the calling instance is hashable."""
    _typ_string = None

    def __init__(self, name, required, strict=DEFAULT_STRICT, default_value=None, docstring=''):
        self.data = WeakKeyDictionary()  # our instance reference dictionary
        # WeakDictionary use is subtle here. A reference to a particular class instance in this dictionary
        # should not be the thing keeping a particular class instance from being destroyed.
        self.name = name
        self.required = (name in required)
        self.strict = strict
        self.default_value = default_value

        self.__doc__ = docstring
        self._format_docstring()

    def _format_docstring(self):
        docstring = self.__doc__
        if docstring is None:
            docstring = ''
        if (self._typ_string is not None) and (not docstring.startswith(self._typ_string)):
            docstring = '{} {}'.format(self._typ_string, docstring)

        suff = self._docstring_suffix()
        if suff is not None:
            docstring = '{} {}'.format(docstring, suff)

        lenstr = self._len_string()
        if lenstr is not None:
            docstring = '{} {}'.format(docstring, lenstr)

        if self.required:
            docstring = '{} {}'.format(docstring, ' **Required.**')
        else:
            docstring = '{} {}'.format(docstring, ' **Optional.**')
        self.__doc__ = docstring

    def _len_string(self):
        minl = getattr(self, 'minimum_length', None)
        maxl = getattr(self, 'minimum_length', None)
        def_minl = getattr(self, '_DEFAULT_MIN_LENGTH', None)
        def_maxl = getattr(self, '_DEFAULT_MAX_LENGTH', None)
        if minl is not None and maxl is not None:
            if minl == def_minl and maxl == def_maxl:
                return None

            lenstr = ' Must have length '
            if minl == def_minl:
                lenstr += '<= {0:d}.'.format(maxl)
            elif maxl == def_maxl:
                lenstr += '>= {0:d}.'.format(minl)
            elif minl == maxl:
                lenstr += ' exactly {0:d}.'.format(minl)
            else:
                lenstr += 'in the range [{0:d}, {1:d}].'.format(minl, maxl)
            return lenstr
        else:
            return None

    def _docstring_suffix(self):
        return None

    def __get__(self, instance, owner):
        """The getter.

        Parameters
        ----------
        instance : object
            the calling class instance
        owner : object
            the type of the class - that is, the actual object to which this descriptor is assigned

        Returns
        -------
        object
            the return value
        """

        if instance is None:
            # this has been access on the class, so return the class
            return self

        fetched = self.data.get(instance, self.default_value)
        if fetched is not None or not self.required:
            return fetched
        else:
            msg = 'Required field {} of class {} is not populated.'.format(self.name, instance.__class__.__name__)
            if self.strict:
                raise AttributeError(msg)
            else:
                logger.debug(msg)  # NB: this is at debug level to not be too verbose
            return fetched

    def __set__(self, instance, value):
        """The setter method.

        Parameters
        ----------
        instance : object
            the calling class instance
        value
            the value to use in setting - the type depends of the specific extension of this base class

        Returns
        -------
        bool
            this base class, and only this base class, handles the required compliance and None behavior and has
            a return. This returns True if this the setting value was None, and False otherwise.
        """

        # NOTE: This is intended to handle this case for every extension of this class. Hence the boolean return,
        # which extensions SHOULD NOT implement. This is merely to follow DRY principles.
        if value is None:
            if self.default_value is not None:
                self.data[instance] = self.default_value
                return True
            elif self.required:
                if self.strict:
                    raise ValueError(
                        'Attribute {} of class {} cannot be assigned '
                        'None.'.format(self.name, instance.__class__.__name__))
                else:
                    logger.debug(  # NB: this is at debuglevel to not be too verbose
                        'Required attribute {} of class {} has been set to None.'.format(
                            self.name, instance.__class__.__name__))
            self.data[instance] = None
            return True
        # note that the remainder must be implemented in each extension
        return False  # this is probably a bad habit, but this returns something for convenience alone

class StringDescriptor(BasicDescriptor):
    """A descriptor for string type"""
    _typ_string = 'str:'

    def __init__(self, name, required, strict=DEFAULT_STRICT, default_value=None, docstring=None):
        super(StringDescriptor, self).__init__(
            name, required, strict=strict, default_value=default_value, docstring=docstring)

    def __set__(self, instance, value):

        if super(StringDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return

        self.data[instance] = parse_str(value, self.name, instance)

class StringListDescriptor(BasicDescriptor):
    """A descriptor for properties for an array type item for specified extension of string"""
    _typ_string = 'List[str]:'
    _DEFAULT_MIN_LENGTH = 0
    _DEFAULT_MAX_LENGTH = 2 ** 32

    def __init__(self, name, required, strict=DEFAULT_STRICT, minimum_length=None, maximum_length=None,
                 default_value=None, docstring=None):
        self.minimum_length = self._DEFAULT_MIN_LENGTH if minimum_length is None else int_func(minimum_length)
        self.maximum_length = self._DEFAULT_MAX_LENGTH if maximum_length is None else int_func(maximum_length)
        if self.minimum_length > self.maximum_length:
            raise ValueError(
                'Specified minimum length is {}, while specified maximum length is {}'.format(
                    self.minimum_length, self.maximum_length))
        super(StringListDescriptor, self).__init__(
            name, required, strict=strict, default_value=default_value, docstring=docstring)

    def __set__(self, instance, value):
        def set_value(new_value):
            if len(new_value) < self.minimum_length:
                msg = 'Attribute {} of class {} is a string list of size {},\n\t' \
                      'and must have length at least {}.'.format(
                        self.name, instance.__class__.__name__, value.size, self.minimum_length)
                if self.strict:
                    raise ValueError(msg)
                else:
                    logger.error(msg)
            if len(new_value) > self.maximum_length:
                msg = 'Attribute {} of class {} is a string list of size {},\n\t' \
                      'and must have length no greater than {}.'.format(
                        self.name, instance.__class__.__name__, value.size, self.maximum_length)
                if self.strict:
                    raise ValueError(msg)
                else:
                    logger.error(msg)
            self.data[instance] = new_value

        if super(StringListDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return

        if isinstance(value, string_types):
            set_value([value, ])
        elif isinstance(value, ElementTree.Element):
            set_value([get_node_value(value), ])
        elif isinstance(value, list):
            if len(value) == 0 or isinstance(value[0], string_types):
                set_value(value)
            elif isinstance(value[0], ElementTree.Element):
                set_value([get_node_value(nod) for nod in value])
        else:
            raise TypeError(
                'Field {} of class {} got incompatible type {}.'.format(
                    self.name, instance.__class__.__name__, type(value)))

class StringEnumDescriptor(BasicDescriptor):
    """A descriptor for enumerated (specified) string type.
    **This implicitly assumes that the valid entries are upper case.**"""
    _typ_string = 'str:'

    def __init__(self, name, values, required, strict=DEFAULT_STRICT, default_value=None, docstring=None):
        self.values = values
        super(StringEnumDescriptor, self).__init__(
            name, required, strict=strict, default_value=default_value, docstring=docstring)
        if (self.default_value is not None) and (self.default_value not in self.values):
            self.default_value = None

    def _docstring_suffix(self):
        suff = ' Takes values in :code:`{}`.'.format(self.values)
        if self.default_value is not None:
            suff += ' Default value is :code:`{}`.'.format(self.default_value)
        return suff

    def __set__(self, instance, value):
        if value is None:
            if self.default_value is not None:
                self.data[instance] = self.default_value
            else:
                super(StringEnumDescriptor, self).__set__(instance, value)
            return

        val = parse_str(value, self.name, instance)

        if val in self.values:
            self.data[instance] = val
        else:
            msg = 'Attribute {} of class {} received {},\n\t' \
                  'but values ARE REQUIRED to be one of {}'.format(
                    self.name, instance.__class__.__name__, value, self.values)
            if self.strict:
                raise ValueError(msg)
            else:
                logger.error(msg)
            self.data[instance] = val

class BooleanDescriptor(BasicDescriptor):
    """A descriptor for boolean type"""
    _typ_string = 'bool:'

    def __init__(self, name, required, strict=DEFAULT_STRICT, default_value=None, docstring=None):
        super(BooleanDescriptor, self).__init__(
            name, required, strict=strict, default_value=default_value, docstring=docstring)

    def __set__(self, instance, value):
        if super(BooleanDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return
        try:
            self.data[instance] = parse_bool(value, self.name, instance)
        except Exception as e:
            logger.error(
                'Failed converting {} of type {} to `bool`\n\t'
                'for field {} of class {} with exception {} - {}\n\t'
                'Setting value to None, which may be against the standard'.format(
                    value, type(value), self.name, instance.__class__.__name__, type(e), e))
            self.data[instance] = None

class IntegerDescriptor(BasicDescriptor):
    """A descriptor for integer type"""
    _typ_string = 'int:'

    def __init__(self, name, required, strict=DEFAULT_STRICT, bounds=None, default_value=None, docstring=None):
        self.bounds = bounds
        super(IntegerDescriptor, self).__init__(
            name, required, strict=strict, default_value=default_value, docstring=docstring)
        if (self.default_value is not None) and not self._in_bounds(self.default_value):
            self.default_value = None

    def _docstring_suffix(self):
        if self.bounds is not None:
            return 'Must be in the range [{}, {}]'.format(*self.bounds)
        return ''

    def _in_bounds(self, value):
        if self.bounds is None:
            return True
        return (self.bounds[0] is None or self.bounds[0] <= value) and \
            (self.bounds[1] is None or value <= self.bounds[1])

    def __set__(self, instance, value):
        if super(IntegerDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return

        try:
            iv = parse_int(value, self.name, instance)
        except Exception as e:
            logger.error(
                'Failed converting {} of type {} to `int`\n\t'
                'for field {} of class {} with exception {} - {}.\n\t'
                'Setting value to None, which may be against the standard'.format(
                    value, type(value), self.name, instance.__class__.__name__, type(e), e))
            self.data[instance] = None
            return

        if self._in_bounds(iv):
            self.data[instance] = iv
        else:
            msg = 'Attribute {} of class {} is required by standard\n\t' \
                  'to take value between {}. Invalid value {}'.format(
                    self.name, instance.__class__.__name__, self.bounds, iv)
            if self.strict:
                raise ValueError(msg)
            else:
                logger.error(msg)
            self.data[instance] = iv

class IntegerEnumDescriptor(BasicDescriptor):
    """A descriptor for enumerated (specified) integer type"""
    _typ_string = 'int:'

    def __init__(self, name, values, required, strict=DEFAULT_STRICT, default_value=None, docstring=None):
        self.values = values
        super(IntegerEnumDescriptor, self).__init__(
            name, required, strict=strict, default_value=default_value, docstring=docstring)
        if (self.default_value is not None) and (self.default_value not in self.values):
            self.default_value = None

    def _docstring_suffix(self):
        return 'Must take one of the values in {}.'.format(self.values)

    def __set__(self, instance, value):
        if super(IntegerEnumDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return

        try:
            iv = parse_int(value, self.name, instance)
        except Exception as e:
            logger.error(
                'Failed converting {} of type {} to `int`\n\t'
                'for field {} of class {} with exception {} - {}.\n\t'
                'Setting value to None, which may be against the standard'.format(
                    value, type(value), self.name, instance.__class__.__name__, type(e), e))
            self.data[instance] = None
            return

        if iv in self.values:
            self.data[instance] = iv
        else:
            msg = 'Attribute {} of class {} must take value in {}.\n\t' \
                  'Invalid value {}.'.format(
                    self.name, instance.__class__.__name__, self.values, iv)
            if self.strict:
                raise ValueError(msg)
            else:
                logger.error(msg)
            self.data[instance] = iv

class IntegerListDescriptor(BasicDescriptor):
    """A descriptor for integer list type properties"""
    _typ_string = 'list[int]:'
    _DEFAULT_MIN_LENGTH = 0
    _DEFAULT_MAX_LENGTH = 2 ** 32

    def __init__(self, name, tag_dict, required, strict=DEFAULT_STRICT,
                 minimum_length=None, maximum_length=None, docstring=None):
        self.child_tag = tag_dict[name]['child_tag']
        self.minimum_length = self._DEFAULT_MIN_LENGTH if minimum_length is None else int_func(minimum_length)
        self.maximum_length = self._DEFAULT_MAX_LENGTH if maximum_length is None else int_func(maximum_length)
        if self.minimum_length > self.maximum_length:
            raise ValueError(
                'Specified minimum length is {}, while specified maximum length is {}'.format(
                    self.minimum_length, self.maximum_length))
        super(IntegerListDescriptor, self).__init__(name, required, strict=strict, docstring=docstring)

    def __set__(self, instance, value):
        def set_value(new_value):
            if len(new_value) < self.minimum_length:
                msg = 'Attribute {} of class {} is an integer list of size {},\n\t' \
                      'and must have size at least {}.'.format(
                        self.name, instance.__class__.__name__, value.size, self.minimum_length)
                if self.strict:
                    raise ValueError(msg)
                else:
                    logger.info(msg)
            if len(new_value) > self.maximum_length:
                msg = 'Attribute {} of class {} is an integer list of size {},\n\t' \
                      'and must have size no larger than {}.'.format(
                        self.name, instance.__class__.__name__, value.size, self.maximum_length)
                if self.strict:
                    raise ValueError(msg)
                else:
                    logger.info(msg)
            self.data[instance] = new_value

        if super(IntegerListDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return

        if isinstance(value, integer_types):
            set_value([value, ])
        elif isinstance(value, ElementTree.Element):
            set_value([int_func(get_node_value(value)), ])
        elif isinstance(value, list):
            if len(value) == 0 or isinstance(value[0], integer_types):
                set_value(value)
            elif isinstance(value[0], ElementTree.Element):
                set_value([int_func(get_node_value(nod)) for nod in value])
        else:
            raise TypeError(
                'Field {} of class {} got incompatible type {}.'.format(
                    self.name, instance.__class__.__name__, type(value)))

class FloatDescriptor(BasicDescriptor):
    """A descriptor for float type properties"""
    _typ_string = 'float:'

    def __init__(self, name, required, strict=DEFAULT_STRICT, bounds=None, default_value=None, docstring=None):
        self.bounds = bounds
        super(FloatDescriptor, self).__init__(
            name, required, strict=strict, default_value=default_value, docstring=docstring)
        if (self.default_value is not None) and not self._in_bounds(self.default_value):
            self.default_value = None

    def _docstring_suffix(self):
        if self.bounds is not None:
            return 'Must be in the range [{}, {}]'.format(*self.bounds)
        return ''

    def _in_bounds(self, value):
        if self.bounds is None:
            return True

        return (self.bounds[0] is None or self.bounds[0] <= value) and \
            (self.bounds[1] is None or value <= self.bounds[1])

    def __set__(self, instance, value):
        if super(FloatDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return

        try:
            iv = parse_float(value, self.name, instance)
        except Exception as e:
            logger.error(
                'Failed converting {} of type {} to `float`\n\t'
                'for field {} of class {} with exception {} - {}.\n\t'
                'Setting value to None, which may be against the standard'.format(
                    value, type(value), self.name, instance.__class__.__name__, type(e), e))
            self.data[instance] = None
            return

        if self._in_bounds(iv):
            self.data[instance] = iv
        else:
            msg = 'Attribute {} of class {}\n\t' \
                  'is required by standard to take value between {}.'.format(
                    self.name, instance.__class__.__name__, self.bounds)
            if self.strict:
                raise ValueError(msg)
            else:
                logger.info(msg)
            self.data[instance] = iv

class FloatListDescriptor(BasicDescriptor):
    """A descriptor for float list type properties"""
    _typ_string = 'list[float]:'
    _DEFAULT_MIN_LENGTH = 0
    _DEFAULT_MAX_LENGTH = 2 ** 32

    def __init__(self, name, tag_dict, required, strict=DEFAULT_STRICT,
                 minimum_length=None, maximum_length=None, docstring=None):
        self.child_tag = tag_dict[name]['child_tag']
        self.minimum_length = self._DEFAULT_MIN_LENGTH if minimum_length is None else int_func(minimum_length)
        self.maximum_length = self._DEFAULT_MAX_LENGTH if maximum_length is None else int_func(maximum_length)
        if self.minimum_length > self.maximum_length:
            raise ValueError(
                'Specified minimum length is {}, while specified maximum length is {}'.format(
                    self.minimum_length, self.maximum_length))
        super(FloatListDescriptor, self).__init__(name, required, strict=strict, docstring=docstring)

    def __set__(self, instance, value):
        def set_value(new_value):
            if len(new_value) < self.minimum_length:
                msg = 'Attribute {} of class {} is an float list of size {},\n\t' \
                      'and must have size at least {}.'.format(
                        self.name, instance.__class__.__name__, value.size, self.minimum_length)
                if self.strict:
                    raise ValueError(msg)
                else:
                    logger.info(msg)
            if len(new_value) > self.maximum_length:
                msg = 'Attribute {} of class {} is a float list of size {},\n\t' \
                      'and must have size no larger than {}.'.format(
                        self.name, instance.__class__.__name__, value.size, self.maximum_length)
                if self.strict:
                    raise ValueError(msg)
                else:
                    logger.info(msg)
            self.data[instance] = new_value

        if super(FloatListDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return

        if isinstance(value, float):
            set_value([value, ])
        elif isinstance(value, ElementTree.Element):
            set_value([float(get_node_value(value)), ])
        elif isinstance(value, list):
            if len(value) == 0 or isinstance(value[0], float):
                set_value(value)
            elif isinstance(value[0], ElementTree.Element):
                set_value([float(get_node_value(nod)) for nod in value])
        else:
            raise TypeError(
                'Field {} of class {} got incompatible type {}.'.format(
                    self.name, instance.__class__.__name__, type(value)))

class ComplexDescriptor(BasicDescriptor):
    """A descriptor for complex valued properties"""
    _typ_string = 'complex:'

    def __init__(self, name, required, strict=DEFAULT_STRICT, default_value=None, docstring=None):
        super(ComplexDescriptor, self).__init__(
            name, required, strict=strict, default_value=default_value, docstring=docstring)

    def __set__(self, instance, value):
        if super(ComplexDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return

        try:
            self.data[instance] = parse_complex(value, self.name, instance)
        except Exception as e:
            logger.error(
                'Failed converting {} of type {} to `complex`\n\t'
                'for field {} of class {} with exception {} - {}.\n\t'
                'Setting value to None, which may be against the standard'.format(
                    value, type(value), self.name, instance.__class__.__name__, type(e), e))
            self.data[instance] = None

class FloatArrayDescriptor(BasicDescriptor):
    """A descriptor for float array type properties"""
    _DEFAULT_MIN_LENGTH = 0
    _DEFAULT_MAX_LENGTH = 2 ** 32
    _typ_string = 'numpy.ndarray[float64]:'

    def __init__(self, name, tag_dict, required, strict=DEFAULT_STRICT, minimum_length=None, maximum_length=None,
                 docstring=None):

        self.child_tag = tag_dict[name]['child_tag']
        self.minimum_length = self._DEFAULT_MIN_LENGTH if minimum_length is None else int_func(minimum_length)
        self.maximum_length = self._DEFAULT_MAX_LENGTH if maximum_length is None else int_func(maximum_length)
        if self.minimum_length > self.maximum_length:
            raise ValueError(
                'Specified minimum length is {}, while specified maximum length is {}'.format(
                    self.minimum_length, self.maximum_length))
        super(FloatArrayDescriptor, self).__init__(name, required, strict=strict, docstring=docstring)

    def __set__(self, instance, value):
        def set_value(new_val):
            if len(new_val) < self.minimum_length:
                msg = 'Attribute {} of class {} is a double array of size {},\n\t' \
                      'and must have size at least {}.'.format(
                        self.name, instance.__class__.__name__, value.size, self.minimum_length)
                if self.strict:
                    raise ValueError(msg)
                else:
                    logger.error(msg)
            if len(new_val) > self.maximum_length:
                msg = 'Attribute {} of class {} is a double array of size {},\n\t' \
                      'and must have size no larger than {}.'.format(
                        self.name, instance.__class__.__name__, value.size, self.maximum_length)
                if self.strict:
                    raise ValueError(msg)
                else:
                    logger.error(msg)
            self.data[instance] = new_val

        if super(FloatArrayDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return

        if isinstance(value, numpy.ndarray):
            if not (len(value) == 1) and (numpy.dtype.name == 'float64'):
                raise ValueError('Only one-dimensional ndarrays of dtype float64 are supported here.')
            set_value(value)
        elif isinstance(value, ElementTree.Element):
            xml_ns = getattr(instance, '_xml_ns', None)
            # noinspection PyProtectedMember
            if hasattr(instance, '_child_xml_ns_key') and self.name in instance._child_xml_ns_key:
                # noinspection PyProtectedMember
                xml_ns_key = instance._child_xml_ns_key[self.name]
            else:
                xml_ns_key = getattr(instance, '_xml_ns_key', None)

            size = int_func(value.attrib['size'])
            child_nodes = find_children(value, self.child_tag, xml_ns, xml_ns_key)
            if len(child_nodes) != size:
                raise ValueError(
                    'Field {} of double array type functionality belonging to class {} got a ElementTree element '
                    'with size attribute {}, but has {} child nodes with tag {}.'.format(
                        self.name, instance.__class__.__name__, size, len(child_nodes), self.child_tag))
            new_value = numpy.empty((size,), dtype=numpy.float64)
            for i, node in enumerate(child_nodes):
                new_value[i] = float(get_node_value(node))
            set_value(new_value)
        elif isinstance(value, list):
            # user or json deserialization
            set_value(numpy.array(value, dtype=numpy.float64))
        else:
            raise TypeError(
                'Field {} of class {} got incompatible type {}.'.format(
                    self.name, instance.__class__.__name__, type(value)))

class DateTimeDescriptor(BasicDescriptor):
    """A descriptor for date time type properties"""
    _typ_string = 'numpy.datetime64:'

    def __init__(self, name, required, strict=DEFAULT_STRICT, docstring=None, numpy_datetime_units='us'):
        self.units = numpy_datetime_units  # s, ms, us, ns are likely choices here, depending on needs
        super(DateTimeDescriptor, self).__init__(name, required, strict=strict, docstring=docstring)

    def __set__(self, instance, value):
        if super(DateTimeDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return
        self.data[instance] = parse_datetime(value, self.name, instance, self.units)

class FloatModularDescriptor(BasicDescriptor):
    """
    A descriptor for float type which will take values in a range [-limit, limit], set using modular
    arithmetic operations
    """
    _typ_string = 'float:'

    def __init__(self, name, limit, required, strict=DEFAULT_STRICT, docstring=None):
        self.limit = float(limit)
        super(FloatModularDescriptor, self).__init__(name, required, strict=strict, docstring=docstring)

    def __set__(self, instance, value):
        if super(FloatModularDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return

        try:
            val = parse_float(value, self.name, instance)
        except Exception as e:
            logger.error(
                'Failed converting {} of type {} to `float`\n\t'
                'for field {} of class {} with exception {} - {}.\n\t'
                'Setting value to None, which may be against the standard'.format(
                    value, type(value), self.name, instance.__class__.__name__, type(e), e))
            self.data[instance] = None
            return

        # do modular arithmetic manipulations
        val = (val % (2 * self.limit))  # NB: % and * have same precedence, so it can be super dumb
        self.data[instance] = val if val <= self.limit else val - 2 * self.limit

class SerializableDescriptor(BasicDescriptor):
    """A descriptor for properties of a specified type assumed to be an extension of Serializable"""

    def __init__(self, name, the_type, required, strict=DEFAULT_STRICT, docstring=None):
        self.the_type = the_type
        self._typ_string = str(the_type).strip().split('.')[-1][:-2] + ':'
        super(SerializableDescriptor, self).__init__(name, required, strict=strict, docstring=docstring)

    def __set__(self, instance, value):
        if super(SerializableDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return

        try:
            self.data[instance] = parse_serializable(value, self.name, instance, self.the_type)
        except Exception as e:
            logger.error(
                'Failed converting {} of type {} to Serializable type {}\n\t'
                'for field {} of class {} with exception {} - {}.\n\t'
                'Setting value to None, which may be against the standard.'.format(
                    value, type(value), self.the_type, self.name, instance.__class__.__name__, type(e), e))
            self.data[instance] = None

class UnitVectorDescriptor(BasicDescriptor):
    """A descriptor for properties of a specified type assumed to be of subtype of Arrayable"""

    def __init__(self, name, the_type, required, strict=DEFAULT_STRICT, docstring=None):
        if not issubclass(the_type, Arrayable):
            raise TypeError(
                'The input type {} for field {} must be a subclass of Arrayable.'.format(the_type, name))
        self.the_type = the_type

        self._typ_string = str(the_type).strip().split('.')[-1][:-2] + ':'
        super(UnitVectorDescriptor, self).__init__(name, required, strict=strict, docstring=docstring)

    def __set__(self, instance, value):
        if super(UnitVectorDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return

        try:
            vec = parse_serializable(value, self.name, instance, self.the_type)
        except Exception as e:
            logger.error(
                'Failed converting {} of type {} to Unit Vector Type type {}\n\t'
                'for field {} of class {} with exception {} - {}.\n\t'
                'Setting value to None, which may be against the standard'.format(
                    value, type(value), self.the_type, self.name, instance.__class__.__name__, type(e), e))
            self.data[instance] = None
            return None

        # noinspection PyTypeChecker
        coords = vec.get_array(dtype=numpy.float64)
        the_norm = norm(coords)
        if the_norm == 0:
            logger.error(
                'The input for field {} is expected to be made into a unit vector.\n\t'
                'In this case, the norm of the input is 0.\n\t'
                'The value is set to None, which may be against the standard.'.format(
                    self.name))
            self.data[instance] = None
        elif the_norm == 1:
            self.data[instance] = vec
        else:
            self.data[instance] = self.the_type.from_array(coords/the_norm)

class ParametersDescriptor(BasicDescriptor):
    """A descriptor for properties of a Parameter type - that is, dictionary"""

    def __init__(self, name, tag_dict, required, strict=DEFAULT_STRICT, docstring=None):
        self.child_tag = tag_dict[name]['child_tag']
        self._typ_string = 'ParametersCollection:'
        super(ParametersDescriptor, self).__init__(name, required, strict=strict, docstring=docstring)

    def __set__(self, instance, value):
        if super(ParametersDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return

        if isinstance(value, ParametersCollection):
            self.data[instance] = value
        else:
            the_inst = self.data.get(instance, None)
            xml_ns = getattr(instance, '_xml_ns', None)
            # noinspection PyProtectedMember
            if hasattr(instance, '_child_xml_ns_key') and self.name in instance._child_xml_ns_key:
                # noinspection PyProtectedMember
                xml_ns_key = instance._child_xml_ns_key[self.name]
            else:
                xml_ns_key = getattr(instance, '_xml_ns_key', None)
            if the_inst is None:
                self.data[instance] = ParametersCollection(
                    collection=value, name=self.name, child_tag=self.child_tag,
                    _xml_ns=xml_ns, _xml_ns_key=xml_ns_key)
            else:
                the_inst.set_collection(value)

class SerializableListDescriptor(BasicDescriptor):
    """A descriptor for properties of a list or array of specified extension of Serializable"""

    def __init__(self, name, child_type, tag_dict, required, strict=DEFAULT_STRICT, docstring=None):
        self.child_type = child_type
        tags = tag_dict[name]
        self.array = tags.get('array', False)
        if self.array:
            raise ValueError(
                'Attribute {} is populated in the `_collection_tags` dictionary with `array`=True. '
                'This is inconsistent with using SerializableListDescriptor.'.format(name))

        self.child_tag = tags['child_tag']
        self._typ_string = 'List[{}]:'.format(str(child_type).strip().split('.')[-1][:-2])
        super(SerializableListDescriptor, self).__init__(name, required, strict=strict, docstring=docstring)

    def __set__(self, instance, value):
        if super(SerializableListDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return

        try:
            self.data[instance] = parse_serializable_list(value, self.name, instance, self.child_type)
        except Exception as e:
            logger.error(
                'Failed converting {} of type {} to serializable list of type {}\n\t'
                'for field {} of class {} with exception {} - {}.\n\t'
                'Setting value to None, which may be against the standard'.format(
                    value, type(value), self.child_type, self.name, instance.__class__.__name__, type(e), e))
            self.data[instance] = None

class SerializableArrayDescriptor(BasicDescriptor):
    """A descriptor for properties of a list or array of specified extension of Serializable"""
    _DEFAULT_MIN_LENGTH = 0
    _DEFAULT_MAX_LENGTH = 2 ** 32

    def __init__(self, name, child_type, tag_dict, required, strict=DEFAULT_STRICT,
                 minimum_length=None, maximum_length=None, docstring=None,
                 array_extension=SerializableArray):
        if not issubclass(array_extension, SerializableArray):
            raise TypeError('array_extension must be a subclass of SerializableArray.')
        self.child_type = child_type
        tags = tag_dict[name]
        self.array = tags.get('array', False)
        if not self.array:
            raise ValueError(
                'Attribute {} is populated in the `_collection_tags` dictionary without `array`=True. '
                'This is inconsistent with using SerializableArrayDescriptor.'.format(name))

        self.child_tag = tags['child_tag']
        self._typ_string = 'numpy.ndarray[{}]:'.format(str(child_type).strip().split('.')[-1][:-2])
        self.array_extension = array_extension

        self.minimum_length = self._DEFAULT_MIN_LENGTH if minimum_length is None else int_func(minimum_length)
        self.maximum_length = self._DEFAULT_MAX_LENGTH if maximum_length is None else int_func(maximum_length)
        if self.minimum_length > self.maximum_length:
            raise ValueError(
                'Specified minimum length is {}, while specified maximum length is {}'.format(
                    self.minimum_length, self.maximum_length))
        super(SerializableArrayDescriptor, self).__init__(name, required, strict=strict, docstring=docstring)

    def __set__(self, instance, value):
        if super(SerializableArrayDescriptor, self).__set__(instance, value):  # the None handler...kinda hacky
            return

        if isinstance(value, self.array_extension):
            self.data[instance] = value
        else:
            xml_ns = getattr(instance, '_xml_ns', None)
            xml_ns_key = getattr(instance, '_xml_ns_key', None)
            the_inst = self.data.get(instance, None)
            if the_inst is None:
                self.data[instance] = self.array_extension(
                    coords=value, name=self.name, child_tag=self.child_tag, child_type=self.child_type,
                    minimum_length=self.minimum_length, maximum_length=self.maximum_length, _xml_ns=xml_ns,
                    _xml_ns_key=xml_ns_key)
            else:
                the_inst.set_array(value)

ext: py

max_stars_repo_name: bombaci-vsc/sarpy

max_issues_repo_name: bombaci-vsc/sarpy

max_forks_repo_name: bombaci-vsc/sarpy

reason: import numpy,from numpy


text:
# archivist.py: Helper functions for Covid19CanadaArchive #
# https://github.com/ccodwg/Covid19CanadaArchive #
# Maintainer: Jean-Paul R. Soucy #

# import modules

## core utilities
import sys
import os
import argparse
import re
import time
from datetime import datetime, timedelta
import pytz  # better time zones
from shutil import copyfile
import tempfile
import csv
import json
from zipfile import ZipFile
from array import *

## other utilities
import pandas as pd # better data processing
import numpy as np # better data processing
from colorit import * # colourful printing

## web scraping
import requests
from selenium import webdriver # requires ChromeDriver and Chromium/Chrome
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup

## Amazon S3
import boto3

## email
import smtplib

# define functions

## misc functions

def parse_args():
    global mode, email, uuid

    # initialize parser with arguments
    parser = argparse.ArgumentParser()
    parser.add_argument("-m", "--mode", choices = ['test', 'prod'], required = True, help = "Run mode: prod or test")
    parser.add_argument("--uuid", nargs = '+', required = False, help = "Specify UUIDs of individual datasets to download")
    parser.add_argument("--no-email", required = False, action = "store_false", dest = "email", help = "If present, no email will be produced at the end of the run.")
    args = parser.parse_args()

    # set run mode
    mode = args.mode
    print('Run mode set to ' + mode + '.')

    # set email mode
    email = args.email
    if email:
        print('An email will be sent at the end of this run.')
    else:
        print('No email will be sent at the end of this run.')

    # report datasets to be downloaded
    uuid = args.uuid
    if uuid:
        print('Specified datasets: ', ', '.join(uuid))
    else:
        print('No datasets specified. Downloading all datasets...')

def get_datetime(tz):
    t = datetime.now(pytz.timezone(tz))
    return t

def print_success_failure():
    global success, failure
    total_files = str(success + failure)
    print(background('Successful downloads: ' + str(success) + '/' + total_files, Colors.blue))
    print(background('Failed downloads: ' + str(failure) + '/' + total_files, Colors.red))

def generate_rerun_code():
    global mode, failure_uuid
    code = 'The following code will rerun failed datasets:\n' + 'python archiver.py -m ' + mode + ' --uuid ' + ' '.join(failure_uuid)
    return code

def find_url(search_url, regex, base_url):
    url = base_url + re.search(regex, requests.get(search_url).text).group(0)
    return url

## functions for Amazon S3

def access_s3(bucket, aws_id, aws_key):
    """Authenticate with AWS and return s3 object.

    Parameters:
    bucket (str): Name of Amazon S3 bucket.
    aws_id (str): ID for AWS.
    aws_key (str): Key for AWS.

    """
    global mode, s3
    print('Authenticating with AWS...')
    ## connect to AWS
    aws = boto3.Session(
        aws_access_key_id=aws_id,
        aws_secret_access_key=aws_key
        )

    ## connect to S3 bucket
    s3 = aws.resource('s3').Bucket(bucket)

    ## confirm authentication was successful
    print('Authentication was successful.')

    ## return s3 object
    return s3

def upload_file(full_name, f_path, uuid, s3_dir=None, s3_prefix=None):
    """Upload local file to Amazon S3.

    Parameters:
    full_name (str): Output filename with timestamp, extension and relative path.
    f_path (str): The path to the local file to upload.
    uuid (str): The UUID of the dataset.
    s3_dir(str): Optional. The directory on Amazon S3.
    s3_prefix (str): Optional. The prefix to the directory on Amazon S3.

    """
    global s3, download_log, success, failure, failure_uuid

    ## generate file name
    f_name = os.path.basename(full_name)
    if (s3_dir):
        f_name = os.path.join(s3_dir, f_name)
    if (s3_prefix):
        f_name = os.path.join(s3_prefix, f_name)
    ## upload file to Amazon S3
    try:
        ## file upload
        s3.upload_file(Filename=f_path, Key=f_name)
        ## append name of file to the log message
        download_log = download_log + 'Success: ' + full_name + '\n'
        print(color('Upload successful: ' + full_name, Colors.blue))
        success+=1
    except Exception as e:
        download_log = download_log + 'Failure: ' + full_name + '\n'
        print(e)
        print(background('Upload failed: ' + full_name, Colors.red))
        failure+=1
        failure_uuid.append(uuid)

## functions for emailing

def send_email(mail_name, mail_pass, mail_to, subject, body, smtp_server, smtp_port):
    """Send email (e.g., a download log).

    Parameters:
    mail_name (str): Email account the log will be sent from.
    mail_pass (str): Email password for the account the log will be sent from.
    mail_to (str): Email the log will be sent to.
    subject (str): Subject line for the email.
    body (str): Body of the email.
    smtp_server (str): SMTP server address.
    smtp_port (int): SMTP server port.

    """
     ## compose message
    email_text = """\
From: %s
To: %s
Subject: %s

%s
""" % (mail_name, mail_to, subject, body)

    ## send message
    try:
        print('Sending message...')
        server = smtplib.SMTP_SSL(smtp_server, smtp_port)
        server.ehlo()
        server.login(mail_name, mail_pass)
        server.sendmail(mail_name, mail_to, email_text)
        server.close()
        print('Message sent!')
    except Exception as e:
        print(e)
        print('Message failed to send.')

## functions for logging

def output_log(download_log, t):
    """Assemble log from current run.

    Parameters:
    download_log (str): Raw text of the download log.
    t (datetime): Date and time script began running (America/Toronto).

    """
    global success, failure

    ## process download log: place failures at the top, successes below
    download_log = download_log.split('\n')
    download_log.sort()
    download_log = '\n'.join(download_log)

    ## count total files
    total_files = str(success + failure)

    ## assemble log
    log = 'Successful downloads : ' + str(success) + '/' + total_files + '\n' + 'Failed downloads: ' + str(failure) + '/' + total_files + '\n' + download_log + '\n\n' + generate_rerun_code()
    log = t.strftime("%Y-%m-%d %H:%M") + '\n\n' + log

    ## return log
    return log

def upload_log(log):
    """Upload the log of file uploads to Amazon S3.

    The most recent log entry is placed in a separate file for easy access.

    Parameters:
    log (str): Log entry from current run.

    """
    global s3
    print("Uploading recent log...")
    try:
        ## write most recent log entry temporarily and upload
        tmpdir = tempfile.TemporaryDirectory()
        log_file = os.path.join(tmpdir.name, 'log.txt')
        with open(log_file, 'w') as local_file:
            local_file.write(log)
        s3.upload_file(Filename=log_file, Key='archive/log_recent.txt')

        ## report success
        print(color('Recent log upload successful!', Colors.green))
    except:
        print(background('Recent log upload failed!', Colors.red))
    print("Appending recent log to full log...")
    try:
        ## read in full log
        tmpdir = tempfile.TemporaryDirectory()
        log_file = os.path.join(tmpdir.name, 'log.txt')
        s3.download_file(Filename=log_file, Key='archive/log.txt')
        with open(log_file, 'r') as full_log:
            full_log = full_log.read()

        ## append recent log to full log
        log = full_log + '\n\n' + log

        ## write log temporarily and upload
        tmpdir = tempfile.TemporaryDirectory()
        log_file = os.path.join(tmpdir.name, 'log.txt')
        with open(log_file, 'w') as local_file:
            local_file.write(log)
        s3.upload_file(Filename=log_file, Key='archive/log.txt')

        ## report success
        print(color('Full log upload successful!', Colors.green))
    except:
        print(background('Full log upload failed!', Colors.red))

## functions for web scraping

def dl_file(url, dir_parent, dir_file, file, ext, uuid, user=False, rand_url=False, verify=True, unzip=False, ab_json_to_csv=False, mb_json_to_csv=False):
    """Download file (generic).

    Used to download most file types (when Selenium is not required). Some files are handled with file-specific code:

    - unzip=True and file='13100781' has unique code.
    - Each instance of ab_json_to_csv=True has unique code.
    - Each instance of mb_json_to_csv=True has unique code.

    Parameters:
    url (str): URL to download file from.
    dir_parent (str): The parent directory. Example: 'other/can'.
    dir_file (str): The file directory ('epidemiology-update').
    file (str): Output file name (excluding extension). Example: 'covid19'
    ext (str): Extension of the output file.
    uuid (str): The UUID of the dataset.
    user (bool): Should the request impersonate a normal browser? Needed to access some data. Default: False.
    rand_url(bool): Should the URL have a number appended as a parameter to prevent caching? Default: False.
    verify (bool): If False, requests will skip SSL verification. Default: True.
    unzip (bool): If True, this file requires unzipping. Default: False.
    ab_json_to_csv (bool): If True, this is an Alberta JSON file embedded in a webpage that should be converted to CSV. Default: False.
    mb_json_to_csv (bool): If True, this is a Manitoba JSON file that that should be converted to CSV. Default: False.

    """
    global mode, download_log, success, failure, failure_uuid, prefix_root

    ## set names with timestamp and file ext
    name = file + '_' + get_datetime('America/Toronto').strftime('%Y-%m-%d_%H-%M')
    full_name = os.path.join(dir_parent, dir_file, name + ext)

    ## download file
    try:

        ## add no-cache headers
        headers = {"Cache-Control": "no-cache", "Pragma": "no-cache"}

        ## some websites will reject the request unless you look like a normal web browser
        ## user is True provides a normal-looking user agent string to bypass this
        if user is True:
            headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:66.0) Gecko/20100101 Firefox/66.0"

        ## add random number to url to prevent caching, if requested
        if rand_url is True:
            url = url + "?randNum=" + str(int(datetime.now().timestamp()))

        ## request URL
        req = requests.get(url, headers=headers, verify=verify)

        ## check if request was successful
        if not req.ok:
            ## print failure
            print(background('Error downloading: ' + full_name, Colors.red))
            failure+=1
            failure_uuid.append(uuid)
            ## write failure to log message
            download_log = download_log + 'Failure: ' + full_name + '\n'
        ## successful request: if mode == test, print success and end
        elif mode == 'test':
            ## print success and write to log
            download_log = download_log + 'Success: ' + full_name + '\n'
            print(color('Test download successful: ' + full_name, Colors.green))
            success+=1
        ## successful request: mode == prod, upload file
        else:
            if unzip:
                ## unzip data
                tmpdir = tempfile.TemporaryDirectory()
                zpath = os.path.join(tmpdir.name, 'zip_file.zip')
                with open(zpath, mode='wb') as local_file:
                    local_file.write(req.content)
                with ZipFile(zpath, 'r') as zip_file:
                    zip_file.extractall(tmpdir.name)
                f_path = os.path.join(tmpdir.name, file + ext)
                if file == '13100781':
                    ## read CSV (informative columns only)
                    data = pd.read_csv(f_path, usecols=['REF_DATE', 'Case identifier number', 'Case information', 'VALUE'])
                    ## save original order of column values
                    col_order = data['Case information'].unique()
                    ## pivot long to wide
                    data = data.pivot(index=['REF_DATE', 'Case identifier number'], columns='Case information', values='VALUE').reset_index()
                    ## use original column order
                    data = data[['REF_DATE', 'Case identifier number'] + col_order.tolist()]
                    ## write CSV
                    data.to_csv(f_path, index=None, quoting=csv.QUOTE_NONNUMERIC)
            elif ab_json_to_csv:
                ## for Alberta JSON data only: extract JSON from webpage, convert JSON to CSV and save as temporary file
                tmpdir = tempfile.TemporaryDirectory()
                f_path = os.path.join(tmpdir.name, file + ext)
                data = re.search("(?<=\"data\"\:)\[\[.*\]\]", req.text).group(0)
                if url == "https://www.alberta.ca/maps/covid-19-status-map.htm":
                    data = BeautifulSoup(data, features="html.parser")
                    data = data.get_text() # strip HTML tags
                    ## this regex may need some tweaking if measures column changes in the future
                    data = re.sub("<\\\/a><\\\/li><\\\/ul>", "", data) # strip remaining tags
                    data = re.sub("(?<=\") ", "", data) # strip whitespace
                    data = re.sub(" (?=\")", "", data) # strip whitespace
                    data = pd.read_json(data).transpose()
                    data = data.rename(columns={0: "", 1: "Region name", 2: "Measures", 3: "Active case rate (per 100,000 population)", 4: "Active cases", 5: "Population"})
                elif url == "https://www.alberta.ca/schools/covid-19-school-status-map.htm":
                    data = re.sub(',"container":.*', "", data) # strip remaining tags
                    data = pd.read_json(data).transpose()
                    data = data.rename(columns={0: "", 1: "Region name", 2: "School status", 3: "Schools details", 4: "num_ord"})
                    data['num_ord'] = data['num_ord'].astype(str).astype(int) # convert to int
                    data[''] = data[''].astype(str).astype(int) # convert to int
                    data = data.sort_values(by=['num_ord', '']) # sort ascending by num_ord and first column (like CSV output on website)
                data = data.to_csv(None, quoting=csv.QUOTE_ALL, index=False) # to match website output: quote all lines, don't terminate with new line
                with open(f_path, 'w') as local_file:
                    local_file.write(data[:-1])
            elif mb_json_to_csv:
                ## for Manitoba JSON data only: convert JSON to CSV and save as temporary file
                tmpdir = tempfile.TemporaryDirectory()
                f_path = os.path.join(tmpdir.name, file + ext)
                data = pd.json_normalize(json.loads(req.content)['features'])
                data.columns = data.columns.str.lstrip('attributes.') # strip prefix
                ## replace timestamps with actual dates
                if 'Date' in data.columns:
                    data.Date = pd.to_datetime(data.Date / 1000, unit='s').dt.date
                data.to_csv(f_path, index=None)
            else:
                ## all other data: write contents to temporary file
                tmpdir = tempfile.TemporaryDirectory()
                f_path = os.path.join(tmpdir.name, file + ext)
                with open(f_path, mode='wb') as local_file:
                    local_file.write(req.content)
            ## upload file
            s3_dir = os.path.join(dir_parent, dir_file)
            upload_file(full_name, f_path, uuid, s3_dir=s3_dir, s3_prefix=prefix_root)
    except Exception as e:
        ## print failure
        print(e)
        print(background('Error downloading: ' + full_name, Colors.red))
        failure+=1
        failure_uuid.append(uuid)
        ## write failure to log message
        download_log = download_log + 'Failure: ' + full_name + '\n'

def load_webdriver(tmpdir, user=False):
    """Load Chromium headless webdriver for Selenium.

    Parameters:
    tmpdir (TemporaryDirectory): A temporary directory for saving files downloaded by the headless browser.
    user (bool): Should the request impersonate a normal browser? Needed to access some data. Default: False.
    """
    options = Options()
    options.binary_location = os.environ['CHROME_BIN']
    options.add_argument("--headless")
    options.add_argument("--disable-dev-shm-usage")
    options.add_argument("--no-sandbox")
    prefs = {'download.default_directory' : tmpdir.name}
    options.add_experimental_option('prefs', prefs)
    if user:
        options.add_argument("user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:66.0) Gecko/20100101 Firefox/66.0")
    driver = webdriver.Chrome(executable_path=os.environ['CHROMEDRIVER_BIN'], options=options)
    return driver

def click_xpath(driver, wait, xpath):
    element = WebDriverWait(driver, timeout=wait).until(
        EC.element_to_be_clickable((By.XPATH, xpath)))
    element.click()
    return driver

def click_linktext(driver, wait, text):
    element = WebDriverWait(driver, timeout=wait).until(
        EC.element_to_be_clickable((By.LINK_TEXT, text)))
    element.click()
    return driver

def html_page(url, dir_parent, dir_file, file, ext, uuid, user=False, js=False, wait=None):
    """Save HTML of a webpage.

    Parameters:
    url (str): URL to screenshot.
    dir_parent (str): The parent directory. Example: 'other/can'.
    dir_file (str): The file directory ('epidemiology-update').
    ext (str): Extension of the output file.
    uuid (str): The UUID of the dataset.
    user (bool): Should the request impersonate a normal browser? Needed to access some data. Default: False.
    js (bool): Is the HTML source rendered by JavaScript?
    wait (int): Used only if js = True. Time in seconds that the function should wait for the page to render. If the time is too short, the source code may not be captured.

    """
    global mode, download_log, success, failure, prefix_root

    ## set names with timestamp and file ext
    name = file + '_' + get_datetime('America/Toronto').strftime('%Y-%m-%d_%H-%M')
    full_name = os.path.join(dir_parent, dir_file, name + ext)

    ## download file
    try:
        ## create temporary directory
        tmpdir = tempfile.TemporaryDirectory()

        ## load webdriver
        driver = load_webdriver(tmpdir, user=user)

        ## load page
        driver.get(url)

        ## special processing
        try:
            if uuid == '9ed0f5cd-2c45-40a1-94c9-25b0c9df8f48':
                # show other figure in tabset
                time.sleep(wait) # allow first figure to load
                driver = click_linktext(driver, wait, 'Tests by Specimen Collection Date') # ID is dynamic
            elif uuid == '66fbe91e-34c0-4f7f-aa94-cf6c14db0158':
                # wait for tab link to be clickable then click
                driver = click_xpath(driver, wait, '/html/body/div[1]/nav/div/ul/li[2]/a')
                # time.sleep(wait); driver.find_element_by_id('complete').get_attribute('innerHTML') # test
            elif uuid == '391d177d-1ea8-45ac-bca4-d9f86733c253':
                # wait for tab link to be clickable then click
                driver = click_xpath(driver, wait, '/html/body/div[1]/nav/div/ul/li[3]/a')
                # time.sleep(wait); driver.find_element_by_id('Title2').get_attribute('innerHTML') # test
            elif uuid == 'effdfd82-7c59-4f49-8445-f1f8f73b6dc2':
                # wait for tab link to be clickable then click
                driver = click_xpath(driver, wait, '/html/body/div[1]/nav/div/ul/li[4]/a')
                driver = click_xpath(driver, wait, '/html/body/div[1]/nav/div/ul/li[4]/ul/li[1]/a')
                # whole population coverage
                driver = click_xpath(driver, wait, '//*[@id="all"]')
                # show all data tables
                elements = driver.find_elements_by_link_text('Data Table')
                for element in elements:
                    element.click()
                # time.sleep(wait); driver.find_element_by_id('VCTitle2').get_attribute('innerHTML') # test
            elif uuid == '454de458-f7b4-4814-96a6-5a426f8c8c60':
                # wait for tab link to be clickable then click
                driver = click_xpath(driver, wait, '/html/body/div[1]/nav/div/ul/li[4]/a')
                driver = click_xpath(driver, wait, '/html/body/div[1]/nav/div/ul/li[4]/ul/li[2]/a')
                # time.sleep(wait); driver.find_element_by_id('VCTitle').get_attribute('innerHTML') # test
        ## prints exception but still proceeds (for now)
        except Exception as e:
            ## print exception
            print(e)

        ## save HTML of webpage
        f_path = os.path.join(tmpdir.name, file + ext)
        if js:
            time.sleep(wait) # complete page load
        with open(f_path, 'w') as local_file:
            local_file.write(driver.page_source)

        ## verify download
        if not os.path.isfile(f_path):
            ## print failure
            print(background('Error downloading: ' + full_name, Colors.red))
            failure+=1
            failure_uuid.append(uuid)
            ## write failure to log message
            download_log = download_log + 'Failure: ' + full_name + '\n'
        ## successful request: if mode == test, print success and end
        elif mode == 'test':
            ## print success and write to log
            download_log = download_log + 'Success: ' + full_name + '\n'
            print(color('Test download successful: ' + full_name, Colors.green))
            success+=1
        ## successful request: mode == prod, prepare files for data upload
        else:
            ## upload file
            s3_dir = os.path.join(dir_parent, dir_file)
            upload_file(full_name, f_path, uuid, s3_dir=s3_dir, s3_prefix=prefix_root)

        ## quit webdriver
        driver.quit()
    except Exception as e:
        ## print failure
        print(e)
        print(background('Error downloading: ' + full_name, Colors.red))
        failure+=1
        failure_uuid.append(uuid)
        ## write failure to log message
        download_log = download_log + 'Failure: ' + full_name + '\n'

def ss_page(url, dir_parent, dir_file, file, ext, uuid, user=False, wait=5, width=None, height=None):
    """Take a screenshot of a webpage.

    By default, Selenium attempts to capture the entire page.

    Parameters:
    url (str): URL to screenshot.
    dir_parent (str): The parent directory. Example: 'other/can'.
    dir_file (str): The file directory ('epidemiology-update').
    ext (str): Extension of the output file.
    uuid (str): The UUID of the dataset.
    user (bool): Should the request impersonate a normal browser? Needed to access some data. Default: False.
    wait (int): Time in seconds that the function should wait. Should be > 0 to ensure the entire page is captured.
    width (int): Width of the output screenshot. Default: None. If not set, the function attempts to detect the maximum width.
    height (int): Height of the output screenshot. Default: None. If not set, the function attempts to detect the maximum height.

    """
    global mode, download_log, success, failure, prefix_root

    ## set names with timestamp and file ext
    name = file + '_' + get_datetime('America/Toronto').strftime('%Y-%m-%d_%H-%M')
    full_name = os.path.join(dir_parent, dir_file, name + ext)

    ## download file
    try:
        ## create temporary directory
        tmpdir = tempfile.TemporaryDirectory()

        ## load webdriver
        driver = load_webdriver(tmpdir, user)

        ## load page and wait
        driver.get(url)
        time.sleep(wait) # wait for page to load

        ## take screenshot
        f_path = os.path.join(tmpdir.name, file + ext)

        ## get total width of the page if it is not set by the user
        if width is None:
            width = driver.execute_script('return document.body.parentNode.scrollWidth')
        ## get total height of the page if it is not set by the user
        if height is None:
            height = driver.execute_script('return document.body.parentNode.scrollHeight')
        ## set window size
        driver.set_window_size(width, height)
        ## take screenshot (and don't stop the script if it fails)
        try:
            driver.find_element_by_tag_name('body').screenshot(f_path) # remove scrollbar

            ## verify screenshot
            if not os.path.isfile(f_path):
                ## print failure
                print(background('Error downloading: ' + full_name, Colors.red))
                failure+=1
                failure_uuid.append(uuid)
                ## write failure to log message if mode == prod
                if mode == 'prod':
                    download_log = download_log + 'Failure: ' + full_name + '\n'
            elif mode == 'test':
                ## print success and write to log
                download_log = download_log + 'Success: ' + full_name + '\n'
                print(color('Test download successful: ' + full_name, Colors.green))
                success+=1
            else:
                ## upload file
                s3_dir = os.path.join(dir_parent, dir_file)
                upload_file(full_name, f_path, uuid, s3_dir=s3_dir, s3_prefix=prefix_root)
        except Exception as e:
            ## print exception
            print(e)
            ## print failure
            print(background('Error downloading: ' + full_name, Colors.red))
            failure+=1
            failure_uuid.append(uuid)
            ## write failure to log message if mode == prod
            if mode == 'prod':
                download_log = download_log + 'Failure: ' + full_name + '\n'

        ## quit webdriver
        driver.quit()
    except Exception as e:
        ## print failure
        print(e)
        print(background('Error downloading: ' + full_name, Colors.red))
        failure+=1
        failure_uuid.append(uuid)
        ## write failure to log message if mode == prod
        if mode == 'prod':
            download_log = download_log + 'Failure: ' + full_name + '\n'

## indexing

def create_index(url_base, bucket, aws_id, aws_key):
    """ Create an index of files in datasets.json stored in the S3 bucket.

    Parameters:
    url_base (str): The base URL to the S3 bucket, used to construct file URLs.
    bucket (str): Name of Amazon S3 bucket.
    aws_id (str): ID for AWS.
    aws_key (str): Key for AWS.

    """
    global s3, prefix_root

    ## temporarily disable pandas chained assignment warning
    pd_option = pd.get_option('chained_assignment') # save previous value
    pd.set_option('chained_assignment', None) # disable

    ## load datasets.json
    with open('datasets.json') as json_file:
        datasets = json.load(json_file)

    ## convert datasets into single dictionary
    ds = {} # create empty dictionary
    for a in datasets: # active and inactive
        for d in datasets[a]:
            for i in range(len(datasets[a][d])):
                ds[datasets[a][d][i]['uuid']] = datasets[a][d][i]

    ## prepare paginator for list of all files in the archive
    paginator = boto3.client(
        's3',
        aws_access_key_id=aws_id,
        aws_secret_access_key=aws_key).get_paginator('list_objects_v2')
    pages = paginator.paginate(Bucket=bucket, Prefix=prefix_root)

    ## create inventory of files in the archive
    inv = []
    for page in pages:
        for obj in page['Contents']:
            inv.append([obj['Key'], obj['Size'], obj['ETag']])
    inv = pd.DataFrame(inv, columns = ['file_path', 'file_size', 'file_etag'])

    ## process inventory
    # calculate other columns
    inv['dir_parent'] = inv['file_path'].apply(lambda x: os.path.dirname(x).split('/')[1:-1])
    inv['dir_parent'] = inv['dir_parent'].apply(lambda x: '/'.join(x))
    inv['dir_file'] = inv['file_path'].apply(lambda x: os.path.dirname(x).split('/')[-1])
    inv['file_name'] = inv['file_path'].apply(lambda x: os.path.basename(x))
    inv['file_timestamp'] = inv['file_name'].str.extract('(?<=_)(\d{4}-\d{2}-\d{2}_\d{2}-\d{2}).*$', expand=True)
    inv['file_date'] = pd.to_datetime(inv['file_timestamp'], format='%Y-%m-%d_%H-%M').dt.date
    inv['file_url'] = url_base + inv['file_path']
    # initialize other columns
    inv['file_date_true'] = inv['file_date'] # set initial values for true date
    inv['file_etag_duplicate'] = np.nan
    # remove directories, log files and supplementary files
    inv = inv[inv['file_name'] != ''] # remove directories
    inv = inv[inv['dir_file'] != prefix_root] # remove log files (stored in root)
    inv = inv[inv['dir_file'] != 'supplementary'] # remove supplementary files
    # keep only necessary columns and reorder
    inv = inv[['dir_parent', 'dir_file', 'file_name', 'file_timestamp', 'file_date', 'file_date_true', 'file_size', 'file_etag', 'file_etag_duplicate', 'file_url']]
    # sort
    inv = inv.sort_values(by=['dir_parent', 'dir_file', 'file_timestamp'])

    ## initialize index and add UUID column
    ind = pd.DataFrame(columns=inv.columns)
    ind.insert(0, 'uuid', '')

    ## calculate true dates and etag duplicates - loop through each dataset
    for key in ds:
        d_p = ds[key]['dir_parent']
        d_f = ds[key]['dir_file']
        # get data
        d = inv[(inv['dir_parent'] == d_p) & (inv['dir_file'] == d_f)]
        # if no data, skip to next dataset
        # (this occurs if a dataset was recently added but has not yet been archived)
        if len(d) == 0:
            continue
        # check if there are multiple hashes on the first date of data
        d_first_date = d[d['file_date'] == d['file_date'].min()].drop_duplicates(['file_etag'])
        if (len(d_first_date) > 1):
            # if there multiple hashes on the first date, assume the earliest file is actually from the previous date
            d.loc[d['file_name'] == d_first_date.iloc[0]['file_name'], 'file_date_true'] = d.loc[d['file_name'] == d_first_date.iloc[0]['file_name'], 'file_date'] - timedelta(days=1)
        # generate list of all possible dates: from first true date to last true date
        d_dates_seq = pd.date_range(d['file_date_true'].min(), d['file_date'].max()).tolist()
        # generate list of all dates in the dataset
        d_dates = d['file_date_true'].unique().tolist()
        # are any expected dates are missing?
        d_dates_missing = np.setdiff1d(d_dates_seq, d_dates)
        if (len(d_dates_missing) > 0):
            # if there are any missing dates, check if there are multiple hashes in the following day
            for j in d_dates_missing:
                d_dates_next = d[d['file_date_true'] == j + timedelta(days=1)].drop_duplicates(['file_etag'])
                if len(d_dates_next) > 1:
                    # if there are more than 0 or 1 hashes on the previous date, assume the earliest hash actually corresponds to the missing day
                    d.loc[d['file_name'] == d_dates_next.iloc[0]['file_name'], 'file_date_true'] = d.loc[d['file_name'] == d_dates_next.iloc[0]['file_name'], 'file_date_true'] - timedelta(days=1)
        # using true date, keep only the final hash of each date ('definitive file' for that date)
        d = d.drop_duplicates(['file_date_true'], keep='last')
        # using hash, mark duplicates appearing after the first instance (e.g., duplicate hashes of Friday value for weekend versions of files updated only on weekdays)
        d['file_etag_duplicate'] = d['file_etag'].duplicated()
        # mark duplicates using 1 and 0 rather than True and False
        d['file_etag_duplicate'] = np.where(d['file_etag_duplicate']==True, 1, 0)
        # finally, add UUID
        d['uuid'] = key
        # save modified index
        ind = ind.append(d)
        # print progress
        print(d_p + '/' + d_f)

    ## reset pandas chained assignment warning option
    pd.set_option('chained_assignment', pd_option) # reset

    ## return index
    return(ind)

def write_index(ind, file_path=None):
    """ Write index locally or upload to Amazon S3.

    If file_path is not provided, the index is uploaded to Amazon S3.

    Parameters:
    index: The index returned by create_index().
    file_path (str): Optional. Path to write file locally.

    """
    global s3, prefix_root

    if file_path is None:
        print('Uploading file index...')
        try:
            ## write file index temporarily and upload to Amazon S3
            tmpdir = tempfile.TemporaryDirectory()
            file_index = os.path.join(tmpdir.name, 'file_index.csv')
            ind.to_csv(file_index, index=False)
            s3.upload_file(Filename=file_index, Key=prefix_root + '/file_index.csv')
            ## report success
            print(color('File index upload successful!', Colors.green))
        except:
            ## report failure
            print(background('File index upload failed!', Colors.red))
    else:
        print('Writing file index...')
        try:
            ## write file index
            ind.to_csv(file_index, index=False)
            ## report success
            print(color('File index upload successful!', Colors.green))
        except:
            ## report failure
            print(background('File index upload failed!', Colors.red))

reason: import numpy


text:
import copy
import logging

import numpy as np

from eit_model.data import EITImage
logger = logging.getLogger(__name__)

def greit_filter(image: EITImage, threshold: float = None, div:float=4.0) -> EITImage:
    image_n = copy.deepcopy(image)
    image_n.data = image.data.copy()
    # print(f"{image_n.data}")
    # print(f"{image.data}")

    # TODO test on normalized data and

    #
    #
    # not normalized data!!

    threshold_max = max(image_n.data)/div
    threshold_min = -max(-image_n.data)/div
    # logger.debug(f"{threshold_max=}{threshold_min=}")
    mask_max = image_n.data > threshold_max
    # logger.debug(f"{mask_max=}")
    image_n.data[mask_max] = 1
    mask_min = image_n.data < threshold_min
    # logger.debug(f"{mask_min=}")
    image_n.data[mask_min] = -1
    image_n.data[~np.logical_or(mask_max, mask_min)] = 0
    # logger.debug(f"{~np.logical_or(mask_max, mask_min)=}")
    return image_n


text:
import os,sys, trimesh, json, pyrender, glob
import numpy as np
import random
import PIL
import matplotlib.pyplot as plt
from tqdm import tqdm
from utils.networks import read_pfm
os.environ['PYOPENGL_PLATFORM'] = 'egl'

def save_camera(filename, pose, K):
    file = open(filename, "wb")

    file.write('extrinsic\n'.encode('utf-8'))
    posetxt = '%f %f %f %f\n%f %f %f %f\n%f %f %f %f\n0 0 0 1\n' % (pose[0,0],pose[0,1],pose[0,2],pose[0,3],
                                                                  pose[1,0],pose[1,1],pose[1,2],pose[1,3],
                                                                  pose[2,0],pose[2,1],pose[2,2],pose[2,3])
    file.write(posetxt.encode('utf-8'))
    file.write('\n'.encode('utf-8'))
    file.write('intrinsic\n'.encode('utf-8'))
    Ktxt = '%f 0 %f\n0 %f %f\n0 0 1\n'%(K[0,0],K[0,2],K[1,1],K[1,2])
    file.write(Ktxt.encode('utf-8'))

    file.close()

def save_pfm(filename, image, scale=1):
    file = open(filename, "wb")
    color = None

    image = np.flipud(image)

    if image.dtype.name != 'float32':
        raise Exception('Image dtype must be float32.')

    if len(image.shape) == 3 and image.shape[2] == 3:  # color image
        color = True
    elif len(image.shape) == 2 or len(image.shape) == 3 and image.shape[2] == 1:  # greyscale
        color = False
    else:
        raise Exception('Image must have H x W x 3, H x W x 1 or H x W dimensions.')

    file.write('PF\n'.encode('utf-8') if color else 'Pf\n'.encode('utf-8'))
    file.write('{} {}\n'.format(image.shape[1], image.shape[0]).encode('utf-8'))

    endian = image.dtype.byteorder

    if endian == '<' or endian == '=' and sys.byteorder == 'little':
        scale = -scale

    file.write(('%f\n' % scale).encode('utf-8'))

    image.tofile(file)
    file.close()

def save_minmax(filename, min, max):
    file = open(filename, "wb")
    file.write("{} {}\n".format(min, max).encode('utf-8'))
    file.close()

def toGLmat(camera_pose):
    yz_flip = np.eye(4, dtype=np.float32)
    #yz_flip[2, 2] = -1 # , yz_flip[2, 2], -1
    camera_pose = yz_flip.dot(camera_pose.T)

    return camera_pose.T

def render_depth(scene, camera_pose, intrinsic, light_pose, size=(256, 256)):
    # campose: cam to world
    camera_pose = toGLmat(camera_pose)
    camera = pyrender.IntrinsicsCamera(intrinsic[0,0], intrinsic[1,1], intrinsic[0, 2],
                                       intrinsic[1, 2], znear=0.05, zfar=100.0)
    scene.add(camera, pose=camera_pose)

    r = pyrender.OffscreenRenderer(size[0], size[1])
    color, depth = r.render(scene)
    r.delete()

    camera_node = list(scene.get_nodes(obj=camera))[0]

    scene.remove_node(camera_node)

    # to opencv format
#     camera_pose[:3,[1,2]] = -camera_pose[:3,[1,2]]
#     camera_pose = np.linalg.inv(camera_pose)
    return color, depth

def mesh_flip_zy(mesh):
    mesh.vertices[:,[1,2]] = mesh.vertices[:,[2,1]]
    mesh.vertices[:,[2]]   = -mesh.vertices[:,[2]]
    return mesh

def find_nearestCam(position, N=10):
    nearest = []
    for p in position:
        nearest.append(np.argsort(np.linalg.norm(position - p, axis=1))[:N+1])
    return np.stack(nearest, axis=0)

def _lookat(eye, center):
    direction = eye - center
    unit_direction = direction / np.linalg.norm(direction)
    phi = np.arccos(unit_direction[2])
    theta = np.arccos(unit_direction[0] / np.sqrt(1 - unit_direction[2] ** 2))
    T = np.array([
        [np.sin(theta), np.cos(theta) * np.cos(phi), np.cos(theta) * np.sin(phi), eye[0]],
        [-np.cos(theta), np.sin(theta) * np.cos(phi), np.sin(theta) * np.sin(phi), eye[1]],
        [0, -np.sin(phi), np.cos(phi), eye[2]],
        [0, 0, 0, 1]
    ])
    return T

from math import sin, cos, pi
def _getSphericalPosition(r, theta, phi, arg=True):
    if arg:
        theta = theta / 180 * pi
        phi = phi / 180 * pi
    x = r * cos(theta) * sin(phi)
    y = r * sin(theta) * sin(phi)
    z = r * cos(phi)
    return np.array([x,y,z])

def main():
    dataset_dir = '../portraits_data/mesh'
    output_dir = './rendered_data'

    mesh_num = 100
    image_num = 100
    W, H = 256, 256
    N = 10 # neighbor number
    center = np.array([0,0,0])
    scale = 0.2
    radius = 2
    focal = 400 #1111.111
    intrinsic = np.array([[focal,0,W/2],[0,focal,H/2],[0,0,1]])
    #------------------------------------------------------

    os.makedirs(f'{output_dir}/cams', exist_ok=True)
    os.makedirs(f'{output_dir}/images', exist_ok=True)
    os.makedirs(f'{output_dir}/pairs', exist_ok=True)
    os.makedirs(f'{output_dir}/depths', exist_ok=True)

    for k in range(1, mesh_num + 1):
        data_path = os.path.join(dataset_dir, str(k // 1000),'%05d.obj' % k)
        if not os.path.isfile(data_path): # Some of the orders are missing/deleted
            continue
        mesh = trimesh.load(data_path)
        mesh = mesh_flip_zy(mesh)

        # Normalize model points to to [-1,1] with maximun range of xyz
        length = max(mesh.extents)
        center = np.mean(mesh.vertices,axis=0) - np.array([0, 0, 0.1])
        mesh.apply_translation(-center)
        mesh.apply_scale(2 / length)

        Mesh = pyrender.Mesh.from_trimesh(mesh)
        scene = pyrender.Scene(ambient_light=[1.0, 1.0, 1.0])
        scene.add(Mesh)

        poses = []
        for _ in range(image_num):
            pos = _getSphericalPosition(radius, theta=random.random() * 180, phi=60 + random.random() * 70)
            center_noise = np.array([random.uniform(-scale, scale),random.uniform(-scale, scale),random.uniform(-scale, scale)])
            new_center = center + center_noise
            # camera pose in openGL
            T = _lookat(pos, new_center)
            poses.append(T)
        poses = np.stack(poses)

        # add light
    #     for l in range(5):
    #         light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=2.2)
    #         scene.add(light, pose=poses[l])

        opengltocv = np.eye(4)
        opengltocv[[1,2],[1,2]] = -1

        # Save poses, images and depths
        for i in tqdm(range(image_num), desc='Progress for Mesh #%05d' % k):
            pose = poses[i]
            color, depth = render_depth(scene=scene, camera_pose=pose, intrinsic=intrinsic, light_pose=poses[0], size=[W,H])
            depth_min, depth_max = np.min([x for x in depth.flatten() if x > 0]), np.max(depth)
            relative_depth = (depth - depth_min) / (depth_max - depth_min)
            pose_cv = np.linalg.inv(pose @ opengltocv)

            # Save camera prameters
            camera_file_path = os.path.join(output_dir, 'cams', 'scan%05d' % k)
            os.makedirs(camera_file_path, exist_ok=True)
            camera_file_name = os.path.join(camera_file_path,'%03d_cam.txt' % i)
            save_camera(camera_file_name, pose_cv, intrinsic)

            # Save relative depth
            depth_file_path = os.path.join(output_dir, 'depths', 'scan%05d' % k)
            os.makedirs(depth_file_path, exist_ok=True)
            depth_file_name = os.path.join(depth_file_path,'depth_map_%03d.pfm' % i)
            save_pfm(depth_file_name, relative_depth)

            # Save minmax depth
            minmax_file_path = os.path.join(output_dir, 'minmaxs', 'scan%05d' % k)
            os.makedirs(minmax_file_path, exist_ok=True)
            minmax_file_name = os.path.join(minmax_file_path,'minmax_map_%03d.txt' % i)
            save_minmax(minmax_file_name, depth_min, depth_max)

            # Save rgba
            color_file_path = os.path.join(output_dir, 'images','scan%05d' % k)
            os.makedirs(color_file_path, exist_ok=True)
            color_file_name = os.path.join(color_file_path,'%03d.png' % i)
            mask = (depth > 0).reshape(H,W,1) * 255
            rgba = np.concatenate([color,mask],axis=2)
            rgba = rgba.astype(np.uint8)
            img = PIL.Image.fromarray(rgba, 'RGBA')
            img.save(color_file_name)

        neighbors = find_nearestCam(poses[:,:3,3], N)
        pairs_path = os.path.join(output_dir, 'pairs', 'scan%05d' % k)
        #np.savetxt(f'{output_dir}/pairs/scan{}.txt'.format(k),neighbors,fmt='%d')
        np.savetxt(pairs_path, neighbors, fmt='%d')

if __name__ == '__main__':
    main()

max_stars_repo_name: shcSteven/mvsnerf

max_issues_repo_name: shcSteven/mvsnerf

max_forks_repo_name: shcSteven/mvsnerf


text:
import tempfile
from os.path import join
from fv3fit.emulation.keras import save_model

import fv3fit.emulation.models
import numpy as np
import pytest
import tensorflow as tf
from fv3fit._shared import SliceConfig
from fv3fit.emulation import transforms
from fv3fit.emulation.layers import ArchitectureConfig
from fv3fit.emulation.layers.architecture import _ARCHITECTURE_KEYS
from fv3fit.emulation.models import MicrophysicsConfig, TransformedModelConfig
from fv3fit.emulation.zhao_carr_fields import Field

def _get_data(shape):

    num = int(np.prod(shape))
    return np.arange(num).reshape(shape).astype(np.float32)

def _get_tensor(shape):
    return tf.convert_to_tensor(_get_data(shape))

def test_Config():

    config = MicrophysicsConfig(
        input_variables=["dummy_in"], direct_out_variables=["dummy_out"]
    )
    assert config.input_variables == ["dummy_in"]
    assert config.direct_out_variables == ["dummy_out"]

def test_Config_from_dict():
    config = MicrophysicsConfig.from_dict(
        dict(input_variables=["dummy_in"], direct_out_variables=["dummy_out"],)
    )
    assert config.input_variables == ["dummy_in"]
    assert config.direct_out_variables == ["dummy_out"]

def test_Config_from_dict_selection_map_sequences():
    config = MicrophysicsConfig.from_dict(
        dict(selection_map=dict(dummy=dict(start=0, stop=2, step=1)))
    )
    assert config.selection_map["dummy"].slice == slice(0, 2, 1)

def test_Config_asdict():
    sl1_kwargs = dict(start=0, stop=10, step=2)
    sl2_kwargs = dict(start=None, stop=25, step=None)
    sel_map = dict(
        dummy_in=SliceConfig(**sl1_kwargs), dummy_out=SliceConfig(**sl2_kwargs)
    )

    original = MicrophysicsConfig(
        input_variables=["dummy_in"],
        direct_out_variables=["dummy_out"],
        selection_map=sel_map,
    )

    config_d = original.asdict()
    assert config_d["selection_map"]["dummy_in"] == sl1_kwargs
    assert config_d["selection_map"]["dummy_out"] == sl2_kwargs

    result = MicrophysicsConfig.from_dict(config_d)
    assert result == original

def test_Config_build():

    config = MicrophysicsConfig(
        input_variables=["dummy_in"], direct_out_variables=["dummy_out"],
    )

    data = _get_data((20, 5))
    m = {"dummy_in": data, "dummy_out": data}
    model = config.build(m)
    output = model(m)
    assert set(output) == {"dummy_out"}

def test_Config_build_residual_w_extra_tends_out():

    config = MicrophysicsConfig(
        input_variables=["dummy_in"],
        residual_out_variables={"dummy_out1": "dummy_in"},
        tendency_outputs={"dummy_out1": "dummy_out1_tendency"},
        architecture=ArchitectureConfig(name="dense"),
    )

    data = _get_data((20, 5))
    m = {"dummy_out1": data, "dummy_in": data, "dummy_out1_tendency": data}
    model = config.build(m)
    output = model(data)
    assert set(model.output_names) == set(output)
    assert set(output) == {"dummy_out1", "dummy_out1_tendency"}

def test_precip_conserving_config():
    factory = fv3fit.emulation.models.ConservativeWaterConfig()

    one = tf.ones((4, 5))

    data = {v: one for v in factory.input_variables + factory.output_variables}
    model = factory.build(data)
    out = model(data)
    precip = out[factory.fields.surface_precipitation.output_name]
    # scalar outputs need a singleton dimension to avoid broadcasting mayhem
    assert tuple(precip.shape) == (one.shape[0], 1)

def test_precip_conserving_output_variables():
    fields = fv3fit.emulation.models.ZhaoCarrFields(
        cloud_water=fv3fit.emulation.models.Field(input_name="a0", output_name="a"),
        specific_humidity=fv3fit.emulation.models.Field(
            input_name="a1", output_name="b"
        ),
        air_temperature=fv3fit.emulation.models.Field(input_name="a2", output_name="c"),
        surface_precipitation=fv3fit.emulation.models.Field(output_name="d"),
    )
    factory = fv3fit.emulation.models.ConservativeWaterConfig(fields=fields)

    assert set(factory.output_variables) == set("abcd")

def test_precip_conserving_extra_inputs():
    extra_names = "abcdef"
    extras = [fv3fit.emulation.models.Field(input_name=ch) for ch in extra_names]

    factory = fv3fit.emulation.models.ConservativeWaterConfig(
        extra_input_variables=extras
    )
    assert set(extra_names) < set(factory.input_variables)

@pytest.mark.parametrize("arch", _ARCHITECTURE_KEYS)
def test_MicrophysicConfig_model_save_reload(arch):

    config = MicrophysicsConfig(
        input_variables=["field_input"],
        direct_out_variables=["field_output"],
        architecture=ArchitectureConfig(name=arch),
    )

    nlev = 15
    data = tf.random.normal((10, nlev))
    sample = {"field_input": data, "field_output": data}

    model = config.build(sample)

    expected = model(sample)

    with tempfile.TemporaryDirectory() as tmpdir:
        model_path = join(tmpdir, "model.tf")
        model.save(model_path, save_format="tf")
        reloaded = tf.keras.models.load_model(model_path, compile=False)

    result = reloaded(sample)
    np.testing.assert_allclose(
        expected["field_output"], result["field_output"], rtol=2e-5
    )

def test_RNN_downward_dependence():

    config = MicrophysicsConfig(
        input_variables=["field_input"],
        direct_out_variables=["field_output"],
        architecture=ArchitectureConfig(name="rnn-v1", kwargs=dict(channels=16)),
    )

    nlev = 15
    data = tf.random.normal((10, nlev))
    sample = {"field_input": data, "field_output": data}
    profile = data[0:1]

    model = config.build(sample)

    with tf.GradientTape() as g:
        g.watch(profile)
        output = model(profile)

    jacobian = g.jacobian(output["field_output"], profile)[0, :, 0]

    assert jacobian.shape == (nlev, nlev)
    for output_level in range(nlev):
        for input_level in range(nlev):
            sensitivity = jacobian[output_level, input_level]
            if output_level > input_level and sensitivity != 0:
                raise ValueError("Downwards dependence violated")

def test_transformed_model_without_transform():
    field = Field("a_out", "a")
    data = {field.input_name: tf.ones((1, 10)), field.output_name: tf.ones((1, 10))}
    config = TransformedModelConfig(ArchitectureConfig("dense"), [field], 900)
    model = config.build(data)
    out = model(data)
    assert set(out) == {field.output_name}

def test_transformed_model_with_transform():
    field = Field("a_out", "a")
    transformed_field = Field("transform_a_out", "transform_a")
    data = {field.input_name: tf.ones((1, 10)), field.output_name: tf.ones((1, 10))}
    model = TransformedModelConfig(
        ArchitectureConfig("dense"), [transformed_field], 900
    ).build(
        data,
        transform=transforms.PerVariableTransform(
            [
                transforms.TransformedVariableConfig(
                    field.input_name,
                    transformed_field.input_name,
                    transforms.LogTransform(),
                ),
                transforms.TransformedVariableConfig(
                    field.output_name,
                    transformed_field.output_name,
                    transforms.LogTransform(),
                ),
            ]
        ),
    )
    out = model(data)
    assert set(out) == {transformed_field.output_name, field.output_name}

def test_save_and_reload_transformed_model(tmpdir):
    field = Field("a_out", "a")
    input = {field.input_name: tf.ones((1, 10))}
    output = {field.output_name: tf.ones((1, 10))}
    data = {**input, **output}
    config = TransformedModelConfig(ArchitectureConfig("dense"), [field], 900)
    model = config.build(data)
    # this will initialize model.call with a signature requiring all the keys in
    # ``data`` even though `a` is the only "true" input
    model(data)
    path = str(tmpdir) + "/model.tf"
    save_model(model, str(tmpdir))
    loaded = tf.keras.models.load_model(path)
    # will often complain since ``input`` is missing the "a_out" field which was
    # passed to ``model`` above.
    loaded.predict(input)

@pytest.mark.xfail
def test_saved_model_jacobian():
    """
    SimpleRNN saving prevents jacobian calculation due to some internal
    metadata missing after loading. Perhaps a tensorflow version upgrade
    fixes?
    """

    config = MicrophysicsConfig(
        input_variables=["field_input"],
        direct_out_variables=["field_output"],
        architecture=ArchitectureConfig(
            name="rnn-v1-shared-weights", kwargs=dict(channels=16)
        ),
    )

    nlev = 15
    data = tf.random.normal((10, nlev))
    sample = {"field_input": data, "field_output": data}
    profile = data[0:1]

    model = config.build(sample)
    with tempfile.TemporaryDirectory() as tmpdir:
        save_path = join(tmpdir, "model.tf")
        model.save(save_path, save_format="tf")
        loaded_model = tf.keras.models.load_model(save_path)

    with tf.GradientTape() as g:
        g.watch(profile)
        output = loaded_model(profile)

    assert g.jacobian(output["field_output"], profile)