text:
######################################################################
# Alchemical Analysis: An open tool implementing some recommended practices for analyzing alchemical free energy calculations
# Copyright 2011-2015 UC Irvine and the Authors
#
# Authors: Pavel Klimovich, Michael Shirts and David Mobley
#
#This library is free software; you can redistribute it and/or
#modify it under the terms of the GNU Lesser General Public
#License as published by the Free Software Foundation; either
#version 2.1 of the License, or (at your option) any later version.
#
#This library is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
#Lesser General Public License for more details.
#
#You should have received a copy of the GNU Lesser General Public
#License along with this library; if not, see <http://www.gnu.org/licenses/>.
######################################################################

import numpy
from glob import glob # for pathname matching

from . import unixlike       # some implemented unixlike commands

#===================================================================================================
# FUNCTIONS: This is the Sire lambda gradient file parser.
#===================================================================================================

def readDataSire(P):
   """Read in Sire's output files; return nsnapshots, lv, dhdlt, and u_klt."""

   class F:
      """This is the object to be built on the filename."""

      def __init__(self, filename):
         self.filename   = filename
         self.skip_lines = 0  # Number of lines from the top that are to be skipped.
         snap_size       = [] # Time from first two snapshots to determine snapshot's size.

         print("Reading metadata from %-*s" % (len_fstring+1, self.filename+';'), end=' ')
         with open(self.filename,'r') as infile:
            for line in infile:

               if line.startswith('#'):
                  self.skip_lines += 1
                  elements = line.split()
                  if 'lambba_val.val' in elements:
                     self.lv = elements[-1]
                     lv.append(elements[-1:])
               else:
                  snap_size.append(float(line.split()[0]))
                  if len(snap_size) > 1:
                     self.snap_size = numpy.diff(snap_size)[0]
                     break
            equilsnapshots  = int(P.equiltime/self.snap_size)
            self.skip_lines += equilsnapshots
            nsnapshots.append(unixlike.wcPy(infile) + 2 - equilsnapshots)
            print("first %s ps (%s snapshots) will be discarded due to equilibration..." % (P.equiltime, equilsnapshots))

      def loadtxtSire(self, state):
         print("Loading in data from %-*s (state %d) ..." % (len_fstring, self.filename, state))
         try: #Numpy 1.1 removes skiprows
            dhdlt[state, :, :nsnapshots[state]] = numpy.genfromtxt(self.filename, dtype=float, skiprows=self.skip_lines, usecols=1)
         except TypeError:
            dhdlt[state, :, :nsnapshots[state]] = numpy.genfromtxt(self.filename, dtype=float, skip_header=self.skip_lines, usecols=1)
         return

   # Preliminaries I-III: Sort the dhdl.xvg files; read in the header; count up the equilibrated snapshots.
   datafile_tuple = P.datafile_directory, P.prefix, P.suffix
   fs = glob('%s/%s*%s' % datafile_tuple)
   K = len(fs)
   if not K:
      raise SystemExit("\nERROR!\nNo files found within directory '%s' with prefix '%s' and suffix '%s': check your inputs." % datafile_tuple)
   len_fstring = max([len(i) for i in fs])

   n_components = 1
   lv           = []
   nsnapshots   = []
   P.lv_names   = ['']
   fs = [ F(filename) for filename in fs ]
   print('')
   fs = sorted(fs, key=lambda f: f.lv)
   lv, nsnapshots = list(zip(*sorted(zip(lv, nsnapshots))))
   lv = numpy.array(lv, float)

   # Preliminaries IV: Load in equilibrated data.
   maxn  = max(nsnapshots)                                   # maximum number of the equilibrated snapshots from any state
   dhdlt = numpy.zeros([K,n_components,int(maxn)], float)    # dhdlt[k,n,t] is the derivative of energy component n with respect to state k of snapshot t
   u_klt = None

   for nf, f in enumerate(fs):
      f.loadtxtSire(nf)
   print('')

   return nsnapshots, lv, dhdlt, u_klt

ext: py

max_stars_repo_name: VincenzoChen/alchemical-analysis

max_issues_repo_name: VincenzoChen/alchemical-analysis

max_forks_repo_name: VincenzoChen/alchemical-analysis

reason: import numpy


text:
from bagua.torch_api.distributed import BaguaModule
from bagua.torch_api.bucket import BaguaBucket
from bagua.torch_api.tensor import BaguaTensor
from bagua.torch_api.algorithms import Algorithm
from bagua.torch_api.communication import get_backend
from typing import List
import torch
import bagua_core as B

from bagua.torch_api import get_world_size
from bagua.torch_api import allgather_inplace, alltoall_inplace, allgather
import torch.distributed as dist
import torch.multiprocessing as mp
# bit packing compression:
import cupy
from cupy._binary import packing
from torch.utils.dlpack import to_dlpack, from_dlpack

import math

class SignSGDAlgorithm(Algorithm):

    def __init__(self, compression_start: int = 100):
        # warmup not implemented yet
        self.compression_start = compression_start
        self.step_id = 0

    def init_operations(
        self,
        bagua_module: BaguaModule,
        bucket: BaguaBucket,
    ):
        """
        concatenate all the tensors of the bucket (cat_tensor),
        compress the tensor (1bit/sign compression),
        communicate via bagua backend all_to_all,
        decompress the received chunk,
        average the chunk,
        compress the chunk average (1bit/sign compression),
        communicate compressed average via bagua backend all_gather,
        decompress the received tensor,
        remove padding,
        update the bucket tensors
        """
        bucket.clear_ops()

        def signsgd_centeralized_communication(*args):

            def onebit_compression(tensor):
                """
                - Compress tensor by taking the sign of each element and pack each sign-bit into a 8bit array
                Input:
                    - tensor: torch.tensor
                Output:
                    - sign_padding: number of padding bits needed at the last element of 8bit-tensor to fit all signbits
                    - compressed_tensor: torch.tensor (dtype=torch.uint8)
                """
                tensor_size = tensor.numel()
                signs = torch.sign(tensor)
                # if tensor element is 0 choose the sign randomly
                signs[signs == 0] = 2 * \
                    torch.randint_like(signs[signs == 0], low=0, high=2) - 1
                signs_binary = signs.add(1.0).bool()
                signs_binary_dlpack = to_dlpack(signs_binary)
                signs_binary_cupy = cupy.fromDlpack(signs_binary_dlpack)
                packed_size = (tensor_size + 7) // 8
                sign_padding = packed_size * 8 - tensor_size
                compressed_signs = cupy.packbits(signs_binary_cupy)
                compressed_signs = compressed_signs.toDlpack()
                compressed_tensor = from_dlpack(compressed_signs).cuda()

                return sign_padding, compressed_tensor

            def onebit_decompression(tensor):
                """
                - decompress tensor by unpacking the uint8 array and transforming the values: (0 -> -1, 1 -> +1)
                Input:
                    - tensor: torch.tensor (dtype=torch.uint8)
                Output:
                    - decompressed: torch.tensor (dtype=torch.float32)
                """
                tensor_cupy = cupy.fromDlpack(to_dlpack(tensor))
                tensor_cupy_unpacked = cupy.unpackbits(tensor_cupy)
                tensor_unpacked = tensor_cupy_unpacked.toDlpack()
                tensor_unpacked = from_dlpack(tensor_unpacked).cuda()
                decompressed = tensor_unpacked.float().mul(2.0).sub(1.0)
                return decompressed

            # get number of devices
            n_workers = get_world_size()

            cat_tensor = torch.empty(
                0, device='cuda:{}'.format(torch.cuda.current_device()))
            # Concatenate the tensors in the bucket
            for tensor in bucket.tensors:
                clone_tensor = tensor.clone().detach()
                cat_tensor = torch.cat(
                    (cat_tensor, clone_tensor.flatten()))
            sign_padding, compressed_tensor = onebit_compression(
                cat_tensor)
            del cat_tensor

            compressed_tensor_size = compressed_tensor.numel()

            # COMMUNICATION
            # compute the chunk size
            chunk_size = math.ceil(compressed_tensor_size / n_workers)
            # add padding to compressed_tensor, so that it can be used by bagua alltoall

            if (compressed_tensor_size % n_workers != 0):
                chunk_padding = (chunk_size * n_workers) - \
                    compressed_tensor_size
                sign_padding += chunk_padding * 8
                compressed_tensor = torch.cat((compressed_tensor, torch.empty(
                    chunk_padding, dtype=torch.uint8, device='cuda:{}'.format(torch.cuda.current_device()))))

            alltoall_inplace(compressed_tensor)

            decompressed = onebit_decompression(compressed_tensor)

            chunk_decompressed = torch.reshape(
                decompressed, (n_workers, chunk_size*8))
            del decompressed
            # AVERAGE
            chunk_avg = torch.mean(chunk_decompressed, 0)

            _, compressed_chunk_avg = onebit_compression(chunk_avg)
            del chunk_avg
            send_tensor = compressed_chunk_avg

            temp = send_tensor.numel()*n_workers
            recv_tensor = torch.empty(
                temp, dtype=torch.uint8, device='cuda:{}'.format(torch.cuda.current_device()))

            allgather(send_tensor, recv_tensor)
            del send_tensor
            decompressed = onebit_decompression(recv_tensor)
            del recv_tensor

            # UPDATE BUCKET
            # remove padding
            padding = decompressed.numel() - sign_padding
            update_tensor = decompressed[0:padding]
            for tensor in bucket.tensors:
                tensor_size = tensor.numel()
                tensor_shape = tensor.shape
                new_tensor = update_tensor[0:tensor_size]
                new_tensor = torch.reshape(new_tensor, tensor_shape)
                tensor.set_(new_tensor)
                update_tensor = update_tensor[tensor_size:]

            # torch.cuda.empty_cache()

        bucket.append_python_op(signsgd_centeralized_communication)

max_stars_repo_name: schragf/bagua

max_issues_repo_name: schragf/bagua

max_forks_repo_name: schragf/bagua

reason: import cupy,from cupy


text:
from matplotlib import pyplot as plt
import isceio as io
import argparse
import numpy as np

def readInputs():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        'path', help='Path to the SLC.')
    args = parser.parse_args()

    return args

def main():
    args = readInputs()
    print(args.path)
    slc = io.load_stack([args.path])[0]

    plt.imshow(10 * np.log(np.abs(slc)**2),
               origin='lower', cmap=plt.cm.Greys_r, vmin=60, vmax=100)
    plt.show()

main()

max_stars_repo_name: rbiessel/CovSAR

max_issues_repo_name: rbiessel/CovSAR

max_forks_repo_name: rbiessel/CovSAR


text:
'''_____Standard imports_____'''
import numpy as np
import matplotlib.pyplot as plt

'''_____Project imports_____'''
from toolbox.maths import unwrap_phase
from toolbox.loadings import load_data
from toolbox.filters import butter_highpass_filter
from toolbox.plottings import plots_signals

class Spectra(object):

    def __init__(self, data_dir, background_dir = None, ref_dir = None, sample_dir = None):

        self.data_dir = data_dir

        self.background_dir = background_dir

        self.ref_dir = ref_dir

        self.sample_dir = sample_dir

    def load_data(self):
        """ This method serve to load the data, i.e, mirror, darf_ref, dark_not,
        dark_sample.

        """
        self.raw = []

        file = open(self.data_dir,'r')

        for line in file:

            self.raw.append(float(line))

        self.raw = np.array(self.raw)

    def get_phase(self):
        """ This method compute the phase of the processed spectra.

        """
        self.phase = unwrap_phase(self.sub_raw)

        self.phase -= self.phase[0]

    def process_data(self, plot=True):
        """ This method compute the processing of data, i.e,
        background removal + high pass filter.

        """

        self.sub_raw = self.raw
        self.sub_raw -= self.sub_raw[0]
        if self.background_dir:
            self.background = load_data(self.background_dir)
            self.sub_raw += self.background

        if self.sample_dir:
            self.sample = load_data(self.sample_dir)
            self.sub_raw -= self.sample

        if self.ref_dir:
            self.ref = load_data(self.ref_dir)
            self.sub_raw -= self.ref

        self.sub_raw = butter_highpass_filter(self.sub_raw,
                                              cutoff=280,
                                              fs=40000,
                                              order=4)

        if plot:

            plots_signals(self.raw,
                          self.sub_raw,
                          [1],
                          [1],
                          [1])
            """
            plots_signals(self.raw,
                          self.sub_raw,
                          self.ref,
                          self.sample,
                          self.background)"""


text:
# coding=utf-8
# Copyright 2020 The Trax Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Mathematical operations."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl import logging

import numpy as np

import tensorflow.compat.v2 as tf

from trax.tf_numpy.numpy import array_creation
from trax.tf_numpy.numpy import array_methods
from trax.tf_numpy.numpy import arrays
from trax.tf_numpy.numpy import dtypes
from trax.tf_numpy.numpy import utils

def dot(a, b):
  """The dot product of two arrays. See numpy.dot for more details.

  This relies on `tf.tensordot` which does not support types int64 and float64.
  So arrays of those types are "unsafely" cast to int32 and float32.

  Args:
    a: array_like. Could be an ndarray, a Tensor or any object that can
      be converted to a Tensor using `tf.convert_to_tensor`.
    b: array_like. Could be an ndarray, a Tensor or any object that can
      be converted to a Tensor using `tf.convert_to_tensor`.

  Returns:
    An ndarray.
  """
  a, b = array_creation._promote_dtype(a, b)
  if utils.isscalar(a) or utils.isscalar(b):
    a = utils.tensor_to_ndarray(tf.expand_dims(a.data, -1))
    b = utils.tensor_to_ndarray(tf.expand_dims(b.data, -1))
    a_axis = b_axis = -1
  else:
    a_axis = -1
    # TODO(agarwal): handle ndim being None when in graph mode.
    b_axis = -2 if b.ndim > 1 else -1
  # TODO(srbs): When the shape of the output is a scalar e.g. when performing
  # a dot-product of two vectors, numpy returns a scalar object and not an
  # instance of ndarray.

  # tensordot/MatMul does not support int64 and float64 so we manually cast to
  # the compatible types. The conversion may be unsafe.
  # TODO(srbs): Figure out why MatMul does not support larger types.
  output_type = None
  if a.dtype == np.int64:
    logging.warning("Unsafe cast to int32.")
    a = utils.tensor_to_ndarray(tf.cast(a.data, tf.int32))
    b = utils.tensor_to_ndarray(tf.cast(b.data, tf.int32))
    output_type = tf.int64
  elif a.dtype == np.float64:
    logging.warning("Unsafe cast to float32.")
    a = utils.tensor_to_ndarray(tf.cast(a.data, tf.float32))
    b = utils.tensor_to_ndarray(tf.cast(b.data, tf.float32))
    output_type = tf.float64

  result_t = tf.tensordot(a.data, b.data, [[a_axis], [b_axis]])
  if output_type:
    result_t = tf.cast(result_t, output_type)
  return utils.tensor_to_ndarray(result_t)

# TODO(wangpeng): Make element-wise ops `ufunc`s
def _bin_op(tf_fun, a, b, promote=True):
  if promote:
    a, b = array_creation._promote_dtype(a, b)
  else:
    a = array_creation.asarray(a)
    b = array_creation.asarray(b)
  return utils.tensor_to_ndarray(tf_fun(a.data, b.data))

@utils.np_doc(np.add)
def add(x1, x2):
  def add_or_or(x1, x2):
    if x1.dtype == tf.bool:
      assert x2.dtype == tf.bool
      return tf.logical_or(x1, x2)
    return tf.add(x1, x2)
  return _bin_op(add_or_or, x1, x2)

@utils.np_doc(np.subtract)
def subtract(x1, x2):
  return _bin_op(tf.subtract, x1, x2)

@utils.np_doc(np.multiply)
def multiply(x1, x2):
  def mul_or_and(x1, x2):
    if x1.dtype == tf.bool:
      assert x2.dtype == tf.bool
      return tf.logical_and(x1, x2)
    return tf.multiply(x1, x2)
  return _bin_op(mul_or_and, x1, x2)

@utils.np_doc(np.true_divide)
def true_divide(x1, x2):
  def _avoid_float64(x1, x2):
    if x1.dtype == x2.dtype and x1.dtype in (tf.int32, tf.int64):
      x1 = tf.cast(x1, dtype=tf.float32)
      x2 = tf.cast(x2, dtype=tf.float32)
    return x1, x2

  def f(x1, x2):
    if x1.dtype == tf.bool:
      assert x2.dtype == tf.bool
      float_ = dtypes.default_float_type()
      x1 = tf.cast(x1, float_)
      x2 = tf.cast(x2, float_)
    if not dtypes.is_allow_float64():
      # tf.math.truediv in Python3 produces float64 when both inputs are int32
      # or int64. We want to avoid that when is_allow_float64() is False.
      x1, x2 = _avoid_float64(x1, x2)
    return tf.math.truediv(x1, x2)
  return _bin_op(f, x1, x2)

divide = true_divide

@utils.np_doc(np.floor_divide)
def floor_divide(x1, x2):
  def f(x1, x2):
    if x1.dtype == tf.bool:
      assert x2.dtype == tf.bool
      x1 = tf.cast(x1, tf.int8)
      x2 = tf.cast(x2, tf.int8)
    return tf.math.floordiv(x1, x2)
  return _bin_op(f, x1, x2)

@utils.np_doc(np.mod)
def mod(x1, x2):
  def f(x1, x2):
    if x1.dtype == tf.bool:
      assert x2.dtype == tf.bool
      x1 = tf.cast(x1, tf.int8)
      x2 = tf.cast(x2, tf.int8)
    return tf.math.mod(x1, x2)
  return _bin_op(f, x1, x2)

remainder = mod

@utils.np_doc(np.divmod)
def divmod(x1, x2):
  return floor_divide(x1, x2), mod(x1, x2)

@utils.np_doc(np.maximum)
def maximum(x1, x2):
  def max_or_or(x1, x2):
    if x1.dtype == tf.bool:
      assert x2.dtype == tf.bool
      return tf.logical_or(x1, x2)
    return tf.math.maximum(x1, x2)
  return _bin_op(max_or_or, x1, x2)

@utils.np_doc(np.minimum)
def minimum(x1, x2):
  def min_or_and(x1, x2):
    if x1.dtype == tf.bool:
      assert x2.dtype == tf.bool
      return tf.logical_and(x1, x2)
    return tf.math.minimum(x1, x2)
  return _bin_op(min_or_and, x1, x2)

# TODO(wangpeng): support broadcasting and 1-D case
@utils.np_doc(np.matmul)
def matmul(x1, x2):
  return _bin_op(tf.matmul, x1, x2)

@utils.np_doc(np.power)
def power(x1, x2):
  return _bin_op(tf.math.pow, x1, x2)

@utils.np_doc(np.float_power)
def float_power(x1, x2):
  return power(x1, x2)

@utils.np_doc(np.arctan2)
def arctan2(x1, x2):
  return _bin_op(tf.math.atan2, x1, x2)

@utils.np_doc(np.nextafter)
def nextafter(x1, x2):
  return _bin_op(tf.math.nextafter, x1, x2)

@utils.np_doc(np.heaviside)
def heaviside(x1, x2):
  def f(x1, x2):
    return tf.where(x1 < 0, tf.constant(0, dtype=x2.dtype),
                    tf.where(x1 > 0, tf.constant(1, dtype=x2.dtype), x2))
  return _bin_op(f, x1, x2)

@utils.np_doc(np.hypot)
def hypot(x1, x2):
  return sqrt(square(x1) + square(x2))

def _pad_left_to(n, old_shape):
  return (1,) * (n - len(old_shape)) + old_shape

@utils.np_doc(np.kron)
def kron(a, b):
  a, b = array_creation._promote_dtype(a, b)
  ndim = max(a.ndim, b.ndim)
  if a.ndim < ndim:
    a = array_methods.reshape(a, _pad_left_to(ndim, a.shape))
  if b.ndim < ndim:
    b = array_methods.reshape(b, _pad_left_to(ndim, b.shape))
  a_reshaped = array_methods.reshape(a, [i for d in a.shape for i in (d, 1)])
  b_reshaped = array_methods.reshape(b, [i for d in b.shape for i in (1, d)])
  out_shape = tuple(np.multiply(a.shape, b.shape))
  return array_methods.reshape(a_reshaped * b_reshaped, out_shape)

@utils.np_doc(np.outer)
def outer(a, b):
  def f(a, b):
    return tf.reshape(a, [-1, 1]) * tf.reshape(b, [-1])
  return _bin_op(f, a, b)

# This can also be implemented via tf.reduce_logsumexp
@utils.np_doc(np.logaddexp)
def logaddexp(x1, x2):
  amax = maximum(x1, x2)
  delta = x1 - x2
  return array_methods.where(
      isnan(delta),
      x1 + x2,  # NaNs or infinities of the same sign.
      amax + log1p(exp(-abs(delta))))

@utils.np_doc(np.logaddexp2)
def logaddexp2(x1, x2):
  amax = maximum(x1, x2)
  delta = x1 - x2
  return array_methods.where(
      isnan(delta),
      x1 + x2,  # NaNs or infinities of the same sign.
      amax + log1p(exp2(-abs(delta))) / np.log(2))

@utils.np_doc(np.polyval)
def polyval(p, x):
  def f(p, x):
    if p.shape.rank == 0:
      p = tf.reshape(p, [1])
    p = tf.unstack(p)
    # TODO(wangpeng): Make tf version take a tensor for p instead of a list.
    y = tf.math.polyval(p, x)
    # If the polynomial is 0-order, numpy requires the result to be broadcast to
    # `x`'s shape.
    if len(p) == 1:
      y = tf.broadcast_to(y, x.shape)
    return y
  return _bin_op(f, p, x)

@utils.np_doc(np.isclose)
def isclose(a, b, rtol=1e-05, atol=1e-08):
  def f(a, b):
    dtype = a.dtype
    if np.issubdtype(dtype.as_numpy_dtype, np.inexact):
      rtol_ = tf.convert_to_tensor(rtol, dtype)
      atol_ = tf.convert_to_tensor(atol, dtype)
      return tf.math.abs(a - b) <= atol_ + rtol_ * tf.math.abs(b)
    else:
      return a == b
  return _bin_op(f, a, b)

def _tf_gcd(x1, x2):
  def _gcd_cond_fn(x1, x2):
    return tf.reduce_any(x2 != 0)
  def _gcd_body_fn(x1, x2):
    # tf.math.mod will raise an error when any element of x2 is 0. To avoid
    # that, we change those zeros to ones. Their values don't matter because
    # they won't be used.
    x2_safe = tf.where(x2 != 0, x2, tf.constant(1, x2.dtype))
    x1, x2 = (tf.where(x2 != 0, x2, x1),
              tf.where(x2 != 0, tf.math.mod(x1, x2_safe),
                       tf.constant(0, x2.dtype)))
    return (tf.where(x1 < x2, x2, x1), tf.where(x1 < x2, x1, x2))
  if (not np.issubdtype(x1.dtype.as_numpy_dtype, np.integer) or
      not np.issubdtype(x2.dtype.as_numpy_dtype, np.integer)):
    raise ValueError("Arguments to gcd must be integers.")
  shape = tf.broadcast_static_shape(x1.shape, x2.shape)
  x1 = tf.broadcast_to(x1, shape)
  x2 = tf.broadcast_to(x2, shape)
  gcd, _ = tf.while_loop(_gcd_cond_fn, _gcd_body_fn,
                         (tf.math.abs(x1), tf.math.abs(x2)))
  return gcd

@utils.np_doc(np.gcd)
def gcd(x1, x2):
  return _bin_op(_tf_gcd, x1, x2)

@utils.np_doc(np.lcm)
def lcm(x1, x2):
  def f(x1, x2):
    d = _tf_gcd(x1, x2)
    # Same as the `x2_safe` trick above
    d_safe = tf.where(d == 0, tf.constant(1, d.dtype), d)
    return tf.where(d == 0, tf.constant(0, d.dtype),
                    tf.math.abs(x1 * x2) // d_safe)
  return _bin_op(f, x1, x2)

def _bitwise_binary_op(tf_fn, x1, x2):
  def f(x1, x2):
    is_bool = (x1.dtype == tf.bool)
    if is_bool:
      assert x2.dtype == tf.bool
      x1 = tf.cast(x1, tf.int8)
      x2 = tf.cast(x2, tf.int8)
    r = tf_fn(x1, x2)
    if is_bool:
      r = tf.cast(r, tf.bool)
    return r
  return _bin_op(f, x1, x2)

@utils.np_doc(np.bitwise_and)
def bitwise_and(x1, x2):
  return _bitwise_binary_op(tf.bitwise.bitwise_and, x1, x2)

@utils.np_doc(np.bitwise_or)
def bitwise_or(x1, x2):
  return _bitwise_binary_op(tf.bitwise.bitwise_or, x1, x2)

@utils.np_doc(np.bitwise_xor)
def bitwise_xor(x1, x2):
  return _bitwise_binary_op(tf.bitwise.bitwise_xor, x1, x2)

@utils.np_doc(np.bitwise_not)
def bitwise_not(x):
  def f(x):
    if x.dtype == tf.bool:
      return tf.logical_not(x)
    return tf.bitwise.invert(x)
  return _scalar(f, x)

def _scalar(tf_fn, x, promote_to_float=False):
  """Computes the tf_fn(x) for each element in `x`.

  Args:
    tf_fn: function that takes a single Tensor argument.
    x: array_like. Could be an ndarray, a Tensor or any object that can
      be converted to a Tensor using `tf.convert_to_tensor`.
    promote_to_float: whether to cast the argument to a float dtype
      (`dtypes.default_float_type`) if it is not already.

  Returns:
    An ndarray with the same shape as `x`. The default output dtype is
    determined by `dtypes.default_float_type`, unless x is an ndarray with a
    floating point type, in which case the output type is same as x.dtype.
  """
  x = array_creation.asarray(x)
  if promote_to_float and not np.issubdtype(x.dtype, np.floating):
    x = x.astype(dtypes.default_float_type())
  return utils.tensor_to_ndarray(tf_fn(x.data))

@utils.np_doc(np.log)
def log(x):
  return _scalar(tf.math.log, x, True)

@utils.np_doc(np.exp)
def exp(x):
  return _scalar(tf.exp, x, True)

@utils.np_doc(np.sqrt)
def sqrt(x):
  return _scalar(tf.sqrt, x, True)

@utils.np_doc(np.abs)
def abs(x):
  return _scalar(tf.math.abs, x)

@utils.np_doc(np.absolute)
def absolute(x):
  return abs(x)

@utils.np_doc(np.fabs)
def fabs(x):
  return abs(x)

@utils.np_doc(np.ceil)
def ceil(x):
  return _scalar(tf.math.ceil, x, True)

@utils.np_doc(np.floor)
def floor(x):
  return _scalar(tf.math.floor, x, True)

@utils.np_doc(np.conj)
def conj(x):
  return _scalar(tf.math.conj, x)

@utils.np_doc(np.negative)
def negative(x):
  return _scalar(tf.math.negative, x)

@utils.np_doc(np.reciprocal)
def reciprocal(x):
  return _scalar(tf.math.reciprocal, x)

@utils.np_doc(np.signbit)
def signbit(x):
  def f(x):
    if x.dtype == tf.bool:
      return tf.fill(x.shape, False)
    return x < 0
  return _scalar(f, x)

@utils.np_doc(np.sin)
def sin(x):
  return _scalar(tf.math.sin, x, True)

@utils.np_doc(np.cos)
def cos(x):
  return _scalar(tf.math.cos, x, True)

@utils.np_doc(np.tan)
def tan(x):
  return _scalar(tf.math.tan, x, True)

@utils.np_doc(np.sinh)
def sinh(x):
  return _scalar(tf.math.sinh, x, True)

@utils.np_doc(np.cosh)
def cosh(x):
  return _scalar(tf.math.cosh, x, True)

@utils.np_doc(np.tanh)
def tanh(x):
  return _scalar(tf.math.tanh, x, True)

@utils.np_doc(np.arcsin)
def arcsin(x):
  return _scalar(tf.math.asin, x, True)

@utils.np_doc(np.arccos)
def arccos(x):
  return _scalar(tf.math.acos, x, True)

@utils.np_doc(np.arctan)
def arctan(x):
  return _scalar(tf.math.atan, x, True)

@utils.np_doc(np.arcsinh)
def arcsinh(x):
  return _scalar(tf.math.asinh, x, True)

@utils.np_doc(np.arccosh)
def arccosh(x):
  return _scalar(tf.math.acosh, x, True)

@utils.np_doc(np.arctanh)
def arctanh(x):
  return _scalar(tf.math.atanh, x, True)

@utils.np_doc(np.deg2rad)
def deg2rad(x):
  def f(x):
    return x * (np.pi / 180.0)
  return _scalar(f, x, True)

@utils.np_doc(np.rad2deg)
def rad2deg(x):
  return x * (180.0 / np.pi)

_tf_float_types = [tf.bfloat16, tf.float16, tf.float32, tf.float64]

@utils.np_doc(np.angle)
def angle(z, deg=False):
  def f(x):
    if x.dtype in _tf_float_types:
      # Workaround for b/147515503
      return tf.where(x < 0, np.pi, 0)
    else:
      return tf.math.angle(x)
  y = _scalar(f, z, True)
  if deg:
    y = rad2deg(y)
  return y

@utils.np_doc(np.cbrt)
def cbrt(x):
  def f(x):
    # __pow__ can't handle negative base, so we use `abs` here.
    rt = tf.math.abs(x) ** (1.0 / 3)
    return tf.where(x < 0, -rt, rt)
  return _scalar(f, x, True)

@utils.np_doc(np.conjugate)
def conjugate(x):
  return _scalar(tf.math.conj, x, True)

@utils.np_doc(np.exp2)
def exp2(x):
  def f(x):
    return 2 ** x
  return _scalar(f, x, True)

@utils.np_doc(np.expm1)
def expm1(x):
  return _scalar(tf.math.expm1, x, True)

@utils.np_doc(np.fix)
def fix(x):
  def f(x):
    return tf.where(x < 0, tf.math.ceil(x), tf.math.floor(x))
  return _scalar(f, x, True)

@utils.np_doc(np.iscomplex)
def iscomplex(x):
  return array_methods.imag(x) != 0

@utils.np_doc(np.isreal)
def isreal(x):
  return array_methods.imag(x) == 0

@utils.np_doc(np.iscomplexobj)
def iscomplexobj(x):
  x = array_creation.asarray(x)
  return np.issubdtype(x.dtype, np.complexfloating)

@utils.np_doc(np.isrealobj)
def isrealobj(x):
  return not iscomplexobj(x)

@utils.np_doc(np.isnan)
def isnan(x):
  return _scalar(tf.math.is_nan, x, True)

def _make_nan_reduction(onp_reduction, reduction, init_val):
  @utils.np_doc(onp_reduction)
  def nan_reduction(a, axis=None, dtype=None, keepdims=False):
    a = array_creation.asarray(a)
    v = array_creation.asarray(init_val, dtype=a.dtype)
    return reduction(array_methods.where(isnan(a), v, a),
                     axis=axis, dtype=dtype, keepdims=keepdims)
  return nan_reduction

nansum = _make_nan_reduction(np.nansum, array_methods.sum, 0)
nanprod = _make_nan_reduction(np.nanprod, array_methods.prod, 1)

@utils.np_doc(np.nanmean)
def nanmean(a, axis=None, dtype=None, keepdims=None):
  a = array_creation.asarray(a)
  if np.issubdtype(a.dtype, np.bool_) or np.issubdtype(a.dtype, np.integer):
    return array_methods.mean(a, axis=axis, dtype=dtype, keepdims=keepdims)
  nan_mask = logical_not(isnan(a))
  normalizer = array_methods.sum(nan_mask, axis=axis, dtype=np.int64,
                                 keepdims=keepdims)
  return nansum(a, axis=axis, dtype=dtype, keepdims=keepdims) / normalizer

@utils.np_doc(np.isfinite)
def isfinite(x):
  return _scalar(tf.math.is_finite, x, True)

@utils.np_doc(np.isinf)
def isinf(x):
  return _scalar(tf.math.is_inf, x, True)

@utils.np_doc(np.isneginf)
def isneginf(x):
  return x == array_creation.full_like(x, -np.inf)

@utils.np_doc(np.isposinf)
def isposinf(x):
  return x == array_creation.full_like(x, np.inf)

@utils.np_doc(np.log2)
def log2(x):
  return log(x) / np.log(2)

@utils.np_doc(np.log10)
def log10(x):
  return log(x) / np.log(10)

@utils.np_doc(np.log1p)
def log1p(x):
  return _scalar(tf.math.log1p, x, True)

@utils.np_doc(np.positive)
def positive(x):
  return _scalar(lambda x: x, x, True)

@utils.np_doc(np.sinc)
def sinc(x):
  def f(x):
    pi_x = x * np.pi
    return tf.where(x == 0, tf.ones_like(x), tf.math.sin(pi_x) / pi_x)
  return _scalar(f, x, True)

@utils.np_doc(np.square)
def square(x):
  return _scalar(tf.math.square, x)

@utils.np_doc(np.diff)
def diff(a, n=1, axis=-1):
  def f(a):
    nd = a.shape.rank
    if (axis + nd if axis < 0 else axis) >= nd:
      raise ValueError("axis %s is out of bounds for array of dimension %s" %
                       (axis, nd))
    if n < 0:
      raise ValueError("order must be non-negative but got %s" % n)
    slice1 = [slice(None)] * nd
    slice2 = [slice(None)] * nd
    slice1[axis] = slice(1, None)
    slice2[axis] = slice(None, -1)
    slice1 = tuple(slice1)
    slice2 = tuple(slice2)
    op = tf.not_equal if a.dtype == tf.bool else tf.subtract
    for _ in range(n):
      a = op(a[slice1], a[slice2])
    return a
  return _scalar(f, a)

def _atleast_nd(n, new_shape, *arys):
  """Reshape arrays to be at least `n`-dimensional.

  Args:
    n: The minimal rank.
    new_shape: a function that takes `n` and the old shape and returns the
      desired new shape.
    arys: ndarray(s) to be reshaped.

  Returns:
    The reshaped array(s).
  """
  def f(x):
    x = array_creation.asarray(x)
    if x.ndim < n:
      x = array_methods.reshape(x, new_shape(n, x.shape))
    return x
  arys = list(map(f, arys))
  if len(arys) == 1:
    return arys[0]
  else:
    return arys

@utils.np_doc(np.atleast_1d)
def atleast_1d(*arys):
  return _atleast_nd(1, _pad_left_to, *arys)

@utils.np_doc(np.atleast_2d)
def atleast_2d(*arys):
  return _atleast_nd(2, _pad_left_to, *arys)

@utils.np_doc(np.atleast_3d)
def atleast_3d(*arys):
  def new_shape(_, old_shape):
    ndim = len(old_shape)
    if ndim == 0:
      return (1, 1, 1)
    elif ndim == 1:
      return (1,) + old_shape + (1,)
    return old_shape + (1,)
  return _atleast_nd(3, new_shape, *arys)

def flip(f):
  def _f(a, b):
    return f(b, a)
  return _f

setattr(arrays.ndarray, '__abs__', absolute)
setattr(arrays.ndarray, '__floordiv__', floor_divide)
setattr(arrays.ndarray, '__rfloordiv__', flip(floor_divide))
setattr(arrays.ndarray, '__mod__', mod)
setattr(arrays.ndarray, '__rmod__', flip(mod))
setattr(arrays.ndarray, '__add__', add)
setattr(arrays.ndarray, '__radd__', flip(add))
setattr(arrays.ndarray, '__sub__', subtract)
setattr(arrays.ndarray, '__rsub__', flip(subtract))
setattr(arrays.ndarray, '__mul__', multiply)
setattr(arrays.ndarray, '__rmul__', flip(multiply))
setattr(arrays.ndarray, '__pow__', power)
setattr(arrays.ndarray, '__rpow__', flip(power))
setattr(arrays.ndarray, '__truediv__', true_divide)
setattr(arrays.ndarray, '__rtruediv__', flip(true_divide))

def _comparison(tf_fun, x1, x2, cast_bool_to_int=False):
  dtype = utils.result_type(x1, x2)
  # Cast x1 and x2 to the result_type if needed.
  x1 = array_creation.asarray(x1, dtype=dtype)
  x2 = array_creation.asarray(x2, dtype=dtype)
  x1 = x1.data
  x2 = x2.data
  if cast_bool_to_int and x1.dtype == tf.bool:
    x1 = tf.cast(x1, tf.int32)
    x2 = tf.cast(x2, tf.int32)
  return utils.tensor_to_ndarray(tf_fun(x1, x2))

@utils.np_doc(np.equal)
def equal(x1, x2):
  return _comparison(tf.equal, x1, x2)

@utils.np_doc(np.not_equal)
def not_equal(x1, x2):
  return _comparison(tf.not_equal, x1, x2)

@utils.np_doc(np.greater)
def greater(x1, x2):
  return _comparison(tf.greater, x1, x2, True)

@utils.np_doc(np.greater_equal)
def greater_equal(x1, x2):
  return _comparison(tf.greater_equal, x1, x2, True)

@utils.np_doc(np.less)
def less(x1, x2):
  return _comparison(tf.less, x1, x2, True)

@utils.np_doc(np.less_equal)
def less_equal(x1, x2):
  return _comparison(tf.less_equal, x1, x2, True)

@utils.np_doc(np.array_equal)
def array_equal(a1, a2):
  def f(a1, a2):
    if a1.shape != a2.shape:
      return tf.constant(False)
    return tf.reduce_all(tf.equal(a1, a2))
  return _comparison(f, a1, a2)

def _logical_binary_op(tf_fun, x1, x2):
  x1 = array_creation.asarray(x1, dtype=np.bool_)
  x2 = array_creation.asarray(x2, dtype=np.bool_)
  return utils.tensor_to_ndarray(tf_fun(x1.data, x2.data))

@utils.np_doc(np.logical_and)
def logical_and(x1, x2):
  return _logical_binary_op(tf.logical_and, x1, x2)

@utils.np_doc(np.logical_or)
def logical_or(x1, x2):
  return _logical_binary_op(tf.logical_or, x1, x2)

@utils.np_doc(np.logical_xor)
def logical_xor(x1, x2):
  return _logical_binary_op(tf.math.logical_xor, x1, x2)

@utils.np_doc(np.logical_not)
def logical_not(x):
  x = array_creation.asarray(x, dtype=np.bool_)
  return utils.tensor_to_ndarray(tf.logical_not(x.data))

setattr(arrays.ndarray, '__invert__', logical_not)
setattr(arrays.ndarray, '__lt__', less)
setattr(arrays.ndarray, '__le__', less_equal)
setattr(arrays.ndarray, '__gt__', greater)
setattr(arrays.ndarray, '__ge__', greater_equal)
setattr(arrays.ndarray, '__eq__', equal)
setattr(arrays.ndarray, '__ne__', not_equal)

@utils.np_doc(np.linspace)
def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=float):
  if dtype:
    dtype = utils.result_type(dtype)
  start = array_creation.asarray(start, dtype=dtype)
  stop = array_creation.asarray(stop, dtype=dtype)
  if num == 0:
    return empty(dtype)
  if num < 0:
    raise ValueError('Number of samples {} must be non-negative.'.format(num))
  step = np.nan
  if endpoint:
    result = tf.linspace(start.data, stop.data, num)
    if num > 1:
      step = (stop - start) / (num - 1)
  else:
    # tf.linspace does not support endpoint=False so we manually handle it
    # here.
    if num > 1:
      step = (stop - start) / num
      result = tf.linspace(start.data, (stop - step).data, num)
    else:
      result = tf.linspace(start.data, stop.data, num)
  if dtype:
    result = tf.cast(result, dtype)
  if retstep:
    return arrays.tensor_to_ndarray(result), step
  else:
    return arrays.tensor_to_ndarray(result)

@utils.np_doc(np.logspace)
def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None):
  if dtype:
    dtype = utils.result_type(dtype)
  result = linspace(start, stop, num=num, endpoint=endpoint)
  result = tf.pow(base, result.data)
  if dtype:
    result = tf.cast(result, dtype)
  return arrays.tensor_to_ndarray(result)

@utils.np_doc(np.ptp)
def ptp(a, axis=None, keepdims=None):
  return (array_methods.amax(a, axis=axis, keepdims=keepdims) -
          array_methods.amin(a, axis=axis, keepdims=keepdims))

@utils.np_doc_only(np.concatenate)
def concatenate(arys, axis=0):
  if not arys:
    raise ValueError('Need at least one array to concatenate.')
  dtype = utils.result_type(*arys)
  arys = [array_creation.asarray(array, dtype=dtype).data for array in arys]
  return arrays.tensor_to_ndarray(tf.concat(arys, axis))

@utils.np_doc_only(np.tile)
def tile(a, reps):
  a = array_creation.asarray(a).data
  reps = array_creation.asarray(reps, dtype=tf.int32).reshape([-1]).data

  a_rank = tf.rank(a)
  reps_size = tf.size(reps)
  reps = tf.pad(
      reps, [[tf.math.maximum(a_rank - reps_size, 0), 0]],
      constant_values=1)
  a_shape = tf.pad(
      tf.shape(a), [[tf.math.maximum(reps_size - a_rank, 0), 0]],
      constant_values=1)
  a = tf.reshape(a, a_shape)

  return arrays.tensor_to_ndarray(tf.tile(a, reps))

max_issues_repo_name: kefirski/trax

max_forks_repo_name: kefirski/trax


text:
"""
Command-line driver example for SMIRKY.

"""

import sys
import string
import time

from optparse import OptionParser # For parsing of command line arguments
import smarty
from openforcefield.utils import utils

import os
import math
import copy
import re
import numpy
from numpy import random

def main():
    # Create command-line argument options.
    usage_string = """\
    Sample over fragment types (atoms, bonds, angles, torsions, or impropers)
    optionally attempting to match created types to an established SMIRFF.
    For all files left blank, they will be taken from this module's
    data/odds_files/ subdirectory.

    usage %prog --molecules molfile --typetag fragmentType
            [--atomORbases AtomORbaseFile --atomORdecors AtomORdecorFile
            --atomANDdecors AtomANDdecorFile --bondORbase BondORbaseFile
            --bondANDdecors BondANDdecorFile --atomIndexOdds AtomIndexFile
            --bondIndexOdds BondIndexFile --replacements substitutions
            --initialtypes initialFragmentsFile --SMIRFF referenceSMIRFF
            --temperature float --verbose verbose
            --iterations iterations --output outputFile]

    example:
    smirky --molecules AlkEthOH_test_filt1_ff.mol2 --typetag Angle

    """
    version_string = "%prog %__version__"
    parser = OptionParser(usage=usage_string, version=version_string)

    parser.add_option("-m", "--molecules", metavar='MOLECULES',
            action="store", type="string", dest='molecules_filename', default=None,
            help="Small molecule set (in any OpenEye compatible file format) containing 'dG(exp)' fields with experimental hydration free energies. This filename can also be an option in this module's data/molecules sub-directory")
    #TODO: ask about the the dG(exp) fields?

    parser.add_option("-T", "--typetag", metavar='TYPETAG',
            action = "store", type="choice", dest='typetag',
            default=None, choices = ['VdW', 'Bond', 'Angle', 'Torsion', 'Improper'],
            help="type of fragment being sampled, options are 'VdW', 'Bond', 'Angle', 'Torsion', 'Improper'")

    parser.add_option('-e', '--atomORbases', metavar="DECORATORS",
            action='store', type='string', dest='atom_OR_bases',
            default = 'odds_files/atom_OR_bases.smarts',
            help="Filename defining atom OR bases and associated probabilities. These are combined with atom OR decorators in SMIRKS, for example in '[#6X4,#7X3;R2:2]' '#6' and '#7' are atom OR bases. (OPTIONAL)")

    parser.add_option("-O", "--atomORdecors", metavar="DECORATORS",
            action='store', type='string', dest='atom_OR_decorators',
            default = 'odds_files/atom_decorators.smarts',
            help="Filename defining atom OR decorators and associated probabilities. These are combined with atom bases in SMIRKS, for example in '[#6X4,#7X3;R2:2]' 'X4' and 'X3' are ORdecorators. (OPTIONAL)")

    parser.add_option('-A', '--atomANDdecors', metavar="DECORATORS",
            action='store', type='string', dest='atom_AND_decorators',
            default='odds_files/atom_decorators.smarts',
            help="Filename defining atom AND decorators and associated probabilities. These are added to the end of an atom's SMIRKS, for example in '[#6X4,#7X3;R2:2]' 'R2' is an AND decorator. (OPTIONAL)")

    parser.add_option('-o', '--bondORbase', metavar="DECORATORS",
            action='store', type='string', dest='bond_OR_bases',
            default='odds_files/bond_OR_bases.smarts',
            help="Filename defining bond OR bases and their associated probabilities. These are OR'd together to describe a bond, for example in '[#6]-,=;@[#6]' '-' and '=' are OR bases. (OPTIONAL)")

    parser.add_option('-a', '--bondANDdecors', metavar="DECORATORS",
            action="store", type='string', dest='bond_AND_decorators',
            default='odds_files/bond_AND_decorators.smarts',
            help="Filename defining bond AND decorators and their associated probabilities. These are AND'd to the end of a bond, for example in '[#6]-,=;@[#7]' '@' is an AND decorator.(OPTIONAL)")

    parser.add_option('-D', '--atomOddsFile', metavar="ODDSFILE",
            action="store", type="string", dest="atom_odds",
            default='odds_files/atom_index_odds.smarts',
            help="Filename defining atom descriptors and probabilities with making changes to that kind of atom. Options for descriptors are integers corresponding to that indexed atom, 'Indexed', 'Unindexed', 'Alpha', 'Beta', 'All'. (OPTIONAL)")

    parser.add_option('-d', '--bondOddsFile', metavar="ODDSFILE",
            action="store", type="string", dest="bond_odds",
            default='odds_files/bond_index_odds.smarts',
            help="Filename defining bond descriptors and probabilities with making changes to that kind of bond. Options for descriptors are integers corresponding to that indexed bond, 'Indexed', 'Unindexed', 'Alpha', 'Beta', 'All'. (OPTIONAL)")

    parser.add_option("-s", "--substitutions", metavar="SUBSTITUTIONS",
            action="store", type="string", dest='substitutions_filename',
            default=None,
            help="Filename defining substitution definitions for SMARTS atom matches. (OPTIONAL).")

    parser.add_option("-f", "--initialtypes", metavar='INITIALTYPES',
            action="store", type="string", dest='initialtypes_filename',
            default=None,
            help="Filename defining initial fragment types. The file is formatted with two columns: 'SMIRKS    typename'. For the default the initial type will be a generic form of the given fragment, for example '[*:1]~[*:2]' for a bond (OPTIONAL)")

    parser.add_option('-r', '--smirff', metavar='REFERENCE',
            action='store', type='string', dest='SMIRFF',
            default=None,
            help="Filename defining a SMIRFF force fielce used to determine reference fragment types in provided set of molecules. It may be an absolute file path, a path relative to the current working directory, or a path relative to this module's data subdirectory (for built in force fields). (OPTIONAL)")

    parser.add_option("-i", "--iterations", metavar='ITERATIONS',
            action="store", type="int", dest='iterations',
            default=150,
            help="MCMC iterations.")

    parser.add_option("-t", "--temperature", metavar='TEMPERATURE',
            action="store", type="float", dest='temperature',
            default=0.1,
            help="Effective temperature for Monte Carlo acceptance, indicating fractional tolerance of mismatched atoms (default: 0.1). If 0 is specified, will behave in a greedy manner.")

    parser.add_option("-p", "--output", metavar='OUTPUT',
            action="store", type="string", dest='outputfile',
            default=None,
            help="Filename base for output information. This same base will be used for all output files created. If None provided then it is set to 'typetag_temperature' (OPTIONAL).")

    parser.add_option('-v', '--verbose', metavar='VERBOSE',
            action='store', type='choice', dest='verbose',
            default=False, choices = ['True', 'False'],
            help="If True prints minimal information to the commandline during iterations. (OPTIONAL)")

    # Parse command-line arguments.
    (option,args) = parser.parse_args()

    # Molecules are required
    if option.molecules_filename is None:
        parser.print_help()
        parser.error("Molecules input files must be specified.")

    verbose = option.verbose == 'True'
    # Load and type all molecules in the specified dataset.
    molecules = utils.read_molecules(option.molecules_filename, verbose=verbose)

    # Parse input odds files
    atom_OR_bases = smarty.parse_odds_file(option.atom_OR_bases, verbose)
    atom_OR_decorators = smarty.parse_odds_file(option.atom_OR_decorators, verbose)
    atom_AND_decorators = smarty.parse_odds_file(option.atom_AND_decorators, verbose)
    bond_OR_bases = smarty.parse_odds_file(option.bond_OR_bases, verbose)
    bond_AND_decorators = smarty.parse_odds_file(option.bond_AND_decorators, verbose)
    atom_odds = smarty.parse_odds_file(option.atom_odds, verbose)
    bond_odds = smarty.parse_odds_file(option.bond_odds, verbose)

    # get initial types if provided, otherwise none
    if option.initialtypes_filename is None:
        initialtypes = None
    else:
        initialtypes = smarty.AtomTyper.read_typelist(option.initialtypes_filename)

    output = option.outputfile
    if output is None:
        output = "%s_%.2e" % ( option.typetag, option.temperature)
    # get replacements
    if option.substitutions_filename is None:
        sub_file = smarty.get_data_filename('odds_files/substitutions.smarts')
    else:
        sub_file = option.substitutions_filename
    replacements = smarty.AtomTyper.read_typelist(sub_file)
    replacements = [ (short, smarts) for (smarts, short) in replacements]

    start_sampler = time.time()
    fragment_sampler = smarty.FragmentSampler(
            molecules, option.typetag, atom_OR_bases, atom_OR_decorators,
            atom_AND_decorators, bond_OR_bases, bond_AND_decorators,
            atom_odds, bond_odds, replacements, initialtypes,
            option.SMIRFF, option.temperature, output)
    # report time
    finish_sampler = time.time()
    elapsed = finish_sampler - start_sampler
    if verbose: print("Creating %s sampler took %.3f s" % (option.typetag, elapsed))

    # Make iterations
    frac_found = fragment_sampler.run(option.iterations, verbose)
    results = fragment_sampler.write_results_smarts_file()
    finished = time.time()
    elapsed = finished - finish_sampler
    per_it = elapsed / float(option.iterations)
    if verbose: print("%i iterations took %.3f s (%.3f s / iteration)" % (option.iterations, elapsed, per_it))
    if verbose: print("Final score was %.3f %%" % (frac_found*100.0))

    # plot results
    plot_file = "%s.pdf" % output
    traj = "%s.csv" % output
    smarty.score_utils.create_plot_file(traj, plot_file, False, verbose)

max_stars_repo_name: openforcefield/smarty

max_issues_repo_name: openforcefield/smarty

max_forks_repo_name: openforcefield/smarty

reason: import numpy,from numpy


text:
# -*- coding: utf-8 -*-
"""
Created on Sat Jul 17 19:38:04 2021

@author: rania
"""

import pandas as pd
import numpy as np

import tensorflow as tf
from transformers import BertTokenizer, TFBertModel
from keras import backend as K
import matplotlib.pyplot as plt

#load data
df = pd.read_excel('training.xlsx')
#split data
train = df[df.split == 0]
validation = df[df.split==1]
test = df[df.split==2]
print('train shape: ',train.shape)
print('test shape: ',test.shape)

#load tokenizer
tokenizer = BertTokenizer.from_pretrained("bert-base-german-cased")
MAXLEN = 192

def preprocess_text(data):
    """ take texts and prepare as input features for BERT
    """
    input_ids = []
    # For every sentence...
    for comment in data:
        encoded_sent = tokenizer.encode_plus(
            text=comment,
            add_special_tokens=True,  # Add `[CLS]` and `[SEP]`
            max_length=MAXLEN,  # Max length to truncate/pad
            pad_to_max_length=True,  # Pad sentence to max length
            return_attention_mask=False,  # attention mask not needed for our task
        )
        # Add the outputs to the lists
        input_ids.append(encoded_sent.get("input_ids"))
    return input_ids

# preprocess data
train_ids = preprocess_text(train["Sentence"])
val_ids = preprocess_text(validation["Sentence"])
test_ids = preprocess_text(test["Sentence"])

train_labels = train['MOS']
test_labels = test['MOS']
val_labels=  validation['MOS']
print(f"Train set: {len(train_ids)}\nTest set: {len(test_ids)}")

# model constant
MAXLEN = MAXLEN
BATCH_SIZE_PER_REPLICA = 16
BATCH_SIZE = 32
EPOCHS = 6
LEARNING_RATE = 1e-5
DATA_LENGTH = len(train)

def create_dataset(
    data_tuple,
    epochs=EPOCHS,
    batch_size=BATCH_SIZE,
    buffer_size=DATA_LENGTH,
    train=False,
):
    dataset = tf.data.Dataset.from_tensor_slices(data_tuple)
    if train:
        dataset = dataset.shuffle(
            buffer_size=buffer_size, reshuffle_each_iteration=True
        ).repeat(epochs)
    dataset = dataset.batch(batch_size)
    return dataset

train = create_dataset(
    (train_ids, train_labels), buffer_size=len(train_ids), train=True
)
test = create_dataset((val_ids, val_labels), buffer_size=len(val_ids))

# loss custom
def root_mean_squared_error(y_true, y_pred):
        return K.sqrt(K.mean(K.square(y_pred - y_true)))

#build the model
def build_model(transformer, max_len=MAXLEN):
    """ add dense layer to pretrained model
    """
    input_word_ids = tf.keras.layers.Input(
        shape=(max_len,), dtype=tf.int32, name="input_word_ids"
    )
    sequence_output = transformer(input_word_ids)[0]
    cls_token = sequence_output[:, 0, :]
    out = tf.keras.layers.Dense(1, activation=None)(cls_token)
    model = tf.keras.models.Model(inputs=input_word_ids, outputs=out)
    model.compile(
        tf.keras.optimizers.Adam(lr=LEARNING_RATE),
        loss=root_mean_squared_error,
        metrics=tf.keras.metrics.RootMeanSquaredError(),
    )
    return model
# load pretrained bert model
transformer_layers = TFBertModel.from_pretrained("bert-base-german-cased")
# build model
model = build_model(transformer_layers, max_len=MAXLEN)
model.summary()

steps_per_epoch = int(np.floor((len(train_ids) / BATCH_SIZE)))
print(
    f"Model Params:\nbatch_size: {BATCH_SIZE}\nEpochs: {EPOCHS}\n"
    f"Step p. Epoch: {steps_per_epoch}\n"
    f"Learning rate: {LEARNING_RATE}"
)
# train the model
hist = model.fit(
    train,
    batch_size=BATCH_SIZE,
    epochs=EPOCHS,
    steps_per_epoch=steps_per_epoch,
    validation_data=test,
    verbose=1
)

#eval the model
eval = create_dataset((test_ids, test_labels), buffer_size=len(val_ids))

loss, RMSE = model.evaluate(eval)

pred = model.predict(eval, batch_size=BATCH_SIZE, verbose=2, use_multiprocessing=True)

print(f'Loss: {loss}')
print(f'RMSE: {RMSE}')

#Plot model training progress
history_dict = hist .history
print(history_dict.keys())

RMSE = history_dict['root_mean_squared_error']
loss = history_dict['loss']
val_RMSE = history_dict['val_root_mean_squared_error']
val_loss = history_dict['val_loss']

epochs = range(1, len(RMSE) + 1)
fig = plt.figure(figsize=(10, 6))
fig.tight_layout()

plt.subplot(2, 1, 1)
plt.plot(epochs, loss, 'b', label='Training loss')
plt.plot(epochs, val_loss, 'r', label='val loss')
plt.title('Training loss')
plt.ylabel('Loss')
plt.legend()

plt.subplot(2, 1, 2)
plt.plot(epochs, RMSE, 'r', label='Training RMSE')
plt.plot(epochs, val_RMSE, 'b', label='val RMSE')
plt.title('Training RMSE')
plt.xlabel('Epochs')
plt.ylabel('RMSE')
plt.legend(loc='lower right')


text:
import numpy as np
import json
import matplotlib.pyplot as plt
from ARIMA.itsmpy import ITSM
from io_support import load_data
from evaluation import smape

def fig_detrend(page, true_data, path=None, p=2):
    """
        Visualizes two figures in one window with trend series and detrend series

        Args:
            page: page name to visualize
                  type: str
            true_data: time series data load by load_data(), [pages, dates, visits]
                       type: list
            path: directory to save the figure, default = None
                  type: str
            p: polynomial order (1=linear, 2=quadratic)
               type: int

    """
    pages, dates, visits = true_data
    series = visits[pages==page,:].flatten()
    time = len(dates)
    itsm = ITSM()
    trendc = itsm.trend(series, p)
    plt.style.use('ggplot')
    plt.figure(figsize = (20,8))
    # trend series
    plt.subplot(1,2,1)
    plt.plot(series, color="#2980B9")
    plt.plot(trendc, color="#EC7063",linewidth=3)
    plt.xticks(np.arange(0,time,time//6), dates[np.arange(0,time,time//6)])
    plt.tick_params(labelcolor = '#239B56', labelsize = 12, width = 3, pad = 12)
    plt.xlabel("Dates", fontsize = 20, labelpad = 14)
    plt.ylabel("Visits", fontsize = 20, labelpad = 14)
    plt.title('Visits with trend components', fontsize = 26, pad = 16)
    plt.legend(['original data', 'trend components'], fontsize=18)
    # detrend series
    plt.subplot(1,2,2)
    plt.plot(series, color="#2980B9")
    plt.plot(series-trendc, color="#16A085")
    plt.xticks(np.arange(0,time,time//6), dates[np.arange(0,time,time//6)])
    plt.tick_params(labelcolor = '#239B56', labelsize = 12, width = 3, pad = 12)
    plt.xlabel("Dates", fontsize = 20, labelpad = 14)
    plt.ylabel("Visits", fontsize = 20, labelpad = 14)
    plt.title('Visits with/without trend components', fontsize = 26, pad = 16)
    plt.legend(['trend', 'detrend'], fontsize=18)
    if path != None:
        plt.savefig(path+'detrend.png')
    plt.show()

def fig_deseasonal(page, true_data, path=None, d=2):
    """
        Visualizes two figures in one window with seasonal series and deseasonal series

        Args:
            page: page name to visualize
                  type: str
            true_data: time series data load by load_data(), [pages, dates, visits]
                       type: list
            path: directory to save the figure, default = None
                  type: str
            d: number of observation per season
               type: int

    """
    pages, dates, visits = true_data
    series = visits[pages==page,:].flatten()
    time = len(dates)
    itsm = ITSM()
    seasonc = itsm.season(series, d)
    plt.style.use('ggplot')
    plt.figure(figsize = (20,8))
    # trend series
    plt.subplot(1,2,1)
    plt.plot(series, color="#2980B9")
    plt.plot(seasonc, color="#EC7063",linewidth=3)
    plt.xticks(np.arange(0,time,time//6), dates[np.arange(0,time,time//6)])
    plt.tick_params(labelcolor = '#239B56', labelsize = 12, width = 3, pad = 12)
    plt.xlabel("Dates", fontsize = 20, labelpad = 14)
    plt.ylabel("Visits", fontsize = 20, labelpad = 14)
    plt.title('Visits with seasonal components', fontsize = 26, pad = 16)
    plt.legend(['original data', 'seasonal components'], fontsize=18)
    # detrend series
    plt.subplot(1,2,2)
    plt.plot(series, color="#2980B9")
    plt.plot(series-seasonc, color="#16A085")
    plt.xticks(np.arange(0,time,time//6), dates[np.arange(0,time,time//6)])
    plt.tick_params(labelcolor = '#239B56', labelsize = 12, width = 3, pad = 12)
    plt.xlabel("Dates", fontsize = 20, labelpad = 14)
    plt.ylabel("Visits", fontsize = 20, labelpad = 14)
    plt.title('Visits with/without seasonal components', fontsize = 26, pad = 16)
    plt.legend(['seasonal', 'deseasonal'], fontsize=18)
    if path != None:
        plt.savefig(path+'deseaonl.png')
    plt.show()

def fig_season_trend(page, true_data, path=None, d=7, p=2):
    """
        Visualizes two figures in one window with seasonal and trend components

        Args:
            page: page name to visualize
                  type: str
            true_data: time series data load by load_data(), [pages, dates, visits]
                       type: list
            path: directory to save the figure, default = None
                  type: str
            d: number of observation per season
               type: int
            p: polynomial order (1=linear, 2=quadratic)
               type: int

    """
    pages, dates, visits = true_data
    series = visits[pages==page,:].flatten()
    time = len(dates)
    itsm = ITSM()
    seasonc = itsm.season(series, d)
    trendc = itsm.trend(series, p)
    ### figure
    plt.style.use('ggplot')
    plt.figure(figsize = (24,9))
    #### seasonal components
    plt.subplot(1,2,1)
    plt.plot(np.arange(time), series, color = "#2980B9")
    plt.plot(np.arange(time), seasonc, color = "#48C9B0", linewidth = 2)
    plt.xticks(np.arange(0,time,time//6), dates[np.arange(0,time,time//6)])
    plt.tick_params(labelcolor = '#239B56', labelsize = 12, width = 3, pad=12)
    plt.xlabel("Dates", fontsize = 20, labelpad = 14)
    plt.ylabel("Visits", fontsize = 20, labelpad = 14)
    plt.legend(['original data', 'seasonal components'], fontsize=18)
    plt.title('Visits with seasonal components', fontsize = 26, pad = 16)
    #### trend components
    plt.subplot(1,2,2)
    plt.plot(np.arange(time), series, color = "#2980B9")
    plt.plot(np.arange(time), trendc, color = "#48C9B0", linewidth = 3)
    plt.xticks(np.arange(0,time,time//6), dates[np.arange(0,time,time//6)])
    plt.tick_params(labelcolor = '#239B56', labelsize = 12, width = 3, pad=12)
    plt.xlabel("Dates", fontsize = 20, labelpad = 14)
    plt.ylabel("Visits", fontsize = 20, labelpad = 14)
    plt.legend(['original data', 'trend components'], fontsize=18)
    plt.title('Visits with trend components', fontsize = 26, pad = 16)
    #### big picture
    plt.suptitle('Page: ' + page, fontsize=30)
    plt.subplots_adjust(top=0.85)
    if path != None:
        plt.savefig(path+'season_trend_fig.png')
    plt.show()

def fig_forecast(page, true_data, train_len, pred, pred2=None, bound=None, path=None):
    """
        Visualizes two figures in one window with long-term and short-term forecasts
        (Only for)

        Args:
            page: page name to visualize
                  type: str
            true_data: time series data load by load_data(), [pages, dates, visits]
                       contains the true values of forecasting dates
                       type: list
            train_len: length of training set of forecasts
                       type: int
            pred: forecasts of the page
                  type: np.array
            pred2: forecasts 2 of the page, default = None
                   type: np.array
            bound: dictionary of upper and lower bounds, default = None
                   tpye: dictionary
            path: directory to save the figure, default = None
                  type: str

    """
    days = len(pred)
    pages, dates, visits = true_data
    series = visits[pages==page, :(train_len+days)].flatten()
    score = smape(series[train_len, (train_len+days)], pred).round(decimals=4)
    if pred != None:
        score2 = smape(series[train_len, (train_len+days)], pred2).round(decimals=4)

    ### figure
    plt.style.use('ggplot')
    plt.figure(figsize = (24,9))
    # long-term
    plt.subplot(1,2,1)
    plt.plot(np.arange((train_len+days)), series[:(train_len+days)], '-o', color = "#2980B9", markersize = 4)
    plt.plot(np.arange(train_len,(train_len+days)), pred, '-o', color = "#EC7063", markersize = 4)
    if pred2 != None:
        plt.plot(np.arange(train_len,(train_len+days)), pred2, '-o', color = "#16A085", markersize = 4)
    plt.xticks(np.arange(0,train_len+days,(train_len+days)//6), dates[np.arange(0,train_len+days,(train_len+days)//6)])
    plt.tick_params(labelcolor='#239B56', labelsize = 12, width = 3, pad=12)
    plt.xlabel("Dates", fontsize = 20, labelpad = 14)
    plt.ylabel("Visits", fontsize = 20, labelpad = 14)
    if pred2 == None:
        plt.legend(['original data', 'forecast'], fontsize=18)
    else:
        plt.legend(['original data', 'forecast 1', 'forecast 2'], fontsize=18)
    plt.title('Visits from ' + dates[0] + ' to ' + dates[(train_len+days-1)], fontsize=26)
    # short-term
    plt.subplot(1,2,2)
    plt.plot(np.aranget(train_len,(train_len+days)), series[train_len:(train_len+days)], '-o', color = "#2980B9", markersize = 4)
    plt.plot(np.arange(train_len,(train_len+days)), pred, '-o', color = "#EC7063", markersize = 4)
    if pred2 != None:
        plt.plot(np.arange(train_len,(train_len+days)), pred2, '-o', color = "#16A085", markersize = 4)
    if bound != None:
        lower = bound['l']
        upper = bound['u']
        plt.plot(np.arange(train_len,(train_len+days)), lower, linestyle = 'dashed', color = "#EC7063")
        plt.plot(np.arange(train_len,(train_len+days)), upper, linestyle = 'dashed', color = "#EC7063")
    plt.title('Visits from ' + dates[train_len] + ' to ' + dates[(train_len+days-1)], fontsize=26)
    plt.xticks(np.arange(train_len,train_len+days,(train_len+days)//6), dates[np.arange(train_len,train_len+days,(train_len+days)//6)])
    plt.xlabel("Dates", fontsize = 20, labelpad = 14)
    plt.ylabel("Visits", fontsize = 20, labelpad = 14)
    plt.tick_params(labelcolor='#239B56', labelsize = 12, width = 3, pad=12)
    if pred2 == None:
        plt.legend(['original data', 'forecast'], fontsize=18)
        plt.suptitle('Page: ' + page + '\n(SMAPE score: ' + str(score), fontsize=30)
    else:
        plt.legend(['original data', 'forecast 1', 'forecast 2'], fontsize=18)
        plt.suptitle('Page: ' + page +
                     '\n(SMAPE score: ' + str(score) + ' for forecast 1, ' + str(score2) + ' for forecast 2)',
                     fontsize=30)
    plt.subplots_adjust(top=0.8)
    if path != None:
        plt.savefig(path+'forecast_fig.png')
    plt.show()

def fig_smape_distribution(s, path=None):
    """
        Visualizes the distribution of SMAPE scores of pages

        Args:
            s: SMAPE scores of pages
               type: np.array
            path: directory to save the figure, default = None
                  type: str

    """
    plt.style.use('ggplot')
    plt.figure(figsize=(18,12))
    plt.hist(s, bins=50, color="#16A085")
    plt.tick_params(labelcolor='#239B56', labelsize=16, width=3, pad=12)
    plt.xlabel("SMAPE score", fontsize=20, labelpad=14)
    plt.ylabel("Frequency", fontsize=20, labelpad=14)
    plt.title('Distribution of SMAPE score', fontsize=26, pad=16)
    if path != None:
        plt.savefig(path+'smape_dist_fig.png')
    plt.show()

max_stars_repo_name: melanieihuei/Web-Traffic-Forecasting

max_issues_repo_name: dsp-uga/Edamame

max_forks_repo_name: dsp-uga/Edamame


text:
# For the experiment of group attributions
# Using factor analysis to decompose attributions

import os
import numpy as np
from os.path import join
from glob import glob
from utils.utils_network import InceptionV1

from utils.utils_activation_maps import create_root_dir, print_result_from_logit
from utils.utils_activation_maps import debug_show_AM_plus_img, decompose_AM_get_group_num
from utils.utils_attrmap import filter_attr_map, filter_attr_map_channel_first, normalize_heatmap
import json

def main():
	model = InceptionV1()
	model.load_graphdef()

	img_paths = []
	corres_attr_classes = []

	# extract valid images from ImageNet before running this code
	for ext in ('*.JPEG', '*.png', '*.jpg'):
		img_paths.extend(glob(join("./data/images/vase_val_images", ext)))
	corres_attr_classes = corres_attr_classes + [['vase']]*len(img_paths)

	# ----------------------------------------

	img_info_name = 'multiobj_img_info'

	# layers = ['conv2d0', 'conv2d1', 'conv2d2',
	#           'mixed3a', 'maxpool4',
	# 		    'mixed4a', 'mixed4b', 'mixed4c', 'mixed4d',
	# 		    'maxpool10', 'mixed5a', 'mixed5b']
	# layer_alpha = [0.9999, 0.9999, 0.9999,
	#                0.9999, 0.9999,
	# 			     0.65, 0.6, 0.6, 0.6,
	# 			     0.65, 0.05, 0.2]
	# layer_alpha = [alpha for alpha in layer_alpha]
	# scale_param = [[80, 120], [80, 120],
	#                [80, 120], [80, 120], [80, 120],
	# 		         [80, 125], [80, 125], [80, 120], [90, 120],
	# 			     [90, 130], [90, 130], [100, 101]]

	# layers = ['mixed4a', 'mixed4b', 'mixed4d', 'maxpool10']
	# layer_alpha = [0.5, 0.5, 0.5, 0.5]
	# layer_alpha = [alpha for alpha in layer_alpha]
	# scale_param = [[110, 140], [110, 140], [110, 140], [120, 140]]

	layers = ['maxpool4', 'mixed4d']
	layer_alpha = [0.3, 0.3]
	layer_alpha = [alpha for alpha in layer_alpha]

	flag1s = ["AShapleyOptf"]
	for flag1 in flag1s:
		neuron_vis(img_paths, corres_attr_classes, layers,
				   flag1=flag1, layer_alpha=layer_alpha, img_info_name=img_info_name)

def neuron_vis(img_paths, corres_attr_classes, layers,
			   flag1=None, layer_alpha=None, img_info_name='img_info'):

	thres_explained_var = 0.3
	decomposition_method = 'FactorAnalysis'

	for i_layer in range(len(layers)):

		attributions = []
		layer = layers[i_layer]
		no_slash_layer_name = ''.join(layer.split('/'))
		if no_slash_layer_name == 'maxpool10':
			no_slash_layer_name = 'mixed4e'
		elif no_slash_layer_name == 'maxpool4':
			no_slash_layer_name = 'mixed3b'
		else:
			pass

		for i in range(len(img_paths)):
			img_name = img_paths[i]
			attr_classes = corres_attr_classes[i]
			attr_class = attr_classes[0]

			root_directory = create_root_dir(img_name, attr_class, "neuronAttr", img_info_name=img_info_name)
			attr_ori = np.load(root_directory + '/' + attr_class + '_' + flag1 + '_' +
						   layer + '.npy')

			attr_squ = np.squeeze(attr_ori)

			attr_filtered, kept_channel_indices = \
				filter_attr_map(attr_squ, filter_with_hm=False, total_thres=layer_alpha[i_layer],
							    filter_model='cumsumMax', flag_kept_channel_list=True)
			attr_filtered = np.maximum(0, attr_filtered)

			attributions.append(attr_filtered)

		attributions = np.stack(attributions, axis=0)
		attributions = attributions.reshape((attributions.shape[0]*attributions.shape[1]*attributions.shape[2],
		                                     attributions.shape[3]))
		attr_corr_max_channel_indices = np.argsort(np.max(attributions, 0))[::-1]

		spatial_factors, channel_factors, n_groups = \
			decompose_AM_get_group_num(decomposition_method, attributions, thres_explained_var)

		# factor analysis
		channel_indices_in_group = np.where(np.max(channel_factors, 0) >= 0.02)[0]

		channel_factors_max_index = channel_factors.argmax(axis=0)

		maxAttr_maxChanFactor_intersect = np.intersect1d(attr_corr_max_channel_indices[:30],
		                                                 channel_indices_in_group)

		short_index = []
		for channel_factors_i in range(n_groups):
			channel_indices_in_one_group_temp = np.squeeze(np.argwhere(channel_factors_max_index == channel_factors_i), axis=1)
			channel_indices_in_one_group = np.intersect1d(channel_indices_in_one_group_temp, channel_indices_in_group)
			short_index.append(channel_indices_in_one_group)
		short_index = [ele for ele in short_index if ele != []]
		save_directory = './experiment/googlenet/' + attr_class
		if not os.path.exists(save_directory):
			os.makedirs(save_directory)

		np.save(save_directory + '/' + no_slash_layer_name + '_groupinfo.npy', short_index, allow_pickle=True)
		aaa = np.load(save_directory + '/' + no_slash_layer_name + '_groupinfo.npy', allow_pickle=True)
		print('Group number is {}'.format(len(short_index)))
		print('Layer {} and class {} is finished'.format(layer, attr_class))
		print()

if __name__ == "__main__":
	# execute only if run as a script
	main()

max_stars_repo_name: GlowingHorse/Attribution-Visualization

max_issues_repo_name: GlowingHorse/Attribution-Visualization

max_forks_repo_name: GlowingHorse/Attribution-Visualization


text:
"""
Provides preprocessing steps to be performed on the timeseries data.
"""
import logging

import numpy as np
import pandas as pd

from solarforecastarbiter import datamodel
from solarforecastarbiter.validation import quality_mapping

logger = logging.getLogger(__name__)

# Titles to refer to counts of preprocessing results
VALIDATION_RESULT_TOTAL_STRING = "TOTAL FLAGGED VALUES DISCARDED"
DISCARD_DATA_STRING = "Values Discarded by Alignment"
UNDEFINED_DATA_STRING = "Undefined Values"

def apply_validation(obs_df, qfilter, handle_func):
    """
    Apply validation steps based on provided filters to the data.

    Parameters
    ----------
    obs_df : pandas.DataFrame
        The observation data with 'value' and 'quality_flag' columns
    qfilter : solarforecastarbiter.datamodel.QualityFlagFilter
    handle_func : function
        Function that handles how `quality_flags` will be used.
        See solarforecastarbiter.metrics.preprocessing.exclude as an
        example.

    Returns
    -------
    validated_obs : pandas.Series
        The validated timeseries data as pandas.Series.
    counts : dict
        Dict where keys are qfilter.quality_flags and values
        are integers indicating the number of points filtered
        for the given flag.
    """
    # List of flags from filter
    if not isinstance(qfilter, datamodel.QualityFlagFilter):
        raise TypeError(f"{qfilter} not a QualityFlagFilter")
    filters = qfilter.quality_flags

    if obs_df.empty:
        return obs_df.value, {f: 0 for f in filters}
    else:
        validation_df = quality_mapping.convert_mask_into_dataframe(
            obs_df['quality_flag'])
        validation_df = validation_df[list(filters)]
        validated_obs = handle_func(obs_df.value, validation_df)
        counts = validation_df.astype(int).sum(axis=0).to_dict()
        return validated_obs, counts

def _resample_event_obs(obs, fx, obs_data):
    """
    Resample the event observation.

    Parameters
    ----------
    obs : datamodel.Observation
        The Observation being resampled.
    fx : datamodel.EventForecast
        The corresponding Forecast.
    obs_data : pd.Series
        Timeseries data of the event observation.

    Returns
    -------
    obs_resampled : pandas.Series
        Timeseries data of the Observation resampled to match the Forecast.

    Raises
    ------
    RuntimeError
        If the Forecast and Observation do not have the same interval length.

    """

    if fx.interval_length != obs.interval_length:
        raise ValueError("Event observation and forecast time-series "
                         "must have matching interval length.")
    else:
        obs_resampled = obs_data

    return obs_resampled

def _validate_event_dtype(ser):
    """
    Validate the event data dtype, converting to boolean values if possible.

    Parameter
    ---------
    ser : pandas.Series
        The event time-series data (observation or forecast).

    Returns
    -------
    ser : pandas.Series
        The event time-series data as boolean values.

    Raises
    ------
    TypeError
        If the event time-series data dtype cannot be converted to boolean.

    """

    if ser.dtype == bool:
        return ser
    elif ser.dtype == int and np.all(np.isin(ser.unique(), [0, 1])):
        return ser.astype(bool)
    elif ser.dtype == float and np.all(np.isin(ser.unique(), [0.0, 1.0])):
        return ser.astype(bool)
    else:
        raise TypeError("Invalid data type for event time-series; unable to "
                        "convert {} to boolean.".format(ser.dtype))

def _resample_obs(obs, fx, obs_data):
    """

    Parameters
    ----------
    obs : datamodel.Observation
        The Observation being resampled.
    fx : datamodel.Forecast
        The corresponding Forecast.
    obs_data : pandas.Series
        Timeseries data of the observation/aggregate after processing the
        quality flag column.

    Returns
    -------
    obs_resampled : pandas.Series
        The observation time-series resampled to match the forecast
        time-series.

    """

    # Resample observation, checking for invalid interval_length and that
    # the Series has data:
    if fx.interval_length > obs.interval_length and not obs_data.empty:
        closed = datamodel.CLOSED_MAPPING[fx.interval_label]
        obs_resampled = obs_data.resample(
            fx.interval_length,
            label=closed,
            closed=closed
        ).agg(["mean", "count"])

        # Drop intervals if too many samples missing
        count_threshold = int(
            fx.interval_length / obs.interval_length * 0.1
        )
        obs_resampled = obs_resampled["mean"].where(
            obs_resampled["count"] >= count_threshold
        )
    else:
        obs_resampled = obs_data

    return obs_resampled

def resample_and_align(fx_obs, fx_data, obs_data, ref_data, tz):
    """
    Resample the observation to the forecast interval length and align to
    remove overlap.

    Parameters
    ----------
    fx_obs : solarforecastarbiter.datamodel.ForecastObservation, solarforecastarbiter.datamodel.ForecastAggregate
        Pair of forecast and observation.
    fx_data : pandas.Series or pandas.DataFrame
        Timeseries data of the forecast.
    obs_data : pandas.Series
        Timeseries data of the observation/aggregate after processing the quality
        flag column.
    ref_data : pandas.Series or pandas.DataFrame or None
        Timeseries data of the reference forecast.
    tz : str
        Timezone to which processed data will be converted.

    Returns
    -------
    forecast_values : pandas.Series or pandas.DataFrame
    observation_values : pandas.Series
    reference_forecast_values : pandas.Series or pandas.DataFrame or None
    results : dict

    Notes
    -----
    In the case where the `interval_label` of the `obs` and `fx` do not match,
    this function currently returns a `ProcessedForecastObservation` object
    with a `interval_label` the same as the `fx`, regardless of whether the
    `interval_length` of the `fx` and `obs` are the same or different.

    Raises
    ------
    ValueError
        If fx_obs.reference_forecast is not None but ref_data is None
        or vice versa
    ValueError
        If fx_obs.reference_forecast.interval_label or interval_length
        does not match fx_obs.forecast.interval_label or interval_length

    Todo
    ----
      * Add other resampling functions (besides mean like first, last, median)
    """  # noqa: E501
    fx = fx_obs.forecast
    obs = fx_obs.data_object
    ref_fx = fx_obs.reference_forecast

    # raise ValueError if intervals don't match
    _check_ref_fx(fx, ref_fx, ref_data)

    # Resample based on forecast type
    if isinstance(fx, datamodel.EventForecast):
        fx_data = _validate_event_dtype(fx_data)
        obs_data = _validate_event_dtype(obs_data)
        obs_resampled = _resample_event_obs(obs, fx, obs_data)
    else:
        obs_resampled = _resample_obs(obs, fx, obs_data)

    # Align (forecast is unchanged)
    # Remove non-corresponding observations and
    # forecasts, and missing periods
    obs_resampled = obs_resampled.dropna(how="any")
    obs_aligned, fx_aligned = obs_resampled.align(fx_data.dropna(how="any"),
                                                  'inner')
    # another alignment step if reference forecast exists.
    # here we drop points that don't exist in all 3 series.
    # could set reference forecast to NaN where missing instead.
    # could set to 0 instead.
    # could build a DataFrame (implicit outer-join), then perform
    # alignment using ['forecast', 'observation'] or
    # ['forecast', 'observation', 'reference'] selections
    if ref_data is not None:
        obs_aligned, ref_fx_aligned = obs_aligned.align(
            ref_data.dropna(how="any"), 'inner')
        fx_aligned = fx_aligned.reindex(obs_aligned.index)
        ref_values = ref_fx_aligned.tz_convert(tz)
    else:
        ref_values = None

    # Determine series with timezone conversion
    forecast_values = fx_aligned.tz_convert(tz)
    observation_values = obs_aligned.tz_convert(tz)

    # prob fx DataFrame needs to be summed across both dimensions
    if isinstance(fx_data, pd.DataFrame):
        undefined_fx = fx_data.isna().sum().sum()
    else:
        undefined_fx = fx_data.isna().sum()

    # Return dict summarizing results
    results = {
        type(fx).__name__ + " " + DISCARD_DATA_STRING:
            len(fx_data.dropna(how="any")) - len(fx_aligned),
        type(obs).__name__ + " " + DISCARD_DATA_STRING:
            len(obs_resampled) - len(observation_values),
        type(fx).__name__ + " " + UNDEFINED_DATA_STRING:
            int(undefined_fx),
        type(obs).__name__ + " " + UNDEFINED_DATA_STRING:
            int(obs_data.isna().sum())
    }

    if ref_data is not None:
        k = type(ref_fx).__name__ + " " + UNDEFINED_DATA_STRING
        results[k] = len(ref_data.dropna(how='any')) - len(ref_fx_aligned)

    return forecast_values, observation_values, ref_values, results

def _check_ref_fx(fx, ref_fx, ref_data):
    if ref_fx is not None and ref_data is None:
        raise ValueError(
            'ref_data must be supplied if fx_obs.reference_forecast is not'
            'None')
    elif ref_fx is None and ref_data is not None:
        raise ValueError(
            'ref_data was supplied but fx_obs.reference_forecast is None')

    if ref_fx is not None:
        if fx.interval_length != ref_fx.interval_length:
            raise ValueError(
                'forecast.interval_length %s must match '
                'reference_forecast.interval_length %s',
                fx.interval_length, ref_fx.interval_length)
        if fx.interval_label != ref_fx.interval_label:
            raise ValueError(
                'forecast.interval_label %s must match '
                'reference_forecast.interval_label %s',
                fx.interval_label, ref_fx.interval_label)
        if isinstance(fx, datamodel.ProbabilisticForecast):
            if fx.axis != ref_fx.axis:
                raise ValueError(
                    'forecast.axis %s must match reference_forecast.axis %s',
                    fx.axis, ref_fx.axis)

def exclude(values, quality_flags=None):
    """
    Return a timeseries with all questionable values removed.
    All NaN values will be removed first and then iff `quality_flag` is set
    (not 0) the corresponding values will also be removed.

    Parameters
    ----------
    values : pandas.Series
        Timeseries values.
    quality_flags : pandas.DataFrame
        Timeseries of quality flags. Default is None.

    Returns
    -------
    pandas.Series :
        Timeseries of values excluding non-quality values.
    """
    # Missing values
    bad_idx = values.isna()

    # Handle quality flags
    if quality_flags is not None:
        consolidated_flag = quality_flags.any(axis=1)
        bad_quality_idx = (consolidated_flag != 0)
        bad_idx = bad_idx | bad_quality_idx

    return values[~bad_idx]

def _merge_quality_filters(filters):
    """Merge any quality flag filters into one single QualityFlagFilter"""
    combo = set()
    for filter_ in filters:
        if isinstance(filter_, datamodel.QualityFlagFilter):
            combo |= set(filter_.quality_flags)
    return datamodel.QualityFlagFilter(tuple(combo))

def process_forecast_observations(forecast_observations, filters, data,
                                  timezone):
    """
    Convert ForecastObservations into ProcessedForecastObservations
    applying any filters and resampling to align forecast and observation.

    Parameters
    ----------
    forecast_observations : list of solarforecastarbiter.datamodel.ForecastObservation, solarforecastarbiter.datamodel.ForecastAggregate
        Pairs to process
    filters : list of solarforecastarbiter.datamodel.BaseFilter
        Filters to apply to each pair.
    data : dict
        Dict with keys that are the Forecast/Observation/Aggregate object
        and values that are the corresponding pandas.Series/DataFrame for
        the object. Keys must also include all Forecast objects assigned
        to the ``reference_forecast`` attributes of the
        ``forecast_observations``.
    timezone : str
        Timezone that data should be converted to

    Returns
    -------
    list of ProcessedForecastObservation
    """  # NOQA
    if not all([isinstance(filter_, datamodel.QualityFlagFilter)
                for filter_ in filters]):
        logger.warning(
            'Only filtering on Quality Flag is currently implemented')
    qfilter = _merge_quality_filters(filters)
    validated_observations = {}
    processed_fxobs = {}
    for fxobs in forecast_observations:
        # validate observation or aggregate data
        if fxobs.data_object not in validated_observations:
            try:
                obs_ser, counts = apply_validation(
                    data[fxobs.data_object],
                    qfilter,
                    exclude)
            except Exception as e:
                logger.error(
                    'Failed to validate data for %s. %s',
                    fxobs.data_object.name, e)
                # store empty data in validated_observations
                preproc_results = (datamodel.PreprocessingResult(
                    name=VALIDATION_RESULT_TOTAL_STRING,
                    count=-1), )
                validated_observations[fxobs.data_object] = (
                    pd.Series([], name='value', index=pd.DatetimeIndex(
                        [], name='timestamp', tz='UTC'), dtype=float),
                    (), preproc_results)
            else:
                # store validated data in validated_observations
                val_results = tuple(datamodel.ValidationResult(flag=k, count=v)
                                    for k, v in counts.items())
                preproc_results = (datamodel.PreprocessingResult(
                    name=VALIDATION_RESULT_TOTAL_STRING,
                    count=len(data[fxobs.data_object]) - len(obs_ser)), )
                validated_observations[fxobs.data_object] = (
                    obs_ser, val_results, preproc_results)

        obs_ser, val_results, preproc_results = (
            validated_observations[fxobs.data_object])

        # resample and align observations to forecast, create
        # ProcessedForecastObservation
        fx_ser = data[fxobs.forecast]
        if fxobs.reference_forecast is not None:
            ref_ser = data[fxobs.reference_forecast]
        else:
            ref_ser = None
        try:
            forecast_values, observation_values, ref_fx_values, results = \
                resample_and_align(fxobs, fx_ser, obs_ser, ref_ser, timezone)
            preproc_results += tuple(datamodel.PreprocessingResult(
                name=k, count=v) for k, v in results.items())
        except Exception as e:
            logger.error(
                'Failed to resample and align data for pair (%s, %s): %s',
                fxobs.forecast.name, fxobs.data_object.name, e)
        else:
            logger.info('Processed data successfully for pair (%s, %s)',
                        fxobs.forecast.name, fxobs.data_object.name)
            name = _name_pfxobs(processed_fxobs.keys(),
                                fxobs.forecast.name)
            processed = datamodel.ProcessedForecastObservation(
                name=name,
                original=fxobs,
                interval_value_type=fxobs.forecast.interval_value_type,
                interval_length=fxobs.forecast.interval_length,
                interval_label=fxobs.forecast.interval_label,
                valid_point_count=len(forecast_values),
                validation_results=val_results,
                preprocessing_results=preproc_results,
                forecast_values=forecast_values,
                observation_values=observation_values,
                reference_forecast_values=ref_fx_values,
                normalization_factor=fxobs.normalization,
                uncertainty=fxobs.uncertainty
            )
            processed_fxobs[name] = processed
    return tuple(processed_fxobs.values())

def _name_pfxobs(current_names, forecast_name, i=1):
    if i > 99:
        logger.warning(
            'Limit of unique names for identically named forecasts reached.'
            ' Aligned pairs may have duplicate names.')
        return forecast_name
    if forecast_name in current_names:
        if i == 1:
            new_name = f'{forecast_name}-{i:02d}'
        else:
            new_name = f'{forecast_name[:-3]}-{i:02d}'
        return _name_pfxobs(current_names, new_name, i + 1)
    else:
        return forecast_name

max_stars_repo_name: awig/solarforecastarbiter-core

max_issues_repo_name: awig/solarforecastarbiter-core

max_forks_repo_name: awig/solarforecastarbiter-core


text:
import os
import sys
import librosa
import numpy as np
import pandas as pd
from joblib import Parallel, delayed
from glob import glob
from tqdm import tqdm

num_cores = 8

def compute_melspec(filename, outdir):
    wav = librosa.load(filename, sr=44100)[0]
    melspec = librosa.feature.melspectrogram(
        wav,
        sr=44100,
        n_fft=128*20,
        hop_length=347*2,
        n_mels=128,
        fmin=20,
        fmax=44100 // 2)
    logmel = librosa.core.power_to_db(melspec)
    np.save(outdir + os.path.basename(filename) + '.npy', logmel)

# Training and validation data
file_list = glob('./data/train/*.wav') + glob('./data/validate/*.wav')
os.makedirs('./data/logmelspec', exist_ok=True)
_ = Parallel(n_jobs=num_cores)(
        delayed(lambda x: compute_melspec(x, './data/logmelspec/'))(x)
        for x in tqdm(file_list))

# Eval data
file_list = glob('./data/audio-eval/*.wav')
os.makedirs('./data/logmelspec-eval', exist_ok=True)
_ = Parallel(n_jobs=num_cores)(
        delayed(lambda x: compute_melspec(x, './data/logmelspec-eval/'))(x)
        for x in tqdm(file_list))

max_stars_repo_name: sainathadapa/urban-sound-tagging

max_forks_repo_name: sainathadapa/urban-sound-tagging


text:
import numpy as np
import seaborn as sns

from analysis.model_peer_info import PeerInfo
from model_loader import ModelLoader
import matplotlib.pyplot as plt

def show_values_on_bars(axs, total):
    def _show_on_single_plot(ax):
        for p in ax.patches:
            _x = p.get_x() + p.get_width() / 2
            _y = p.get_y() + p.get_height()
            value = '{:.1f}%'.format(100 * p.get_height() / total)
            ax.text(_x, _y, value, ha="center")

    if isinstance(axs, np.ndarray):
        for idx, ax in np.ndenumerate(axs):
            _show_on_single_plot(ax)
    else:
        _show_on_single_plot(axs)

def plot():
    sns.set_theme()

    def calc_hop_count(peer_id: str, peer_infos: dict[str, PeerInfo], hop_count: int) -> int:
        peer_info = peer_infos[peer_id]
        if peer_info.discovered_from == "":
            return hop_count
        return calc_hop_count(peer_info.discovered_from, peer_infos, hop_count + 1)

    hop_count_distribution = []

    measurements = ModelLoader.open("../data")
    for measurement in measurements:
        for span in measurement.provider.spans:
            if span.type != "ADD_PROVIDER":
                continue
            hop_count = calc_hop_count(span.peer_id, measurement.provider.peer_infos, 0)
            hop_count_distribution += [hop_count]

    fig, ax = plt.subplots(figsize=(15, 6))

    sns.histplot(ax=ax, x=hop_count_distribution, bins=np.arange(0, 10))

    ax.set_xticks(np.arange(0, 10))
    ax.set_xlabel("Number of Hops")
    ax.set_ylabel("Count (log scale)")
    ax.set_yscale('log')

    ax.title.set_text(
        f"Number of Hops to Discover a Peer that was Selected to Store a Provider Record (Sample Size {len(hop_count_distribution)})")

    plt.tight_layout()

    show_values_on_bars(ax, len(hop_count_distribution))

    plt.savefig("../plots/hop_distribution.png")
    plt.show()

if __name__ == '__main__':
    plot()

max_stars_repo_name: dennis-tra/optimistic-provide

max_issues_repo_name: dennis-tra/optimistic-provide

max_forks_repo_name: dennis-tra/optimistic-provide


text:
"""Main application"""
from .model_trainers import ModelTrainer
from .predictors import Predictor
from .evaluation import ModelEvaluator
from .individual_importance import IndividualImportanceCalculator
from .model_grouping import ModelGrouper
from .subsetters import Subsetter
from .protected_groups_generators import ProtectedGroupsGenerator, ProtectedGroupsGeneratorNoOp
from .utils import filename_friendly_hash
import logging
from collections import namedtuple

import numpy
import pandas

TaskBatch = namedtuple('TaskBatch', ['parallelizable', 'tasks', 'description'])

class ModelTrainTester(object):
    def __init__(
        self,
        matrix_storage_engine,
        model_trainer,
        model_evaluator,
        individual_importance_calculator,
        predictor,
        subsets,
        protected_groups_generator,
        cohort_hash=None,
        replace=True
    ):
        self.matrix_storage_engine = matrix_storage_engine
        self.model_trainer = model_trainer
        self.model_evaluator = model_evaluator
        self.individual_importance_calculator = individual_importance_calculator
        self.predictor = predictor
        self.subsets = subsets
        self.replace = replace
        self.protected_groups_generator = protected_groups_generator or ProtectedGroupsGeneratorNoOp()
        self.cohort_hash = cohort_hash

    def generate_task_batches(self, splits, grid_config, model_comment=None):
        train_test_tasks = []
        logging.info("Generating train/test tasks for %s splits", len(splits))
        for split in splits:
            train_store = self.matrix_storage_engine.get_store(split["train_uuid"])
            train_tasks = self.model_trainer.generate_train_tasks(
                grid_config=grid_config,
                misc_db_parameters=dict(test=False, model_comment=model_comment),
                matrix_store=train_store
            )

            for test_matrix_def, test_uuid in zip(
                split["test_matrices"], split["test_uuids"]
            ):
                test_store = self.matrix_storage_engine.get_store(test_uuid)

                for train_task in train_tasks:
                    train_test_tasks.append(
                        {
                            "test_store": test_store,
                            "train_store": train_store,
                            "train_kwargs": train_task,
                        }
                    )
        return self.order_and_batch_tasks(train_test_tasks)

    def order_and_batch_tasks(self, tasks):
        batches = (
            TaskBatch(
                parallelizable=True,
                tasks=[],
                description="Baselines or simple classifiers (e.g. DecisionTree, SLR)"
            ),
            TaskBatch(
                parallelizable=False,
                tasks=[],
                description="Heavyweight classifiers with n_jobs set to -1."
            ),
            TaskBatch(
                parallelizable=True,
                tasks=[],
                description="All classifiers not found in one of the other batches (e.g. gradient boosting)."
            ),
        )

        for task in tasks:
            if task['train_kwargs']['class_path'].startswith('triage.component.catwalk.baselines') \
                    or task['train_kwargs']['class_path'] in (
                    'triage.component.catwalk.estimators.classifiers.ScaledLogisticRegression',
                    'sklearn.tree.DecisionTreeClassifier',
                    'sklearn.dummy.DummyClassifier'
                    ):
                # First priority: baselines or simple, effective classifiers
                batches[0].tasks.append(task)
            elif task['train_kwargs']['parameters'].get('n_jobs', None) == -1:
                # Second priority: heavyweight classifiers that we use the whole machines for
                batches[1].tasks.append(task)
            else:
                # Last priority: Everything else. Maybe these are slow/non-parallelizable
                batches[2].tasks.append(task)
        logging.info("Split train/test tasks into three task batches. - each batch has models from all splits")
        for batch_num, batch in enumerate(batches, 1):
            logging.info("Batch %s: %s (%s tasks total)", batch_num, batch.description, len(batch.tasks))
        return batches

    def process_all_batches(self, task_batches):
        # In the simple loop version here we ignore parallelizability and do everything serially
        for batch in task_batches:
            for task in batch.tasks:
                self.process_task(**task)

    def process_task(self, test_store, train_store, train_kwargs):
        logging.info("Beginning train task %s", train_kwargs)

        # If the matrices and train labels are OK, train and test the model!
        with self.model_trainer.cache_models(), test_store.cache(), train_store.cache():
            # will cache any trained models until it goes out of scope (at the end of the task)
            # this way we avoid loading the model pickle again for predictions

            # If the train or test design matrix empty, or if the train store only
            # has one label value, skip training the model.
            if train_store.empty:
                logging.warning(
                    """Train matrix for split %s was empty,
                no point in training this model. Skipping
                """,
                    train_store.uuid
                )
                return
            if len(train_store.labels.unique()) == 1:
                logging.warning(
                    """Train Matrix for split %s had only one
                unique value, no point in training this model. Skipping
                """,
                    train_store.uuid,
                )
                return
            if test_store.empty:
                logging.warning(
                    """Test matrix for uuid %s
                was empty, no point in generating predictions. Not processing train/test task.
                """,
                    test_store.uuid,
                )
                return

            model_id = self.model_trainer.process_train_task(**train_kwargs)
            if not model_id:
                logging.warning("No model id returned from ModelTrainer.process_train_task, "
                                "training unsuccessful. Not attempting to test")
                return
            logging.info("Trained task %s and got model id %s", train_kwargs, model_id)
            as_of_dates = test_store.as_of_dates
            logging.info(
                "Testing and scoring model id %s with test matrix %s. "
                "as_of_times min: %s max: %s num: %s",
                model_id,
                test_store.uuid,
                min(as_of_dates),
                max(as_of_dates),
                len(as_of_dates),
            )

            self.individual_importance_calculator.calculate_and_save_all_methods_and_dates(
                model_id, test_store
            )

            # Generate predictions for the testing data then training data
            for store in (test_store, train_store):
                predictions_proba = numpy.array(None)
                protected_df = None
                if self.replace:
                    logging.info(
                        "Replace flag set; generating new predictions and evaluations for"
                        "matrix %s-%s, and model %s",
                        store.uuid,
                        store.matrix_type,
                        model_id
                    )

                    predictions_proba = self.predictor.predict(
                        model_id,
                        store,
                        misc_db_parameters=dict(),
                        train_matrix_columns=train_store.columns(),
                    )

                for subset in self.subsets:
                    if self.replace or self.model_evaluator.needs_evaluations(
                        store, model_id, filename_friendly_hash(subset)
                    ):
                        logging.info(
                            "Evaluating matrix %s-%s, subset %s, and model %s",
                            store.uuid,
                            store.matrix_type,
                            filename_friendly_hash(subset),
                            model_id,
                        )

                        if not predictions_proba.any():
                            logging.info(
                                "Generating new predictions for"
                                "matrix %s-%s, and model %s to make evaluation",
                                store.uuid,
                                store.matrix_type,
                                model_id
                            )

                            predictions_proba = self.predictor.predict(
                                model_id,
                                store,
                                misc_db_parameters=dict(),
                                train_matrix_columns=train_store.columns(),
                            )
                        if protected_df is None:
                            protected_df = self.protected_groups_generator.as_dataframe(
                                as_of_dates=store.as_of_dates,
                                cohort_hash=self.cohort_hash,
                            )

                        self.model_evaluator.evaluate(
                            predictions_proba=predictions_proba,
                            matrix_store=store,
                            model_id=model_id,
                            subset=subset,
                            protected_df=protected_df
                        )

                    else:
                        logging.info(
                            "The evaluations needed for matrix %s-%s, subset %s, and "
                            "model %s are all present"
                            "in db from a previous run (or none needed at all), so skipping!",
                            store.uuid,
                            store.matrix_type,
                            filename_friendly_hash(subset),
                            model_id
                        )
                self.predictor.update_db_with_ranks(model_id, store.uuid, store.matrix_type)

__all__ = (
    "IndividualImportanceCalculator",
    "ModelEvaluator",
    "ModelGrouper"
    "ModelTrainer",
    "Predictor",
    "ModelTrainTester",
    "Subsetter",
)

max_stars_repo_name: adunmore/triage

max_issues_repo_name: adunmore/triage

max_forks_repo_name: adunmore/triage