text:
"""
    Full example of using TF-Yarn to run Keras on YARN.
"""
import logging
logging.basicConfig(level="INFO") # noqa
import sys
import getpass
import gzip
import subprocess
import os
import skein
import numpy as np

from functools import partial
from datetime import datetime
from tensorflow import keras

import cluster_pack
from tf_yarn import event, TaskSpec, Experiment, run_on_yarn
import winequality

USER = getpass.getuser()

"""
1. Download winequality-*.csv from the Wine Quality dataset at UCI
   ML repository
   (https://archive.ics.uci.edu/ml/datasets/Wine+Quality).
2. Upload it to HDFS
3. Pass a full URI to either of the CSV files to the example
"""
WINE_EQUALITY_FILE = f"{cluster_pack.get_default_fs()}/user/{USER}/tf_yarn_test/winequality-red.csv"

"""
Output path of the learned model on hdfs
"""
HDFS_DIR = (f"{cluster_pack.get_default_fs()}/user/{USER}"
            f"/tf_yarn_test/tf_yarn_{int(datetime.now().timestamp())}")

def main():
    def experiment_fn() -> Experiment:
        # To mitigate issue https://github.com/tensorflow/tensorflow/issues/32159 for tf >= 1.15
        import tensorflow as tf

        def convert_to_tensor(x, y):
            return (tf.convert_to_tensor(value=list(x.values()), dtype=tf.float32),
                    tf.convert_to_tensor(value=y, dtype=tf.int32))

        def train_input_fn():
            dataset = winequality.get_dataset(WINE_EQUALITY_FILE, split="train")
            return (dataset.map(convert_to_tensor)
                    .shuffle(1000)
                    .batch(128)
                    .repeat())

        def eval_input_fn():
            dataset = winequality.get_dataset(WINE_EQUALITY_FILE, split="test")
            return (dataset.map(convert_to_tensor)
                    .shuffle(1000)
                    .batch(128))

        model = keras.Sequential()
        model.add(keras.layers.Dense(units=300, activation="relu", input_shape=(11,)))
        model.add(keras.layers.Dense(units=100, activation="relu"))
        model.add(keras.layers.Dense(units=10, activation="softmax"))
        model.summary()
        model.compile(loss='sparse_categorical_crossentropy',
                      optimizer="sgd",
                      metrics=['accuracy'])

        config = tf.estimator.RunConfig(model_dir=HDFS_DIR)
        estimator = tf.keras.estimator.model_to_estimator(model, config=config)
        return Experiment(
            estimator,
            tf.estimator.TrainSpec(
                train_input_fn,
                max_steps=1000),
            tf.estimator.EvalSpec(
                eval_input_fn,
                steps=10,
                start_delay_secs=0,
                throttle_secs=30))

    # forcing call to model_to_estimator._save_first_checkpoint l457
    # https://github.com/tensorflow/estimator/blob/ \
    # 1d55f01d8af871a35ef83fc3354b9feaa671cbe1/tensorflow_estimator/python/estimator/keras.py
    # otherwise there is a race condition
    # when all workers try to save the first checkpoint at the same time
    experiment_fn()

    pyenv_zip_path, env_name = cluster_pack.upload_env()
    editable_requirements = cluster_pack.get_editable_requirements()
    run_on_yarn(
        pyenv_zip_path,
        experiment_fn,
        task_specs={
            "chief": TaskSpec(memory="2 GiB", vcores=4),
            "worker": TaskSpec(memory="2 GiB", vcores=4, instances=4),
            "ps": TaskSpec(memory="2 GiB", vcores=4, instances=2),
            "evaluator": TaskSpec(memory="2 GiB", vcores=1)
        },
        files={
            **editable_requirements,
            os.path.basename(winequality.__file__): winequality.__file__,
        }
    )

if __name__ == "__main__":
    main()

max_stars_repo_name: fhoering/tf-yarn

max_issues_repo_name: fhoering/tf-yarn

max_forks_repo_name: fhoering/tf-yarn


text:
"""
Example for a specialized analysis class.

It includes two algorithms for specific analysis routines.
And it provides standard interface functions modified for the specific analysis routine like report.
"""

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from scipy import stats

from locan.analysis.analysis_base import _Analysis

#
#### The algorithms
#
# First there is an algorithm to compute something given a set of point coordinates.
# Second there might be an alternative algorithm doing the same thing.
# Both can be tested with some simple point data.
#
# The user will not have to use this particular function directly.
#

def _algorithm_1(data=None, limits=(0, 10)):
    """ Provides a list of data values. data would be input data that is currently not used."""
    results = [i for i in range(*limits)]  # some complicated algorithm
    results = pd.DataFrame.from_dict(dict(a=results))
    return results

def _algorithm_2(data=None, n_sample=100, seed=None):
    """ Provides random normal distributed data. data would be input data that is currently not used."""
    np.random.seed(seed)
    dict_ = {"a": np.random.normal(size=n_sample), "b": np.random.normal(size=n_sample)}
    results = pd.DataFrame.from_dict(dict_)
    return results

#
##### The base analysis class
#
# Now we want a class implementing this algorithm to be used with locdata. Also the results from this algorithm should
# be reused in some visual representation that is specific for this analysis routine.
# Therefore we have a class that holds results and organizes metadata and provides the specific plotting routine.
#

# class: analysis.analysis_base._Analysis

#
# This specific analysis classes inherit from _Analysis.
#
# The classes for each particular algorithm are defined as:
#

class AnalysisExampleAlgorithm_1(_Analysis):
    """
    Example for an analysis class implementing algorithm_1.
    Compute some data and provide a plot and histogram with secondary data (e.g. from fitting plot or histogram).

    This is a specialized analysis class implementing an example analysis routine. For illustrating the analysis
    procedure it only takes a LocData object, creates some random data as result and
    provides plots and a report of the results.

    Parameters
    ----------
    meta : locan.analysis.metadata_analysis_pb2.AMetadata
        Metadata about the current analysis routine.
    kwargs : dict
        Parameter that are passed to the algorithm.

    Attributes
    ----------
    count : int
        A counter for counting instantiations.
    parameter : dict
        A dictionary with all settings for the current computation.
    meta : locan.analysis.metadata_analysis_pb2.AMetadata
        Metadata about the current analysis routine.
    results : numpy.ndarray, pandas.DataFrame
        Computed results.
    """

    count = 0

    def __init__(self, meta=None, limits=(0, 10)):
        super().__init__(meta=meta, limits=limits)

    def compute(self, locdata=None):
        """
        Run the computation.

        Parameters
        ----------
        locdata : LocData
          Localization data that might be clustered.

        Returns
        -------
        Analysis class
          Returns the Analysis class object (self).
        """
        data = locdata  # take certain elements from locdata
        self.results = _algorithm_1(
            data=data, **self.parameter
        )  # some complicated algorithm
        return self

    def plot(self, ax=None):
        plot(self, ax)

    def plot_2(self, ax=None, bins="auto", normed=True, log=False, fit=True):
        plot_2(self, ax, bins, normed, log, fit)

    def plot_histogram_fit(self, ax=None):
        plot_histogram_fit(self, ax)

    def report(self, path=None):
        report(self, path)

class AnalysisExampleAlgorithm_2(_Analysis):
    """
    Example for an analysis class implementing algorithm_2.
    Compute some data and provide a plot and histogram with secondary data (e.g. from fitting plot or histogram).

    This is a specialized analysis class implementing an example analysis routine. For illustrating the analysis
    procedure it only takes a LocData object, creates some random data as result and
    provides plots and a report of the results.

    Parameters
    ----------
    meta : locan.analysis.metadata_analysis_pb2.AMetadata
        Metadata about the current analysis routine.
    kwargs : dict
        Parameter that are passed to the algorithm.

    Attributes
    ----------
    count : int
        A counter for counting instantiations.
    parameter : dict
        A dictionary with all settings for the current computation.
    meta : locan.analysis.metadata_analysis_pb2.AMetadata
        Metadata about the current analysis routine.
    results : numpy.ndarray, pandas.DataFrame
        Computed results.
    """

    count = 0

    def __init__(self, meta=None, n_sample=100, seed=None):
        super().__init__(meta=meta, n_sample=n_sample, seed=seed)

    def compute(self, locdata=None):
        """
        Run the computation.

        Parameters
        ----------
        locdata : LocData
          Localization data that might be clustered.

        Returns
        -------
        Analysis class
          Returns the Analysis class object (self).
        """
        data = locdata  # take certain elements from locdata
        self.results = _algorithm_2(
            data=data, **self.parameter
        )  # some complicated algorithm
        return self

    def plot(self, ax=None):
        plot(self, ax)

    def plot_2(self, ax=None, bins="auto", density=True, log=False, fit=True):
        plot_2(self, ax, bins, density, log, fit)

    def plot_histogram_fit(self, ax=None):
        plot_histogram_fit(self, ax)

    def report(self, path=None):
        report(self, path)

#
#### Interface functions
#
# Now we have a class structure with results that can be further processed.
# Secondary results from e.g. fit procedures are added to the analysis class as new attributes.
#

def plot(self, ax=None):
    """
    A specialized plot to give a standardized visualization of results.
    """
    if ax is None:
        ax = plt.gca()

    ax.plot(self.results)
    ax.set(title="Normal Data", xlabel=property, ylabel="PDF")

    return ax

def plot_2(self, ax=None, bins="auto", density=True, log=False, fit=True):
    """
    A specialized plot to give a standardized visualization of results - in this case a histogram of results.
    """
    if ax is None:
        fig = plt.figure(figsize=(8, 3))
        ax = fig.subplots(nrows=1, ncols=2)
        plt.subplots_adjust(wspace=0)

    # create histogram on first axes
    hist, bins, _ = ax[0].hist(
        self.results.values,
        bins=bins,
        density=density,
        log=log,
        label=list(self.results),
    )
    ax[0].set(title="Normal Data", xlabel="property", ylabel="PDF")

    # create legend and results text on second axes
    h, l = ax[0].get_legend_handles_labels()
    ax[1].legend(
        h,
        l,
        loc="upper left",
        bbox_to_anchor=(0, 1),
        title="Legend",
        frameon=False,
        borderaxespad=0,
    )

    ax[1].set_axis_off()

    # fit distributions
    if fit:
        plot_histogram_fit(self, ax=ax)

    return ax

def plot_histogram_fit(self, ax=None):
    """
    A specialized plot to give a standardized visualization of results - in this case a histogram of results.
    """
    if ax is None:
        fig = plt.figure(figsize=(8, 3))
        ax = fig.subplots(nrows=1, ncols=2)
        plt.subplots_adjust(wspace=0)

    # fit distributions
    loc, scale = fit_histogram(self, data=self.results["a"].values, id_="a")

    # plot fit
    x_values = np.linspace(
        stats.norm.ppf(0.01, loc=loc, scale=scale),
        stats.norm.ppf(0.99, loc=loc, scale=scale),
        100,
    )
    ax[0].plot(
        x_values,
        stats.norm.pdf(x_values, loc=loc, scale=scale),
        "r-",
        lw=3,
        alpha=0.6,
        label="norm pdf",
    )

    # present fit results
    ax[1].text(0, 0.5, "Fit Results:")
    ax[1].text(
        0,
        0.5,
        "center: " + str(loc) + "\n" + "sigma: " + str(scale),
        horizontalalignment="left",
        verticalalignment="top",
        transform=ax[1].transAxes,
        clip_on=False,
    )

    ax[1].set_axis_off()

    return ax

def fit_histogram(self, data, id_):

    # MLE fit of distribution on data
    loc, scale = stats.norm.fit(data)

    attribute_center = id_ + "_center"
    attribute_sigma = id_ + "_sigma"
    self.attribute_center = loc
    self.attribute_sigma = scale

    return loc, scale

# there will be other specific visualization methods for other analysis routines.

def report(self, path=None):
    """
    Provide a report that is either displayed or saved as pdf.
    The report is a figure summarizing all visual representations. It is arranged specifically for a particular
    analysis routine.
    """
    fig = plt.figure(figsize=(8.3, 11.7))
    ax = fig.subplots(nrows=3, ncols=2)

    # provide the axes elements (i.e. the plots)
    self.plot(ax=ax[0][0])
    self.plot_2(ax=ax[1][0:2])

    # adjust figure layout
    plt.tight_layout()

    # save figure as pdf
    if path is not None:
        plt.savefig(
            fname=path,
            dpi=None,
            facecolor="w",
            edgecolor="w",
            orientation="portrait",
            papertype=None,
            format=None,
            transparent=False,
            bbox_inches=None,
            pad_inches=0.1,
            frameon=None,
        )

    return ax

ext: py

max_stars_repo_name: super-resolution/Locan

max_issues_repo_name: super-resolution/Locan

max_forks_repo_name: super-resolution/Locan

reason: import numpy,from scipy


text:
# Copyright (c) 2019-2020, NVIDIA CORPORATION.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pytest
import cupy as cp
from cusignal.test.utils import array_equal
import cusignal
import numpy as np
from scipy import signal

@pytest.mark.parametrize('num_samps', [2**14])
@pytest.mark.parametrize('fs', [1.0, 1e6])
@pytest.mark.parametrize('nperseg', [1024, 2048])
def test_csd(num_samps, fs, nperseg):
    cpu_x = np.random.rand(num_samps)
    cpu_y = np.random.rand(num_samps)
    gpu_x = cp.asarray(cpu_x)
    gpu_y = cp.asarray(cpu_y)

    cpu_csd = signal.csd(cpu_x, cpu_y, fs, nperseg=nperseg)
    gpu_csd = cp.asnumpy(cusignal.csd(gpu_x, gpu_y, fs, nperseg=nperseg))

    assert array_equal(cpu_csd, gpu_csd)

@pytest.mark.parametrize('num_samps', [2**14])
@pytest.mark.parametrize('fs', [1.0, 1e6])
@pytest.mark.parametrize('nperseg', [1024, 2048])
def test_csd_complex(num_samps, fs, nperseg):
    cpu_x = np.random.rand(num_samps) + 1j * np.random.rand(num_samps)
    cpu_y = np.random.rand(num_samps) + 1j * np.random.rand(num_samps)
    gpu_x = cp.asarray(cpu_x)
    gpu_y = cp.asarray(cpu_y)

    cpu_csd = signal.csd(cpu_x, cpu_y, fs, nperseg=nperseg)
    gpu_csd = cp.asnumpy(cusignal.csd(gpu_x, gpu_y, fs, nperseg=nperseg))

    assert array_equal(cpu_csd, gpu_csd)

@pytest.mark.parametrize('num_samps', [2**14])
@pytest.mark.parametrize('fs', [1.0, 1e6])
@pytest.mark.parametrize('window', ['flattop', 'nuttall'])
@pytest.mark.parametrize('scaling', ['spectrum', 'density'])
def test_periodogram(num_samps, fs, window, scaling):
    cpu_sig = np.random.rand(num_samps)
    gpu_sig = cp.asarray(cpu_sig)

    cpu_periodogram = signal.periodogram(cpu_sig, fs, window=window,
                                         scaling=scaling)
    gpu_periodogram = cp.asnumpy(
        cusignal.periodogram(gpu_sig, fs, window=window, scaling=scaling)
    )

    assert array_equal(cpu_periodogram, gpu_periodogram)

@pytest.mark.parametrize('num_samps', [2**14])
@pytest.mark.parametrize('fs', [1.0, 1e6])
@pytest.mark.parametrize('window', ['flattop', 'nuttall'])
@pytest.mark.parametrize('scaling', ['spectrum', 'density'])
def test_periodogram_complex(num_samps, fs, window, scaling):
    cpu_sig = np.random.rand(num_samps) + 1j * np.random.rand(num_samps)
    gpu_sig = cp.asarray(cpu_sig)

    cf, cpu_periodogram = signal.periodogram(
        cpu_sig, fs, window=window, scaling=scaling
    )
    gf, gpu_periodogram = cusignal.periodogram(
        gpu_sig, fs, window=window, scaling=scaling
    )
    gpu_periodogram = cp.asnumpy(gpu_periodogram)

    assert array_equal(cpu_periodogram, gpu_periodogram)

@pytest.mark.parametrize('num_samps', [2**14])
@pytest.mark.parametrize('fs', [1.0, 1e6])
@pytest.mark.parametrize('nperseg', [1024, 2048])
def test_welch(num_samps, fs, nperseg):
    cpu_sig = np.random.rand(num_samps)
    gpu_sig = cp.asarray(cpu_sig)

    cf, cPxx_spec = signal.welch(cpu_sig, fs, nperseg=nperseg)
    gf, gPxx_spec = cusignal.welch(gpu_sig, fs, nperseg=nperseg)
    gPxx_spec = cp.asnumpy(gPxx_spec)

    assert array_equal(cPxx_spec, gPxx_spec)

@pytest.mark.parametrize('num_samps', [2**14])
@pytest.mark.parametrize('fs', [1.0, 1e6])
@pytest.mark.parametrize('nperseg', [1024, 2048])
def test_welch_complex(num_samps, fs, nperseg):
    cpu_sig = np.random.rand(num_samps) + 1j * np.random.rand(num_samps)
    gpu_sig = cp.asarray(cpu_sig)

    cf, cPxx_spec = signal.welch(cpu_sig, fs, nperseg=nperseg)
    gf, gPxx_spec = cusignal.welch(gpu_sig, fs, nperseg=nperseg)
    gPxx_spec = cp.asnumpy(gPxx_spec)

    assert array_equal(cPxx_spec, gPxx_spec)

@pytest.mark.parametrize('num_samps', [2**14])
@pytest.mark.parametrize('fs', [1.0, 1e6])
def test_spectrogram(num_samps, fs):
    cpu_sig = np.random.rand(num_samps)
    gpu_sig = cp.asarray(cpu_sig)

    cf, ct, cPxx_spec = signal.spectrogram(cpu_sig, fs)
    gf, gt, gPxx_spec = cusignal.spectrogram(gpu_sig, fs)
    gPxx_spec = cp.asnumpy(gPxx_spec)

    assert array_equal(cPxx_spec, gPxx_spec)

@pytest.mark.parametrize('num_samps', [2**14])
@pytest.mark.parametrize('fs', [1.0, 1e6])
def test_spectrogram_complex(num_samps, fs):
    cpu_sig = np.random.rand(num_samps) + 1j * np.random.rand(num_samps)
    gpu_sig = cp.asarray(cpu_sig)

    cf, ct, cPxx_spec = signal.spectrogram(cpu_sig, fs)
    gf, gt, gPxx_spec = cusignal.spectrogram(gpu_sig, fs)
    gPxx_spec = cp.asnumpy(gPxx_spec)

    assert array_equal(cPxx_spec, gPxx_spec)

@pytest.mark.parametrize('num_samps', [2**14])
@pytest.mark.parametrize('fs', [1.0, 1e6])
@pytest.mark.parametrize('nperseg', [1024, 2048])
def test_coherence(num_samps, fs, nperseg):
    cpu_x = np.random.rand(num_samps)
    cpu_y = np.random.rand(num_samps)
    gpu_x = cp.asarray(cpu_x)
    gpu_y = cp.asarray(cpu_y)

    cf, cpu_coherence = signal.coherence(cpu_x, cpu_y, fs, nperseg=nperseg)
    gf, gpu_coherence = cusignal.coherence(gpu_x, gpu_y, fs, nperseg=nperseg)
    gpu_coherence = cp.asnumpy(gpu_coherence)

    assert array_equal(cpu_coherence, gpu_coherence)

@pytest.mark.parametrize('num_samps', [2**14])
@pytest.mark.parametrize('fs', [1.0, 1e6])
@pytest.mark.parametrize('nperseg', [1024, 2048])
def test_coherence_complex(num_samps, fs, nperseg):
    cpu_x = np.random.rand(num_samps) + 1j * np.random.rand(num_samps)
    cpu_y = np.random.rand(num_samps) + 1j * np.random.rand(num_samps)
    gpu_x = cp.asarray(cpu_x)
    gpu_y = cp.asarray(cpu_y)

    cf, cpu_coherence = signal.coherence(cpu_x, cpu_y, fs, nperseg=nperseg)
    gf, gpu_coherence = cusignal.coherence(gpu_x, gpu_y, fs, nperseg=nperseg)
    gpu_coherence = cp.asnumpy(gpu_coherence)

    assert array_equal(cpu_coherence, gpu_coherence)

@pytest.mark.parametrize('num_samps', [2**14])
@pytest.mark.parametrize('fs', [1.0, 1e6])
@pytest.mark.parametrize('nperseg', [1024, 2048])
def test_stft(num_samps, fs, nperseg):
    cpu_sig = np.random.rand(num_samps)
    gpu_sig = cp.asarray(cpu_sig)

    cf, ct, cpu_stft = signal.stft(cpu_sig, fs, nperseg=nperseg)
    gf, gt, gpu_stft = cusignal.stft(gpu_sig, fs, nperseg=nperseg)
    gpu_stft = cp.asnumpy(gpu_stft)

    assert array_equal(cpu_stft, gpu_stft)

@pytest.mark.parametrize('num_samps', [2**14])
@pytest.mark.parametrize('fs', [1.0, 1e6])
@pytest.mark.parametrize('nperseg', [1024, 2048])
def test_stft_complex(num_samps, fs, nperseg):
    cpu_sig = np.random.rand(num_samps) + 1j * np.random.rand(num_samps)
    gpu_sig = cp.asarray(cpu_sig)

    cf, ct, cpu_stft = signal.stft(cpu_sig, fs, nperseg=nperseg)
    gf, gt, gpu_stft = cusignal.stft(gpu_sig, fs, nperseg=nperseg)
    gpu_stft = cp.asnumpy(gpu_stft)

    assert array_equal(cpu_stft, gpu_stft)

max_stars_repo_name: raydouglass/cusignal

max_issues_repo_name: raydouglass/cusignal

max_forks_repo_name: raydouglass/cusignal

reason: import numpy,from scipy,import cupy


text:
from styx_msgs.msg import TrafficLight
import rospy
from sensor_msgs.msg import Image
import sys
import os
sys.path.append("../../../keras-yolo3")
from yolo import YOLO
import cv2
from PIL import Image as PILImage
import numpy as np

class TLClassifier(object):
    def __init__(self):
        #TODO load classifier
        self.frameCounter = 0

        rospy.loginfo("Creating YOLO object.")
        self.yolo = YOLO()
        rospy.loginfo("Create success YOLO object")

        with open("/home/student/Desktop/workspace/debug.csv", "w") as f:
            f.writelines("frame,subframe,result,green_area_h_mean,yellow_area_h_mean,red_area_h_mean,green_area_r_mean,yellow_area_r_mean,red_area_r_mean,result2\n")

    def get_classification(self, cv_image):
        """Determines the color of the traffic light in the image

        Args:
            image (cv::Mat): image containing the traffic light

        Returns:
            int: ID of traffic light color (specified in styx_msgs/TrafficLight)

        """
        tl_state = TrafficLight.GREEN
        tl_state2 = TrafficLight.GREEN

        cv_image = cv2.resize(cv_image, (416,416))
        cv_image = cv2.cvtColor(cv_image, cv2.COLOR_BGR2RGB)
        origin_image = PILImage.fromarray(cv_image)
        origin_image = origin_image.convert('RGB')

        target_image = origin_image.copy()

        det_image, results = self.yolo.detect_image(target_image)
        #det_image.save('/home/student/Desktop/workspace/Det{:0>4}.bmp'.format(self.frameCounter))
        #rospy.loginfo(det_image)

        subFrCounter = 0
        clf_results = None
        finaljudge = []
        for ret in results:
            predicted_class = ret[0]
            score = ret[1]
            left = ret[2]
            top = ret[3]
            right = ret[4]
            bottom = ret[5]
            #print("Result:"+ret[0])

            # Crop traffic light area
            tl_image = None
            if(predicted_class == 'traffic light'):
                #print('Traffic light:found')
                tl_image = origin_image.crop((left, top, right, bottom))
                #tl_image.save('/home/student/Desktop/workspace/crop{:0>8}.png'.format(self.frameCounter))

            # Judge traffic light color
            isRedSign = False
            curr_judge = None
            if(tl_image):

                # Convert color space
                tl_cv_image = np.asarray(tl_image.copy().convert("HSV"))
                h_image = tl_cv_image[:, :, 0]
                s_image = tl_cv_image[:, : ,1]
                v_image = tl_cv_image[:, :, 2]

                tl_cv_imageRGB = np.asarray(tl_image.copy().convert("RGB"))
                r_image = tl_cv_imageRGB[:, :, 0]
                g_image = tl_cv_imageRGB[:, :, 1]
                b_image = tl_cv_imageRGB[:, :, 2]

                #cv2.imwrite('/home/student/Desktop/workspace/crop_h_{:0>8}.bmp'.format(self.frameCounter),h_image)
                #cv2.imwrite('/home/student/Desktop/workspace/crop_s_{:0>8}.bmp'.format(self.frameCounter),s_image)
                #cv2.imwrite('/home/student/Desktop/workspace/crop_v_{:0>8}.bmp'.format(self.frameCounter),v_image)

                ############ method 1 : mean ###############
                tl_top = 0
                tl_bottom = tl_height = h_image.shape[0]

                # crop red area
                area_height = tl_height // 3
                red_top = tl_top
                red_bottom = area_height
                red_area_h_image = h_image[red_top:red_bottom, :]
                red_area_r_image = r_image[red_top:red_bottom, :]
                #print("h_image:{}".format(h_image.shape))
                #print("red_area_h_image:{}".format(red_area_h_image.shape))
                #print("red area top:{},bottom:{}  area height:{}".format(red_top,red_bottom,area_height))

                # crop yellow area
                yellow_top = red_bottom + 1
                yellow_bottom = yellow_top + area_height
                yellow_area_h_image = h_image[yellow_top:yellow_bottom, :]
                yellow_area_r_image = r_image[yellow_top:yellow_bottom, :]

                # crop green area
                green_top = yellow_bottom + 1
                green_bottom = green_top + area_height
                green_area_h_image = h_image[green_top:green_bottom, :]
                green_area_r_image = h_image[green_top:green_bottom, :]

                red_area_h_mean = red_area_h_image.mean()
                yellow_area_h_mean = yellow_area_h_image.mean()
                green_area_h_mean = green_area_h_image.mean()

                red_area_r_mean = red_area_r_image.mean()
                yellow_area_r_mean = yellow_area_r_image.mean()
                green_area_r_mean = green_area_r_image.mean()

                tl_state = TrafficLight.GREEN
                tl_state_str = "GREEN"
                if ((red_area_h_mean > yellow_area_h_mean)&(red_area_h_mean > green_area_h_mean)):
                    tl_state = TrafficLight.RED
                    #print("State:RED({})  ({:.0f},{:.0f},{:.0f})".format(tl_state,green_area_h_mean,yellow_area_h_mean,red_area_h_mean))
                    tl_state_str = "RED"
                #else:
                    #print("State:GREEN({})  ({:.0f},{:.0f},{:.0f})".format(tl_state,green_area_h_mean,yellow_area_h_mean,red_area_h_mean))

                ############ method 2 : histogram ###############
                tl_cv_imageRGB32 = cv2.resize(tl_cv_imageRGB,(32,32))

                r_hist = np.histogram(tl_cv_imageRGB32[5:10,14:19,0], bins=32, range=(0, 255))
                g_hist = np.histogram(tl_cv_imageRGB32[15:18,12:17,1], bins=32, range=(0, 255))
                b_hist = np.histogram(tl_cv_imageRGB32[25:28,12:17,2], bins=32, range=(0, 255))

                bin_edges = r_hist[1]
                bin_centers = (bin_edges[1:]  + bin_edges[0:len(bin_edges)-1])/2

                fullest_r = np.argmax(r_hist[0])
                fullest_g = np.argmax(g_hist[0])
                fullest_b = np.argmax(b_hist[0])
                feature = [fullest_r, fullest_g ,fullest_b]

                predColor = np.argmax(feature)

                tl_state_str2 = "GREEN"
                tl_state2 = TrafficLight.GREEN
                if(predColor==0 or predColor==1):
                    tl_state_str2 = "RED"
                    tl_state2 = TrafficLight.RED

                ############ for multiple tl detection ###############
                curr_judge = tl_state2
                if(subFrCounter==0):
                    clf_results = [curr_judge]
                else:
                    clf_results.append(curr_judge)

                det_image.save('/home/student/Desktop/workspace/Det{:0>4}_{}.bmp'.format(self.frameCounter,tl_state_str2))

                with open("/home/student/Desktop/workspace/debug.csv", "a") as f:
                    f.writelines("{},{},{},{},{},{},{},{},{},{}\n".format(self.frameCounter,
                                                                    subFrCounter,
                                                                    tl_state_str,
                                                                    green_area_h_mean,
                                                                    yellow_area_h_mean,
                                                                    red_area_h_mean,
                                                                    green_area_r_mean,
                                                                    yellow_area_r_mean,
                                                                    red_area_r_mean,
                                                                    tl_state_str2))

                #cv2.imwrite('/home/student/Desktop/workspace/redArea{:0>8}_{}_{:0>3}.bmp'.format(self.frameCounter, tl_state,red_area_h_mean),red_area_h_image)
                #cv2.imwrite('/home/student/Desktop/workspace/yellowArea{:0>8}_{}_{:0>3}.bmp'.format(self.frameCounter, tl_state,yellow_area_h_mean),yellow_area_h_image)
                #cv2.imwrite('/home/student/Desktop/workspace/greenArea{:0>8}_{}_{:0>3}.bmp'.format(self.frameCounter, tl_state,green_area_h_mean),green_area_h_image)
                #tl_image.save('/home/student/Desktop/workspace/crop{:}_Obj{}_{}.bmp'.format(self.frameCounter, subFrCounter,tl_state_str))

                subFrCounter += 1
        ######### Majority vote ###########
        if (subFrCounter == 1):
            finaljudge = clf_results
            print("final:RED")
        elif (subFrCounter > 1):
            red_count = clf_results.count(TrafficLight.RED)
            green_count = clf_results.count(TrafficLight.GREEN)
            if(red_count >= green_count):
                finaljudge = TrafficLight.RED
                print("Majority vote! red:{},green:{},final:RED".format(red_count,green_count))
            else:
                finaljudge = TrafficLight.GREEN
                print("Majority vote! red:{},green:{},final:GREEN".format(red_count,green_count))
        self.frameCounter += 1

        #TODO implement light color prediction
        return finaljudge

    def pil2cv(self, image):

        new_image = np.array(image, dtype=np.uint8)
        if new_image.ndim == 2:
            pass
        elif new_image.shape[2] == 3:
            new_image = cv2.cvtColor(new_image, cv2.COLOR_RGB2BGR)
        elif new_image.shape[2] == 4:
            new_image = cv2.cvtColor(new_image, cv2.COLOR_RGBA2BGRA)
        return new_image

reason: import numpy


text:
import sys
sys.path.insert(1,"../../../")
import h2o
from tests import pyunit_utils

import random
import numpy as np
from sklearn import preprocessing
from sklearn.cluster import KMeans
from h2o.estimators.kmeans import H2OKMeansEstimator

def emptyclusKmeans():
  # Connect to a pre-existing cluster
  # connect to localhost:54321

  #Log.info("Importing ozone.csv data...\n")
  ozone_sci = np.loadtxt(pyunit_utils.locate("smalldata/glm_test/ozone.csv"), delimiter=',', skiprows=1)
  ozone_h2o = h2o.import_file(path=pyunit_utils.locate("smalldata/glm_test/ozone.csv"))

  ncent = 10
  nempty = random.randint(1,ncent/2)
  initial_centers = [[41,190,67,7.4],
                     [36,118,72,8],
                     [12,149,74,12.6],
                     [18,313,62,11.5],
                     [23,299,65,8.6],
                     [19,99,59,13.8],
                     [8,19,61,20.1],
                     [16,256,69,9.7],
                     [11,290,66,9.2],
                     [14,274,68,10.9]]
  for i in random.sample(range(ncent-1), nempty):
    initial_centers[i] = [100*i for z in range(1,len(initial_centers[0])+1)]

  initial_centers_sci = np.asarray(initial_centers)
  initial_centers = zip(*initial_centers)

  initial_centers_h2o = h2o.H2OFrame(initial_centers)

  #Log.info("Initial cluster centers:")
  print "H2O initial centers:"
  initial_centers_h2o.show()
  print "scikit initial centers:"
  print initial_centers_sci

  # H2O can handle empty clusters and so can scikit
  #Log.info("Check that H2O can handle badly initialized centers")
  km_sci = KMeans(n_clusters=ncent, init=initial_centers_sci, n_init=1)
  km_sci.fit(preprocessing.scale(ozone_sci))
  print "scikit final centers"
  print km_sci.cluster_centers_

  km_h2o = H2OKMeansEstimator(k=ncent, user_points=initial_centers_h2o, standardize=True)
  km_h2o.train(x = range(ozone_h2o.ncol), training_frame=ozone_h2o)
  print "H2O final centers"
  print km_h2o.centers()

if __name__ == "__main__":
  pyunit_utils.standalone_test(emptyclusKmeans)
else:
  emptyclusKmeans()


text:
"""
The module `preprocessing.text` includes functions and classes to preprocess text data
    - tokenize and process unstructued text data into sequence data
    - load word embeddings
    - normalize text

"""

import re
import string
import warnings
import numpy as np

from kaggle_learn.utils import convert_to_numpy_1d_array
from sklearn.base import BaseEstimator, TransformerMixin
from nltk.corpus import stopwords
from nltk.stem.snowball import PorterStemmer, SnowballStemmer
from keras.preprocessing import text, sequence

warnings.filterwarnings("ignore")

def process_text_to_sequence(X_train, X_test, **kwargs):
    """
    Process text data (array) to equal length sequences use keras

    Parameters
    ----------
    X_train : np.array with shape (m, )
    X_test  : np.array with shape (n, )
    kwargs  : other parameters needed

    Returns
    -------

    """
    max_features = kwargs.get("max_features", 10000)
    max_len = kwargs.get("max_len", 50)

    tokenizer = text.Tokenizer(num_words=max_features, lower=True, split=" ",
                               filters="!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n",
                               char_level=False, oov_token=None)
    tokenizer.fit_on_texts(list(X_train) + list(X_test))

    # process text to sequence
    X_train_sequence = tokenizer.texts_to_sequences(X_train)
    X_test_sequence = tokenizer.texts_to_sequences(X_test)

    # truncate / padding
    X_train_sequence_pad = sequence.pad_sequences(X_train_sequence, maxlen=max_len)
    X_test_sequence_pad = sequence.pad_sequences(X_test_sequence, maxlen=max_len)

    return dict(X_train_sequence=X_train_sequence,
                X_test_sequence=X_test_sequence,
                X_train_sequence_pad=X_train_sequence_pad,
                X_test_sequence_pad=X_test_sequence_pad,
                tokenizer=tokenizer)

def load_pretrained_word_embeddings(embedding_path, tokenizer, **kwargs):
    """
    Load pretrained word embeddings

    Parameters
    ----------
    embedding_path : str, example: "./embeddings/glove.840B.300d/glove.840B.300d.txt"
    tokenizer      : keras tokenizer, return from process_text_to_sequence
    kwargs         : other parameters needed

    Returns
    -------

    """
    embedding_size = kwargs.get("embedding_size", 300)
    max_features = kwargs.get("max_features", 10000)

    def _get_coefs(word, *arr):
        return word, np.asarray(arr, dtype="float32")

    embeddings_index = dict(_get_coefs(*o.strip().rsplit(" ")) for o in open(embedding_path, encoding="utf-8", errors="ignore"))
    word_index = tokenizer.word_index
    num_words = min(max_features, len(word_index))
    embeddings_matrix = np.zeros((num_words, embedding_size))
    for word, i in word_index.items():
        i -= 1
        if i >= max_features:
            continue
        embeddings_vector = embeddings_index.get(word)
        # oov or not
        if embeddings_vector is not None:
            embeddings_matrix[i] = embeddings_vector

        # make zero as start
        word_index[word] -= 1

    return dict(embeddings_matrix=embeddings_matrix,
                word_index=word_index)

def clean_regex(x, pattern_mapping):
    """ Clean text based on pre-defined regular expressions """
    for old_pattern, new_pattern in pattern_mapping.items():
        x = re.sub(old_pattern, new_pattern, x)
    return x

def separate_punctuation(x, punctuations):
    """ Add space around pre-defined punctuations """
    for p in punctuations:
        x = x.replace(p, f" {p} ")
    return x

def remove_words(x, words):
    """ Remove pre-defined words """
    return " ".join([w for w in x.split() if w not in words])

def strip_space(x):
    """ Strip extra spaces """
    return " ".join([s.strip() for s in x.split()])

def stemming(x, method="snowball"):
    """ Apply stemming on text """
    __builtin_stemmer = {"snowball": SnowballStemmer, "porter": PorterStemmer}
    stemmer = __builtin_stemmer[method]("english")
    return " ".join([stemmer.stem(w) for w in x.split()])

class RegexCleaner(TransformerMixin):
    """ Clean text based on pre-defined regular expressions """
    def __init__(self, pattern_mapping=None, strip_space=True):
        self.pattern_mapping = {} if pattern_mapping is None else pattern_mapping
        self.strip_space = strip_space

    def _replace_text(self, text):
        for old_pattern, new_pattern in self.pattern_mapping.items():
            text = re.sub(old_pattern, new_pattern, text)

        if self.strip_space:
            text = " ".join([s.strip() for s in text.split()])

        return text

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        X = convert_to_numpy_1d_array(X)
        output = list(map(self._replace_text, X))
        return np.array(output).reshape(-1, 1)

class PunctuationSeparator(TransformerMixin):
    """ Add space around pre-defined punctuations """
    def __init__(self, punctuations=None, strip_space=True):
        self.punctuations = string.punctuation if punctuations is None else punctuations
        self.strip_space = strip_space

    def _add_separator(self, text):
        for p in self.punctuations:
            text = text.replace(p, f" {p} ")

        if self.strip_space:
            text = " ".join([s.strip() for s in text.split()])

        return text

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        X = convert_to_numpy_1d_array(X)
        output = list(map(self._add_separator, X))
        return np.array(output).reshape(-1, 1)

class StopwordsRemover(TransformerMixin):
    """ Remove stopwords """
    def __init__(self, stop_words=None):
        self.stop_words = stopwords.words("english") if stop_words is None else stop_words

    def _remove_words(self, text):
        return " ".join([w for w in text.split() if w not in self.stop_words])

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        X = convert_to_numpy_1d_array(X)
        output = list(map(self._remove_words, X))
        return np.array(output).reshape(-1, 1)

class Stemmer(TransformerMixin):
    """ Apply stemming on text """
    __builtin_stemmer = {"snowball": SnowballStemmer, "porter": PorterStemmer}

    def __init__(self, method="snowball"):
        self.stemmer = self.__builtin_stemmer[method]("english")

    def _stemming(self, text):
        return " ".join([self.stemmer.stem(w) for w in text.split()])

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        X = convert_to_numpy_1d_array(X)
        output = list(map(self._stemming, X))
        return np.array(output).reshape(-1, 1)


text:
import numpy as np
from denoising.utils import *
from scipy.signal import wiener
from denoising import _batch_algorithm_implementation
from tqdm import tqdm

def wiener_filter(noisy_images: np.ndarray, noise_std_dev: float, show_progress:bool = False) -> np.ndarray:
    """
    Params:
    noisy_images: receive noisy_images with shape (IMG, M, N, C),
    where IMG is the quantity of noisy images
    with dimensions MxN and
    C represents the color dimensions, i.e. if the images are colored,
    C = 3 (RGB), otherwise if C = 1, is grayscale.

    noise_std_dev: the standart deviation from noise.
    """

    validate_array_input(noisy_images)
    validate_if_noise_std_dev_is_a_float(noise_std_dev)

    filtered_images = []

    if show_progress:
        for i in tqdm(range(noisy_images.shape[0])):
            filtered_images.append(
                wiener(
                    noisy_images[i, :,:,0],
                    mysize=3, # window dimensions -> (3,3).
                    noise=(noise_std_dev**2), # the noise-power or variance from noise.
                )
            )
    else:
        for i in range(noisy_images.shape[0]):
            filtered_images.append(
                wiener(
                    noisy_images[i, :,:,0],
                    mysize=3, # window dimensions -> (3,3).
                    noise=(noise_std_dev**2), # the noise-power or variance from noise.
                )
            )

    filtered_images = np.array(filtered_images)

    return filtered_images

max_stars_repo_name: darlannakamura/denoising-dbt

max_issues_repo_name: darlannakamura/denoising-dbt

max_forks_repo_name: darlannakamura/denoising-dbt


text:
"""
Copyright 2019 Jannik Busse

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
License. You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
language governing permissions and limitations under the License.

File description:

Module that contains the NumpyPointCloud class, which effectively wraps together a pointcloud and the description of
it's data fields for precise adressing, addition, replacement and deletion of data columns.

Field naming conventions: Start with a capital letter and separate words with underscores. Examples following.
    "X", "Y", "Z":              Point coordinates
    "Nx", "Ny", "Nz":           Normal vector components
    "Sigma":                    Noise, computed from the smalles eigenvalue in normal computation
    "Intensity":                Reflection intensity
    "Classification":           Point class
    "C2C_absolute_distances":   Distance from a point to it's nearest neigbor

# Example
import numpy as np
from modules.np_pointcloud import NumpyPointCloud

## prepare numpy cloud
numpy_cloud = np.array([[1.1, 2.1, 3.1],
                        [1.2, 2.2, 3.2],
                        [1.3, 2.3, 3.3],
                        [1.4, 2.4, 3.4],
                        [1.5, 2.5, 3.5],
                        [1.6, 2.6, 3.6]] )

## create NumpyPointCloud from numpy cloud values
np_pointcloud = NumpyPointCloud (numpy_cloud, ["X", "Y", "Z"])
print (np_pointcloud )
print ("Has Field Classification: " + str (np_pointcloud.has_fields ("Classification" )))

## add data
np_pointcloud.add_fields ([0, 0, 0, 20, 0, 0], "Classification")
np_pointcloud.add_fields (np.random.uniform (-1, 1, size=(6, 3)), ["Nx", "Ny", "Nz"] )
print (np_pointcloud )
print ("Has Field Classification, Normals: " + str (np_pointcloud.has_fields (["Nx", "Classification", "Ny", "Nz"] )))

## delete data
np_pointcloud.delete_fields ("Classification")
np_pointcloud.delete_fields (["Nx", "Ny", "Nz"])
print (np_pointcloud )
"""

import numpy as np
import warnings

class NumpyPointCloud (object ):
    """
    A Class that contains a 2-D numpy.ndarray and field labels describing the content of the arrays columns for easy
    access

    Field naming conventions: Start with a capital letter and separate words with underscores. Examples following.
        "X", "Y", "Z":              Point coordinates
        "Nx", "Ny", "Nz":           Normal vector components
        "Sigma":                    Noise, computed from the smalles eigenvalue in normal computation
        "Intensity":                Reflection intensity
        "Classification":           Point class
        "C2C_absolute_distances":   Distance from a point to it's nearest neigbor

    Functions:
        get_fields (field_labels_list ):    Extract a copy of one or multiple fields
        has_fields (field_labels_list ):    See if requested fields are in this cloud
        get_xyz_coordinates ():             shorthand for self.get_fields (["X", "Y", "Z"] )
        get_normals ():                     shorthand for self.get_fields (["Nx", "Ny", "Nz"] )
        get_normals_and_sigma ():           shorthand for self.get_fields (["Nx", "Ny", "Nz", "Sigma"] )
        add_fields (field_data, field_labels_list, replace=False ):     Add one or multiple fields
        delete_fields (field_labels_list, warn=True ):                  Delete one or multiple fields
    """

    def __init__(self, numpy_ndarray=None, field_labels_list=[] ):
        """
        Constructor. Creates a cloud consisting of an numpy.ndarray data matrix and a list of field labels to describe
        the data contained in each matrix column.

        Input:
            numpy_ndarray: (np.ndarray)         Input Cloud. Minimum size is (n, 3) for the X, Y, Z point data
            field_labels_list: (list(string))   Labels of the columns of this cloud, describing the type of data
        """

        # Assign a copy of labels and data, remove any spaces around the labels
        self.field_labels = [label.strip () for label in field_labels_list].copy ()
        self.points = numpy_ndarray.copy ()
        #self.shape = self.points.shape  # wraps np.ndarray.shape, this has to be updated with changes
        # (and does not work reliably)

    def __get_indices (self, field_labels_list ):
        """Returns all indices of fields in this cloud that correspond to the requested labels"""

        if (type (field_labels_list) is str):
            field_labels_list = [field_labels_list]

        indices = []
        for field in field_labels_list:
            indices.append (self.field_labels.index(field ))

        return indices

    def has_fields (self, field_labels_list ):
        """Checks if all given field names are in this cloud"""

        # cast string into list
        if (type (field_labels_list) is str):
            field_labels_list = [field_labels_list]

        # See if there is only one entry in the list
        return all (label in self.field_labels for label in field_labels_list )

    def get_fields (self, field_labels_list ):
        """
        Extract one or multiple columns (fields) of this cloud by name. Returns a copy.

        Input:
            field_labels_list: (list(string))    The names of the fields to be returned, in a list

        Output:
            data (numpy.ndarray):   The requested column(s). If one single column, it's shape will be 1 dimensional
        """

        if (type (field_labels_list) is str):
            field_labels_list = [field_labels_list]

        # check if all requested_fields are contained in this cloud
        if (all (field in self.field_labels for field in field_labels_list )):

            # add all indices that correspond to the requested labels
            indices = self.__get_indices (field_labels_list )

        # if something is missing raise an error, so no lengthy computations are done with incomplete data
        else:
            raise ValueError ("This Cloud is missing one or more requested fields: "
                              + str(field_labels_list)
                              + ".\nCloud fields are: " + str(self.field_labels ))

        # return a copy of the requested data
        data = self.points[:, indices].copy ()

        # if only one column is requested, reshape it to 1-D, so boolean comparison doesn't throw an error
        if (data.shape[1] == 1):
            data = data[:, 0]

        return data

    def get_xyz_coordinates (self ):
        """Get the (X, Y, Z) coordinates of this cloud (usually the first three fields)"""
        return self.get_fields (["X", "Y", "Z"] )

    def get_normals (self ):
        """Get the three fields containing the normals of this cloud (Nx, Ny, Nz)"""
        return self.get_fields (["Nx", "Ny", "Nz"] )

    def get_normals_and_sigma (self ):
        """Get the three normal vector fields and the Sigma field containing the noise from normal estimation"""
        return self.get_fields (["Nx", "Ny", "Nz", "Sigma"] )

    def add_fields (self, field_data, field_labels_list, replace=False ):
        """
        Add one or more data fields to this cloud, optionally replacing existing fields.

        Input:
            field_data: (np.ndarray)            Data in the shape of (n, m), where n = number of points in cloud
            field_labels_list: (list(string))   Labels of field_data, length of list = m
            replace: (boolean)                  If True, fields already contained in this cloud are replaced

        Output:
            self: (NumpyPointCloud)             The object of this cloud (optional)
        """

        # # input checks
        # check type of input
        if (type (field_labels_list ) is str ):
            field_labels_list = [field_labels_list]
        if (type (field_data ) is not np.ndarray ):
            field_data = np.array (field_data)  # try casting

        # check dimesions of input data
        if (len (field_data.shape ) < 2 ):
            field_data = field_data.reshape (-1, 1)
        if (self.points.shape[0] != field_data.shape[0] ):
            raise ValueError ("The number of points in the input data is different from this cloud."
                             + "\nNumber of input points: " + str (field_data.shape[0] )
                             + "\nNumber of cloud points: " + str (self.points.shape[0] ))

        # check number of labels
        if (len (field_labels_list ) != field_data.shape[1] ):
            raise ValueError ("Supplied data has " + str (field_data.shape[1] )
                              + " fields, but there were " + str (len (field_labels_list )) + " Labels supplied." )

        # # addition
        # check if any field to be added is already in the cloud
        # create a truth_array that shows which of the labels in field_labels_list are contained in self.field_labels
        truth_array = [new_field_name in self.field_labels for new_field_name in field_labels_list]     # is a list
        if (any(truth_array ) and not replace ):
            raise ValueError ("This Cloud already has fields named " + str (np.array(field_labels_list)[truth_array])
                              + ".\nIf you want to replace the data, set replace=True."
                              + "\nCloud fields are: " + str(self.field_labels ))
        # replace
        elif (any(truth_array ) and replace):
            # replacing the fields already present
            # prepare field_labels_list for indexing and get the indices of the fields already present in the cloud
            field_labels_list = np.array (field_labels_list )
            indices = self.__get_indices (field_labels_list[truth_array] )

            # replace the existing fields with the new data
            self.points[:, indices] = field_data[:, truth_array]

            # concatenate the rest of the fields (which are not already contained in the cloud)
            reversed_truth_array = [not boolean for boolean in truth_array]
            self.points = np.concatenate ((self.points, field_data[:, reversed_truth_array]), axis=1 )
            self.field_labels += field_labels_list[reversed_truth_array].tolist ()

        # simple add
        else:
            self.points = np.concatenate ((self.points, field_data), axis=1 )
            self.field_labels += field_labels_list

        # update shape (does not work reliably)
        #self.shape = self.points.shape

        return self

    def delete_fields (self, field_labels_list, warn=True ):
        """
        Delete one or multiple columns of this cloud by name.

        Input:
            field_labels_list: (list(string))   The names of the fields to be deleted, in a list
            warn: (boolean)                     If True, prints a warning if fields to be deleted could not be found

        Output:
            self: (NumpyPointCloud)             The object of this cloud (optional)
        """

        if (type (field_labels_list) is str):
            field_labels_list = [field_labels_list]

        # check if all requested_fields are contained in this cloud
        truth_array = [field in self.field_labels for field in field_labels_list]
        if (all (truth_array )):
            indices = self.__get_indices (field_labels_list )

        else:

            # issue a warning
            if (warn ):
                message = str ("This Cloud is missing one of the fields to be deleted: "
                              + str(field_labels_list )
                              + ".\nCloud fields are: " + str(self.field_labels )
                              + "\nDeleting fields: " + str(np.array (field_labels_list )[truth_array] ))
                warnings.warn (message )

            # cut away the labels that don't exist and delete the remaining fields
            field_labels_list = np.array (field_labels_list)[truth_array].tolist ()     # convert to array and back
            indices = self.__get_indices (field_labels_list )

        # remove columns and their labels
        self.points = np.delete(self.points, indices, axis=1 )
        for field_to_delete in field_labels_list:
            self.field_labels.remove (field_to_delete )

        # update shape  (does not work reliably)
        #self.shape = self.points.shape

        return self

    def __str__(self ):
        """
        This Method is called when an object of this class is printed.
        It returns a nicely formatted string containing the labels and points.
        """

        # change np printoptions to make linebreaks happen later and suppress -3e10 display style
        np.set_printoptions(precision=6, linewidth=120, suppress=True )
        string = ("\n" + str (self.field_labels ) + "\n" + str (self.points ))
        np.set_printoptions ()

        return string

max_stars_repo_name: Agentvm/icp-pointcloud

max_issues_repo_name: Agentvm/icp-pointcloud

max_forks_repo_name: Agentvm/icp-pointcloud

reason: import numpy,from numpy


text:
import numpy as np
from sklearn.svm import LinearSVR
from sparse_ho.models import SVR
from sparse_ho.forward import get_beta_jac_iterdiff
from sklearn.datasets import make_regression
from sparse_ho.implicit_forward import get_beta_jac_fast_iterdiff
from sparse_ho.criterion import CV
from sparse_ho.forward import Forward
from sparse_ho.implicit_forward import ImplicitForward
from sparse_ho.ho import grad_search
from sparse_ho.utils import Monitor

n_samples = 100
n_features = 100
n_active = 100
SNR = 5.0
tol = 1e-16
C = 2.0
log_C = np.log(C)
epsilon = 0.05
log_epsilon = np.log(epsilon)

max_iter = 5000
X_train, y_train, beta_star = make_regression(
    shuffle=False, random_state=15, n_samples=n_samples, n_features=n_features,
    n_informative=n_features, n_targets=1, coef=True)

X_val, y_val, beta_star = make_regression(
    shuffle=False, random_state=125, n_samples=n_samples, n_features=n_features,
    n_informative=n_features, n_targets=1, coef=True)

model = SVR(X_train, y_train, log_C, log_epsilon, max_iter=max_iter, tol=tol)

def get_v(mask, dense):
    return 2 * (X_val[:, mask].T @ (
        X_val[:, mask] @ dense - y_val)) / X_val.shape[0]

def test_beta_jac():
    supp1, dense1, jac1 = get_beta_jac_iterdiff(
        X_train, y_train, np.array([log_C, log_epsilon]), tol=tol,
        model=model, compute_jac=True, max_iter=max_iter)

    dual = np.zeros(2 * n_samples)
    dual[supp1] = dense1
    primal = X_train.T @ (dual[0:n_samples] - dual[n_samples:(2 * n_samples)])
    clf = LinearSVR(
        epsilon=epsilon, fit_intercept=False, C=C, tol=1e-12, max_iter=max_iter)
    clf.fit(X_train, y_train)

    supp2, dense2, jac2 = get_beta_jac_fast_iterdiff(
        X_train, y_train, np.array([log_C, log_epsilon]), None, None,
        get_v, tol=tol, model=model, tol_jac=1e-16, max_iter=max_iter, niter_jac=10000)
    import ipdb; ipdb.set_trace()
    assert np.allclose(primal, clf.coef_)
    assert np.allclose(dense1, dense2)
    assert np.all(supp1 == supp2)
    assert np.allclose(jac1, jac2, atol=1e-3)

def test_val_grad():
    #######################################################################
    # Not all methods computes the full Jacobian, but all
    # compute the gradients
    # check that the gradient returned by all methods are the same

    criterion = CV(X_val, y_val, model)
    algo = Forward(criterion)
    val_fwd, grad_fwd = algo.get_val_grad(
        np.array([log_C, log_epsilon]), tol=tol)

    criterion = CV(X_val, y_val, model)
    algo = ImplicitForward(criterion, tol_jac=1e-16, n_iter_jac=1000)
    val_imp_fwd, grad_imp_fwd = algo.get_val_grad(
        np.array([log_C, log_epsilon]), tol=tol)
    # import ipdb; ipdb.set_trace()
    assert np.allclose(val_fwd, val_imp_fwd)
    assert np.allclose(grad_fwd, grad_imp_fwd, atol=1e-3)

def test_grad_search():

    n_outer = 2
    criterion = CV(X_val, y_val, model, X_test=None, y_test=None)
    monitor1 = Monitor()
    algo = Forward(criterion)
    grad_search(
        algo, np.array([np.log(C), log_epsilon]), monitor1, n_outer=n_outer,
        tol=1e-16)

    criterion = CV(X_val, y_val, model, X_test=None, y_test=None)
    monitor3 = Monitor()
    algo = ImplicitForward(criterion, tol_jac=1e-6, n_iter_jac=1000)
    grad_search(
        algo, np.array([np.log(C), log_epsilon]), monitor3, n_outer=n_outer,
        tol=1e-16)

    # [np.linalg.norm(grad) for grad in monitor1.grads]
    # [np.exp(alpha) for alpha in monitor1.log_alphas]

    # assert np.allclose(
    #     np.array(monitor1.log_alphas), np.array(monitor3.log_alphas))
    # assert np.allclose(
    #     np.array(monitor1.grads), np.array(monitor3.grads))
    # assert np.allclose(
    #     np.array(monitor1.objs), np.array(monitor3.objs))
    # assert not np.allclose(
    #     np.array(monitor1.times), np.array(monitor3.times))

if __name__ == '__main__':
    # test_beta_jac()
    test_val_grad()
    test_grad_search()


text:
#! /bin/python
# -*- coding: utf-8 -*-  #
"""
Created on Tue Oct 08

@author: wcgrizolli
"""

#import dxchange

from pywinspec import SpeFile, test_headers

from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
from matplotlib import cm
import numpy as np

import sys

sys.path.append('/home/grizolli/workspace/pythonWorkspace/wgTools/')

import wgTools as wgt

import wavepy.utils as wpu

import scipy

from scipy.ndimage import gaussian_filter, uniform_filter, maximum_filter

wpu._mpl_settings_4_nice_graphs()

#==============================================================================
# preamble
#==============================================================================

# Flags
saveFigFlag = True

# useful constants
rad2deg = np.rad2deg(1)
deg2rad = np.deg2rad(1)
NAN = float('Nan')  # not a number alias

from scipy import constants
hc = constants.value('inverse meter-electron volt relationship') # hc

#==============================================================================
# %% Experimental Values
#==============================================================================

pixelsize = 13.5e-6
dist2detector = 375.5e-3
phenergy = 778.00

wavelength = hc/phenergy
kwave = 2*np.pi/wavelength

#==============================================================================
# %% Load SPE
#==============================================================================

fname = wgt.selectFile('*SPE', 3)
spe_file = SpeFile(fname)

#==============================================================================
# %% Crop ROI
#==============================================================================

#idx4crop = [635, 1993, 841, 1792]
#img = wpu.crop_matrix_at_indexes(spe_file.data[0], idx4crop)

img, idx4crop = wpu.crop_graphic_image(spe_file.data[0], verbose=False)
print(idx4crop)

img = wpu.pad_to_make_square(img, mode='edge')

# %% Plot Detector coordinates

xx, yy = wpu.realcoordmatrix(img.shape[1], pixelsize,
                             img.shape[0], pixelsize)

plt.figure(figsize=plt.figaspect(.6))
plt.contourf(xx*1e3, yy*1e3, img/np.nanmax(img), 201, cmap='plasma', vmin=0.1)
plt.xlabel(r'$x$ [mm]')
plt.ylabel(r'$y$ [mm]')
plt.title(r'Data ')
plt.colorbar()
plt.show(block=True)

#xVec = wpu.realcoordvec(img.shape[1], pixelsize)
#yVec = wpu.realcoordvec(img.shape[0], pixelsize)

# %%

#
xx, yy = wpu.realcoordmatrix(img.shape[1], 1,
                             img.shape[0], 1)
#wpu.plot_profile(xx, yy, img/np.nanmax(img))
#==============================================================================
# %% FFT
#==============================================================================

from numpy.fft import *

qx, qy = wpu.realcoordmatrix(img.shape[1], pixelsize/dist2detector*kwave,
                             img.shape[0], pixelsize/dist2detector*kwave)

fftimg = ifftshift(fft2(fftshift(img)))*pixelsize*pixelsize
abs_fftimg = np.abs(fftimg)
abs_fftimg -= np.min(abs_fftimg)
norm_abs_fftimg = abs_fftimg/np.max(abs_fftimg)

log_abs_fftimg = np.log(norm_abs_fftimg + np.finfo(float).eps)

# %%

plt.figure(figsize=plt.figaspect(.6))
plt.contourf(qx*1e-6, qy*1e-6, log_abs_fftimg, 201, cmap='plasma', vmin=-10, vmax=-.5)
plt.xlabel(r'$q_x$ [$ \mu m^{-1} $]')
plt.ylabel(r'$q_y$ [$ \mu m^{-1}$]')
plt.title(r'log of module FFT ')
plt.colorbar()
plt.show(block=True)

#==============================================================================
# %% Mask Angle
#==============================================================================

def create_mask_angle(angle, delAngle, shape, indexing='xy'):

    if indexing == 'xy':
        ii, jj = np.mgrid[shape[0]:0:-1,0:shape[1]]
        angle = - angle
    elif indexing == 'ij':
        ii, jj = np.mgrid[0:shape[0],0:shape[1]]

    ii -= shape[0] // 2
    jj -= shape[1] // 2

    #mask = 1.0*(np.logical_and(np.arctan2(ii,jj)*np.rad2deg(1) < angle + delAngle - 180.00,
    #                           np.arctan2(ii,jj)*np.rad2deg(1) > angle - delAngle - 180.00))

    #mask = 1.0*(np.logical_and(np.arctan(ii/jj)*np.rad2deg(1) < angle + delAngle,
    #                           np.arctan(ii/jj)*np.rad2deg(1) > angle - delAngle))

    mask = 1.0*(np.logical_and(np.arctan2(ii,jj)*np.rad2deg(1) < angle + delAngle,
                               np.arctan2(ii,jj)*np.rad2deg(1) > angle - delAngle) +
                np.logical_and(np.arctan2(ii,jj)*np.rad2deg(1) < 180. + angle + delAngle,
                               np.arctan2(ii,jj)*np.rad2deg(1) > 180. + angle - delAngle)+
                np.logical_and(np.arctan2(ii,jj)*np.rad2deg(1) < -180. + angle + delAngle,
                               np.arctan2(ii,jj)*np.rad2deg(1) > -180. + angle - delAngle))

    #mask = 1.0*(np.logical_and(jj/ii < np.tan(angle + delAngle),
    #                           jj/ii > np.tan(angle - delAngle)) +
    #            np.logical_and(jj/ii < np.tan(180. + angle + delAngle),
    #                           jj/ii > np.tan(180. + angle - delAngle)))

    mask[mask>.5] = 1.000
    mask[mask<.5] = np.nan

    return mask

plotThis = log_abs_fftimg
plotThis = plotThis[::-1,:]

# Select angle

#joio = wpu.graphical_select_point_idx(plotThis, verbose=True)
#jo = int(joio[0])
#io = int(joio[1])
#angle = np.arctan2(abs_fftimg.shape[0]//2 - io, jo - abs_fftimg.shape[1]//2)*np.rad2deg(1)

angle = -21.4061120849

print('angle = ' + str(angle))

# %%
mask_angle = create_mask_angle(angle, 1, abs_fftimg.shape, indexing = 'xy')

print('oi 1346')

# %% peaks

def create_mask_peaks2DOLD(array2D, threshold=None, order=3):

    import scipy.signal

    if threshold is not None:
        mask_threshold = wpu.nan_mask_threshold(array2D, threshold=.001)
    else:
        mask_threshold = array2D*0.0 +1.0

    idx_x_axis_0, idx_y_axis_0 = scipy.signal.argrelmax(array2D*mask_threshold,
                                                        axis=0, order = order)
    idx_x_axis_1, idx_y_axis_1 = scipy.signal.argrelmax(array2D*mask_threshold,
                                                        axis=1, order = order)

    peaks_axis0 = np.zeros(np.shape(array2D))
    peaks_axis0[idx_x_axis_0[:], idx_y_axis_0[:]] = 1.0
    peaks_axis1 = np.zeros(np.shape(array2D))
    peaks_axis1[idx_x_axis_1[:], idx_y_axis_1[:]] = 1.0

    return peaks_axis0, peaks_axis1

# %%

def create_mask_peaks2D(array2D, order=1, mode='clip', srn_threshold=0.0):

    array2D = np.pad(array2D[order:-order,order:-order], order, 'edge')
    # make our life easier by making the edges peak free

    idx_axis_0 = scipy.signal.argrelmax(array2D, axis=0, order=order, mode=mode)
    idx_axis_1 = scipy.signal.argrelmax(array2D, axis=1, order=order, mode=mode)

    peaks_axis0 = np.zeros(np.shape(array2D))
    peaks_axis0[idx_axis_0] = 1.0
    peaks_axis1 = np.zeros(np.shape(array2D))
    peaks_axis1[idx_axis_1] = 1.0

    snr0 = np.zeros(np.shape(array2D))
    snr1 = np.zeros(np.shape(array2D))

    snr0[idx_axis_0] = np.abs(array2D[idx_axis_0[0], idx_axis_0[1]] / \
                              np.mean((array2D[idx_axis_0[0] - order, idx_axis_0[1]],
                              array2D[idx_axis_0[0] + order, idx_axis_0[1]])))

    snr1[idx_axis_1] = np.abs(array2D[idx_axis_1[0], idx_axis_1[1]] / \
                              np.mean((array2D[idx_axis_1[0], idx_axis_1[1]],
                              array2D[idx_axis_1[0], idx_axis_1[1]+ order])))

    srn = (snr0 + snr1)/2
    mask_snr = np.where(srn > srn_threshold, 1, 0)

    return np.where(peaks_axis0*peaks_axis1*mask_snr >= 0.5), srn*peaks_axis0*peaks_axis1

# %%

for srn_threshold in [1, 1.5, 2, 3, 5]:

    fig = plt.figure(figsize=plt.figaspect(.6))

    plotThis = log_abs_fftimg

    plt.contourf(qx, qy, plotThis, 101, cmap='plasma', vmin=-10, vmax=5)

    [idx_angle_i, idx_angle_j], srn = create_mask_peaks2D(gaussian_filter(norm_abs_fftimg, 2),
                                                     order=4, srn_threshold=srn_threshold)

    #[idx_angle_i, idx_angle_j], srn =  create_mask_peaks2D(norm_abs_fftimg,
    #                                                       order=5, srn_threshold=2.0)

    plt.plot(qx[idx_angle_i,idx_angle_j],
             qy[idx_angle_i,idx_angle_j], 'bx', ms=10, mew=2)

    plt.show(block=False)

# %%

fig = plt.figure(figsize=plt.figaspect(.6))

plotThis = maximum_filter(srn, 3)

plotThis[plotThis > 500] = NAN

plt.contourf(qx, qy, plotThis, 101, cmap='plasma')

plt.colorbar()
plt.show(block=True)

# %%

plt.figure()
plotThis = log_abs_fftimg
plt.contourf(qx*1e-3, qy*1e-3, plotThis, 101, cmap='plasma', vmin=-10, vmax=5)

idx_angle_i, idx_angle_j = np.where(peaks_mask0*peaks_mask1>.5)
plt.plot(qx[idx_angle_i,idx_angle_j]*1e-3,
         qy[idx_angle_i,idx_angle_j]*1e-3, 'bx', ms=10, mew=2)

plt.title('FFT Image, Log Scale')
plt.colorbar()
plt.show(block=True)

print('oi')

# %%

fig = plt.figure(figsize=plt.figaspect(.6))

plt.contourf(qx*1e-6, qy*1e-6, log_abs_fftimg, 201, cmap='plasma', vmin=-10, vmax=-.5)
plt.xlabel(r'$q_x$ [$ \mu m^{-1} $]')
plt.ylabel(r'$q_y$ [$ \mu m^{-1}$]')
plt.title(r'log of module FFT ')
plt.colorbar()

idx_angle_i, idx_angle_j = np.where(peaks_mask0*peaks_mask1*mask_angle>.5)
plt.plot(qx[idx_angle_i,idx_angle_j]*1e-6,
         qy[idx_angle_i,idx_angle_j]*1e-6, 'bo', ms=10, mew=2, mfc="None")

plt.show(block=True)

# %%

import skimage.filters
import scipy.ndimage

scipy.ndimage.uniform_filter

# %%

vec_q = np.sqrt(qx[idx_angle_i,idx_angle_j]**2 + \
                qy[idx_angle_i,idx_angle_j]**2)*np.sign(qy[idx_angle_i,idx_angle_j])

#intensity = abs_fftimg[idx_angle_i,idx_angle_j]
#intensity = gaussian_filter(abs_fftimg, 2.5)[idx_angle_i,idx_angle_j]

intensity = scipy.ndimage.uniform_filter(abs_fftimg, 5)[idx_angle_i,idx_angle_j]

# %%

plt.figure()

plt.plot(vec_q, intensity, '-x', ms=10, mew=2)
plt.plot(-vec_q, intensity*1.1, '-x', ms=10, mew=2)

plt.xlabel(r'q [$ m^{-1}$]')
plt.title('FFT Image, Log Scale')
plt.show(block=True)

# %%

rho_x, rho_y = wpu.reciprocalcoordmatrix(qx.shape[1], qy[1, 0] - qy[0,0],
                                         qy.shape[0], qx[0, 1] - qx[0,0])

rho_x *= 2*np.pi
rho_y *= 2*np.pi

vec_rho = np.sqrt(rho_x[idx_angle_i,idx_angle_j]**2 + \
                  rho_y[idx_angle_i,idx_angle_j]**2)*np.sign(rho_y[idx_angle_i,idx_angle_j])

# %%
plt.figure()

intensity = scipy.ndimage.uniform_filter(norm_abs_fftimg, 3)[idx_angle_i,idx_angle_j]
plt.plot(vec_rho*1e6, intensity, '-xg', ms=10, mew=2)

intensity = scipy.ndimage.uniform_filter(norm_abs_fftimg, 0)[idx_angle_i,idx_angle_j]
plt.plot(vec_rho*1e6, intensity, '-xb', ms=10, mew=2)

print(vec_rho*1e6)

for i in range(intensity.size):

    print('{:.3f}, \t {:2.4g}'.format(vec_rho[i]*1e6, intensity[i]*1e2))

intensity = scipy.ndimage.uniform_filter(norm_abs_fftimg, 5)[idx_angle_i,idx_angle_j]
plt.plot(vec_rho*1e6, intensity, '-xr', ms=5, mew=2)

#plt.plot(-vec_rho*1e6/1.5, intensity*1.1, '-x', ms=10, mew=2)

plt.xlabel(r'$\rho$  [$\mu m$]')
plt.title('FFT Image, Log Scale')
plt.show(block=True)

# %%

exit()

# %%
c_j_over_c_0 = np.array([4.465, 3.976, 3.13, 3.024, 3.113, 2.308, 1.781])

rho = np.array([1.5,   3. ,   4.5,   6. ,   7.5,   9. ,  10.5])

S_j = np.array([0.992, 0.968, 0.96, 0.81, 0.817, 0.576, 0.791])

sigma = 4

intensity_temp = c_j_over_c_0/S_j

# %%
plt.figure()

#
#plt.plot(rho, c_j_over_c_0, '-or', ms=10, mew=2)

#plt.plot(rho, S_j, '-ob', ms=10, mew=2)
plt.plot(rho, intensity_temp, '-kx', ms=10, mew=2)

plt.xlabel(r'$\rho$  [$\mu m$]')
plt.show(block=False)

# %%

max_stars_repo_name: APS-XSD-OPT-Group/wavepytools

max_issues_repo_name: APS-XSD-OPT-Group/wavepytools

max_forks_repo_name: APS-XSD-OPT-Group/wavepytools

reason: import numpy,from numpy,import scipy,from scipy


text:
import numpy as np
import sys
import random
from typing import Dict, List, Optional

sys.path.append(".") # will make "utils" callable from root
sys.path.append("..") # will make "utils" callable from simulators

from poke_env.environment.field import Field
from poke_env.environment.side_condition import SideCondition
from poke_env.environment.status import Status
from poke_env.environment.weather import Weather
from poke_env.environment.pokemon_type import PokemonType
from poke_env.environment.move_category import MoveCategory
from poke_env.environment.target_type import TargetType
from poke_env.environment.volatile_status import VolatileStatus
from poke_env.environment.battle import Battle
from bots.random_doubles_player import RandomDoublesPlayer

from poke_env.player.battle_order import DoubleBattleOrder, DefaultBattleOrder, BattleOrder

from helpers.doubles_utils import *

""" ============== Creating a player that moves based on Heuristics ==============
This is not a finished bot

Possible options:
- Optimize for my mons move first
- Optimize for the least damage taken (e.g. if another mon moves first, assume worst case scenario)
- Optimize for KOs
- Optimize for sweeps
- Assume worst-case scenario pokemon is brought as a counter, and update when this isnt the case

What I plan to do:
- Take a "click buttons" team
- Set up rule-based way to determine the position I want to be in
- Every turn, check if I am in the position or I can get in that position
- If I am:
    - Click Buttons
- If I'm not:
    - If I can get there && I can get there with an acceptable cost:
        - Do it
    - Otherwise
        - Identify why I'm not in that position
        - Fix it, without compromising worst-case scenario
"""
class HeuristicsPlayer(Player):

    def choose_move(self, battle):
        return DefaultBattleOrder()

    # Get Random Team
    def teampreview(self, battle):
        return "/team " + "".join(random.sample(list(map(lambda x: str(x+1), range(0, len(battle.team)))), k=4))

max_stars_repo_name: caymansimpson/reuniclusVGC

max_issues_repo_name: caymansimpson/reuniclusVGC

max_forks_repo_name: caymansimpson/reuniclusVGC


text:
from ophyd.scaler import ScalerCH
from ophyd import Component, Device
from ophyd import EpicsSignal, EpicsSignalRO
from apstools.devices import SRS570_PreAmplifier
import logging

logger = logging.getLogger(__name__)

class Keithley6485(Device):
    value = Component(EpicsSignalRO, ":read")
#    refresh = Component(EpicsSignal, ":read.SCAN", write_pv=":read.SCAN")
    rate = Component(EpicsSignal, ":rate", write_pv=":rateSet", kind="config", string=True)
    range= Component(EpicsSignalRO,':range', kind="config", string=True)
    autorange = Component(EpicsSignal, ":rangeAuto", write_pv=":rangeAutoSet", kind="config", string=True)
    autoulimit = Component(EpicsSignal, ":rangeAutoUlimit", write_pv=":rangeAutoUlimitSet", kind="config", string=True)
    autollimit = Component(EpicsSignal, ":rangeAutoLlimit", write_pv=":rangeAutoLlimitSet", kind="config", string=True)
    zerocheck = Component(EpicsSignal, ":zeroCheck", write_pv= ":zeroCheckSet", kind="config", string=True)
    zerocorrect = Component(EpicsSignal, ":zeroCorrect", write_pv= ":zeroCorrectSet", kind="config", string=True)
    medianfilter = Component(EpicsSignal, ":medianFilter", write_pv= ":medianFilterSet", kind="config", string=True)
    medianfilterrank = Component(EpicsSignal, ":medianFilterRank", write_pv= ":medianFilterRankSet", kind="config", string=True)
    digitalfilter = Component(EpicsSignal, ":digitalFilter", write_pv= ":digitalFilterSet", kind="config", string=True)
    filtercount = Component(EpicsSignal, ":digitalFilterCount", write_pv= ":digitalFilterCountSet", kind="config", string=True)
    filtercontrol = Component(EpicsSignal, ":digitalFilterControl", write_pv= ":digitalFilterControlSet", kind="config", string=True)

class MyKeithley(Device):
    ca1 = Component(Keithley6485, "1")
    ca2 = Component(Keithley6485, "2")
    ca3 = Component(Keithley6485, "3")
    ca4 = Component(Keithley6485, "4")
    ca5 = Component(Keithley6485, "5")
    ca6 = Component(Keithley6485, "6")
    ca7 = Component(Keithley6485, "7")
    ca8 = Component(Keithley6485, "8")
    ca9 = Component(Keithley6485, "9")
    ca11 = Component(Keithley6485, "11")
    ca12 = Component(Keithley6485, "12")
    ca13 = Component(Keithley6485, "13")
    ca14 = Component(Keithley6485, "14")
    ca15 = Component(Keithley6485, "15")
keithley_objects = MyKeithley("29idb:ca", name="keithley_objects")

# from epics import  caput, caget
# from IEX_29id.utils.strings import ClearStringSeq
# from IEX_29id.utils.exp import BL_Mode_Read, BL_ioc
# from IEX_29id.utils.misc import RangeUp
# from time import sleep
# import numpy as np

# def ca2flux(ca,hv=None,p=1):
#     curve=LoadResponsivityCurve()
#     responsivity=curve[:,0]
#     energy=curve[:,1]
#     charge = 1.602e-19
#     if hv is None:
#         hv=caget('29idmono:ENERGY_SP')
#         print("\nCalculating flux for:")
#         print("   hv = %.1f eV" % hv)
#         print("   ca = %.3e Amp" % ca)
#     eff=np.interp(hv,energy,responsivity)
#     flux = ca/(eff*hv*charge)
#     if p is not None:
#         print("Flux = %.3e ph/s\n" % flux)
#     return flux

# def LoadResponsivityCurve():
#     FilePath='/home/beams/29IDUSER/Documents/User_Macros/Macros_29id/IEX_Dictionaries/'
#     FileName="DiodeResponsivityCurve"
#     data = np.loadtxt(FilePath+FileName, delimiter=' ', skiprows=1)
#     return data

# def reset_keithley(keithley_ioc,keithley_num,rate="Slow"):
#     pv="29id"+keithley_ioc+":ca"+str(keithley_num)
#     caput(pv+":reset.PROC",1)
#     caput(pv+":digitalFilterSet","Off")
#     caput(pv+":medianFilterSet","Off")
#     caput(pv+":zeroCheckSet",0)
#     caput(pv+":rangeAuto",1)
#     caput(pv+":rateSet",rate)
#     caput(pv+":rangeAutoUlimit","20mA")
#     caput(pv+":read.SCAN",".5 second")

# def reset_all_keithley(rate="Slow"):
#     for i in [1,2,3,4,5,9,10,12,13,14,15]:
#         reset_keithley("b",i,rate)
#     for i in [1,2]:
#         reset_keithley("c",i,rate)
#     #for i in [1,2,3,4,5]:
#     for i in [2,3,4]:
#         reset_keithley("d",i,rate)
#     caput("29idb:ca5:read.SCAN","Passive")    # CA5 in passive
#     print("\nAll the current amplifiers have been reset; ca5 set to passive.")

# def keithley_live_strseq(scanIOC):              # do we need to add 29idb:ca5 ???
#     n=7
#     pvstr="29id"+scanIOC+":userStringSeq"+str(n)
#     ClearStringSeq(scanIOC,n)
#     caput(pvstr+".DESC","CA_Live_"+scanIOC)
#     n=len(Detector_List(scanIOC))
#     for (i,list) in enumerate(Detector_List(scanIOC)):
#         pvCA_read='29id'+list[0]+':ca'+str(list[1])+':read.SCAN CA NMS'
#         pvCA_avg='29id'+list[0]+':ca'+str(list[1])+':digitalFilterSet PP NMS'

#         caput(pvstr+".LNK"+str(i+1),pvCA_avg)
#         caput(pvstr+".STR" +str(i+1),"Off")

#         if n+1+i < 10:
#             caput(pvstr+".LNK" +str(n+1+i),pvCA_read)
#             caput(pvstr+".STR" +str(n+1+i),".5 second")
#             caput(pvstr+".WAIT"+str(n+1+i),"After"+str(n))
#         elif n+1+i == 10:
#             caput(pvstr+".LNKA",pvCA_read)
#             caput(pvstr+".STRA",".5 second")
#             caput(pvstr+".WAITA","After"+str(n))
# #    if scanIOC == 'Kappa':

# #        caput(pvstr+".LNK" +str(2*n+1),'29idMZ0:scaler1.CONT CA NMS')
# #        caput(pvstr+".STR" +str(2*n+1),"AutoCount")
# #        caput(pvstr+".WAIT"+str(2*n+1),"After"+str(2*n))

#     return pvstr+".PROC"

# def Detector_List(scanIOC):
#     """
#     Define the detector used for:
#         keithley_live_strseq()
#         Detector_Triggers_StrSeq()
#         BeforeScan_StrSeq() => puts everybody in passive
#         CA_Average()
#     WARNING: can't have more than 5 otherwise keithley_live_strseq gets angry.
#     """

#     BL_mode=BL_Mode_Read()[0]
#     if scanIOC == "ARPES":
#         CA_list=[["c",1],["b",15],["b",4],["b",13]]
#     elif scanIOC == "Kappa":
#         CA_list=[["d",2],["d",3],["d",4],["b",14]]
#     elif scanIOC == "RSoXS":
#         CA_list=[["d",3],["d",4],["d",5],["b",14],]
#     else:
#         CA_list=[]
# #    if BL_mode == 1:
# #        CA_list=[["b",1],["b",2],["b",3],["b",4],["b",5]] #JM was here
# #        CA_list=[["b",15],["d",2],["d",3],["d",4],["b",14]]
#     return CA_list

# def CA_Name(ca_ioc,ca_num):    #{motor,position}
#     ca={}
#     ca["b"] = {4:'Slit1A',13:'Slit3C',14:'MeshD',15:'DiodeC'}
#     ca["c"] = {1:'TEY'   ,2:'Diode'}
#     ca["d"] = {1:'APD'   ,2:'TEY',  3:'D-3',  4:'D-4',5:'RSoXS Diode'}
#     try:
#         name=ca[ca_ioc][ca_num]
#     except:
#         name=""
#     return name

# def CA_Autoscale(ca_ioc,ca_num,On_Off='On',gain=7):
#     """
#     On_Off= 'On' => Turns On the Autoscale; gain is irrelevant.
#     On_Off= 'Off' => Turns Off the Autoscale with gain below:
#             0 = 2nA
#             1 = 20nA
#             2 = 200nA
#             3 = 2uA
#             4 = 20uA
#             5 = 200uA
#             6 = 2mA
#             7 = 20mA
#     """
#     pv="29id"+ca_ioc+":ca"+str(ca_num)
#     caput(pv+":rangeAutoSet",On_Off)
#     sleep(0.5)
#     caput(pv+":rangeSet",gain)
#     print(pv,"Autoscale",On_Off)
#     if On_Off == 'Off':
#             sleep(1)
#             print("Gain set to:",caget(pv+":range",as_string=True))


text:
import pdb
import argparse
import os
import time
import logging
from random import uniform
from datetime import datetime
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim
import torch.utils.data
import models
from torch.autograd import Variable
from data import get_dataset
from preprocess import get_transform
from utils import *
from ast import literal_eval
from torch.nn.utils import clip_grad_norm
from math import ceil
import numpy as np
import scipy.optimize as sciopt
import warnings
from sklearn import random_projection as rp
import re
#import matplotlib.pyplot as plt

model_names = sorted(name for name in models.__dict__
                     if name.islower() and not name.startswith("__")
                     and callable(models.__dict__[name]))

parser = argparse.ArgumentParser(description='PyTorch ConvNet Training')

parser.add_argument('--results_dir', metavar='RESULTS_DIR',
                    default='./TrainingResults', help='results dir')
parser.add_argument('--save', metavar='SAVE', default='',
                    help='saved folder')
parser.add_argument('--dataset', metavar='DATASET', default='cifar10',
                    help='dataset name or folder')
parser.add_argument('--model', '-a', metavar='MODEL', default='resnet',
                    choices=model_names,
                    help='model architecture: ' +
                    ' | '.join(model_names) +
                    ' (default: alexnet)')
parser.add_argument('--input_size', type=int, default=None,
                    help='image input size')
parser.add_argument('--model_config', default='',
                    help='additional architecture configuration')
parser.add_argument('--type', default='torch.cuda.FloatTensor',
                    help='type of tensor - e.g torch.cuda.HalfTensor')
parser.add_argument('--gpus', default='0',
                    help='gpus used for training - e.g 0,1,3')
parser.add_argument('-j', '--workers', default=8, type=int, metavar='N',
                    help='number of data loading workers (default: 8)')
parser.add_argument('--epochs', default=200, type=int, metavar='N',
                    help='number of total epochs to run')
parser.add_argument('--start-epoch', default=0, type=int, metavar='N',
                    help='manual epoch number (useful on restarts)')
parser.add_argument('-b', '--batch-size', default=1000, type=int,
                    metavar='N', help='mini-batch size (default: 2048)')
parser.add_argument('-mb', '--mini-batch-size', default=100, type=int,
                    help='mini-mini-batch size (default: 128)')
parser.add_argument('--save_all', dest='save_all', action='store_true',
                    help='save all better checkpoints')
parser.add_argument('--no-save_all', dest='save_all', action='store_false',
                    help='save all better checkpoints')
parser.set_defaults(save_all=False)
parser.add_argument('--augment', dest='augment', action='store_true',
                    help='data augment')
parser.add_argument('--no-augment', dest='augment', action='store_false',
                    help='data augment')
parser.set_defaults(augment=True)
parser.add_argument('--optimizer', default='SGD', type=str, metavar='OPT',
                    help='optimizer function used')
parser.add_argument('--lr', '--learning_rate', default=0.001, type=float,
                    metavar='LR', help='initial learning rate')
parser.add_argument('--momentum', default=0.9, type=float, metavar='M',
                    help='momentum')
parser.add_argument('--weight-decay', '--wd', default=1e-4, type=float,
                    metavar='W', help='weight decay (default: 1e-4)')
parser.add_argument('--sharpness-smoothing', '--ss', default=0.0, type=float,
                    metavar='SS', help='sharpness smoothing (default: 0)')
parser.add_argument('--anneal-index', '--ai', default=0.55, type=float,
                    metavar='AI', help='Annealing index of noise (default: 0.55)')
parser.add_argument('--tanh-scale', '--ts', default=10., type=float,
                    metavar='TS', help='scale of transition in tanh')
parser.add_argument('--smoothing-type', default='tanh', type=str, metavar='ST',
                    help='The type of chaning smoothing noise: constant, anneal, or tanh')
parser.add_argument('--print-freq', '-p', default=10, type=int,
                    metavar='N', help='print frequency (default: 10)')
parser.add_argument('--resume', default='', type=str, metavar='PATH',
                    help='path to latest checkpoint (default: none)')
parser.add_argument('-e', '--evaluate', type=str, metavar='FILE',
                    help='evaluate model FILE on validation set')
parser.add_argument('--epsilon', default=0.0005, type=float,
                    help='epsilon to contrain the box size for sharpness measure')
parser.add_argument('-m', '--manifolds', default=0, type=int, metavar='M',
                    help='The dimensionality of manifolds to measure sharpness. (0: full-space)')
parser.add_argument('-t', '--times', default=1, type=int, metavar='T',
                    help='Times to average over for sharpness')
def main():
    #torch.manual_seed(123)
    global args, best_prec1
    best_prec1 = 0
    args = parser.parse_args()

    if args.evaluate:
        args.results_dir = '/tmp'
    if args.save is '':
        args.save = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
    save_path = os.path.join(args.results_dir, args.save)
    if not os.path.exists(save_path):
        os.makedirs(save_path)
    else:
      raise OSError('Directory {%s} exists. Use a new one.' % save_path)

    setup_logging(os.path.join(save_path, 'log.txt'))
    results_file = os.path.join(save_path, 'results.%s')
    results = ResultsLog(results_file % 'csv', results_file % 'html')

    logging.info("saving to %s", save_path)
    logging.debug("run arguments: %s", args)

    if 'cuda' in args.type:
        #torch.cuda.manual_seed_all(123)
        args.gpus = [int(i) for i in args.gpus.split(',')]
        torch.cuda.set_device(args.gpus[0])
        cudnn.benchmark = True
        if len(args.gpus) != 1:
          raise NotImplementedError('Please use one gpu.')
    else:
        args.gpus = None

    if args.batch_size != args.mini_batch_size:
      args.mini_batch_size = args.batch_size
      warnings.warn('--mini-batch-size is enforced to be set as --batch-size {}'.format(args.mini_batch_size),
                    RuntimeWarning)

    # create model
    logging.info("creating model %s", args.model)
    model = models.__dict__[args.model]
    model_config = {'input_size': args.input_size, 'dataset': args.dataset}

    if args.model_config is not '':
        model_config = dict(model_config, **literal_eval(args.model_config))

    model = model(**model_config)
    logging.info("created model with configuration: %s", model_config)

    # optionally resume from a checkpoint
    if args.evaluate:
        if not os.path.isfile(args.evaluate):
            parser.error('invalid checkpoint: {}'.format(args.evaluate))
        checkpoint = torch.load(args.evaluate, map_location=lambda storage, loc: storage)
        model.load_state_dict(checkpoint['state_dict'])
        logging.info("loaded checkpoint '%s' (epoch %s)",
                     args.evaluate, checkpoint['epoch'])
    else:
      raise ValueError("Please specify the path of evaluated model")

    num_parameters = sum([l.nelement() for l in model.parameters()])
    logging.info("number of parameters: %d", num_parameters)

    # Data loading code
    default_transform = {
        'train': get_transform(args.dataset,
                               input_size=args.input_size, augment=args.augment),
        'eval': get_transform(args.dataset,
                              input_size=args.input_size, augment=False)
    }
    transform = getattr(model, 'input_transform', default_transform)

    # define loss function (criterion) and optimizer
    criterion = getattr(model, 'criterion', nn.CrossEntropyLoss)()
    criterion.type(args.type)
    model.type(args.type)

    val_data = get_dataset(args.dataset, 'val', transform['eval'])
    val_loader = torch.utils.data.DataLoader(
        val_data,
        batch_size=args.batch_size, shuffle=False,
        num_workers=args.workers, pin_memory=True)

#    for k in model.state_dict():
#        if re.match('.*weight.*', k):
#            plt.figure()
#            plt.hist(model.state_dict()[k].cpu().numpy().reshape(-1), bins='auto')
#            plt.title(k)
#    plt.show()

    val_result = validate(val_loader, model, criterion, 0)
    val_loss, val_prec1, val_prec5 = [val_result[r]
                                      for r in ['loss', 'prec1', 'prec5']]
    logging.info('\nValidation Loss {val_loss:.4f} \t'
                 'Validation Prec@1 {val_prec1:.3f} \t'
                 'Validation Prec@5 {val_prec5:.3f} \n'
                 .format(val_loss=val_loss,
                         val_prec1=val_prec1,
                         val_prec5=val_prec5))
    sharpnesses= []
    for time in range(args.times):
        sharpness = get_sharpness(val_loader, model, criterion, manifolds=args.manifolds)
        sharpnesses.append(sharpness)
        logging.info('sharpness {} = {}'.format(time,sharpness))
    logging.info('sharpnesses = {}'.format(str(sharpnesses)))
    _std = np.std(sharpnesses)*np.sqrt(args.times)/np.sqrt(args.times-1)
    _mean = np.mean(sharpnesses)
    logging.info(u'mean sharpness = {sharpness:.4f}\u00b1{err:.4f}'.format(sharpness=_mean,err=_std))

    return

def forward(data_loader, model, criterion, epoch=0, training=True, optimizer=None):
    if args.gpus and len(args.gpus) > 1:
        model = torch.nn.DataParallel(model, args.gpus)

    data_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    end = time.time()
    grad_vec = None
    if training:
      optimizer = torch.optim.SGD(model.parameters(), 1.0)
      optimizer.zero_grad()  # only zerout at the beginning

    for i, (inputs, target) in enumerate(data_loader):
        # measure data loading time
        data_time.update(time.time() - end)
        if args.gpus is not None:
            target = target.cuda(async=True)
        input_var = Variable(inputs.type(args.type), volatile=not training)
        target_var = Variable(target)

        # compute output
        if not training:
            output = model(input_var)
            loss = criterion(output, target_var)

            # measure accuracy and record loss
            prec1, prec5 = accuracy(output.data, target_var.data, topk=(1, 5))
            losses.update(loss.data[0], input_var.size(0))
            top1.update(prec1[0], input_var.size(0))
            top5.update(prec5[0], input_var.size(0))

        else:
            mini_inputs = input_var.chunk(args.batch_size // args.mini_batch_size)
            mini_targets = target_var.chunk(args.batch_size // args.mini_batch_size)

            for k, mini_input_var in enumerate(mini_inputs):

                mini_target_var = mini_targets[k]
                output = model(mini_input_var)
                loss = criterion(output, mini_target_var)

                prec1, prec5 = accuracy(output.data, mini_target_var.data, topk=(1, 5))
                losses.update(loss.data[0], mini_input_var.size(0))
                top1.update(prec1[0], mini_input_var.size(0))
                top5.update(prec5[0], mini_input_var.size(0))

                # compute gradient and do SGD step
                loss.backward()

            #optimizer.step() # no step in this case

    # reshape and averaging gradients
    if training:
      for p in model.parameters():
        p.grad.data.div_(len(data_loader))
        if grad_vec is None:
          grad_vec = p.grad.data.view(-1)
        else:
          grad_vec = torch.cat((grad_vec, p.grad.data.view(-1)))

    #logging.info('{phase} - \t'
    #             'Loss {loss.avg:.4f}\t'
    #             'Prec@1 {top1.avg:.3f}\t'
    #             'Prec@5 {top5.avg:.3f}'.format(
    #              phase='TRAINING' if training else 'EVALUATING',
    #              loss=losses, top1=top1, top5=top5))

    return {'loss': losses.avg,
            'prec1': top1.avg,
            'prec5': top5.avg}, grad_vec

def train(data_loader, model, criterion, epoch, optimizer):
    # switch to train mode
    raise NotImplementedError('train functionality is changed. Do not use it!')

def validate(data_loader, model, criterion, epoch):
    # switch to evaluate mode
    model.eval()
    res, _ = forward(data_loader, model, criterion, epoch,
                   training=False, optimizer=None)
    return res

def get_minus_cross_entropy(x, data_loader, model, criterion, training=False):
  if type(x).__module__ == np.__name__:
    x = torch.from_numpy(x).float()
    x = x.cuda()
  # switch to evaluate mode
  model.eval()

  # fill vector x of parameters to model
  x_start = 0
  for p in model.parameters():
    psize = p.data.size()
    peltnum = 1
    for s in psize:
      peltnum *= s
    x_part = x[x_start:x_start+peltnum]
    p.data = x_part.view(psize)
    x_start += peltnum

  result, grads = forward(data_loader, model, criterion, 0,
                 training=training, optimizer=None)
  #print ('get_minus_cross_entropy {}!'.format(-result['loss']))
  return (-result['loss'], None if grads is None else grads.cpu().numpy().astype(np.float64))

def get_sharpness(data_loader, model, criterion, manifolds=0):

  # extract current x0
  x0 = None
  for p in model.parameters():
    if x0 is None:
      x0 = p.data.view(-1)
    else:
      x0 = torch.cat((x0, p.data.view(-1)))
  x0 = x0.cpu().numpy()

  # get current f_x
  f_x0, _ = get_minus_cross_entropy(x0, data_loader, model, criterion)
  f_x0 = -f_x0
  logging.info('min loss f_x0 = {loss:.4f}'.format(loss=f_x0))

  # get the bounds
  epsilon = args.epsilon
  # find the minimum
  if 0==manifolds:
    x_min = np.reshape(x0 - epsilon * (np.abs(x0) + 1), (x0.shape[0], 1))
    x_max = np.reshape(x0 + epsilon * (np.abs(x0) + 1), (x0.shape[0], 1))
    bounds = np.concatenate([x_min, x_max], 1)
    func = lambda x: get_minus_cross_entropy(x, data_loader, model, criterion, training=True)
    init_guess = x0
  else:
    warnings.warn("Small manifolds may not be able to explore the space.")
    assert(manifolds<=x0.shape[0])
    #transformer = rp.GaussianRandomProjection(n_components=manifolds)
    #transformer.fit(np.random.rand(manifolds, x0.shape[0]))
    #A_plus = transformer.components_
    #A = np.linalg.pinv(A_plus)
    A_plus = np.random.rand(manifolds, x0.shape[0])*2.-1.
    # normalize each column to unit length
    A_plus_norm = np.linalg.norm(A_plus, axis=1)
    A_plus = A_plus / np.reshape(A_plus_norm, (manifolds,1))
    A = np.linalg.pinv(A_plus)
    abs_bound = epsilon * (np.abs(np.dot(A_plus, x0))+1)
    abs_bound = np.reshape(abs_bound, (abs_bound.shape[0], 1))
    bounds = np.concatenate([-abs_bound, abs_bound], 1)
    def func(y):
      floss, fg = get_minus_cross_entropy(x0 + np.dot(A, y), data_loader, model, criterion, training=True)
      return floss, np.dot(np.transpose(A), fg)
    #func = lambda y: get_minus_cross_entropy(x0+np.dot(A, y), data_loader, model, criterion, training=True)
    init_guess = np.zeros(manifolds)

  #rand_selections = (np.random.rand(bounds.shape[0])+1e-6)*0.99
  #init_guess = np.multiply(1.-rand_selections, bounds[:,0])+np.multiply(rand_selections, bounds[:,1])

  minimum_x, f_x, d = sciopt.fmin_l_bfgs_b(
    func,
    init_guess,
    maxiter=10,
    bounds=bounds,
    #factr=10.,
    #pgtol=1.e-12,
    disp=1)
  f_x = -f_x
  logging.info('max loss f_x = {loss:.4f}'.format(loss=f_x))
  sharpness = (f_x - f_x0)/(1+f_x0)*100

  # recover the model
  x0 = torch.from_numpy(x0).float()
  x0 = x0.cuda()
  x_start = 0
  for p in model.parameters():
      psize = p.data.size()
      peltnum = 1
      for s in psize:
          peltnum *= s
      x_part = x0[x_start:x_start + peltnum]
      p.data = x_part.view(psize)
      x_start += peltnum

  return sharpness

if __name__ == '__main__':
    main()

reason: import numpy,import scipy


text:
import pennylane as qml
import numpy as np
from tqdm import tqdm
import os

from utils.smallmodel_functions import accuracy, prediction_single, loader, cost, classifier, weights_save, accuracy_single
class SmallModel:
    def __init__(self, dataset, alpha, save_name=None, is_aux=False):
        self.x_train, self.y_train = loader(dataset, crazy=True)
        self.dataset=dataset
        if save_name:
            self.dataset = save_name
        self.opt = qml.NesterovMomentumOptimizer(.5)
        self.weights = 0.01 * np.random.randn(5,5)
        self.alpha = alpha
        self.is_aux = is_aux
    def train(self, batch_size, epochs):
        accs = []
        for i in range(epochs):
            batch_index = np.random.randint(0, len(self.x_train), (batch_size,))
            x_train_batch = self.x_train[batch_index]
            y_train_batch = np.array(self.y_train)[batch_index]
            self.weights = self.opt.step(lambda v: cost(v, x_train_batch, y_train_batch, self.alpha), self.weights)
            accs.append(accuracy_single(y_train_batch, [prediction_single(classifier(self.weights,features=x)) for x in x_train_batch]))
            weights_save(self.weights, self.dataset, i, batch_size, accs[-1], accs[-1], False)
            if i % 50 ==0:
                print("Model: " + self.dataset + " Epoch " + str(i) + " Acc: " + str(np.average(accs)))
                accs = []
    def get_acc(self, batch_size):
        #guesses = [prediction(classifier(self.weights, features=x), num_classes=self.num_classes) for x in self.eval_batch]
        batch_index  = np.random.randint(0, len(self.x_train), (batch_size,))
        x_train_batch = self.x_train[batch_index]
        y_train_batch = self.y_train[batch_index]
        best_weights_path = os.path.join('weights', 'MNIST', 'splits', self.dataset, 'weights.npy')
        best_weights = np.load(best_weights_path)
        acc = accuracy(y_train_batch, [prediction(classifier(best_weights,features=x)) for x in x_train_batch])
        return acc
    def eval(self, images):
        best_weights_path = os.path.join('weights', 'MNIST', 'splits', self.dataset, 'weights.npy')
        best_weights = np.load(best_weights_path)
        guesses = [prediction_single(classifier(best_weights,features=x)) for x in images]
        return guesses

max_stars_repo_name: SilverEngineered/Quilt

max_issues_repo_name: SilverEngineered/Quilt

max_forks_repo_name: SilverEngineered/Quilt