text:
#!/usr/bin/env python
# coding=utf-8
import inspect
import logging
import simplejson as json
from pathlib import Path
from importlib import import_module

import numpy as np
import pandas as pd
import netCDF4 as nc4

from ioos_qc.config import Config
from ioos_qc.utils import GeoNumpyDateEncoder, cf_safe_name
from ioos_qc.results import collect_results

L = logging.getLogger(__name__)  # noqa

class BaseStore:

    def save(self, *args, **kwargs):
        """
        Serialize results to a store. This could save file or publish messages.
        """

class PandasStore(BaseStore):
    """Store results in a dataframe"""

    def __init__(self, results, axes: dict = None):
        self.results = list(results)
        self.collected_results = collect_results(self.results, how='list')
        self.axes = axes or {
            't': 'time',
            'z': 'z',
            'y': 'lat',
            'x': 'lon'
        }

    def save(self, write_data: bool = False, include: list = None, exclude: list = None) -> pd.DataFrame:

        df = pd.DataFrame()

        for cr in self.collected_results:

            # Add time axis
            if self.axes['t'] not in df and cr.tinp is not None and cr.tinp.size != 0:
                L.info(f"Adding column {self.axes['t']} from stream {cr.stream_id}")
                df[self.axes['t']] = cr.tinp

            # Add z axis
            if self.axes['z'] not in df and cr.zinp is not None and cr.zinp.size != 0:
                L.info(f"Adding columm {self.axes['z']} from stream {cr.stream_id}")
                df[self.axes['z']] = cr.zinp

            # Add x axis
            if self.axes['x'] not in df and cr.lon is not None and cr.lon.size != 0:
                L.info(f"Adding columm {self.axes['x']} from stream {cr.stream_id}")
                df[self.axes['x']] = cr.lon

            # Add x axis
            if self.axes['y'] not in df and cr.lat is not None and cr.lat.size != 0:
                L.info(f"Adding columm {self.axes['y']} from stream {cr.stream_id}")
                df[self.axes['y']] = cr.lat

            # Inclusion list, skip everything not defined
            if include is not None and (cr.function not in include and cr.stream_id not in include and cr.test not in include):
                continue

            # Exclusion list, skip everything defined
            if exclude is not None and (cr.function in exclude or cr.stream_id in exclude or cr.test in cr.test in include):
                continue

            # Add data column
            if write_data and cr.stream_id not in df:
                df[cr.stream_id] = cr.data

            # Add QC results column
            column_name = cf_safe_name(f'{cr.stream_id}.{cr.package}.{cr.test}')
            if column_name not in df:
                df[column_name] = cr.results
            else:
                L.warning(f"Found duplicate QC results column: {column_name}, skipping.")

        return df

class CFNetCDFStore(BaseStore):

    def __init__(self, results, axes=None, **kwargs):
        self.results = list(results)
        self.collected_results = collect_results(self.results, how='list')
        self.axes = axes or {
            't': 'time',
            'z': 'z',
            'y': 'lat',
            'x': 'lon'
        }

    def save(self, path_or_ncd, dsg, config: Config, dsg_kwargs: dict = {}, write_data: bool = False, include: list = None, exclude: list = None):
        ps = PandasStore(self.results, self.axes)
        df = ps.save(write_data=write_data, include=include, exclude=exclude)

        # Write a new file
        attrs = {}
        for cr in self.collected_results:

            column_name = cf_safe_name(f'{cr.stream_id}.{cr.package}.{cr.test}')

            # Set the ancillary variables
            if cr.stream_id not in attrs:
                attrs[cr.stream_id] = {
                    'ancillary_variables': column_name
                }
            else:
                # Update the source ancillary_variables
                existing = getattr(attrs[cr.stream_id], 'ancillary_variables', '').split(' ')
                existing += [column_name]
                attrs[cr.stream_id] = ' '.join(list(set(existing))).strip()

            # determine standard name and long name. These should be defined on each test function
            # https://github.com/cf-convention/cf-conventions/issues/216
            standard_name = getattr(cr.function, 'standard_name', 'quality_flag')
            long_name = getattr(cr.function, 'long_name', 'Quality Flag')

            # Get flags from module attribute called FLAGS
            flags = getattr(inspect.getmodule(cr.function), 'FLAGS')
            varflagnames = [ d for d in flags.__dict__ if not d.startswith('__') ]
            varflagvalues = [ getattr(flags, d) for d in varflagnames ]

            # Set QC variable attributes
            if column_name not in attrs:
                attrs[column_name] = {
                    'standard_name': standard_name,
                    'long_name': long_name,
                    'flag_values': np.byte(varflagvalues),
                    'flag_meanings': ' '.join(varflagnames),
                    'valid_min': np.byte(min(varflagvalues)),
                    'valid_max': np.byte(max(varflagvalues)),
                    'ioos_qc_module': cr.package,
                    'ioos_qc_test': cr.test,
                    'ioos_qc_target': cr.stream_id,
                }
                # If there is only one context we can write variable specific configs
                if len(config.contexts) == 1:
                    varconfig = config.contexts[0].streams[cr.stream_id].config[cr.package][cr.test]
                    varconfig = json.dumps(varconfig, cls=GeoNumpyDateEncoder, allow_nan=False, ignore_nan=True)
                    attrs[column_name]['ioos_qc_config'] = varconfig
                    if config.contexts[0].region:
                        attrs[column_name]['ioos_qc_region'] = json.dumps(config.contexts[0].region, cls=GeoNumpyDateEncoder, allow_nan=False, ignore_nan=True)
                    if config.contexts[0].window.starting or config.contexts[0].window.ending:
                        attrs[column_name]['ioos_qc_window'] = json.dumps(config.contexts[0].window, cls=GeoNumpyDateEncoder, allow_nan=False, ignore_nan=True)

        if len(config.contexts) > 1:
            # We can't represent these at the variable level, so make one global config
            attrs['ioos_qc_config'] = json.dumps(config.config, cls=GeoNumpyDateEncoder, allow_nan=False, ignore_nan=True)

        dsg_kwargs = {
            **dsg_kwargs,
            **{
                'attributes': attrs
            }
        }

        # pocean requires these default columns, which should be removed as a requirement
        # in pocean.
        df['station'] = 0
        df['trajectory'] = 0
        df['profile'] = 0
        ncd = dsg.from_dataframe(df, path_or_ncd, axes=self.axes, **dsg_kwargs)
        return ncd

class NetcdfStore:

    def save(self, path_or_ncd, config, results):
        """
        Updates the given netcdf with test configuration and results.
        If there is already a variable for a given test, it will update that variable with the latest results.
        Otherwise, it will create a new variable.

        :param path_or_ncd: path or netcdf4 Dataset in which to store results
        :param results: output of run()
        """
        try:
            ncd = None
            should_close = True
            if isinstance(path_or_ncd, (str, Path)):
                ncd = nc4.Dataset(str(path_or_ncd), 'a')
            elif isinstance(path_or_ncd, nc4.Dataset):
                ncd = path_or_ncd
                should_close = False
            else:
                return ValueError('Input is not a valid file path or Dataset')

            for vname, qcobj in results.items():

                if vname not in ncd.variables:
                    L.warning(f'{vname} not found in the Dataset, skipping')
                    continue

                source_var = ncd.variables[vname]

                for modu, tests in qcobj.items():

                    try:
                        testpackage = import_module('ioos_qc.{}'.format(modu))
                    except ImportError:
                        L.error('No ioos_qc test package "{}" was found, skipping.'.format(modu))
                        continue

                    # Keep track of the test names so we can add to the source's
                    # ancillary_variables at the end
                    qcvar_names = []
                    for testname, testresults in tests.items():

                        # Try to find a qc variable that matches this config
                        qcvars = ncd.get_variables_by_attributes(
                            ioos_qc_module=modu,
                            ioos_qc_test=testname,
                            ioos_qc_target=vname
                        )
                        if not qcvars:
                            qcvarname = cf_safe_name(vname + '.' + modu + '.' + testname)
                        else:
                            if len(qcvars) > 1:
                                names = [ v.name for v in qcvars ]
                                L.warning('Found more than one QC variable match: {}'.format(names))
                            # Use the last one found
                            qcvarname = qcvars[-1].name

                        # Get flags from module attribute called FLAGS
                        flags = getattr(testpackage, 'FLAGS')
                        varflagnames = [ d for d in flags.__dict__ if not d.startswith('__') ]
                        varflagvalues = [ getattr(flags, d) for d in varflagnames ]

                        if qcvarname not in ncd.variables:
                            v = ncd.createVariable(qcvarname, np.byte, source_var.dimensions)
                        else:
                            v = ncd[qcvarname]
                        qcvar_names.append(qcvarname)

                        # determine standard name
                        # https://github.com/cf-convention/cf-conventions/issues/216
                        try:
                            testfn = getattr(testpackage, testname)
                            standard_name = testfn.standard_name
                            long_name = testfn.long_name
                        except AttributeError:
                            standard_name = 'quality_flag'
                            long_name = 'Quality Flag'

                        # write to netcdf
                        v[:] = testresults
                        v.setncattr('standard_name', standard_name)
                        v.setncattr('long_name', long_name)
                        v.setncattr('flag_values', np.byte(varflagvalues))
                        v.setncattr('flag_meanings', ' '.join(varflagnames))
                        v.setncattr('valid_min', np.byte(min(varflagvalues)))
                        v.setncattr('valid_max', np.byte(max(varflagvalues)))
                        v.setncattr('ioos_qc_module', modu)
                        v.setncattr('ioos_qc_test', testname)
                        v.setncattr('ioos_qc_target', vname)
                        # If there is only one context we can write variable specific configs
                        if len(config.contexts) == 1:
                            varconfig = config.contexts[0].streams[vname].config[modu][testname]
                            varconfig = json.dumps(varconfig, cls=GeoNumpyDateEncoder, allow_nan=False, ignore_nan=True)
                            v.setncattr('ioos_qc_config', varconfig)
                            v.setncattr('ioos_qc_region', json.dumps(config.contexts[0].region, cls=GeoNumpyDateEncoder, allow_nan=False, ignore_nan=True))
                            v.setncattr('ioos_qc_window', json.dumps(config.contexts[0].window, cls=GeoNumpyDateEncoder, allow_nan=False, ignore_nan=True))

                # Update the source ancillary_variables
                existing = getattr(source_var, 'ancillary_variables', '').split(' ')
                existing += qcvar_names
                source_var.ancillary_variables = ' '.join(list(set(existing))).strip()

            if len(config.contexts) > 1:
                # We can't represent these at the variable level, so make one global config
                ncd.setncattr(
                    'ioos_qc_config',
                    json.dumps(config.config, cls=GeoNumpyDateEncoder, allow_nan=False, ignore_nan=True)
                )

        finally:
            if ncd and should_close is True:
                ncd.close()


text:
import os
import random
import shutil
from pathlib import Path

import numpy as np
import torch
from sacred.config.custom_containers import ReadOnlyDict
from sacred.observers import FileStorageObserver, MongoObserver

from utils import loggers

ROOT = Path(__file__).parents[1]

def set_seed(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

def try_snapshot_files(_run):
    if _run.observers:
        for obs in _run.observers:
            if isinstance(obs, FileStorageObserver):
                for source_file, _ in _run.experiment_info['sources']:
                    os.makedirs(os.path.dirname(f'{obs.dir}/source/{source_file}'), exist_ok=True)
                    obs.save_file(source_file, f'source/{source_file}')
                shutil.rmtree(f'{obs.basedir}/_sources')
                break

def add_observers(ex, config, fileStorage=True, MongoDB=True, db_name="default"):
    if fileStorage:
        observer_file = FileStorageObserver(config["g"]["model_dir"])
        ex.observers.append(observer_file)

    if MongoDB:
        try:
            observer_mongo = MongoObserver(url="localhost:{}".format(config["g"]["mongo_port"]), db_name=db_name)
            ex.observers.append(observer_mongo)
        except ModuleNotFoundError:
            # Ignore Mongo Observer
            pass

def post_hook(ex, NAME):
    @ex.config_hook
    def config_hook(config, command_name, logger):
        g = config["g"]
        if command_name in ["train", "test"]:
            if config["split"] == -1:
                raise ValueError("Argument `split` is required! For example: `split=0` ")

            add_observers(ex, config, fileStorage=g["fileStorage"], MongoDB=g["mongodb"], db_name=NAME)
            ex.logger = loggers.get_global_logger(name=NAME)
        return config

class MapConfig(ReadOnlyDict):
    """
    A wrapper for dict. This wrapper allow users to access dict value by `dot` operation.
    For example, you can access `cfg["split"]` by `cfg.split`, which makes the code more clear.

    Notice that the result object is a sacred.config.custom_containers.ReadOnlyDict, which is
    a read-only dict for preserving the configuration.

    Parameters
    ----------
    obj: ReadOnlyDict
        Configuration dict.
    """
    __getattr__ = dict.__getitem__
    __setattr__ = dict.__setitem__
    __delattr__ = dict.__delitem__

    def __init__(self, obj, **kwargs):
        new_dict = {}
        if isinstance(obj, dict):
            for k, v in obj.items():
                if isinstance(v, dict):
                    new_dict[k] = MapConfig(v)
                else:
                    new_dict[k] = v
        else:
            raise TypeError(f"`obj` must be a dict, got {type(obj)}")
        super(MapConfig, self).__init__(new_dict, **kwargs)

def get_pth(exp_dir, ckpt):
    ckpt_name = [ckpt, "bestckpt.pth", "ckpt.pth"]
    for name in ckpt_name:
        if name is None:
            continue
        ckpt_path = exp_dir / name
        if ckpt_path.exists() and ckpt_path.is_file():
            return ckpt_path
    return False

def try_possible_ckpt_names(base, exp_id, ckpt=None):
    if isinstance(exp_id, int) and exp_id >= 0:
        # Search running results from `base` directory (with specific tag)
        ckpt_path = get_pth(base / str(exp_id), ckpt)
        if ckpt_path:
            return ckpt_path, exp_id

        # Search running results from `model_dir` (without specific tag)
        for exp_dir in base.parent.glob("*/[0-9]*"):
            if exp_dir.is_dir() and int(exp_dir.name) == exp_id:
                ckpt_path = get_pth(exp_dir, ckpt)
                return ckpt_path, exp_id

    # Use ckpt directly
    ckpt_file = Path(ckpt)
    if ckpt_file.exists():
        return ckpt_file, 99999999

    return False, 99999999

def find_snapshot(cfg, exp_id=-1, ckpt=None):
    """ Find experiment checkpoint """
    if ckpt is None:
        ckpt = cfg.ckpt

    base = Path(cfg.g.model_dir) / str(cfg.tag)
    ckpt_path = try_possible_ckpt_names(base, exp_id, ckpt)
    if ckpt_path[0]:
        return ckpt_path

    # Guess the maximum exp_id as the desired exp_id
    all_exp_ids = [int(dir_name.name) for dir_name in base.glob("[0-9]*") if dir_name.is_dir()]
    if len(all_exp_ids) > 0:
        exp_id = max(all_exp_ids)
        ckpt_path = try_possible_ckpt_names(base, exp_id, ckpt)
        if ckpt_path[0]:
            return ckpt_path

    # Import readline module to improve the experience of input
    # noinspection PyUnresolvedReferences
    import readline

    while True:
        ckpt_path = Path(input("Cannot find checkpoints. Please input:"))
        if ckpt_path.exists():
            return ckpt_path, 99999999

ext: py

reason: import numpy


text:
# this takes an scb file and mutliple scb files by chopping it up into
# smaller files

import struct
import sys
import os

DEFAULT_OUTPUT = 'output.scb'
import numpy as np

def readHeader( inFile ):
    '''Reads the header of the scb file and returns the following data:
        version # (as a number)
        '''
    version = float( inFile.read( 4 )[:-1] )
    print "Version", version
    agtCount = struct.unpack( 'i', inFile.read( 4 ) )[0]
    stepSize = 0.0
    ids = None

    if ( version == 2.0 ):
        stepSize = struct.unpack( 'f', inFile.read( 4 ) )[0]
        idReadSize = agtCount * 4
        ids = inFile.read( idReadSize )
    print agtCount
    print stepSize

    return version, agtCount, stepSize, ids

def writeHeader( file, version, agentCount, timeStep=0.0, ids=None ):
    '''Writes the scb header to the file'''
    if ( version == 1 ):
        file.write( '1.0\x00' )
        file.write( struct.pack( 'i', agentCount ) )
    if ( version == 2 ):
        file.write( '2.0\x00' )
        file.write( struct.pack( 'i', agentCount ) )
        file.write( struct.pack( 'f', timeStep ) )
        file.write( ids )

def sliceSCB( inFile, outBase, count ):
    '''Reads the input scb file, inFile, and separates it into count,
    roughly equally sized, files.  All files have a name in the format:
    outBase##.scb'''
    FLOAT_SIZE = 4      # bytes per float
    AGENT_SIZE = 3      # floats per agent
    version, agtCount, stepSize, ids = readHeader( inFile )
    dataPos = inFile.tell()
    framesize = agtCount * FLOAT_SIZE * AGENT_SIZE

    # pass 1, count the number of frames
    frameCount = 0
    data = inFile.read( framesize )
    while ( data != '' ):
        frameCount += 1
        data = inFile.read( framesize )

    counts = np.zeros( count, dtype=np.int ) + frameCount
    counts /= count
    counts[0] += frameCount - np.sum( counts )
    padding = int( np.ceil( np.log10( count ) ) )
    print "Total frames:", frameCount
    print "Frame counts:", counts

    # pass 2, write out the files
    inFile.seek( dataPos )
    for i in range( count ):
        fName = '{0:s}{1:0{2}d}.scb'.format( outBase, i, padding )
        f = open( fName, 'wb' )
        writeHeader( f, version, agtCount, stepSize, ids )
        f.write( inFile.read( framesize * counts[i] ) )
        f.close()

def usage():
    print "Divide an scb file -- slice it into multiple scb files"
    print
    print "scbSlice -in filename <-out filename> -n # "
    print
    print "  -in filename     - the name of the file to truncate"
    print "                     must be valid scb file"
    print "  -n #             - an integer, the number of pieces"
    print "                     to chop the scb file into"
    print "  -out filename    - the name of the output file."
    print "                     if not provided, output file defaults to %s" % ( DEFAULT_OUTPUT )
    sys.exit(1)

def outBaseName( fileName ):
    '''Given the filename indicated, produces a basename for the output.scb
    the base name is unadorned by extensions.'''
    base, ext = os.path.splitext( fileName )
    if ( ext == '.scb' ):
        return base
    else:
        return fileName

def main():
    """Determine the input file, output file and number of frames"""
    from commandline import SimpleParamManager

    # parse command-line arguments
    try:
        pMan = SimpleParamManager( sys.argv[1:], {'in':'', 'out':DEFAULT_OUTPUT, 'n':0 } )
    except IOError:
        usage()

    inName = pMan[ 'in' ]
    outName = pMan[ 'out' ]
    count = int( pMan[ 'n' ] )

    if ( count < 2 ):
        print "ERROR! Can't chop the file into %d pieces" % ( count )
        usage()
    try:
        inFile = open( inName, 'rb' )
    except IOError:
        print "ERROR! Couldn't open input file %s" % ( inName )
        usage()

    outName = outBaseName( outName )

    sliceSCB( inFile, outName, count )

if __name__ == '__main__':
    main()


text:
import numpy as np
from copy import deepcopy
from .base import Representation as base_Representation
from .exceptions import IncompleteArguments

AVAILABLE_REPR = ['Binary', 'Integer', 'Permutation', 'Real']

def Representation(genotype: str, **kwargs):
    """
    Function that returns a Representation object based on the indicated genotype.

    Call to the function:

        Representation(genotype: str, **kwargs) -> base.Representation

    Available genotypes:

    Parameters (genotype = 'binary')
    ----------
    length: int
        Number of bits.

    For more info help(beagle.Binary)

    Parameters (genotype = 'integer')
    ----------
    value_coding: dict
        Dictionary with the characteristic as a key and the coding value (integer) as a value.

    replacement: bool
        Value indicating if there can be repeated characteristics. By default False.

    For more info help(beagle.Integer)

    Parameters (genotype = 'permutation')
    ----------
        events: list
            List with the different events on which you want to find the optimal order.

        restrictions: list
            List of functions that must take a list of values and return True if the order falls within the range
            of possibilities or False if there is some restriction for that combination.

    For more info help(beagle.Permutation)

    Parameters (genotype = 'real')
    ----------
        bounds: list
            List of tuples indicating the lower and upper limit for each possible value.

    For more info help(beagle.RealValue)

    Returns
    --------
    :return base.Representation
        Subclass of base.representation that will depend on the indicated genotype.
    """
    if genotype.lower() == 'binary':
        if kwargs.get('length', -1) == -1 or not isinstance(kwargs['length'], int):
            raise IncompleteArguments(Binary.args(), str(Binary(-1)))

        return Binary(kwargs.get('length', -1))

    elif genotype.lower() == 'integer':
        if kwargs.get('value_coding', -1) == -1 or not isinstance(kwargs['value_coding'], dict):
            raise IncompleteArguments(Integer.args(), str(Integer({'None': None})))

        return Integer(kwargs.get('value_coding', {'None': None}))

    elif genotype.lower() == 'permutation':
        if kwargs.get('events', -1) == -1 or not isinstance(kwargs['events'], list):
            raise IncompleteArguments(Permutation.args(), str(Permutation([None])))

        return Permutation(kwargs.get('events', [None]), kwargs.get('restrictions', None))

    elif genotype.lower() == 'real':
        if kwargs.get('bounds', -1) == -1 or not isinstance(kwargs['bounds'], list):
            raise IncompleteArguments(RealValue.args(), str(RealValue([(None, None)])))
        if isinstance(kwargs['bounds'], list):
            for e in kwargs['bounds']:
                if not isinstance(e, tuple):  # limits not indicated as tuple
                    raise IncompleteArguments(RealValue.args(), str(RealValue([(None, None)])))
                if len(e) != 2:               # tuple length smaller than 2
                    raise IncompleteArguments(RealValue.args(), str(RealValue([(None, None)])))
                if e[0] >= e[1]:              # lower limit smaller than upper limit.
                    raise IncompleteArguments(RealValue.args(), str(RealValue([(None, None)])))

        return RealValue(kwargs.get('bounds', [(None, None)]))

    else:
        raise TypeError('Genotype representation %s not implemented. Available representations: %s'
                        % (genotype, ', '.join(AVAILABLE_REPR)))

class RealValue(base_Representation):
    """
    Class to represent real-value numbers. Indicated as a list of tuples which indicate the range in which
    each value can be found.
    """

    def __init__(self, bounds: list, initialization: bool = True):
        """
        __init__(bounds: list, initialization: bool = True)

        Parameters
        -----------
        :param bounds: list
            List of tuples indicating the lower and upper limit for each value.

        :param initialization: bool
             Indicates whether the individual will call the initialization function to perform a random initialization.
        """
        self._bounds = bounds
        self._values = []
        if initialization:
            self.initialization()

    def __repr__(self):
        return "real"

    def __str__(self):
        return self.__repr__()

    def __getitem__(self, index: int):
        return self._values[index]

    def __setitem__(self, index: int, value: float):
        self._values[index] = value

    def __len__(self):
        return len(self._bounds)

    @property
    def kwargs(self):
        return {'bounds': self._bounds}

    @property
    def permissible_values(self):
        """
        Returns a list of tuples indicating the range that each possible value can take.

        Returns
        --------
        :return: list
            List of tuples of length 2.
        """
        return self._bounds

    @property
    def values(self):
        return self._values

    @values.setter
    def values(self, values: dict):
        self._values = values

    @classmethod
    def args(cls):
        """
        Class method that returns the arguments required for the initialization of the class.

        Returns
        -------
        :return list
            List of tuples indicating the name of the parameter and its type or instructions.
        """
        return [('bounds', 'list of tuples indicating lower and upper limits')]

    def initialization(self):
        """
        Initializes an array of real-values.
        """
        self._values = [np.random.uniform(low=lower, high=upper) for lower, upper in self._bounds]

class Permutation(base_Representation):
    """
    Class for permutation representation. This type of representation allows to find the order in which a sequence
    of events should occur. This class represents the permutation as a fixed set of values represented as integers.
    """

    def __init__(self, events: list, restrictions: list = None, initialization: bool = True):
        """
        __init__(events: list, restrictions: list = None, initialization: bool = True)

        Parameters
        -----------
        :param events: list
            List with the different events on which you want to find the optimal order.

        :param restrictions: list
            List of functions that must take a list of values and return True if the order falls within the range
            of possibilities or False if there is some restriction for that combination.

        :param initialization: bool
             Indicates whether the individual will call the initialization function to perform a random initialization.
        """
        self._events = events

        if restrictions is None:
            self._restrictions = []
        else:
            for restriction in restrictions:
                if not callable(restriction):
                    raise TypeError("restrictions argument must be a list of callable terms.")
            self._restrictions = restrictions  # list of functions that return boolean values

        self._values = []

        if initialization:
            self.initialization()

    def __repr__(self):
        return "permutation"

    def __str__(self):
        return self.__repr__()

    def __getitem__(self, index: int):
        return self._values[index]

    def __setitem__(self, index: int, value: int):
        self._values[index] = value

    def __len__(self):
        return len(self._events)

    @property
    def kwargs(self):
        return {'events': self._events, 'restrictions': self._restrictions}

    @property
    def values(self):
        return self._values

    @values.setter
    def values(self, values):
        self._values = values

    @property
    def permissible_values(self):
        """
        Returns the list of possible events of the representation.

        Returns
        --------
        :return: list
            List of possible events.
        """
        return self._events

    @property
    def restrictions(self):
        return self._restrictions

    @values.setter
    def values(self, values):
        self._values = values

    @classmethod
    def args(cls):
        """
        Class method that returns the arguments required for the initialization of the class.

        Returns
        -------
        :return list
            List of tuples indicating the name of the parameter and its type or instructions.
        """
        return [('events', 'list')]

    def is_valid(self, values: list):
        """
        It checks if the order of the values passed as parameters is valid based on the restrictions indicated
        by the user in the constructor.

        Parameters
        ----------
        :param values: list
            List of values.

        Returns
        --------
        :return bool
            True if the order is valid, otherwise False.
        """
        if len(self._restrictions) == 0:
            return True

        for restriction in self._restrictions:
            if not restriction(values):
                return False

        return True

    def initialization(self):
        """
        Initializes a new random order based on permissible values and restrictions.
        """
        new_order = deepcopy(self._events)
        np.random.shuffle(new_order)
        while not self.is_valid(new_order):
            np.random.shuffle(new_order)

        self._values = new_order

class Integer(base_Representation):
    """
    Class for integer representations. Adequate representation for a problem in which we want to look for the best
    subset of variables within a wide range of options. In this representation the genotype consist of and array
    of integer values of variable length.
    """

    def __init__(self, value_coding: dict, replacement: bool = False, initialization: bool = True):
        """
        __init__(value_coding: dict, replacement: bool = False, initialization: bool = True)

        Parameters
        ----------
        :param value_coding: dict
            Dictionary with the characteristic as a key and the coding value (integer) as a value.

        :param replacement: bool
            Value indicating if there can be repeated characteristics. By default False.

        :param initialization: bool
             Indicates whether the individual will call the initialization function to perform a random initialization.

        """
        self._value_coding = value_coding
        self._replacement = replacement
        self._values = []
        if initialization:
            self.initialization()

    def __repr__(self):
        return "integer"

    def __str__(self):
        return self.__repr__()

    def __getitem__(self, index):
        return self._values[index]

    def __setitem__(self, index, value):
        self._values[index] = value

    def __len__(self):
        return len(self._values)

    @property
    def kwargs(self):
        return {'value_coding': self._value_coding, 'replacement': self._replacement}

    @property
    def permissible_values(self):
        """
        It returns a list with the coding values of the variables in the form of integers.

        Returns
        --------
        :return: list
            List of integer values.
        """
        return list(self._value_coding.values())

    @property
    def replacement(self):
        """
        Returns
        --------
        :return: bool
            True if a variable can be repeated, otherwise False.
        """
        return self._replacement

    @property
    def values(self):
        return self._values

    @property
    def value_coding(self):
        """
        Returns
        --------
        :return: dict
            Dictionary with the variable and its codification.
        """
        return self._value_coding

    @values.setter
    def values(self, values):
        self._values = values

    @classmethod
    def args(cls):
        """
        Class method that returns the arguments required for the initialization of the class.

        Returns
        -------
        :return list
            List of tuples indicating the name of the parameter and its type or instructions.
        """
        return [('value_coding', 'dict')]

    def initialization(self):
        """
        Initializes a random array of variable length using the available values from value_coding used for
        codification.
        """
        self._values = np.random.choice(
            list(self._value_coding.values()),  # possible values
            size=np.random.randint(low=2, high=len(self._value_coding)),  # variable size from 2 to all values
            replace=self._replacement  # Either repeated values are allowed or not
        )

class Binary(base_Representation):
    """
    Class for binary representations. In this type of representation the genotype consists of an array of binary values.
    """

    def __init__(self, length: int, initialization: bool = True):
        """
        __init__(length: int, initialization: bool = True)

        Parameters
        ----------
        :param length: int
            Number of bits.

        :param initialization: bool
             Indicates whether the individual will call the initialization function to perform a random initialization.
        """
        self._length = length
        self._values = []
        if initialization:
            self.initialization()

    def __repr__(self):
        return "binary"

    def __str__(self):
        return self.__repr__()

    def __getitem__(self, index):
        return self._values[index]

    def __setitem__(self, index, value):
        self._values[index] = value

    def __len__(self):
        return self._length

    @property
    def kwargs(self):
        return {'length': self._length}

    @property
    def values(self):
        return self._values

    @values.setter
    def values(self, values):
        self._values = values

    @property
    def permissible_values(self):
        return None

    @classmethod
    def args(cls):
        """
        Class method that returns the arguments required for the initialization of the class.

        Returns
        -------
        :return list
            List of tuples indicating the name of the parameter and its type or instructions.
        """
        return [('length', 'int')]

    def initialization(self):
        """
        Initializes a random binary array of fixed length.
        """
        self._values = np.random.randint(low=0, high=2, size=self._length)

max_stars_repo_name: FernandoGaGu/beagle

max_issues_repo_name: FernandoGaGu/beagle

max_forks_repo_name: FernandoGaGu/beagle


text:
# -*- coding: utf-8 -*-

from imagepy.core.engine import Simple
from imagepy import ImagePlus
from imagepy.ui.canvasframe import CanvasFrame
from imagepy.core.manager import ConfigManager
from imagepy.ui.panelconfig import ParaDialog
import numpy as np
from imagepy import IPy

def add(recent, v):
    if v in recent:
        idx = recent.index(v)
        recent.insert(0, recent.pop(idx))
    else:
        recent.insert(0, v)
    if len(recent)>5:
        del recent[5:]

class ScaleDialog(ParaDialog):
    def para_check(self, para, key):
        if key=='recent' and para[key] != 'Recent':
            k, u = para[key].split(' - ')
            para['k'], para['unit'] = float(k), u
            self.reset()

class Plugin(Simple):
    title = 'Scale And Unit'
    note = ['all']
    recent = []

    para = {'k':1.0, 'unit':'pix', 'kill':False, 'recent':'Recent'}
    view = [(float, (0,1000000), 2, 'per', 'k', 'pix'),
            (str, 'unit', 'unit',''),
            (list, [], str, 'commen', 'recent', ''),
            (bool, 'kill scale', 'kill')]

    def show(self):
        self.dialog = ScaleDialog(IPy.get_window(), self.title)
        self.dialog.init_view(self.view, self.para)
        self.dialog.set_handle(lambda x:self.preview(self.ips, self.para))
        return self.dialog.ShowModal()

    def load(self, ips):
        self.recent = ConfigManager.get('recent-units')
        if self.recent == None : self.recent = ['Recent']
        else: self.recent.insert(0, 'Recent')
        self.view[2] = (list, self.recent, str, 'commen', 'recent', '')
        if ips.unit==None:
            self.para['K'],self.para['unit'] = (1, 'pix')
        else: self.para['k'], self.para['unit'] = ips.unit
        return True

    def run(self, ips, imgs, para = None):
        if para['kill'] : ips.unit=(1,'pix')
        else :
            ips.unit = (para['k'], para['unit'])
            self.recent.pop(0)
            add(self.recent, '%s - %s'%(para['k'], para['unit']))
            ConfigManager.set('recent-units', self.recent)


text:
#Loading dependencies
import numpy as np
from collections import Counter
from .BaseMRRPlan import *

class SHMActions(BaseMRRPlan):
	def __init__(self, **params):
		super().__init__(**params)
		'''Dummy module for SHM analysis

		This module is developed per request of reviewers who
		required implementation of SHM in GIAMS.
		'''
		self.randomize_mrr()

	def set_mrr(self, new_mrr):
		'''Set the MRR'''
		if np.shape(new_mrr) != (self.settings.n_elements, self.settings.n_steps*self.settings.dt):
			raise ValueError(f"Expected shape of mrr is {(self.settings.n_elements, self.settings.n_steps*self.settings.dt)}"\
								f"But {new_mrr.shape} was given")
		self.mrr = new_mrr

	def randomize_mrr(self):
		'''Randomly initialize the mrr'''
		self.mrr = np.random.randint(2, size=(self.settings.n_elements, self.settings.dt*self.settings.n_steps))
		return self.mrr

	def mrr_to_decimal(self, mrr_binary = None):
		'''Converting the binray representation to decicaml representations'''
		if mrr_binary is None:
			mrr = self.mrr
		else:
			mrr = mrr_binary

		self.mrr_decoded = []
		for element_idx in range (self.settings.n_elements):
			element_mrr = []
			for j in range(0, len(mrr[element_idx]), 2):
				element_mrr.append(int(str(int(mrr[element_idx][j]*10+ mrr[element_idx][j+1])), 2))
			self.mrr_decoded.append(element_mrr)

		return self.mrr_decoded

	def mrr_to_binary(self, decoded_mrr):
		'''Converting the decimal representation to decimal representations'''
		self.mrr = []
		for i in range(len(decoded_mrr)):
			temp_mrr = []
			for val in decoded_mrr[i]:
				for binar in bin(val)[2:]:
					temp_mrr.append(binar)
			self.mrr.append(temp_mrr)

		return self.mrr

	def check_policy(self):
		'''Checking if a policy is acceptable'''
		return True

	def update_mrr(self, observation):
		'''Dummy model for updating the mrr model based on an observations'''
		self.mrr = mrr


text:
# -*-coding:utf-8-*-
# @version: 0.0.1
# License: MIT

import unittest
import sys
import configparser
import random
import pandas as pd
import numpy as np

sys.path.append("../../../..")

from xes_ml_arch.src.analysis import analysis
from xes_ml_arch.src.cross_validation import cross_validation
from xes_ml_arch.src.ml import train_model_ml
from xes_ml_arch.src.ml_utils import runstatus

class TestAnalysis(unittest.TestCase):
    def setUp(self):
        self.conf = configparser.ConfigParser()
        self.conf_try = configparser.ConfigParser()
        self.conf_no = configparser.ConfigParser()
        self.conf_try1 = configparser.ConfigParser()
        self.conf_try2 = configparser.ConfigParser()
        self.conf_try3 = configparser.ConfigParser()

        self.conf_file = "./config/ml.conf"
        self.conf.read(self.conf_file)
        self.conf_try.read("./config/ml_try.conf")
        self.conf_no.read("./conf/ml_online.conf")

        self.conf_try1.read("./config/ml_try1.conf")
        self.conf_try2.read("./config/ml_try2.conf")
        self.conf_try3.read("./config/ml_try3.conf")

        self.columns = ["col%s" % x for x in range(10)]
        # self.x = self.columns[:9]
        # self.y = self.columns[9]
        self.get_data()
        self.ins = train_model_ml.TrainML(conf=self.conf,xeasy_log_path = '../log/log.conf')
        self.ins_try = train_model_ml.TrainML(conf=self.conf_try,xeasy_log_path = '../log/log.conf')
        self.ins_no = train_model_ml.TrainML(conf=self.conf_no,xeasy_log_path = '../log/log.conf')
        self.ins_try1 = train_model_ml.TrainML(conf=self.conf_try1,xeasy_log_path = '../log/log.conf')
        self.ins_try2 = train_model_ml.TrainML(conf=self.conf_try2,xeasy_log_path = '../log/log.conf')
        self.ins_try3 = train_model_ml.TrainML(conf=self.conf_try3,xeasy_log_path = '../log/log.conf')

    def test_set_data(self):
        self.assertTrue(self.ins.set_data(self._data))
        self.assertTrue(self.ins_try.set_data(self._data))

    def test_start(self):
        if self.ins.init() == runstatus.RunStatus.SUCC:
            self.assertTrue(self.ins.set_data(self._data))
            self.assertTrue(self.ins.start())
            self.assertTrue(self.ins.set_data(self._data_no))
            self.assertFalse(self.ins.start())
            self.assertTrue(self.ins.set_data(self._data_tt))
            self.assertFalse(self.ins.start())
        self.assertFalse(self.ins_try.init())
        self.assertFalse(self.ins_try.start())
        self.assertFalse(self.ins_no.init())
        self.assertFalse(self.ins_no.start())

    def test_init(self):
        assert (self.ins.init(), runstatus.RunStatus.SUCC)
        self.assertFalse(self.ins_try1.init())
        self.assertFalse(self.ins_try1.start())
        self.assertFalse(self.ins_try2.init())
        self.assertFalse(self.ins_try2.start())
        self.assertFalse(self.ins_try3.init())
        self.assertFalse(self.ins_try3.start())

    def test_set_analysis_ins(self):
        ana = analysis.Analysis(log_path=  '../log/log.conf')
        noa = None
        self.ins.set_analysis_ins(ana)
        try:
            self.ins.set_analysis_ins(noa)
        except:
            pass

    def test_set_cross_validation(self):
        cv = cross_validation.Cross_Validation(log_path=  '../log/log.conf')
        nocv = None
        self.ins.set_cross_validation(cv)
        try:
            self.ins.set_cross_validation(nocv)
        except:
            pass

    def test_init_cv_ins(self):
        self.assertTrue(self.ins._init_cv_ins())
        self.assertFalse(self.ins_no._init_cv_ins())

    def test_train_handle(self):
        self.ins = train_model_ml.TrainML(conf=self.conf,xeasy_log_path = '../log/log.conf')
        self.assertTrue(self.ins.init())
        self.assertTrue(self.ins.set_data(self._data))
        self.assertTrue(self.ins.start())

    def get_data(self):
        self._data = pd.DataFrame(
            [[int(random.random() * 100) for x in range(10)] for y in range(10000)],
            columns=self.columns)
        self._data["col9"] = np.random.choice([0, 1], size=10000)
        self._data_no = pd.DataFrame(
            [[int(random.random() * 100) for x in range(10)] for y in range(10000)],
            columns=self.columns)
        self._data_tt = pd.DataFrame(
            [[int(random.random() * 100) for x in range(4)] for y in range(10000)],
            columns=['col1', 'col2', 'col3', 'col4'])

if __name__ == '__main__':
    unittest.main()


text:
from __future__ import division, print_function, absolute_import

import os
import functools
import numpy as np
import tensorflow as tf
import tqdm
import datasets
from models import models_factory
from layers import *
import click
from datetime import datetime
from time import process_time

def get_time_str():
    #https://www.programiz.com/python-programming/datetime/current-datetime
    # datetime object containing current date and time
    now = datetime.now()
    return now.strftime("%d_%m_%Y_%H_%M_%S")

def get_trainable_str(trainable_kernel):
    trainable_str = ""
    if trainable_kernel.lower()=="true":
        trainable_str = "_trainable"

    return trainable_str

def delete_file_if_exists(file):
    if os.path.exists(file):
        os.remove(file)
    else:
        print("The file does not exist: " + file)

def train_and_evaluate(datasetname,n_classes,batch_size,
         model_name, kernel, trainable_kernel, cp, dp, gamma,pooling_method,
         epochs,lr,keep_prob,weight_decay,
         log_dir, num_folds, fold, target_accuracies=[0.98]):

    print('Using the ' + datasetname + ' dataset.')

    trainable_str = get_trainable_str(trainable_kernel)

    if trainable_kernel.lower()=='true':
        print('Using a trainable kernel.')
        trainable_kernel = True
    else:
        trainable_kernel = False

    # dataset
    train_dataset, train_samples = datasets.get_dataset(datasetname, batch_size, k_folds=num_folds, fold=fold)
    validate_dataset, _ = datasets.get_dataset(datasetname, batch_size, subset="validate", shuffle=False, k_folds=num_folds, fold=fold)

    #Network
    kernel_fn = get_kernel(kernel, cp=cp, dp=dp, gamma=gamma, trainable=trainable_kernel, lambda_param=0.1)

    model = models_factory.get_model(model_name,
                      num_classes=n_classes,
                      keep_prob=keep_prob,
                      kernel_fn=kernel_fn,
                      pooling=pooling_method)

    #Train optimizer, loss
    nrof_steps_per_epoch = (train_samples//batch_size)
    boundries = [nrof_steps_per_epoch*10, nrof_steps_per_epoch*15]
    values = [lr, lr*0.1, lr*0.01]
    lr_schedule = tf.keras.optimizers.schedules.PiecewiseConstantDecay(\
                    boundries,
                    values)
    optimizer = tf.keras.optimizers.SGD(learning_rate=lr_schedule, momentum=0.9)
    loss_fn = tf.keras.losses.SparseCategoricalCrossentropy()

    #metrics
    train_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy()
    validate_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy()

    #Train step
    @tf.function
    def train_step(x,labels):
        with tf.GradientTape() as t:
            logits = model(x, training=True)
            loss = loss_fn(labels, logits)

        gradients = t.gradient(loss,model.trainable_variables)
        optimizer.apply_gradients(zip(gradients, model.trainable_variables))
        return loss, logits

    #Run

    ep_cnt = tf.Variable(initial_value=0, trainable=False, dtype=tf.int64)

    dt_string = get_time_str()

    #Summary writers
    train_summary_writer = tf.summary.create_file_writer(os.path.join(log_dir,
                                                                      'summaries',
                                                                      'train',
                                                                      kernel + trainable_str,
                                                                      dt_string))
    validate_summary_writer = tf.summary.create_file_writer(os.path.join(log_dir,
                                                                         'summaries',
                                                                         'validate',
                                                                         kernel + trainable_str,
                                                                         dt_string))

    best_validate_accuracy = 0
    best_train_accuracy = 0
    converge_times = np.zeros(len(target_accuracies))
    total_time_training = 0

    training_accuracies_np_array = []
    validate_accuracies_np_array = []

    for ep in tqdm.trange(epochs, desc='Epoch Loop'):
        if ep < ep_cnt:
            continue

            # update epoch counter
        ep_cnt.assign_add(1)

        # train for an epoch
        for step, (x,y) in enumerate(train_dataset):
            if len(x.shape)==3:
                x = tf.expand_dims(x,3)
            time_start = process_time()
            loss, logits = train_step(x,y)#train step
            time_stop = process_time()
            total_time_training = total_time_training + (time_stop - time_start)

            # Log every 100 batch
            if (step + 1) % 100 == 0:
                train_acc = 0
                validate_acc = 0
                with train_summary_writer.as_default():
                    for x_batch, y_batch in train_dataset:
                        if len(x_batch.shape)==3:
                            x_batch = tf.expand_dims(x_batch, 3)
                        train_logits = model(x_batch, training=False)
                        # Update train metrics
                        train_acc_metric(y_batch, train_logits)
                    train_acc = train_acc_metric.result()
                    train_acc_metric.reset_states()
                    tf.summary.scalar("accuracy", train_acc, step=(step + 1) + ep * nrof_steps_per_epoch)
                    training_accuracies_np_array.append(float(train_acc))

                with validate_summary_writer.as_default():
                    for x_batch, y_batch in validate_dataset:
                        if len(x_batch.shape)==3:
                            x_batch = tf.expand_dims(x_batch, 3)
                        validate_logits = model(x_batch, training=False)
                        # Update validate metrics
                        validate_acc_metric(y_batch, validate_logits)

                    validate_acc = validate_acc_metric.result()
                    validate_acc_metric.reset_states()
                    tf.summary.scalar("accuracy", validate_acc, step=(step + 1) + ep * nrof_steps_per_epoch)
                    validate_accuracies_np_array.append(float(validate_acc))

                print('[Step {}] train acc: {}'.format(step + 1, float(train_acc)))
                print('[Step {}] validate acc: {}'.format(step + 1, float(validate_acc)))

                if best_train_accuracy <= float(train_acc):
                    print("got a better train accuracy, updating validate accuracy")
                    best_validate_accuracy = float(validate_acc)
                    best_train_accuracy = float(train_acc)
                print("best validate accuracy so far: " + str(best_validate_accuracy))

                for i in range(len(target_accuracies)):
                    if target_accuracies[i] <= best_validate_accuracy and converge_times[i] == 0:
                        converge_times[i] = total_time_training
                        print("new converge time of: " + str(converge_times[i]))
                        print("for convergance of: " + str(target_accuracies[i]))

    training_accuracies_np_array = np.array(training_accuracies_np_array)
    np.save(os.path.join(log_dir,'summaries','train',kernel + trainable_str,dt_string,'training_accuracies_np_array'),
            training_accuracies_np_array)
    validate_accuracies_np_array = np.array(validate_accuracies_np_array)
    np.save(os.path.join(log_dir,'summaries','validate',kernel + trainable_str,dt_string,'validate_accuracies_np_array'),
            validate_accuracies_np_array)

    return best_validate_accuracy, converge_times

#args
@click.command()
##Data args
@click.option("-d","--datasetname", default="mnist", type=click.Choice(['cifar10','mnist', 'fashion_mnist']))
@click.option("--n_classes", default=10)
##Training args
@click.option('--model_name', default='lenetKNN')
@click.option('--kernel', default='polynomial')
@click.option('--trainable_kernel', default='False')
@click.option('--pooling_method', default='max')
@click.option('--cp', default=1.0)
@click.option('--dp', default=3.0)
@click.option('--gamma', default=1.0)
@click.option("--batch_size", default=50)
@click.option("--epochs", default=20)
@click.option("--lr", default=0.003)
@click.option("--keep_prob", default=1.0)
@click.option("--weight_decay", default=0.0)
@click.option("--lr_search", default='False')
##logging args
@click.option("-o","--base_log_dir", default="logs")
def main(datasetname,n_classes,batch_size,
         model_name, kernel, trainable_kernel, cp, dp, gamma,pooling_method,
         epochs,lr,keep_prob,weight_decay,lr_search,
         base_log_dir):

    log_dir = os.path.join(os.path.expanduser(base_log_dir),
                           "{}".format(datasetname))
    os.makedirs(log_dir, exist_ok=True)

    #Fix TF random seed
    tf.random.set_seed(1777)

    trainable_str = get_trainable_str(trainable_kernel)

    if lr_search.lower()=='true':
        accuracies = [[],[],[],[],[]]
        learning_rates = [[],[],[],[],[]]
        lr_search_log_path = os.path.join(log_dir,'summaries','learning_rate_searches',
                                          kernel + trainable_str,get_time_str())
        lr_search_validate_summary_writer = tf.summary.create_file_writer(os.path.join(lr_search_log_path))
        current_lr = 0.2

        current_lr_step = 0
        num_folds = 5
        target_accuracies=[0.85,0.86,0.87,0.88,0.89,0.90,0.91,0.92,0.93,0.94,0.95,0.96,0.97,0.98,0.99]
        recorded_converge_times=[]
        for i in range(len(target_accuracies)):
            recorded_converge_times.append([[],[],[],[],[]])
        while current_lr >= 0.0002:
            for fold in range(num_folds):
                print("current learning rate: " + str(current_lr))
                print("current fold: " + str(fold))
                best_accuracy, converge_times = train_and_evaluate(datasetname,n_classes,batch_size,
                                                   model_name, kernel, trainable_kernel, cp, dp, gamma,pooling_method,
                                                   epochs,current_lr,keep_prob,weight_decay,
                                                   log_dir, num_folds, fold, target_accuracies)
                print("best validation accuracy: " + str(best_accuracy) + " with a learning rate of: " + str(current_lr))
                with lr_search_validate_summary_writer.as_default():
                    tf.summary.scalar("accuracy", best_accuracy, step=current_lr_step)
                accuracies[fold].append(best_accuracy)
                learning_rates[fold].append(current_lr)
                for i in range(len(target_accuracies)):
                    recorded_converge_times[i][fold].append(converge_times[i])

            current_lr = current_lr / 2
            current_lr_step = current_lr_step + 1

            lr_folder = "lr_" + str(current_lr_step)
            if not os.path.exists(os.path.join(lr_search_log_path,lr_folder)):
                os.makedirs(os.path.join(lr_search_log_path,lr_folder))
            np_accuracies = np.array(accuracies)
            np.save(os.path.join(lr_search_log_path,lr_folder,'numpy_accuracies'),np_accuracies)
            np_learning_rates = np.array(learning_rates)
            np.save(os.path.join(lr_search_log_path,lr_folder,'numpy_learning_rates'),np_learning_rates)

            for i in range(len(target_accuracies)):
                recorded_converge_time = np.array(recorded_converge_times[i])
                np.save(os.path.join(lr_search_log_path,lr_folder,
                                     'numpy_target_convergence_'+str(int(target_accuracies[i]*100))),
                        recorded_converge_time)

    else:
        train_and_evaluate(datasetname,n_classes,batch_size,
                           model_name, kernel, trainable_kernel, cp, dp, gamma,pooling_method,
                           epochs,lr,keep_prob,weight_decay,
                           log_dir, 7, 0)

if __name__=="__main__":
    main()

max_stars_repo_name: NickPerezCarletonUniversity/Kervolution

max_issues_repo_name: NickPerezCarletonUniversity/Kervolution

max_forks_repo_name: NickPerezCarletonUniversity/Kervolution


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
    This file defines a fourier gaussian filter that can be used in experiments and evaluations.
"""

__author__ = "John Hoff"
__email__ = "john.hoff@braindonor.net"
__copyright__ = "Copyright 2020, John Hoff"
__license__ = "MIT License"
__version__ = "1.0.0"

import numpy as np
from scipy import fftpack
from scipy.ndimage import fourier_gaussian
from skopt.space import Real

class FourierGaussianFilter(object):
    """
    This filter will apply a fourier ellipsoid transformation with the indicated sigma.
    """
    def __init__(self):
        pass

    @staticmethod
    def get_default_dimensions():
        return [2.0]

    @staticmethod
    def get_dimensions():
        return [
            Real(low=0.5, high=30., name='sigma')
        ]

    @staticmethod
    def transform_image(dimensions, image):
        sigma = dimensions[0]

        image = image.copy() / 255.
        for channel in range(image.shape[2]):
            image[:, :, channel] = fftpack.ifft2(fourier_gaussian(fftpack.fft2(image[:, :, channel]), sigma)).real
        return np.clip(image * 255., 0., 255.)

max_stars_repo_name: theBraindonor/adversarial-filters

max_issues_repo_name: theBraindonor/adversarial-filters

max_forks_repo_name: theBraindonor/adversarial-filters

reason: import numpy,from scipy


text:
# -*- coding: utf-8 -*-
"""

@author: Rachel Schlossman

"""

from math import pi
from pyomo.environ import Constraint, Objective, Var, ConstraintList
from pyomo.environ import *
from pyomo.dae import *
import numpy as np
from pyomo.util.model_size import build_model_size_report
import matplotlib.pyplot as plt

from PlanarVehicle import PlanarVehicle
from CVX_obstacles import CVX_obstacles
from utilities.saveOptimizationVariables import saveOptimizationVariables
from utilities.loadOptimizationVariables import loadOptimizationVariables
from utilities.VarContainer import VarContainer
from utilities.plotter import plotResults

RAD_TO_DEG = 180 / pi
DEG_TO_RAD = pi / 180

def main():

    # Initial and final conditions
    x0=0
    y0=0
    theta0=0
    v0=0
    omega0=0
    xf=4
    yf=8
    thetaf=None
    vf=None
    omegaf=None

    for iter_num in range(2):

        # Create planar vehicle
        veh = PlanarVehicle(x0=x0, y0=y0, theta0=theta0, v0=v0, omega0=omega0, \
                            xf=xf, yf=yf, thetaf=thetaf,  vf=vf, omegaf=omegaf)

        # No obstacle in first iteration
        if iter_num == 0:
            CVX_obstacles(veh.m, obstacles_present=False)
        else:
            CVX_obstacles(veh.m, obstacles_present=True)

        # Boundary Conditions
        veh.m.bcCon         = ConstraintList(rule= veh.BCs)

        # Phase 1
        n1 = [1] # flag for Phase 1
        veh.m.ForceControlCon1     = Constraint(n1, veh.m.tau1, rule=veh.ForceControl)
        veh.m.TorqueControlCon1    = Constraint(n1, veh.m.tau1, rule=veh.TorqueControl)
        veh.m.ForceControlDot1     = Constraint(n1, veh.m.tau1, rule=veh.ForceControlDot)
        veh.m.TorqueControlDot1    = Constraint(n1, veh.m.tau1, rule=veh.TorqueControlDot)
        veh.m.xdotCon1             = Constraint(n1, veh.m.tau1, rule=veh.xdot)
        veh.m.ydotCon1             = Constraint(n1, veh.m.tau1, rule=veh.ydot)
        veh.m.thetadotCon1         = Constraint(n1, veh.m.tau1, rule=veh.thetadot)
        veh.m.vdotCon1             = Constraint(n1, veh.m.tau1, rule=veh.vdot)
        veh.m.omegadotCon1         = Constraint(n1, veh.m.tau1, rule=veh.omegadot)
        veh.m.JdotCon1             = Constraint(n1, veh.m.tau1, rule=veh.JdotTotalTime)

        # Phase 2
        n2 = [2] # flag for Phase 2
        veh.m.ForceControlCon2     = Constraint(n2, veh.m.tau2, rule=veh.ForceControl)
        veh.m.TorqueControlCon2    = Constraint(n2, veh.m.tau2, rule=veh.TorqueControl)
        veh.m.ForceControlDot2     = Constraint(n2, veh.m.tau2, rule=veh.ForceControlDot)
        veh.m.TorqueControlDot2    = Constraint(n2, veh.m.tau2, rule=veh.TorqueControlDot)
        veh.m.xdotCon2             = Constraint(n2, veh.m.tau2, rule=veh.xdot)
        veh.m.ydotCon2             = Constraint(n2, veh.m.tau2, rule=veh.ydot)
        veh.m.thetadotCon2         = Constraint(n2, veh.m.tau2, rule=veh.thetadot)
        veh.m.vdotCon2             = Constraint(n2, veh.m.tau2, rule=veh.vdot)
        veh.m.omegadotCon2         = Constraint(n2, veh.m.tau2, rule=veh.omegadot)
        veh.m.JdotCon2             = Constraint(n2, veh.m.tau2, rule=veh.JdotTotalTime)

        veh.m.obj  = Objective(expr = veh.m.J1[1] + veh.m.J2[1], sense=minimize)

        # Warm start initial guess from previous solution
        if iter_num > 0:
            loadOptimizationVariables(veh, myPyomoVars)

        # Solve optimization problem
        solver = SolverFactory('ipopt')
        solver.options["halt_on_ampl_error"] = "yes"
        solver.options["tol"] = 1e-6
        solver.options["max_iter"] = 2000
        solver.options["linear_solver"] = "ma27"
        solver.options["ma27_pivtol"] = 1e-6
        solver.options["linear_scaling_on_demand"] = "yes"

        results = solver.solve(veh.m, tee=True)

        # Save all data for next warm start
        _, myPyomoVars = saveOptimizationVariables(veh)

        trajectory_vars = VarContainer(veh.m)
        if iter_num == 1:
            plotResults(trajectory_vars, veh.m)

    plt.show()
    return veh.m

if __name__ == '__main__':
    m = main()

reason: import numpy,from pyomo


text:
"""
MIT License

Copyright (c) 2019 Sadeep Jayasumana

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

import numpy as np
from PIL import Image

# Pascal VOC color palette for labels
_PALETTE = [0, 0, 0,
            128, 0, 0,
            0, 128, 0,
            128, 128, 0,
            0, 0, 128,
            128, 0, 128,
            0, 128, 128,
            128, 128, 128,
            64, 0, 0,
            192, 0, 0,
            64, 128, 0,
            192, 128, 0,
            64, 0, 128,
            192, 0, 128,
            64, 128, 128,
            192, 128, 128,
            0, 64, 0,
            128, 64, 0,
            0, 192, 0,
            128, 192, 0,
            0, 64, 128,
            128, 64, 128,
            0, 192, 128,
            128, 192, 128,
            64, 64, 0,
            192, 64, 0,
            64, 192, 0,
            192, 192, 0]

def get_label_image(probs):
    """
    Returns the label image (PNG with Pascal VOC colormap) given the label probabilities.

    Args:
        probs:  Label probabilities in (h, w, l) format where l = number of labels

    Returns:
        PIL image of matrix shape (h, w)
    """

    labels = probs.argmax(axis=2).astype('uint8')
    label_im = Image.fromarray(labels, 'P')
    label_im.putpalette(_PALETTE)
    return label_im

def softmax(x):
    e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
    q = e_x / e_x.sum(axis=-1, keepdims=True)
    return np.ascontiguousarray(q)  # TODO(sadeep) Improve this?


text:
"""Tensorflow specific common functions."""
from typing import List

import tensorflow as tf
from numpy import ndarray

def _to_tensor(arr: ndarray) -> tf.Tensor:
    """Convert a numpy array to tensorflow tensor."""
    return tf.constant(arr)

def nt_layers_list() -> List:
    return []

def reshape_tensor(h: tf.Tensor, new_shape:List[int]) -> tf.Tensor:
    return tf.reshape(h,new_shape)

max_stars_repo_name: CooKey-Monster/spiral

max_issues_repo_name: CooKey-Monster/spiral

max_forks_repo_name: CooKey-Monster/spiral

reason: from numpy


text:
import os, sys, copy, time, copy
import numpy as np
from tqdm.notebook import tqdm
from IPython.utils import io
from concurrent.futures import ProcessPoolExecutor
sys.path.append(os.path.abspath('../optimize'))
from src.optimization import optimize
from easydict import EasyDict as edict
import matplotlib.pyplot as plt

def optimize_silent(config):
    try:
        with io.capture_output() as captured:
            X, F, H = optimize(edict(config))
    except Exception as e:
        print('Error in optimize', e)
        X, F, H = None, None, None
    print('.', end='', flush=True)
    return X, F, H

def optimize_parallel(configs, n_cores=None):
    if n_cores is None:
        n_cores = os.cpu_count()
    start = time.time()
    with ProcessPoolExecutor(max_workers=n_cores) as executor:
        data = list(executor.map(optimize_silent, configs))
    print(f'\nFinished {len(configs)} in {round(time.time()-start, 5)} seconds.')
    return data

def optimize_repeat(configs, n_repeats, n_cores=None):
    with_repeats = []
    for conf in configs:
        for seed in range(n_repeats):
            out = f"{conf.out}_s{seed}" if conf.out else None
            print(out)
            with_repeats.append(dictadd(conf, seed=seed, out=out))
    results = optimize_parallel(with_repeats, n_cores)
    return [ results[i*n_repeats:(i+1)*n_repeats ] for i in range(len(configs)) ]

def dictadd(d, **kwarks):
    d = edict(copy.deepcopy(d))
    for k, v in kwarks.items():
        assert k in d, (k, d)
        d[k] = v
    return d

def plot_final_hv(data, xlabel, x):
    hv_histories = [ d[1] for d in data ]
    y = np.zeros(len(hv_histories))
    err = np.zeros(len(hv_histories))
    for i in range(len(hv_histories)):
        final_hvs = [ hv_histories[i][j][-1][-1] for j in range(len(results[i])) ]
        y[i] = np.mean(final_hvs)
        err[i] = np.std(final_hvs)
    plt.fill_between(x, y-err, y+err, alpha=.5)
    plt.plot(x, y)
    plt.xlabel(xlabel)
    plt.ylabel('Hypervolume')

def plot_repeats(data, title=None, log=True):
    hv_histories = [ d[2] for d in data ]
    hv_histories = np.array(hv_histories)

    if log:
        print(title)
        print('\tavg final hv', round(hv_histories[:, -1, -1].mean(), 5))
        print('\tavg final pf', np.mean( [ len(d[0]) for d in data ]))

    n_steps = hv_histories.shape[2]
    for phase_i in range(hv_histories.shape[1]):
        x = np.arange(n_steps)
        y = hv_histories[:, phase_i].mean(axis=0)
        err = hv_histories[:, phase_i].std(axis=0)
        plt.fill_between(x, y-err, y+err, alpha=.5)
        plt.plot(x, y, label=f'Phase {phase_i}')
    if title:
        plt.title(title)
    plt.xlabel('Generations')
    plt.ylabel('Hypervolume')
    plt.legend(loc='lower right')

default_config = edict({
    "state": None,
    "out": None,
    "seed": 1,
    "feasinfeas":False,
    "feasinfeas_2N":False,
    "max_start_tiles":400,
    "n_districts": 8,
    "n_gens": 12000,
    "pop_size": 300,
    "metrics":[
      "polsby_popper",
      "competitiveness",
      "efficiency_gap"
    ],
    "equality_constraint":0.05,
    "novelty":False,
    "NSGA3":False,
    "dont_fix_seeds":False,
    "pp_constraint": None,
    "nov_params": {}
})

max_stars_repo_name: joel-simon/Antimander

max_issues_repo_name: joel-simon/Antimander

max_forks_repo_name: joel-simon/Antimander


text:
import sys
import numpy as np
import pandas as pd
from sqlalchemy import create_engine
import nltk
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import fbeta_score, make_scorer
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer
from sklearn.multioutput import MultiOutputClassifier
import pickle
nltk.download('punkt')
nltk.download('wordnet')

def load_data(database_filepath):
	"""
	Loads data from sqlite database and divides it into labels and target.

	Input: sqlite database filepath

	Output: X - Feature Array
			Y - Target Array
			category_names - names of categories in Y
	"""
	engine = create_engine('sqlite:///' + database_filepath)
	df = pd.read_sql_query('SELECT * FROM DisasterResponseTable', engine)

	X = df['message']
	Y = df.iloc[:, 4:]
	category_names = Y.columns.values

	return X.values, Y.values, category_names

def tokenize(text):
	"""
	Returns tokenized->lemmatized->case_lowered text

	Input: text in string format

	Output: list of normalized tokens
	"""

	tokens = nltk.word_tokenize(text)
	lemmatizer = nltk.WordNetLemmatizer()

	clean_tokens = []
	for tok in tokens:
		clean_tok = lemmatizer.lemmatize(tok).lower().strip()
		clean_tokens.append(clean_tok)

	return clean_tokens

def build_model():
	"""
	Making a pipeline and using grid-search to optimize it.

	Input: None

	Output: Returns a model
	"""

	pipeline = Pipeline([
		('vect', CountVectorizer(tokenizer=tokenize)),
		('tfidf', TfidfTransformer()),
		('clf', MultiOutputClassifier(RandomForestClassifier()))
	])

	parameters = {
		'vect__max_df': [0.5, 1.2],
		'clf__estimator__n_estimators': [10, 50],
		'clf__estimator__min_samples_split': [2, 3, 4],
		'clf__estimator__criterion': ['entropy', 'gini']
	}

	return GridSearchCV(pipeline, param_grid=parameters, verbose=3, n_jobs = -1)

def evaluate_model(model, X_test, Y_test, category_names):
	"""
	Displays precision, recall, f1-score, and support.

	Input: 	model
			X_test - Features from test set
			Y_test - Target values of X_test
			category_names - list of target values

	Output: None, prints the scores.
	"""

	Y_predict = model.predict(X_test)
	Y_predict_t = Y_predict.T

	Y_actual = Y_test.T

	for i, pred in enumerate(Y_predict_t):
		print(category_names[i])
		print(classification_report(Y_actual[i], pred))

def save_model(model, model_filepath):
	"""
	Saves model as pickle file.

	Input: model, location
	Output: None, saves model to the location.
	"""

	with open(model_filepath, 'wb') as file:
		pickle.dump(model, file)

def main():
	'''
	Main function that loads data, trains model, and saves it.
	'''

	if len(sys.argv) == 3:
		database_filepath, model_filepath = sys.argv[1:]
		print('Loading data...\n    DATABASE: {}'.format(database_filepath))
		X, Y, category_names = load_data(database_filepath)
		X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)

		print('Building model...')
		model = build_model()

		print('Training model...')
		model.fit(X_train, Y_train)

		print('Evaluating model...')
		evaluate_model(model, X_test, Y_test, category_names)

		print('Saving model...\n    MODEL: {}'.format(model_filepath))
		save_model(model, model_filepath)

		print('Trained model saved!')

	else:
		print('Please provide the filepath of the disaster messages database '\
			  'as the first argument and the filepath of the pickle file to '\
			  'save the model to as the second argument. \n\nExample: python '\
			  'train_classifier.py ../data/DisasterResponse.db classifier.pkl')

if __name__ == '__main__':
	main()


text:
import time

import numpy
from matplotlib import pyplot

import dnn_utils_v2
import dnn_app_utils_v3
import test_cases_v4a

def initialize_parameters_deep(layer_dims, seed=3):

    numpy.random.seed(seed)
    parameters = {}
    L = len(layer_dims)  # number of layers in the network

    for l in range(1, L):
        # parameters["W" + str(l)] = numpy.random.randn(layer_dims[l], layer_dims[l-1]) * 0.01
        parameters["W" + str(l)] = numpy.random.randn(layer_dims[l], layer_dims[l-1]) / numpy.sqrt(layer_dims[l-1])
        parameters["b" + str(l)] = numpy.zeros((layer_dims[l], 1))

    assert (parameters['W' + str(l)].shape == (layer_dims[l], layer_dims[l - 1]))
    assert (parameters['b' + str(l)].shape == (layer_dims[l], 1))

    return parameters

def linear_forward(A, W, b):

    Z = numpy.dot(W, A) + b

    assert (Z.shape == (W.shape[0], A.shape[1]))
    cache = (A, W, b)

    return Z, cache

def linear_activation_forward(A_prev, W, b, activation):

    Z, linear_cache = linear_forward(A_prev, W, b)

    if activation == "sigmoid":
        A, activation_cache = dnn_utils_v2.sigmoid(Z)
    elif activation == "relu":
        A, activation_cache = dnn_utils_v2.relu(Z)

    assert (A.shape == (W.shape[0], A_prev.shape[1]))
    cache = (linear_cache, activation_cache)

    return A, cache

def L_model_forward(X, parameters):

    caches = []
    A = X
    L = len(parameters) // 2

    for l in range(1, L):
        W = parameters["W" + str(l)]
        b = parameters["b" + str(l)]
        A, cache = linear_activation_forward(A, W, b, 'relu')
        caches.append(cache)

    W = parameters["W" + str(L)]
    b = parameters["b" + str(L)]
    AL, cache = linear_activation_forward(A, W, b, 'sigmoid')
    caches.append(cache)

    assert (AL.shape == (1, X.shape[1]))

    return AL, caches

def compute_cost(AL, Y):

    m = Y.shape[1]
    cost = -(1/m) * numpy.sum(Y * numpy.log(AL) + (1-Y) * numpy.log(1-AL))

    cost = numpy.squeeze(cost)  # To make sure your cost's shape is what we expect (e.g. this turns [[17]] into 17).
    assert (cost.shape == ())

    return cost

def linear_backward(dZ, cache):

    A_prev, W, b = cache
    m = A_prev.shape[1]

    dW = (1/m) * numpy.dot(dZ, A_prev.T)
    db = (1/m) * numpy.sum(dZ, axis=1, keepdims=True)
    dA_prev = numpy.dot(W.T, dZ)

    assert (dA_prev.shape == A_prev.shape)
    assert (dW.shape == W.shape)
    assert (db.shape == b.shape)

    return dA_prev, dW, db

def linear_activation_backward(dA, cache, activation):
    linear_cache, activation_cache = cache

    if activation == "sigmoid":
        dZ = dnn_utils_v2.sigmoid_backward(dA, activation_cache)
    elif activation == "relu":
        dZ = dnn_utils_v2.relu_backward(dA, activation_cache)

    dA_prev, dW, db = linear_backward(dZ, linear_cache)

    return dA_prev, dW, db

def L_model_backward(AL, Y, caches):

    grads = {}
    L = len(caches)
    m = AL.shape[1]
    Y = Y.reshape(AL.shape)  # after this line, Y is the same shape as AL
    dAL = - (numpy.divide(Y, AL) - numpy.divide(1 - Y, 1 - AL))  # derivative of cost with respect to AL

    dA = dAL
    grads["dA" + str(L - 1)], grads["dW" + str(L)], grads["db" + str(L)] =\
        linear_activation_backward(dA, caches[L-1], "sigmoid")

    for l in reversed(range(L-1)):
        grads["dA" + str(l)], grads["dW" + str(l+1)], grads["db" + str(l+1)] =\
            linear_activation_backward(grads["dA" + str(l+1)], caches[l], "relu")

    return grads

def update_parameters(parameters, grads, learning_rate):

    L = len(parameters) // 2  # number of layers in the neural network

    for l in range(1, L+1):
        parameters["W" + str(l)] = parameters["W" + str(l)] - learning_rate * grads["dW" + str(l)]
        parameters["b" + str(l)] = parameters["b" + str(l)] - learning_rate * grads["db" + str(l)]

    return parameters

def two_layer_model(X, Y, layers_dims, learning_rate=0.0075, num_iterations=3000, print_cost=False):
    numpy.random.seed(1)
    grads = {}
    costs = []  # to keep track of the cost
    m = X.shape[1]  # number of examples
    (n_x, n_h, n_y) = layers_dims

    parameters = dnn_app_utils_v3.initialize_parameters(n_x, n_h, n_y)

    W1 = parameters["W1"]
    b1 = parameters["b1"]
    W2 = parameters["W2"]
    b2 = parameters["b2"]

    for i in range(num_iterations):
        A1, cache1 = linear_activation_forward(X, W1, b1, "relu")
        A2, cache2 = linear_activation_forward(A1, W2, b2, "sigmoid")

        cost = compute_cost(A2, Y)
        dA2 = - (numpy.divide(Y, A2) - numpy.divide(1 - Y, 1 - A2))

        dA1, dW2, db2 = linear_activation_backward(dA2, cache2, "sigmoid")
        dA0, dW1, db1 = linear_activation_backward(dA1, cache1, "relu")

        grads['dW1'] = dW1
        grads['db1'] = db1
        grads['dW2'] = dW2
        grads['db2'] = db2

        parameters = update_parameters(parameters, grads, learning_rate)
        ### END CODE HERE ###

        # Retrieve W1, b1, W2, b2 from parameters
        W1 = parameters["W1"]
        b1 = parameters["b1"]
        W2 = parameters["W2"]
        b2 = parameters["b2"]

        # Print the cost every 100 training example
        if print_cost and i % 100 == 0:
            print("Cost after iteration {}: {}".format(i, numpy.squeeze(cost)))
        if print_cost and i % 100 == 0:
            costs.append(cost)

        # plot the cost

    pyplot.plot(numpy.squeeze(costs))
    pyplot.ylabel('cost')
    pyplot.xlabel('iterations (per hundreds)')
    pyplot.title("Learning rate =" + str(learning_rate))
    pyplot.show()

    return parameters

# def L_layer_model(X, Y, layers_dims, learning_rate=0.0075, num_iterations = 3000, print_cost=False):#lr was 0.009
#     numpy.random.seed(1)
#     costs = []  # keep track of cost
#     parameters = initialize_parameters_deep(layers_dims)
#
#     for i in range(num_iterations):
#         AL, caches = L_model_forward(X, parameters)
#         cost = compute_cost(AL, Y)
#         grads = L_model_backward(AL, Y, caches)
#         update_parameters(parameters, grads, learning_rate)
#
#         # Print the cost every 100 training example
#         if print_cost and i % 100 == 0:
#             print("Cost after iteration %i: %f" % (i, cost))
#         if print_cost and i % 100 == 0:
#             costs.append(cost)
#
#     # plot the cost
#     pyplot.plot(numpy.squeeze(costs))
#     pyplot.ylabel('cost')
#     pyplot.xlabel('iterations (per hundreds)')
#     pyplot.title("Learning rate =" + str(learning_rate))
#     pyplot.show()
#
#     return parameters

def L_layer_model(X, Y, layers_dims, learning_rate=0.0075, num_iterations=3000, print_cost=False):  # lr was 0.009
    """
    Implements a L-layer neural network: [LINEAR->RELU]*(L-1)->LINEAR->SIGMOID.

    Arguments:
    X -- data, numpy array of shape (num_px * num_px * 3, number of examples)
    Y -- true "label" vector (containing 0 if cat, 1 if non-cat), of shape (1, number of examples)
    layers_dims -- list containing the input size and each layer size, of length (number of layers + 1).
    learning_rate -- learning rate of the gradient descent update rule
    num_iterations -- number of iterations of the optimization loop
    print_cost -- if True, it prints the cost every 100 steps

    Returns:
    parameters -- parameters learnt by the model. They can then be used to predict.
    """

    numpy.random.seed(1)
    costs = []  # keep track of cost

    # Parameters initialization. (≈ 1 line of code)
    ### START CODE HERE ###
    parameters = initialize_parameters_deep(layers_dims, 1)
    ### END CODE HERE ###

    # Loop (gradient descent)
    for i in range(0, num_iterations):

        # Forward propagation: [LINEAR -> RELU]*(L-1) -> LINEAR -> SIGMOID.
        ### START CODE HERE ### (≈ 1 line of code)
        AL, caches = L_model_forward(X, parameters)
        ### END CODE HERE ###

        # Compute cost.
        ### START CODE HERE ### (≈ 1 line of code)
        cost = compute_cost(AL, Y)
        ### END CODE HERE ###

        # Backward propagation.
        ### START CODE HERE ### (≈ 1 line of code)
        grads = L_model_backward(AL, Y, caches)
        ### END CODE HERE ###

        # Update parameters.
        ### START CODE HERE ### (≈ 1 line of code)
        update_parameters(parameters, grads, learning_rate)
        ### END CODE HERE ###

        # Print the cost every 100 training example
        if print_cost and i % 100 == 0:
            print("Cost after iteration %i: %f" % (i, cost))
        if print_cost and i % 100 == 0:
            costs.append(cost)

    # plot the cost
    pyplot.plot(numpy.squeeze(costs))
    pyplot.ylabel('cost')
    pyplot.xlabel('iterations (per hundreds)')
    pyplot.title("Learning rate =" + str(learning_rate))
    pyplot.show()

    return parameters

def prediction(parameters, X):

    pass
    # return prediction

def optimize():

    pass

def nn_model():

    pass

def test_case_initialize_paramters():
    parameters = initialize_parameters_deep([5, 4, 3])
    print("W1 = " + str(parameters["W1"]))
    print("b1 = " + str(parameters["b1"]))
    print("W2 = " + str(parameters["W2"]))
    print("b2 = " + str(parameters["b2"]))

def test_case_linear_forward():
    A, W, b = test_cases_v4a.linear_forward_test_case()

    Z, linear_cache = linear_forward(A, W, b)
    print("Z = " + str(Z))

def test_case_linear_activation_forward():
    A_prev, W, b = test_cases_v4a.linear_activation_forward_test_case()

    A, linear_activation_cache = linear_activation_forward(A_prev, W, b, activation="sigmoid")
    print("With sigmoid: A = " + str(A))

    A, linear_activation_cache = linear_activation_forward(A_prev, W, b, activation="relu")
    print("With ReLU: A = " + str(A))

def test_case_L_model_forward():
    X, parameters = test_cases_v4a.L_model_forward_test_case_2hidden()
    AL, caches = L_model_forward(X, parameters)
    print("AL = " + str(AL))
    print("Length of caches list = " + str(len(caches)))

def test_case_compute_cost():
    Y, AL = test_cases_v4a.compute_cost_test_case()
    print("cost = " + str(compute_cost(AL, Y)))

def test_case_linear_backward():
    # Set up some test inputs
    dZ, linear_cache = test_cases_v4a.linear_backward_test_case()

    dA_prev, dW, db = linear_backward(dZ, linear_cache)
    print("dA_prev = " + str(dA_prev))
    print("dW = " + str(dW))
    print("db = " + str(db))

def test_case_linear_activation_backward():
    dAL, linear_activation_cache = test_cases_v4a.linear_activation_backward_test_case()

    dA_prev, dW, db = linear_activation_backward(dAL, linear_activation_cache, activation="sigmoid")
    print("sigmoid:")
    print("dA_prev = " + str(dA_prev))
    print("dW = " + str(dW))
    print("db = " + str(db) + "\n")

    dA_prev, dW, db = linear_activation_backward(dAL, linear_activation_cache, activation="relu")
    print("relu:")
    print("dA_prev = " + str(dA_prev))
    print("dW = " + str(dW))
    print("db = " + str(db))

def test_case_L_model_backward():
    AL, Y_assess, caches = test_cases_v4a.L_model_backward_test_case()
    grads = L_model_backward(AL, Y_assess, caches)
    test_cases_v4a.print_grads(grads)

def test_case_update_paramters():
    parameters, grads = test_cases_v4a.update_parameters_test_case()
    parameters = update_parameters(parameters, grads, 0.1)

    print("W1 = " + str(parameters["W1"]))
    print("b1 = " + str(parameters["b1"]))
    print("W2 = " + str(parameters["W2"]))
    print("b2 = " + str(parameters["b2"]))

def test_case_two_layer_model():
    train_x_orig, train_y, test_x_orig, test_y, classes = dnn_app_utils_v3.load_data()
    # Reshape the training and test examples
    train_x_flatten = train_x_orig.reshape(train_x_orig.shape[0],
                                           -1).T  # The "-1" makes reshape flatten the remaining dimensions
    test_x_flatten = test_x_orig.reshape(test_x_orig.shape[0], -1).T

    # Standardize data to have feature values between 0 and 1.
    train_x = train_x_flatten / 255.
    test_x = test_x_flatten / 255.

    ### CONSTANTS DEFINING THE MODEL ####
    n_x = 12288  # num_px * num_px * 3
    n_h = 7
    n_y = 1
    layers_dims = (n_x, n_h, n_y)

    parameters = two_layer_model(train_x, train_y, layers_dims=(n_x, n_h, n_y), num_iterations=2500, print_cost=True)

def test_case_L_layer_model():
    train_x_orig, train_y, test_x_orig, test_y, classes = dnn_app_utils_v3.load_data()
    # Reshape the training and test examples
    train_x_flatten = train_x_orig.reshape(train_x_orig.shape[0],
                                           -1).T  # The "-1" makes reshape flatten the remaining dimensions
    test_x_flatten = test_x_orig.reshape(test_x_orig.shape[0], -1).T

    # Standardize data to have feature values between 0 and 1.
    train_x = train_x_flatten / 255.
    test_x = test_x_flatten / 255.

    ### CONSTANTS ###
    layers_dims = [12288, 20, 7, 5, 1]  # 4-layer model
    parameters = L_layer_model(train_x, train_y, layers_dims, num_iterations = 2500, print_cost = True)
    predictions_train = dnn_app_utils_v3.predict(train_x, train_y, parameters)
    predictions_test = dnn_app_utils_v3.predict(test_x, test_y, parameters)

def main():
    # test_case_initialize_paramters()
    # test_case_linear_forward()
    # test_case_linear_activation_forward()
    # test_case_L_model_forward()
    # test_case_compute_cost()
    # test_case_linear_backward()
    # test_case_linear_activation_backward()
    # test_case_L_model_backward()
    # test_case_update_paramters()
    # test_case_two_layer_model()
    test_case_L_layer_model()

start = time.time()
main()
end = time.time()
total_time = end - start
print("%s: Total time = %f seconds" % (time.strftime("%Y/%m/%d-%H:%M:%S"), total_time))

max_stars_repo_name: franpena-kth/learning-deep-learning

max_issues_repo_name: franpena-kth/learning-deep-learning

max_forks_repo_name: franpena-kth/learning-deep-learning


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Script to conduct CAV control trial. Run in the terminal with a
sys.argv argument specifying the trial type (either ECO-AND ("ECO-AND")
or human driver ("HUMAN"). Other environmental parameters can be
modified in the constants.

Example:
    $ python main.py 'ECO-AND'
    $ python main.py 'HUMAN'

Questions? sherwinl@bu.edu, BU CODES Lab
"""
import sys
import numpy as np
from controller import config as cfg
from controller import acceleration as acc
from helper.scale import mcity_to_lab, lab_to_mcity
from helper.trajectory import liveplot

# Environment constants
DELAY = 5  # Initial delay before robot moves
TG = 10  # Green interval
TR = 10  # Red interval
SGR = 1  # Starting light (green=1, red=0)
D0 = 0.5  # Fraction of time remaining of starting light interval
L = 70  # Road segment length
vM = 13  # Maximum velocity
vm = 2.78  # Minimum velocity
aM = 2.5  # Maximum acceleration
am = -2.9  # Minimum acceleration
T = TG+TR # Light cycle period
e = 0.05 # Start/end proximity threshold

# Starting light interval
if SGR == 1:
    T_SGR = TG
else:
    T_SGR = TR

# Road segment start/endpoints
y0 = -4.4
yf = y0 - mcity_to_lab(L)

if __name__== "__main__":
    # Initialize global variables
    cfg.init(thresh=e, road_len=L, ctrl_env_start=y0,
             ctrl_env_end=yf, max_vel=vM, min_vel=vm,
             max_acc=aM, min_acc=am, green=TG, red=TR,
             start_frac=D0, start_len=T_SGR,
             start_light=SGR)

    # Initialize instance of acceleration controller
    acc_obj = acc.AccControl(robot='create3',
                             accel=0,
                             vel=0.1,
                             zero_hd=0,
                             delay=10,
                             trial='ECO-AND')

    # Initialize control with constant velocity
    #acc_obj = acc.AccControl(robot=sys.argv[1],
    #                         accel=float(sys.argv[2]),
    #                         vel=float(sys.argv[3]),
    #                         zero_hd=float(sys.argv[4]),
    #                         delay=int(sys.argv[5]).
    #                         trial=sys.argv[6],)

    acc_obj.control()
    liveplot(acc_obj.x,acc_obj.y,
             cfg.X_LABLIM,cfg.Y_LABLIM,
             'Robot Trajectory')

max_stars_repo_name: sherwinhlee/robotics

max_issues_repo_name: sherwinhlee/robotics

max_forks_repo_name: sherwinhlee/robotics


text:
#!/usr/bin/python
"""
This script extracts data from ccx dat-files.
Extended from the dat2txt.py helper script by
https://github.com/mkraska/CalculiX-Examples/tree/master/Scripts

"""
import sys
import re
import glob
import os
import argparse
import logging
import textwrap
import numpy as np

def dat2txt(filename, verbose=False):
    """Read CalculiX .DAT output file and write analysis results to .TXT file.

    Parameters
    ----------
    filename
        CalculiX ouput .DAT file.
    verbose
        Verbose output.

    Returns
    -------
    resname
        results name
    group
        results group name
    time_points
        number of time steps
    """

    if verbose:
        logging.basicConfig(level=logging.INFO)

    # variables initialization
    resname = ''
    group = ''
    data = {}
    pH = re.compile(' (.+) for .*set\\s(\\S+) and time  (.+)')
    skip = 0  # if empty lines are to be skipped
    body = 0  # if data lines are expected
    nev = 0  # number of eigenvalue file
    stop_on_empty = 0  # empty line triggers end of body
    time_points = 0

    # open job file
    logging.info('Reading file {}.'.format(filename))
    f = open(filename, "r")

    for line in f:
        if skip:  # if the line is known to be useless
            skip = skip - 1
            continue
        line = line.replace("(", "")
        line = line.replace(")", "")
        line = line.replace("\n", "")
        b = pH.match(line)
        if b:  # a result header was found
            resname = b.group(1)
            group = b.group(2)
            # print(group)
            res = resname + "_" + group
            time = float(b.group(3))
            if not (res in data.keys()):  # new result type
                res_filename = res + ".txt"
                data[res] = open(res_filename, "w")
            data[res].write("\n" + str(time) + " ")
            line_end = " "
            body = 1
            stop_on_empty = 0
        elif line.startswith("     E I G E N V A L U E   O U T P U T"):
            # eigenvalue data
            nev += 1
            res = "Eigenvalues_" + str(nev)
            # print(res)
            data[res] = open(res + ".txt", "w")
            data[res].write("# mode  EV  Re(f)_rad/time Re(f)_cycles/time Im(f)_rad/time\n")
            line_end = "\n"
            body = 1
            skip = 5
            stop_on_empty = 1
        elif line.startswith("     P A R T I C I P A T I O N   F A C T O R S"):
            # eigenvalue data
            res = "Eigenvalues_PF_" + str(nev)
            print(res)
            data[res] = open(res + ".txt", "w")
            data[res].write("# mode  UX  UY UZ RX RY RZ\n")
            line_end = "\n"
            body = 1
            skip = 3
            stop_on_empty = 1
        elif line.startswith("     E F F E C T I V E   M O D A L   M A S S"):
            # eigenvalue data
            res = "Eigenvalues_MM_" + str(nev)
            print(res)
            data[res] = open(res + ".txt", "w")
            data[res].write("# mode  UX  UY UZ RX RY RZ\n")
            line_end = "\n"
            body = 1
            skip = 3
            stop_on_empty = 1
        else:
            if body == 1:
                if not "force" in line and not "center" in line:
                    data[res].write(line + line_end)
                    if line.strip():
                        time_points += 1
                if stop_on_empty and line == "":
                    body = 0

    for name in data.keys():
        data[name].close()

    logging.info('Written {} for group {} to file: {}.\nTotal time points: {}.\n'.format(resname, group, res_filename, time_points))
    return resname, group, time_points

def main():
    description = textwrap.dedent('''\
                Script to process CalculiX Finite Element (FE) output .DAT files.
                Reads CalculiX FE analysis results and writes a .TXT file of the output variables and time steps.
                ''')

    parser = argparse.ArgumentParser(description=description, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('job', type=str, help='Input CalculiX job name.')
    # parser.add_argument('j', '--jobname', type=str, default=None, help='<CalculiX job name.')
    # parser.add_argument('fileout', type=str, default=None, help='<Required> Output filename (Abaqus .INP).')
    parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='Verbose output.')
    parser.set_defaults(job=None, verbose=True)

    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.INFO)

    # processing command line arguments, get the jobname
    if args.job is None:
        logging.info("No jobname given.\n")
        files = glob.glob("*.dat")
        if len(files) == 1:
            logging.info('File {} found'.format(files[0]))
            job = files[0]
        else:
            logging.exception('Available data files: {}'.format(*f) for f in files)
            quit()
    else:
        job = args.job + ".dat"

    # run dat2txt
    resname, group, time_points = dat2txt(job)

if __name__ == '__main__':
    main()

# names = ['time', 'fx', 'fy', 'fz']
# filename = '/home/gianthk/PycharmProjects/CT2FE/test_data/steel_foam/total force fx,fy,fz_NODES_B.txt'
# data2 = pd.read_table(filename, delim_whitespace=True, names=names, index_col=0)

max_issues_repo_name: gianthk/ciclope

max_forks_repo_name: gianthk/ciclope


text:
from __future__ import absolute_import
import math
import numpy as np
from . import ahp_fast
from ektelo.algorithm import estimate_engine
from ektelo import util

""" Wrapper to support n-dimensional data using original AHP implementation
"""

class ahp_engine(estimate_engine.estimate_engine):
    """Estimate a dataset by assuming it is uniform over the domain; total number of records derived privately according to epsilon.
        1D data and workload only. Fast Cython version"""

    def __init__(self, ratio = 0.85, eta = 0.35, short_name = 'AHP' ):
        self.init_params = util.init_params_from_locals(locals())
        self._ratio = ratio
        self._eta = eta
        self.short_name = short_name

    def Run(self, Q, x, epsilon, seed):
        assert seed is not None, 'seed must be set'

        prng = np.random.RandomState(seed)

        assert len(x.shape)==1, '%s is defined for 1D data only' % self.__class__.__name__
        res = ahp_fast.ahpr(x, epsilon, self._ratio, self._eta, prng, debug=False)
        return res
'''
Canonical name:     AHP (ND)
Additional aliases: -
Reference:          [X. Zhang, R. Chen, J. Xu, X. Meng, and Y. Xie. Towards accurate histogram publication under differential privacy. ICDM, 2014.](http://epubs.siam.org/doi/abs/10.1137/1.9781611973440.68)
Invocation:         ektelo.algorithm.ahp.ahpND_engine()
Implementation:     DPComp team
'''
class ahpND_engine(estimate_engine.estimate_engine):
    """ahp engine that works for any dimension"""
    def __init__(self, ratio=0.85, eta=0.35, short_name = 'AHP'):
        self.init_params = util.init_params_from_locals(locals())
        self._ratio = ratio
        self._eta = eta
        self.short_name = short_name

    def Run(self, Q, x, epsilon, seed):
        assert seed is not None, 'seed must be set'

        engine = ahp_engine(self._ratio, self._eta)
        shape = x.shape
        hatx1d = engine.Run(Q, np.reshape(x, x.size), epsilon, seed)  # call AHP on 1d array
        hatx1d = np.array(hatx1d) # might come back as a list
        return np.reshape(hatx1d, shape)
'''
Canonical name:     AHP* (ND)
Additional aliases: AHP_ADP
Reference:          -
Invocation:         ektelo.algorithm.ahp.ahpND_adaptive_engine()
Implementation:     DPComp team
'''
class ahpND_adaptive_engine(estimate_engine.estimate_engine):
    """Adaptive version assumes true scale known"""

    def __init__(self, index = 'DEFAULT1D', short_name = 'AHP*'):
        self.init_params = util.init_params_from_locals(locals())

        self._vSEP = Lfunction[index]['vSEP']
        self._vRatio = Lfunction[index]['vRatio']
        self._vEta = Lfunction[index]['vEta']
        self.short_name = short_name

    def Run(self, Q, x, epsilon, seed):

        #split part of the privacy budget to calculate SEP and select parameters
        assert seed is not None, 'seed must be set'

        Ntotal = x.sum()

        SEP =  Ntotal * epsilon
        self._ratio = float(np.interp(SEP,self._vSEP ,self._vRatio))
        self._eta = float(np.interp(SEP,self._vSEP ,self._vEta))

        ahp_nd_engine = ahpND_engine(self._ratio, self._eta)

        return ahp_nd_engine.Run(Q, x, epsilon, seed)

Lfunction ={ \
            #1D
            #POWER1
            'DEFALUT1D':{\
                'vSEP':[0, 50.0, 100.0,500.0,1000.0,10000.0,500000.0,1000000.0,5000000.0,10000000.0,100000000.0,1e+20],
                'vRatio':[0.7, 0.7, 0.7, 0.9, 0.9, 0.8, 0.7, 0.5, 0.5, 0.3, 0.3, 0.3],
                'vEta':[0.2, 0.2, 0.1, 0.2, 0.1, 0.1, 0.3, 0.1, 0.2, 0.1, 0.3, 0.3]},
            #2D
            #TDNORMAL1
            'DEFALUT2D':{\
                'vSEP':[0,500.0,1000.0,5000.0,10000.0,100000.0,500000.0,1000000.0,5000000.0,10000000.0,100000000.0,1e+20],
                'vRatio':[0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.7, 0.5, 0.7, 0.5, 0.5],
                'vEta':[0.5, 0.5, 0.1, 0.1, 0.1, 0.3, 0.7, 0.1, 0.1, 0.9, 0.1, 0.1]},

}

max_stars_repo_name: dpcomp-org/ektelo

max_issues_repo_name: ektelo/ektelo

max_forks_repo_name: dpcomp-org/ektelo