text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division

""" Resampling and convolution functionality for spectra. """

__author__ = "Andy Casey <arc@ast.cam.ac.uk>"

__all__ = ("resample", "resample_and_convolve")

import logging
import numpy as np
from scipy import sparse

from sick.utils import lru_cache

logger = logging.getLogger("sick")

LRU_SIZE = 25

def resample_and_convolve(old_wavelengths, new_wavelengths, new_resolution,
    old_resolution=np.inf, threshold=5):

    N, M = (new_wavelengths.size, old_wavelengths.size)
    threshold_pixels = threshold/np.diff(old_wavelengths).max()

    # Calculate the width of the kernel at each point.
    # [TODO] should this actually be squared???
    fwhms = (new_wavelengths/new_resolution)**2
    if np.isfinite(old_resolution):
        assert old_resolution > new_resolution
        fwhms -= (new_wavelengths/old_resolution)**2

    # 2.355 ~= 2 * sqrt(2*log(2))
    sigmas = fwhms/2.3548200450309493
    N_kernel_pixels = int(np.ceil(sigmas[-1] * threshold_pixels))
    integer_offsets = old_wavelengths.searchsorted(new_wavelengths)

    # For +/- N_kernel_pixels at each point, calculate the kernel and retain
    # the indices.
    ios = integer_offsets[0]
    indices = np.tile(np.arange(ios - N_kernel_pixels,
        ios + N_kernel_pixels), N).reshape(N, 2 * N_kernel_pixels)
    indices[1:, :] += np.cumsum(np.diff(integer_offsets)).reshape(-1, 1)
    indices = np.clip(indices, 0, M - 1)

    x_indices = indices.flatten()
    y_indices = np.repeat(np.arange(N), 2 * N_kernel_pixels)

    # Calculate the kernel at each point.
    pdf = np.exp(
        -(old_wavelengths[indices].T - new_wavelengths)**2/(2*sigmas**2))
    pdf /= pdf.sum(axis=0)

    return sparse.coo_matrix((pdf.T.flatten(), (x_indices, y_indices)),
        shape=(M, N))

def resample(old_wavelengths, new_wavelengths):
    """
    Resample a spectrum to a new wavelengths map while conserving total flux.

    :param old_wavelengths:
        The original wavelengths array.

    :type old_wavelengths:
        :class:`numpy.array`

    :param new_wavelengths:
        The new wavelengths array to resample onto.

    :type new_wavelengths:
        :class:`numpy.array`
    """

    data = []
    old_px_indices = []
    new_px_indices = []
    for i, new_wl_i in enumerate(new_wavelengths):

        # These indices should span just over the new wavelength pixel.
        indices = np.unique(np.clip(
            old_wavelengths.searchsorted(new_wavelengths[i:i + 2], side="left")\
                + [-1, +1], 0, old_wavelengths.size - 1))
        N = np.ptp(indices)

        if N == 0:
            # 'Fake' pixel.
            data.append(np.nan)
            new_px_indices.append(i)
            old_px_indices.extend(indices)
            continue

        # Sanity checks.
        assert (old_wavelengths[indices[0]] <= new_wl_i \
            or indices[0] == 0)
        assert (new_wl_i <= old_wavelengths[indices[1]] \
            or indices[1] == old_wavelengths.size - 1)

        fractions = np.ones(N)

        # Edges are handled as fractions between rebinned pixels.
        _ = np.clip(i + 1, 0, new_wavelengths.size - 1)
        lhs = old_wavelengths[indices[0]:indices[0] + 2]
        rhs = old_wavelengths[indices[-1] - 1:indices[-1] + 1]
        fractions[0]  = (lhs[1] - new_wavelengths[i])/np.ptp(lhs)
        fractions[-1] = (new_wavelengths[_] - rhs[0])/np.ptp(rhs)

        # Being binned to a single pixel. Prevent overflow from fringe cases.
        fractions = np.clip(fractions, 0, 1)
        fractions /= fractions.sum()

        data.extend(fractions)
        new_px_indices.extend([i] * N) # Mark the new pixel indices affected.
        old_px_indices.extend(np.arange(*indices)) # And the old pixel indices.

    return sparse.csc_matrix((data, (old_px_indices, new_px_indices)),
        shape=(old_wavelengths.size, new_wavelengths.size))

def _fast_resample(old_wavelengths, new_wavelengths):

    # Map which pixels go to which index, assuming that the from_wavelengths
    # scale is linear.

    N, M = (new_wavelengths.size, old_wavelengths.size)
    pixel_edges = np.hstack([
        new_wavelengths[0] - (new_wavelengths[1] - new_wavelengths[0])/2.,
        new_wavelengths[:-1] + np.diff(new_wavelengths)/2.,
        new_wavelengths[-1] + (new_wavelengths[-1] - new_wavelengths[-2])/2.])

    fractional_indices_scale = (pixel_edges - old_wavelengths[0]) \
        * M/np.ptp(old_wavelengths)
    int_fractional_indices_scale = fractional_indices_scale.astype(int)

    upper_diagonals = 1 - fractional_indices_scale[:-1] % 1
    lower_diagonals = fractional_indices_scale[1:] % 1
    central_diagonal_pixel_width = np.clip(
        np.diff(int_fractional_indices_scale) - 1, 0, None)

    normalised_scale = 1.0/(upper_diagonals + central_diagonal_pixel_width \
        + lower_diagonals)

    # If the to_wavelength is linear then the normalised scale per pixel
    # would be the same for every pixel. Let's not assume that..
    px_normalised_scale = np.repeat(normalised_scale,
        central_diagonal_pixel_width)

    values = np.hstack([
        upper_diagonals * normalised_scale,
        lower_diagonals * normalised_scale,
        np.ones(central_diagonal_pixel_width.sum()) * px_normalised_scale])

    _ = np.arange(N)
    y_indices = \
        np.clip(np.hstack([_, _, np.repeat(_, central_diagonal_pixel_width)]),
            0, N - 1)
    x_indices = np.clip(np.hstack([
        int_fractional_indices_scale[:-1],
        int_fractional_indices_scale[1:],
        1 + np.repeat(int_fractional_indices_scale[:-1],
            central_diagonal_pixel_width)
        ]), 0, M - 1)

    return sparse.coo_matrix((values, (x_indices, y_indices)), shape=(M, N))

class _BoxFactory(object):

    """
    For producing binning (box) matrices quickly on the fly.
    """

    def __init__(self, to_wavelengths, from_wavelengths, linear_tolerance=1e-3):

        self.to_wavelengths = to_wavelengths
        self.from_wavelengths = from_wavelengths
        self.N, self.M = (to_wavelengths.size, from_wavelengths.size)
        self._scale = self.M/np.ptp(self.from_wavelengths)
        if not linear_tolerance >= np.std(np.diff(from_wavelengths)):
            logger.warn("from wavelengths scale might be non-linear")

    @lru_cache(maxsize=LRU_SIZE, tol=6)
    def __call__(self, z=0, **kwargs):
        """
        Return a binning matrix for the given redshift based on the original
        wavelengths provided when the class was initiated.
        """
        """
        wavelengths = self.to_wavelengths * (1 + z)

        # Map which pixels go to which index, assuming that the from_wavelengths
        # scale is linear.
        pixel_edges = np.hstack([
            wavelengths[0] - (wavelengths[1] - wavelengths[0])/2.,
            wavelengths[:-1] + np.diff(wavelengths)/2.,
            wavelengths[-1] + (wavelengths[-1] - wavelengths[-2])/2.])

        fractional_indices_scale = (pixel_edges - self.from_wavelengths[0]) \
            * self._scale
        int_fractional_indices_scale = fractional_indices_scale.astype(int)

        upper_diagonals = 1 - fractional_indices_scale[:-1] % 1
        lower_diagonals = fractional_indices_scale[1:] % 1
        central_diagonal_pixel_width = np.diff(int_fractional_indices_scale) - 1

        normalised_scale = 1.0/(upper_diagonals + central_diagonal_pixel_width \
            + lower_diagonals)

        # If the to_wavelength is linear then the normalised scale per pixel
        # would be the same for every pixel. Let's not assume that..
        px_normalised_scale = np.repeat(normalised_scale,
            central_diagonal_pixel_width)

        values = np.hstack([
            upper_diagonals * normalised_scale,
            lower_diagonals * normalised_scale,
            np.ones(central_diagonal_pixel_width.sum()) * px_normalised_scale])

        _ = np.arange(self.N)
        y_indices = np.clip(
            np.hstack([_, _, np.repeat(_, central_diagonal_pixel_width)]),
                0, self.N - 1)
        x_indices = np.clip(np.hstack([
            int_fractional_indices_scale[:-1],
            int_fractional_indices_scale[1:],
            1 + np.repeat(int_fractional_indices_scale[:-1],
                central_diagonal_pixel_width)
            ]), 0, self.M - 1)

        return sparse.coo_matrix((values, (x_indices, y_indices)),
            shape=(self.M, self.N))
        """

        new_wavelengths = self.to_wavelengths * (1 + z)

        data = []
        old_px_indices = []
        new_px_indices = []
        for i, new_wl_i in enumerate(new_wavelengths):

            # These indices should span just over the new wavelength pixel.
            indices = np.unique(np.clip(
                self.from_wavelengths.searchsorted(new_wavelengths[i:i + 2], side="left")\
                    + [-1, +1], 0, self.from_wavelengths.size - 1))
            N = np.ptp(indices)

            if N == 0:
                # 'Fake' pixel.
                data.append(np.nan)
                new_px_indices.append(i)
                old_px_indices.extend(indices)
                continue

            # Sanity checks.
            assert (self.from_wavelengths[indices[0]] <= new_wl_i \
                or indices[0] == 0)
            assert (new_wl_i <= self.from_wavelengths[indices[1]] \
                or indices[1] == self.from_wavelengths.size - 1)

            fractions = np.ones(N)

            # Edges are handled as fractions between rebinned pixels.
            _ = np.clip(i + 1, 0, new_wavelengths.size - 1)
            lhs = self.from_wavelengths[indices[0]:indices[0] + 2]
            rhs = self.from_wavelengths[indices[-1] - 1:indices[-1] + 1]
            fractions[0]  = (lhs[1] - new_wavelengths[i])/np.ptp(lhs)
            fractions[-1] = (new_wavelengths[_] - rhs[0])/np.ptp(rhs)

            # Being binned to a single pixel. Prevent overflow from fringe cases.
            fractions = np.clip(fractions, 0, 1)
            fractions /= fractions.sum()

            data.extend(fractions)
            new_px_indices.extend([i] * N) # Mark the new pixel indices affected.
            old_px_indices.extend(np.arange(*indices)) # And the old pixel indices.

        return sparse.csc_matrix((data, (old_px_indices, new_px_indices)),
            shape=(self.from_wavelengths.size, new_wavelengths.size))

class _BlurryBoxFactory(object):

    """
    For producing convolution (Blurring) and rebinning (Box) matrices quickly on
    the fly.
    """

    def __init__(self, to_wavelengths, from_wavelengths, from_resolution=None,
        threshold=5):

        self.to_wavelengths = to_wavelengths
        self.from_wavelengths = from_wavelengths

        self.from_resolution = from_resolution
        self._threshold_pixels = threshold/np.diff(self.from_wavelengths).max()

        self.N, self.M = (to_wavelengths.size, from_wavelengths.size)

    @lru_cache(maxsize=LRU_SIZE, tol=[0, 6])
    def __call__(self, resolution, z=0, **kwargs):
        """
        Return a binning matrix for the given resolution and optional redshift,
        based on the original wavelengths provided when the class was initiated.

        :param resolution:
            Spectral resolving power.

        :param z: [optional]
            The redshift.
        """

        # Calculate the redshifted wavelengths.
        wavelengths = self.to_wavelengths * (1 + z)

        # Calculate the width of the kernel at each point.
        fwhms = (wavelengths/resolution)**2
        if self.from_resolution:
            fwhms -= (wavelengths/self.from_resolution)**2

        # 2.355 ~= 2 * sqrt(2*log(2))
        sigmas = fwhms/2.3548200450309493
        N_kernel_pixels = int(np.ceil(sigmas[-1] * self._threshold_pixels))
        integer_offsets = self.from_wavelengths.searchsorted(wavelengths)

        # For +/- N_kernel_pixels at each point, calculate the kernel and retain
        # the indices.
        ios = integer_offsets[0]
        indices = np.tile(np.arange(ios - N_kernel_pixels,
            ios + N_kernel_pixels), self.N).reshape(self.N, 2 * N_kernel_pixels)
        indices[1:, :] += np.cumsum(np.diff(integer_offsets)).reshape(-1, 1)
        indices = np.clip(indices, 0, self.M - 1)

        x_indices = indices.flatten()
        y_indices = np.repeat(np.arange(self.N), 2 * N_kernel_pixels)

        # Calculate the kernel at each point.
        pdf = np.exp(
            -(self.from_wavelengths[indices].T - wavelengths)**2/(2*sigmas**2))
        pdf /= pdf.sum(axis=0)

        return sparse.coo_matrix((pdf.T.flatten(), (x_indices, y_indices)),
            shape=(self.M, self.N))

max_stars_repo_name: andycasey/sick

max_issues_repo_name: andycasey/sick

max_forks_repo_name: andycasey/sick

reason: import numpy,from scipy


text:
import os
import os.path as op
import numpy as np
import matplotlib.pyplot as plt

from swann.utils import get_config, derivative_fname

from mne import Epochs

def plot_erp(rawf, raw, event, events, bl_events, l_freq=None, h_freq=30,
             picks=None, overwrite=False):
    ''' Plots event-related potentials for given data
    Parameters
    ----------
    rawf : pybids.BIDSlayout file
        The object containing the raw data.
    raw : mne.io.Raw
        The raw data object.
    event : str
        The name of the event (e.g. `Response`).
    events : np.array(n_events, 3)
        The events from mne.events_from_annotations or mne.find_events
        corresponding to the event and trials that are described by the name.
    bl_events: np.array(n_events, 3)
        The events from mne.events_from_annotations or mne.find_events
        corresponding to the baseline for the event and trials
        that are described by the name.
    l_freq : float
        Low frequency to use to filter the data.
    h_freq : float
        High frequency to use to filter the data.
    picks : None | list of str
        The names of the channels to plot.
    '''
    config = get_config()
    raw = raw.copy()
    raw.load_data()
    plotf = derivative_fname(rawf, 'plots/erps',
                             'event-{}_erp'.format(event), config['fig'])
    if not op.isdir(op.dirname(plotf)):
        os.makedirs(op.dirname(plotf))
    if op.isfile(plotf) and not overwrite:
        print('erp plot for {} already exists, '
              'use `overwrite=True` to replot'.format(event))
        return
    epochs = Epochs(raw, events, tmin=config['tmin'], baseline=None,
                    tmax=config['tmax'], preload=True)
    bl_epochs = Epochs(raw, bl_events, tmin=config['baseline_tmin'],
                       baseline=None, tmax=config['baseline_tmax'],
                       preload=True)
    evoked_data = np.median(bl_epochs._data, axis=0)
    epochs._data -= np.median(evoked_data, axis=1)[:, np.newaxis]
    evoked = epochs.average()
    if l_freq is not None and h_freq is not None:
        evoked = evoked.filter(l_freq=l_freq, h_freq=h_freq)
    fig = evoked.plot(picks=picks, show=False)
    fig.suptitle('Event-Related Potential for the {} Event'.format(event))
    fig.savefig(plotf, dpi=300)
    plt.close(fig)

ext: py

max_stars_repo_name: alexrockhill/SwannLabResources

max_issues_repo_name: alexrockhill/SwannLabResources

max_forks_repo_name: alexrockhill/SwannLabResources

reason: import numpy


text:
from dataclasses import dataclass
import xarray as xr
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
import numpy as np
from typing import List

from src.configure import params_from_file
from src.analysis import load_data, relative_improvement
from src.evaluation_geographic import GeographicValidation
from src.evaluation_plots import plot_histogram
from src.inference import Mask

@dataclass
class ScoreData():

    percentile: xr.DataArray = None
    ifs: xr.DataArray = None
    dnn: xr.DataArray = None

class PlotSpatialHSSScores():

    def __init__(self, data: ScoreData, percentile: float, configs: dict, out_path: str, plot_percentiles=True):

        self.data = data
        self.configs = configs
        self.percentile = percentile
        self.mask_threshold = -1
        self.fname = f'{out_path}geographic_{percentile}th_percentiles_and_hss.png'
        self.in_path = params_from_file('paths')
        self.plot_percentiles = plot_percentiles
        self.figsize = (19,10)

    def get_coordinates(self):

        ds = xr.open_dataset(f'{self.in_path.dataset_path}/{self.in_path.dataset_training}')
        self.lats = ds.latitude
        self.lons = ds.longitude

    def plot_geographic_percentiles(self):

        eval = GeographicValidation(self.lats, self.lons,
                                    orography_flag=False,
                                    mask_threshold=None,
                                    clean_threshold=None,
                                    show_coordinates=False)
        data = self.data.percentile
        eval.plot_single('Percentile',  data, data,
                         configs=self.configs, single_plot=False)

    def plot_ifs_geographic_hss_scores(self):

        eval = GeographicValidation(self.lats, self.lons,
                               orography_flag=False,
                               mask_threshold=self.mask_threshold,
                               clean_threshold=None,
                               show_coordinates=False
                              )
        metric_name = 'HSS'

        data = self.data.ifs
        self.configs['HSS']['title'] = None
        self.configs['HSS']['cbar_title'] = 'IFS HSS'
        eval.plot_single(metric_name,  data, data,
                         configs=self.configs, single_plot=False)

    def plot_hss_geographic_hss_scores(self):

        eval = GeographicValidation(self.lats, self.lons,
                               orography_flag=False,
                               mask_threshold=self.mask_threshold,
                               clean_threshold=None,
                               show_coordinates=False
                              )
        metric_name = 'HSS'

        self.configs['HSS']['title'] = None
        self.configs['HSS']['cbar_title'] = 'DNN (WMSE-MS-SSIM) HSS'

        data = self.data.dnn
        eval.plot_single(metric_name, data, data,
                 configs=self.configs, single_plot=False)

    def plot(self):

        self.get_coordinates()

        plt.rcParams.update({'font.size': 11})

        if self.plot_percentiles:
            plt.figure(figsize=self.figsize, dpi=300)

            ax1 = plt.subplot(311)
            ax1.annotate("a", ha="center", va="center", size=13,
                     xy=(0.985, 0.945), xycoords=ax1,
                     bbox=dict(boxstyle="square,pad=0.3", fc="white", ec="k", lw=1))

            self.plot_geographic_percentiles()

        else:
            plt.figure(figsize=self.figsize, dpi=300)

        if self.plot_percentiles:
            ax2 = plt.subplot(312)
            annotate = "b"

        else:
            ax2 = plt.subplot(211)
            annotate = "a"

        ax2.annotate(annotate, ha="center", va="center", size=13,
        xy=(0.985, 0.945), xycoords=ax2,
        bbox=dict(boxstyle="square,pad=0.3", fc="white", ec="k", lw=1))

        self.plot_ifs_geographic_hss_scores()

        if self.plot_percentiles:
            ax3 = plt.subplot(313)
            annotate = "c"

        else:
            ax3 = plt.subplot(212)
            annotate = "b"

        ax3.annotate(annotate, ha="center", va="center", size=13,
        xy=(0.985, 0.945), xycoords=ax3,
        bbox=dict(boxstyle="square,pad=0.3", fc="white", ec="k", lw=1))

        self.plot_hss_geographic_hss_scores()
        plt.tight_layout()
        if self.fname is not None:
            print(self.fname)
            plt.savefig(self.fname, dpi=300, format='png', bbox_inches='tight')
        plt.show()

class PlotSingleFrames():

    def __init__(self, dnn_data, ifs_data, trmm_data,
                 timestamps=['2012-07-16T00', '2013-07-16T00', '2014-07-16T00']):

        self.dnn_data = dnn_data
        self.ifs_data = ifs_data
        self.trmm_data = trmm_data
        self.min_precipitation_threshold_in_mm_per_3hours = 0.1
        path = '/path/to/training_dataset'

        self.ds = xr.open_dataset(path, chunks={'time': 1})
        self.lats = self.ds.latitude
        self.lons = self.ds.longitude

        self.color_map = 'YlGnBu'
        self.vmin = 0
        self.vmax = 11.5
        self.timestamps = timestamps
        self.fname = f'/path/to/figures/single_frames.png'

    def get_time_indeces(self):

        first_test_set_index = np.where(self.ds.time ==  np.datetime64(f'2012-06-01T00:00:00.000000000'))[0][0]
        index_2012 = np.where(self.ds.time ==  np.datetime64(f'{self.timestamps[0]}:00:00.000000000'))[0][0] - first_test_set_index
        index_2013 = np.where(self.ds.time ==  np.datetime64(f'{self.timestamps[1]}:00:00.000000000'))[0][0] - first_test_set_index
        index_2014 = np.where(self.ds.time ==  np.datetime64(f'{self.timestamps[2]}:00:00.000000000'))[0][0] - first_test_set_index

        self.time_idx = [index_2012, index_2013, index_2014]

    def get_data(self):
        self.get_time_indeces()
        self.dnn_frames = []
        self.ifs_frames = []
        self.trmm_frames = []

        for t in self.time_idx:
            self.dnn_frames.append(np.where(self.dnn_data[t] <
                                   self.min_precipitation_threshold_in_mm_per_3hours,
                                   0, self.dnn_data[t]))

            self.ifs_frames.append(np.where(self.ifs_data[t] <
                                   self.min_precipitation_threshold_in_mm_per_3hours,
                                   0, self.ifs_data[t]))

            self.trmm_frames.append(np.where(self.trmm_data[t] <
                                   self.min_precipitation_threshold_in_mm_per_3hours,
                                   0, self.trmm_data[t]))

    def single_plot(self, data,
                    show_cbar=True,
                    show_latitude_labels=True,
                    show_longitude_labels=True,
                    ):
        m = Basemap(llcrnrlon=self.lons[0], llcrnrlat=self.lats[0],
                    urcrnrlon=self.lons[-1], urcrnrlat=self.lats[-1],
                    projection='merc', lon_0=0, lat_0=20, resolution='c')

        m.drawparallels([-30, 0, 30],
                        labels=[show_latitude_labels, 0, 0, 0], linewidth=.5)
        m.drawmeridians([-120, -60, 0, 60, 120],
                        labels=[0, 0, 0, show_longitude_labels], linewidth=.5)

        m.drawcoastlines()
        Lon, Lat = np.meshgrid(self.lons, self.lats)
        x, y = m(Lon, Lat)
        m.pcolormesh(x, y, data, vmin=self.vmin,  vmax=self.vmax, cmap=self.color_map)

        if show_cbar:

            plt.colorbar(fraction=0.016, pad=0.04,  extend='both', label=r'Precipitation [mm/3hr]')

    def plot(self):

        self.get_data()

        plt.figure(figsize=(15, 6))
        plt.rcParams.update({'font.size': 11})
        for i in range(len(self.time_idx)):

            plt.subplot(3,3,1+3*i)
            plt.title(f'IFS, time: {self.timestamps[i]}')
            self.single_plot(self.ifs_frames[i], show_cbar=False,
                             show_latitude_labels=True,
                             show_longitude_labels=i==2)

            plt.subplot(3,3,2+3*i)
            plt.title(f'DNN (WMSE-MS-SSIM), time: {self.timestamps[i]}')
            self.single_plot(self.dnn_frames[i], show_cbar=False,
                             show_latitude_labels=False,
                             show_longitude_labels=i==2)

            plt.subplot(3,3,3+3*i)
            plt.title(f'TRMM, time: {self.timestamps[i]}')
            self.single_plot(self.trmm_frames[i], show_cbar=True,
                             show_latitude_labels=False,
                             show_longitude_labels=i==2)

        plt.tight_layout()

        if self.fname is not None:
            print(self.fname)
            plt.savefig(self.fname, dpi=300, format='png')
        plt.show()

class ScoresPerPercentile():

    def __init__(self):
        self.heidke_skill_score = {}
        self.f1 = {}
        self.critical_success_index = {}
        self.false_alarm_ratio = {}
        self.probability_of_detection = {}

    def set_data(self, data, percentile):
        self.heidke_skill_score[percentile] = data['heidke_skill_score'][0]
        self.f1[percentile] = data['f1'][0]
        self.critical_success_index[percentile] = data['critical_success_index'][0]
        self.false_alarm_ratio[percentile] = data['false_alarm_ratio'][0]
        self.probability_of_detection[percentile] = data['probability_of_detection'][0]

@dataclass
class ModelTestSetData():

    dnn = None
    dnn_mssim = None
    dnn_weigthed = None
    linear = None
    qm = None
    raw_ifs = None

class PlotHistograms():

    def __init__(self, path='/path/to/models/'):

        self.path = path
        self.model_file_names = {
                  'dnn_weighted': 'dnn_weighted.npy',
                  'dnn_mssim':    'dnn_mssim.npy',
                  'dnn':          'dnn.npy',
                  'qm':           'qm.npy',
                  'linear':       'linear.npy'}

    def load_data(self):

        fname = f"{self.path}/{self.model_file_names['dnn_weighted']}"
        self.dnn_weighted = np.load(fname)[0]

        fname = f"{self.path}/{self.model_file_names['dnn']}"
        self.dnn_mse = np.load(fname)[0]

        fname = f"{self.path}/{self.model_file_names['dnn']}"
        self.trmm = np.load(fname)[1]

        fname = f"{self.path}/{self.model_file_names['dnn']}"
        self.ifs = np.load(fname)[2]

        fname = f"{self.path}/{self.model_file_names['dnn_mssim']}"
        self.dnn_mssim = np.load(fname)[0]

        fname = f"{self.path}/{self.model_file_names['qm']}"
        self.qm = np.load(fname)[0]

        fname = f"{self.path}/{self.model_file_names['linear']}"
        self.linear = np.load(fname)[0]

    def mask_data(self):

        self.ifs = Mask(self.ifs).apply()
        self.trmm = Mask(self.trmm).apply()
        self.linear = Mask(self.linear).apply()
        self.dnn_mse = Mask(self.dnn_mse).apply()
        self.dnn_mssim = Mask(self.dnn_mssim).apply()
        self.dnn_weighted = Mask(self.dnn_weighted).apply()
        self.qm = Mask(self.qm).apply()

    def plot(self,
             log_plot=True,
             show_plot=True,
             x_max=None,
             fname=None,
             legend_position='upper center',
             masked=False
             ):

        plot_histogram(self.trmm, self.ifs, model_name='IFS', color='tab:blue')
        plot_histogram(self.trmm, self.trmm, model_name='TRMM', color='black')
        plot_histogram(self.trmm, self.linear, model_name='Ridge regr.', color='tab:orange')
        plot_histogram(self.trmm, self.dnn_mse, model_name='DNN (MSE)', color='tab:green')
        plot_histogram(self.trmm, self.dnn_mssim, model_name='DNN (MS-SSIM)', color='tab:purple')
        plot_histogram(self.trmm, self.dnn_weighted, model_name='DNN (WMSE-MS-SSIM)', color='tab:red')
        plot_histogram(self.trmm, self.qm, model_name='Quantile map.', color='tab:brown')

        plt.legend(loc=legend_position)

        if log_plot:
            plt.yscale('log')

        if x_max is not None:
            plt.xlim(0, x_max)

        plt.xlabel(f'Rainfall [mm/3h]')
        plt.ylabel('Relative frequency')
        plt.grid()

        if fname is not None:
            plt.savefig(fname, dpi=300, format='pdf')

        if show_plot:
            plt.show()

class PlotCategoricalScores():

    def __init__(self, in_path: str, percentiles: List[str], model_names: List[str]):

        self.in_path = in_path
        self.percentiles = percentiles
        self.model_names = model_names
        self.models = ModelTestSetData()
        self.scores_per_percentile = ScoresPerPercentile

    def load_results(self):
        for name in self.model_names:
            scores_precentile = self.scores_per_percentile()
            if name == 'dnn_mssim': tmp = scores_precentile
            for percentile in self.percentiles:
                fname = f"{self.in_path}/{name}_percentile_{percentile}"
                data = load_data(fname)

                scores_precentile.set_data(data, percentile)

            vars(self.models)[name] = scores_precentile

    def get_results(self):
        return self.models

    def single_plot(self, metric_name: str, file_name=None):
        plt.figure(figsize=(7, 5))
        self.plot(metric_name)
        if file_name is not None:
            plt.savefig(file_name, dpi=300, format='pdf')
        plt.show()

    def plot(self, metric_name: str):

        colors = {'dnn_weighted': 'tab:red',
                  'dnn_mssim': 'tab:purple',
                  'dnn': 'tab:green',
                  'ifs': 'tab:blue',
                  'linear': 'tab:orange',
                  'qm': 'tab:brown'}

        label = {'dnn_weighted': 'DNN (WMSE-MS-SSIM)',
                 'dnn_mssim': 'DNN (MS-SSIM)',
                 'dnn': 'DNN (MSE)',
                 'ifs': 'IFS',
                 'linear': 'Ridge regr.',
                 'qm': 'Quantile map.'}

        metric_name_dict = {'heidke_skill_score':   'HSS',
                            'f1': 'F1',
                            'critical_success_index':   'CSI',
                            'false_alarm_ratio': 'FAR',
                            'probability_of_detection': 'POD'
        }

        for model in vars(self.models):
            scores = vars(self.models)[model]
            score_dict = sorted(getattr(scores, metric_name).items())

            percentiles, scores = zip(*score_dict)
            percentiles = [float(p) for p in percentiles]

            plt.plot(percentiles, scores, color=colors[model],
                     marker='o', markersize=5, linewidth=2, label=label[model])

        plt.xticks(list(plt.xticks()[0])[:-2] + [99])
        plt.xlim(75, 100)
        #plt.legend(loc='lower left')
        plt.legend()
        plt.grid()
        plt.ylabel(metric_name_dict[metric_name])
        plt.xlabel('Percentile')

    def plot_relative_hss_improvement(self):

        colors = {'dnn_weighted': 'tab:red',
                  'dnn_mssim': 'tab:purple',
                  'dnn': 'tab:green',
                  'ifs': 'tab:blue',
                  'linear': 'tab:orange',
                  'qm': 'tab:brown'}

        label = {'dnn_weighted': 'DNN (WMSE-MS-SSIM)',
                 'dnn_mssim': 'DNN (MS-SSIM)',
                 'dnn': 'DNN (MSE)',
                 'ifs': 'IFS',
                 'linear': 'Ridge regr.',
                 'qm': 'Quantile map.'}

        metric_name = 'heidke_skill_score'
        scores = vars(self.models)['ifs']
        score_dict = sorted(getattr(scores, metric_name).items())
        percentiles, ifs_scores = zip(*score_dict)

        for model in vars(self.models):
            scores = vars(self.models)[model]
            score_dict = sorted(getattr(scores, metric_name).items())

            percentiles, scores = zip(*score_dict)
            percentiles = [float(p) for p in percentiles]
            relative_scores = relative_improvement(scores, ifs_scores)
            plt.plot(percentiles, relative_scores, color=colors[model],
                     marker='o', markersize=5, linewidth=2, label=label[model])

        plt.xticks(list(plt.xticks()[0])[:-2] + [99])
        plt.xlim(75, 100)
        plt.legend()
        plt.grid()
        plt.ylabel('Relative improvement [%]')
        plt.xlabel('Percentile')

    def plot_summary(self, file_name=None):

        plt.figure(figsize=(16,10))
        plt.rcParams.update({'font.size': 11})

        ax1 = plt.subplot(221)

        self.plot('heidke_skill_score')
        ax1.annotate("a", ha="center", va="center", size=13,
                     xy=(0.965, 0.945), xycoords=ax1,
                     bbox=dict(boxstyle="square,pad=0.25", fc="white", ec="k", lw=1))
        plt.ylim(-.07,0.45)

        ax2 = plt.subplot(222)
        plt.ylim(-75,500)

        ax2.annotate("b", ha="center", va="center", size=13,
                     xy=(0.965, 0.945), xycoords=ax2,
                     bbox=dict(boxstyle="square,pad=0.25", fc="white", ec="k", lw=1))

        self.plot_relative_hss_improvement()

        histplot = PlotHistograms()

        ax1 = plt.subplot(223)
        ax1.annotate("c", ha="center", va="center", size=13,
                     xy=(0.965, 0.945), xycoords=ax1,
                     bbox=dict(boxstyle="square,pad=0.25", fc="white", ec="k", lw=1))

        histplot.plot(log_plot=False, show_plot=False, x_max=20)

        ax2 = plt.subplot(224)
        histplot.plot(log_plot=True, show_plot=False, x_max=None)

        ax2.annotate("d", ha="center", va="center", size=13,
                     xy=(0.965, 0.945), xycoords=ax2,
                     bbox=dict(boxstyle="square,pad=0.25", fc="white", ec="k", lw=1))

        if file_name is not None:
            plt.savefig(file_name, dpi=300, bbox_inches='tight', format='pdf')
        plt.show()


text:
import csv
import math

import cv2
import numpy as np
from keras import Model
from keras.layers import Activation, Conv2D, Cropping2D, Dense, Flatten, Input, Lambda
from keras.layers.normalization import BatchNormalization
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle

def get_csv_data(csv_path):
    lines = []
    with open(csv_path) as csv_file:
        reader = csv.reader(csv_file)
        for line in reader:
            lines.append(line)
    return lines

def add_training_data(name, angle, images, angles):
    # cv2 reads images as BGR
    image = cv2.cvtColor(cv2.imread(name), cv2.COLOR_BGR2RGB)
    images.append(image)
    angles.append(angle)
    images.append(np.fliplr(image))
    angles.append(-angle)

def generator(samples, batch_size=32):
    n_samples = len(samples)
    while True:
        shuffle(samples)
        for offset in range(0, n_samples, batch_size):
            batch_samples = samples[offset:offset + batch_size]
            images = []
            angles = []
            for batch_sample in batch_samples:
                center = batch_sample[0].split('/')[-1]
                left = batch_sample[1].split('/')[-1]
                right = batch_sample[2].split('/')[-1]
                angle = float(batch_sample[3])
                add_training_data(center, angle, images, angles)
                add_training_data(left, angle + 0.2, images, angles)
                add_training_data(right, angle - 0.2, images, angles)
            X_train = np.array(images)
            y_train = np.array(angles)
            yield shuffle(X_train, y_train)

def yuv_conversion(x):
    import tensorflow as tf
    # Simulator and the generator send images as RGB
    return tf.image.rgb_to_yuv(x)

def define_model(input_shape):
    inputs = Input(shape=input_shape, name='input')
    x = Cropping2D(cropping=((50, 20), (0, 0)),
                   input_shape=input_shape)(inputs)
    x = Lambda(yuv_conversion)(x)
    x = BatchNormalization()(x)
    x = Conv2D(24, (5, 5), strides=(2, 2), kernel_initializer='he_normal')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = Conv2D(36, (5, 5), strides=(2, 2), kernel_initializer='he_normal')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = Conv2D(48, (5, 5), strides=(2, 2), kernel_initializer='he_normal')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = Conv2D(64, (3, 3), kernel_initializer='he_normal')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = Conv2D(64, (3, 3), kernel_initializer='he_normal')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = Flatten()(x)
    x = Dense(100, kernel_initializer='he_normal')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = Dense(50, kernel_initializer='he_normal')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    x = Dense(10, kernel_initializer='he_normal')(x)
    x = BatchNormalization()(x)
    x = Activation('relu')(x)
    outputs = Dense(1, kernel_initializer='he_normal')(x)
    return Model(inputs, outputs)

def main():
    samples_c1_kb = get_csv_data('./course1_kb/driving_log.csv')
    samples_c1_mouse = get_csv_data('./course1_mouse/driving_log.csv')
    samples = samples_c1_kb + samples_c1_mouse
    train_samples, validation_samples = train_test_split(
        samples, test_size=0.2)
    epochs = 1
    batch_size = 32
    train_generator = generator(train_samples, batch_size=batch_size)
    validation_generator = generator(validation_samples, batch_size=batch_size)

    row, col, ch = 160, 320, 3
    model = define_model((row, col, ch))
    model.summary()

    model.compile(loss='mse', optimizer='adam')
    model.fit_generator(
        train_generator,
        steps_per_epoch=math.ceil(
            len(train_samples) /
            batch_size),
        validation_data=validation_generator,
        validation_steps=math.ceil(
            len(validation_samples) /
            batch_size),
        epochs=epochs,
        verbose=1)

    model.save('model.h5')

if __name__ == "__main__":
    main()


text:
# Copyright 2018 Alexander Matthews
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pickle
import numpy as np
from scipy.stats import norm

from matplotlib import pylab as plt
from IPython import embed

import shared
import defaults

import comparison_bias_four

results = pickle.load( open(comparison_bias_four.results_file_name,'rb' ) )

log_pred_densities_nn = results['log_densities_NN']
log_pred_densities_gp = results['log_densities_GP']
param_trackers = results['param_trackers']
pred_trackers = results['pred_trackers']
energies = results['energies']
X_train = results['X_train']
Y_train = results['Y_train'].data.numpy()
X_test = results['X_test']
Y_test = results['Y_test'].data.numpy()

[fig, axes] = plt.subplots(1,1, figsize=(4.5,4.5))
#fig = plt.figure()
#ax1 = fig.add_subplot(1,2,1, adjustable='box', aspect=1)
#ax2 = fig.add_subplot(1,2,2, adjustable='box', aspect=1)
#plt.

shared_limits = [-8., 0.]
plot_range  = np.linspace( *shared_limits, 100)
axes.plot(plot_range, plot_range,'k')
axes.plot(log_pred_densities_gp, log_pred_densities_nn, 'bo')
axes.set_xlabel('Gaussian process log density')
axes.set_ylabel('Neural network log density')
axes.axis('square')
axes.set_xlim( shared_limits )
axes.set_ylim( shared_limits )
random_point_index = 9

plt.savefig('../figures/comparison_bias_four_a.pdf')

[fig, axesB] = plt.subplots(1,1, figsize=(4.5,4.5))
axesB.hist(pred_trackers[50:,random_point_index],30,normed=True)
test_point = np.atleast_2d(X_test[random_point_index,:])

#add gp density to plot.
gp_model = shared.get_gp_model(X_train,Y_train,input_dim=X_train.shape[1],depth=defaults.shared_depth)
pred_mean, pred_var = gp_model.predict_f( test_point )
pred_std = np.sqrt(pred_var)
plot_range = [-3. , 2.]
x_points = np.linspace(*plot_range,100)
densities = norm.pdf( x_points, loc=pred_mean, scale = pred_std )
axesB.plot( x_points, densities.flatten() )
axesB.set_xlabel('Function value')
axesB.set_ylabel('Density')
plt.savefig('../figures/comparison_bias_four_b.pdf')
embed()
plt.show()


text:
#!/usr/bin/python3

import os
import sys

import numpy as np
from PIL import Image

np.set_printoptions(threshold=sys.maxsize)

script_dir = os.path.dirname(__file__)

image_dimensions = (7, 7)

if __name__ == "__main__":
    print("EECS 332 - MP#1 - Andrew Quinn random bmp file generator")

    if not os.path.exists(os.path.join(script_dir, "test_images")):
        os.makedirs(os.path.join(script_dir, "test_images"))
    if not os.path.exists(os.path.join(script_dir, "test_results")):
        os.makedirs(os.path.join(script_dir, "test_results"))

    print(os.path.join(script_dir, "test_images"))

    for i in range(0, 10):
        new_bmp = np.random.choice(a=[False, True], size=image_dimensions)
        print(new_bmp)
        im = Image.fromarray(new_bmp)

        file_index = str(i).zfill(3)
        im.save(
            os.path.join(
                script_dir, "test_images", "test_image_{}.bmp".format(file_index)
            )
        )


text:
from typing import List, Tuple, Dict
import numpy as np
from scipy import stats

def get_metrics(
    y_pred=None,
    y_true=None,
    metrics: List[str] = ['Accuracy'],
    classes: List[str] = None,
) -> Dict:

    if isinstance(y_pred, np.ndarray) is False:
        y_pred = np.array(y_pred)
    if isinstance(y_true, np.ndarray) is False:
        y_true = np.array(y_true)

    results = {}

    for metric in metrics:

        if metric == 'Accuracy':
            results[metric] = accuracy(y_true, y_pred)

        elif metric == 'Precision':
            results[metric] = np.zeros(len(classes))
            for i, label in enumerate(classes):
                results[metric][i] = (precision_score(
                    y_true, y_pred, label=i))

        elif metric == 'Recall':
            results[metric] = np.zeros(len(classes))
            for i, label in enumerate(classes):
                results[metric][i] = (recall_score(
                    y_true, y_pred, label=i))

        elif metric == 'F1':
            results[metric] = np.zeros(len(classes))
            for i, label in enumerate(classes):
                results[metric][i] = (f1_score(
                    y_true, y_pred, label=i))

        elif metric == 'Confusion Matrix':
            results[metric] = confusion_matrix(
                y_true, y_pred, labels=classes)
    return results

def accuracy(y_true, y_pred):
    return np.mean(y_true == y_pred)

def precision_score(y_true, y_pred, label: int):
    """
    Computes the precision score for a given set of labels.
    Precision score = true_pos / (true_pos + false_pos)
    Args:
        y_true: The true labels.
        y_pred: The predicted labels.
        label: The label to consider.
    Returns:
        Precision Score.
    """
    true_pos = np.sum(np.logical_and(y_true == label, y_pred == label))
    return true_pos / (np.sum(y_pred == label) + 1e-7)

def recall_score(y_true, y_pred, label: int):
    """
    Computes the recall score for a given set of labels.
    Recall score = true_pos / (true_pos + false_neg)
    Args:
        y_true: The true labels.
        y_pred: The predicted labels.
        label: The label to consider.
    Returns:
        Recall Score.
    """
    true_pos = np.sum(np.logical_and(y_true == label, y_pred == label))
    return true_pos / np.sum(y_true == label)

def f1_score(y_true, y_pred, label: int):
    """
    Computes the F1 score for a given set of labels.
    F1 score = 2 * (precision * recall) / (precision + recall)
    Args:
        y_true: The true labels.
        y_pred: The predicted labels.
        label: The label to consider.
    Returns:
        F1 score.
    """
    precision = precision_score(y_true, y_pred, label)
    recall = recall_score(y_true, y_pred, label)
    return 2 * (precision * recall) / (precision + recall + 1e-7)

def confusion_matrix(y_true, y_pred, labels: List = []):
    """
    Computes the confusion matrix for a given set of labels.
    Args:
        y_true: The true labels.
        y_pred: The predicted labels.
        labels: The list of labels to consider.
    Returns:
        The confusion matrix. (np.ndarray)
    """
    matrix = np.zeros((len(labels), len(labels)))
    for label, label_name in enumerate(labels):
        true_pos = np.sum(np.logical_and(y_true == label, y_pred == label))
        matrix[label, label] = true_pos
        for i, other_label in enumerate(labels):
            if i != label:
                matrix[label, i] = np.sum(
                    np.logical_and(y_true == label, y_pred == i))
    return matrix

def find_ci_interval(
    data: np.ndarray,
    confidence=0.95
) -> Tuple[float, float, float]:
    """
    Finds the confidence interval for a given dataset of a estimator.
    """
    if isinstance(data, np.ndarray) is False:
        data = np.array(data)
    n = len(data)
    mean, std = data.mean(axis=0), data.std(axis=0)
    h = std * stats.t.ppf((1+confidence)/2., n-1)
    return mean, (mean-h, mean+h)


text:
#!/usr/bin/python
########################################################################################################################
#
# Copyright (c) 2014, Regents of the University of California
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
# following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following
#   disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
#    following disclaimer in the documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
########################################################################################################################

"""GridBasedLayoutGenerator utility functions for users"""
__author__ = "Jaeduk Han"
__maintainer__ = "Jaeduk Han"
__email__ = "jdhan@eecs.berkeley.edu"
__status__ = "Prototype"

#import struct
#import math
#from math import *
import numpy as np
from copy import deepcopy

def generate_boundary(laygen, objectname_pfix, placement_grid,
                      devname_bottom, devname_top, devname_left, devname_right,
                      shape_bottom=None, shape_top=None, shape_left=None, shape_right=None,
                      transform_bottom=None, transform_top=None, transform_left=None, transform_right=None,
                      origin=np.array([0, 0])):
    """generate a boundary structure to resolve boundary design rules"""
    pg = placement_grid
    #parameters
    if shape_bottom == None:
        shape_bottom = [np.array([1, 1]) for d in devname_bottom]
    if shape_top == None:
        shape_top = [np.array([1, 1]) for d in devname_top]
    if shape_left == None:
        shape_left = [np.array([1, 1]) for d in devname_left]
    if shape_right == None:
        shape_right = [np.array([1, 1]) for d in devname_right]
    if transform_bottom == None:
        transform_bottom = ['R0' for d in devname_bottom]
    if transform_top == None:
        transform_top = ['R0' for d in devname_top]
    if transform_left == None:
        transform_left = ['R0' for d in devname_left]
    if transform_right == None:
        transform_right = ['R0' for d in devname_right]

    #bottom
    dev_bottom=[]
    dev_bottom.append(laygen.place("I" + objectname_pfix + 'BNDBTM0', devname_bottom[0], pg, xy=origin,
                      shape=shape_bottom[0], transform=transform_bottom[0]))
    for i, d in enumerate(devname_bottom[1:]):
        dev_bottom.append(laygen.relplace("I" + objectname_pfix + 'BNDBTM'+str(i+1), d, pg, dev_bottom[-1].name,
                                          shape=shape_bottom[i+1], transform=transform_bottom[i+1]))
    dev_left=[]
    dev_left.append(laygen.relplace("I" + objectname_pfix + 'BNDLFT0', devname_left[0], pg, dev_bottom[0].name, direction='top',
                                    shape=shape_left[0], transform=transform_left[0]))
    for i, d in enumerate(devname_left[1:]):
        dev_left.append(laygen.relplace("I" + objectname_pfix + 'BNDLFT'+str(i+1), d, pg, dev_left[-1].name, direction='top',
                                        shape=shape_left[i+1], transform=transform_left[i+1]))
    dev_right=[]
    dev_right.append(laygen.relplace("I" + objectname_pfix + 'BNDRHT0', devname_right[0], pg, dev_bottom[-1].name, direction='top',
                                     shape=shape_right[0], transform=transform_right[0]))
    for i, d in enumerate(devname_right[1:]):
        dev_right.append(laygen.relplace("I" + objectname_pfix + 'BNDRHT'+str(i+1), d, pg, dev_right[-1].name, direction='top',
                                         shape=shape_right[i+1], transform=transform_right[i+1]))
    dev_top=[]
    dev_top.append(laygen.relplace("I" + objectname_pfix + 'BNDTOP0', devname_top[0], pg, dev_left[-1].name, direction='top',
                                   shape=shape_top[0], transform=transform_top[0]))
    for i, d in enumerate(devname_top[1:]):
        dev_top.append(laygen.relplace("I" + objectname_pfix + 'BNDTOP'+str(i+1), d, pg, dev_top[-1].name,
                                       shape=shape_top[i+1], transform=transform_top[i+1]))
    return [dev_bottom, dev_top, dev_left, dev_right]

def generate_power_rails(laygen, routename_tag, layer, gridname, netnames=['VDD', 'VSS'], direction='x',
                         start_coord=0, end_coord=0, route_index=None, via_index=None, generate_pin=True):
    """generate power rails"""
    rail_list=[]
    for netidx, netname in enumerate(netnames):
        rail_sub_list=[]
        for rcnt, ridx in enumerate(route_index[netidx]):
            if direction=='x': rxy0=np.array([[start_coord, ridx], [end_coord, ridx]])
            if direction=='y': rxy0=np.array([[ridx, start_coord], [ridx, end_coord]])
            if generate_pin == True:
                if netname.endswith(':'): #remove colon from netname
                    pn=netname[:-1] + routename_tag + str(rcnt)
                else:
                    pn=netname + routename_tag + str(rcnt)
                p=laygen.pin(name=pn, layer=layer, xy=rxy0, gridname=gridname, netname=netname)
                rail_sub_list.append(p)
            else:
                r=laygen.route(None, layer, xy0=rxy0[0], xy1=rxy0[1], gridname0=gridname)
                rail_sub_list.append(r)
            if not via_index==None: #via generation
                for vidx in via_index[netidx]:
                    if direction=='x': vxy0=np.array([vidx, ridx])
                    else: vxy0=np.array([ridx, vidx])
                    laygen.via(None, vxy0, gridname=gridname)
        rail_list.append(rail_sub_list)
    return rail_list

def generate_power_rails_from_rails_xy(laygen, routename_tag, layer, gridname, netnames=['VDD', 'VSS'], direction='x',
                                       input_rails_xy=None, generate_pin=True,
                                       overwrite_start_coord=None, overwrite_end_coord=None,
                                       offset_start_coord=None, offset_end_coord=None,
                                       overwrite_num_routes=None,
                                       overwrite_start_index=None, overwrite_end_index=None,
                                       offset_start_index=0, offset_end_index=0):
    """generate power rails from pre-existing power rails in upper/lower layer.
       the pre-existing rail information is provided as xy array
    """
    route_index=[]
    via_index=[]
    for netidx, netname in enumerate(netnames):
        sub_via_index=[]
        for i, irxy in enumerate(input_rails_xy[netidx]):
            if direction == 'x':
                #boundary estimation
                if netidx==0 and i==0: #initialize
                    start_coord=irxy[0][0]
                    end_coord=irxy[0][0]
                    route_index_start=min((irxy[0][1], irxy[1][1]))
                    route_index_end=max((irxy[0][1], irxy[1][1]))
                else:
                    if start_coord > irxy[0][0]: start_coord=irxy[0][0]
                    if end_coord < irxy[0][0]: end_coord=irxy[0][0]
                    rist=min((irxy[0][1], irxy[1][1]))
                    ried=max((irxy[0][1], irxy[1][1]))
                    if route_index_start < rist: route_index_start = rist
                    if route_index_end > ried: route_index_end = ried
                sub_via_index.append(irxy[0][0])
            else:
                #boundary estimation
                if netidx==0 and i==0: #initialize
                    start_coord=irxy[0][1]
                    end_coord=irxy[0][1]
                    route_index_start=min((irxy[0][0], irxy[1][0]))
                    route_index_end=max((irxy[0][0], irxy[1][0]))
                else:
                    if start_coord > irxy[0][1]: start_coord=irxy[0][1]
                    if end_coord < irxy[0][1]: end_coord=irxy[0][1]
                    rist=min((irxy[0][0], irxy[1][0]))
                    ried=max((irxy[0][0], irxy[1][0]))
                    if route_index_start < rist: route_index_start = rist
                    if route_index_end > ried: route_index_end = ried
                sub_via_index.append(irxy[0][1])
        via_index.append(np.array(sub_via_index))
    #offset route index if necessary
    route_index_start+=offset_start_index
    route_index_end+=offset_end_index
    #overwrite route index if necessary
    if not overwrite_start_index==None:
        route_index_start=overwrite_start_index
    if not overwrite_end_index==None:
        route_index_end=overwrite_end_index
    #change number of routes if necessary
    if not overwrite_num_routes==None:
        route_index_end=route_index_start + overwrite_num_routes
    #route index
    for netidx, netname in enumerate(netnames):
        sub_route_index=[]
        for ri in range(int((route_index_end - route_index_start + 1)/len(netnames))):
            sub_route_index += [route_index_start + netidx + len(netnames)*ri]
        route_index.append(np.array(sub_route_index))
    #offset start/end coordinates if necessary
    if not offset_start_coord==None:
        start_coord+=offset_start_coord
    if not offset_end_coord==None:
        end_coord+=offset_end_coord
    #overwrite start/end coordinates if necessary
    if not overwrite_start_coord==None:
        start_coord=overwrite_start_coord
    if not overwrite_end_coord==None:
        end_coord=overwrite_end_coord
    return generate_power_rails(laygen, routename_tag=routename_tag, layer=layer, gridname=gridname, netnames=netnames, direction=direction,
                                start_coord=start_coord, end_coord=end_coord, route_index=route_index, via_index=via_index, generate_pin=generate_pin)

def generate_power_rails_from_rails_rect(laygen, routename_tag, layer, gridname, netnames=['VDD', 'VSS'], direction='x',
                                         input_rails_rect=None, generate_pin=True,
                                         overwrite_start_coord=None, overwrite_end_coord=None, overwrite_num_routes=None,
                                         overwrite_start_index=None, overwrite_end_index=None,
                                         offset_start_coord=None, offset_end_coord=None,
                                         offset_start_index=0, offset_end_index=0):
    """generate power rails from pre-existing power rails in upper/lower layer.
       the pre-existing rail information is provided as rect
    """
    xy=[]
    for netidx, netname in enumerate(netnames):
        sub_xy=[]
        for i, ir in enumerate(input_rails_rect[netidx]):
            sub_xy.append(laygen.get_rect_xy(ir.name, gridname))
        xy.append(np.array(sub_xy))
    return generate_power_rails_from_rails_xy(laygen, routename_tag, layer, gridname, netnames=netnames, direction=direction,
                                              input_rails_xy=xy, generate_pin=generate_pin,
                                              overwrite_start_coord=overwrite_start_coord, overwrite_end_coord=overwrite_end_coord,
                                              offset_start_coord=offset_start_coord, offset_end_coord=offset_end_coord,
                                              overwrite_num_routes=overwrite_num_routes,
                                              overwrite_start_index=overwrite_start_index, overwrite_end_index=overwrite_end_index,
                                              offset_start_index=offset_start_index, offset_end_index=offset_end_index)

def generate_power_rails_from_rails_inst(laygen, routename_tag, layer, gridname, netnames=['VDD', 'VSS'], direction='x',
                                         input_rails_instname=None, input_rails_pin_prefix=['VDD', 'VSS'], generate_pin=True,
                                         overwrite_start_coord=None, overwrite_end_coord=None, overwrite_num_routes=None,
                                         overwrite_start_index=None, overwrite_end_index=None,
                                         offset_start_coord=None, offset_end_coord=None,
                                         offset_start_index=0, offset_end_index=0):
    """generate power rails from pre-existing power rails in upper/lower layer.
       the pre-existing rail information is provided as inst / pin prefix
    """
    xy=[]
    pdict=laygen.get_inst_pin_xy(None, None, gridname)
    #iname=input_rails_instname
    if not isinstance(input_rails_instname, list):
        input_rails_instname=[input_rails_instname]
    for pfix in input_rails_pin_prefix:
        sub_xy=[]
        for iname in input_rails_instname:
            for pn, p in pdict[iname].items():
                if pn.startswith(pfix):
                    sub_xy.append(p)
            xy.append(sub_xy)
    return generate_power_rails_from_rails_xy(laygen, routename_tag, layer, gridname, netnames=netnames, direction=direction,
                                              input_rails_xy=xy, generate_pin=generate_pin,
                                              overwrite_start_coord=overwrite_start_coord, overwrite_end_coord=overwrite_end_coord,
                                              overwrite_num_routes=overwrite_num_routes,
                                              overwrite_start_index=overwrite_start_index, overwrite_end_index=overwrite_end_index,
                                              offset_start_coord=offset_start_coord, offset_end_coord=offset_end_coord,
                                              offset_start_index=offset_start_index, offset_end_index=offset_end_index)

def generate_grids_from_xy(laygen, gridname_input, gridname_output, xy, xy_grid_type=None):
    """generate route grids combining a pre-existing grid and xy-array
    it will create a new array by copying the given grid and update part of entries from xy-lists
    """
    #copy original database
    gi=laygen.get_grid(gridname_input)
    bnd=deepcopy(gi.xy)
    #xgrid = deepcopy(gi.get_xgrid())
    #ygrid = deepcopy(gi.get_ygrid())
    #xwidth = deepcopy(gi.get_xwidth())
    #ywidth = deepcopy(gi.get_ywidth())
    #_viamap = gi.get_viamap()
    xgrid = deepcopy(gi.xgrid)
    ygrid = deepcopy(gi.ygrid)
    xwidth = deepcopy(gi.xwidth)
    ywidth = deepcopy(gi.ywidth)
    _viamap = gi.viamap
    vianame = list(_viamap.keys())[0] #just pickig one via; should be fixed
    #figure out routing direction
    if xy_grid_type==None:
        if abs(xy[0][0][0]-xy[0][1][0]) > abs(xy[0][0][1]-xy[0][1][1]): #aspect ratio
            xy_grid_type = 'ygrid'
        else:
            xy_grid_type = 'xgrid'
    #extract grid information from xy list
    if xy_grid_type== 'xgrid':
        xgrid=[]
        xwidth=[]
        for xy0 in xy:
            #xgrid.append(0.5 * (xy0[0][0] + xy0[1][0]))
            xgrid_new=0.5 * (xy0[0][0] + xy0[1][0])
            if xgrid_new not in xgrid:
                xgrid.append(xgrid_new)
                xwidth.append(abs(xy0[0][0] - xy0[1][0]))
        #sort
        xwidth = [x for (y, x) in sorted(zip(xgrid, xwidth))]
        xgrid.sort()
        xgrid = np.array(xgrid)
        xwidth = np.array(xwidth)
        bnd[1][0] = max(xgrid)+min(ygrid)
    if xy_grid_type== 'ygrid':
        ygrid=[]
        ywidth=[]
        for xy0 in xy:
            #ygrid.append(0.5 * (xy0[0][1] + xy0[1][1]))
            ygrid_new=0.5 * (xy0[0][1] + xy0[1][1])
            if ygrid_new not in ygrid:
                ygrid.append(ygrid_new)
                ywidth.append(abs(xy0[0][1] - xy0[1][1]))
        #sort
        ywidth = [x for (y, x) in sorted(zip(ygrid, ywidth))]
        ygrid.sort()
        ygrid=np.array(ygrid)
        ywidth=np.array(ywidth)
        bnd[1][1]=max(ygrid)+min(ygrid)
    # viamap
    viamap = {vianame: []}
    for x in range(len(xgrid)):
        for y in range(len(ygrid)):
            viamap[vianame].append([x, y])
    viamap[vianame] = np.array(viamap[vianame])
    # add grid information
    laygen.grids.add_route_grid(name=gridname_output, libname=None, xy=bnd, xgrid=xgrid, ygrid=ygrid, xwidth=xwidth,
                                ywidth=ywidth, viamap=viamap)
    #laygen.grids.display()

def generate_grids_from_inst(laygen, gridname_input, gridname_output, instname,
                           inst_pin_prefix=['VDD', 'VSS'], xy_grid_type=None):
    """generate route grids combining a pre-existing grid and inst pins
        it will create a new array by copying the given grid and update part of entries from xy coordinates of pins
    """
    if not isinstance(instname, list):
        instname=[instname]

    xy = []
    for inn in instname:
        inst = laygen.get_inst(name=inn)
        #inst.display()
        t = laygen.templates.get_template(inst.cellname, libname=inst.libname)
        xy0 = inst.xy
        for p in t.pins:
            for pfix in inst_pin_prefix:
                if p.startswith(pfix):
                    xy_new=xy0 + t.pins[p]['xy']
                    xy.append(xy_new)
        generate_grids_from_xy(laygen, gridname_input, gridname_output, xy, xy_grid_type=xy_grid_type)
    #inst = laygen.get_inst(name=instname)
    #t = laygen.templates.get_template(inst.cellname, libname=template_libname)
    #xy0 = inst.xy
    #xy = []
    #for p in t.pins:
    #    for pfix in inst_pin_prefix:
    #        if p.startswith(pfix):
    #            xy.append(xy0 + t.pins[p]['xy'])
    #generate_grids_from_xy(laygen, gridname_input, gridname_output, xy, xy_grid_type=xy_grid_type)

def generate_grids_from_template(laygen, gridname_input, gridname_output, template_name, template_libname,
                                 template_pin_prefix=['VDD', 'VSS'], xy_grid_type=None, offset=np.array([0, 0])):
    """generate route grids combining a pre-existing grid and template pins
        it will create a new array by copying the given grid and update part of entries from xy coordinates of pins
    """
    t = laygen.templates.get_template(template_name, libname=template_libname)
    xy = []
    for p in t.pins:
        for pfix in template_pin_prefix:
            if p.startswith(pfix):
                xy.append(offset+t.pins[p]['xy'])
    generate_grids_from_xy(laygen, gridname_input, gridname_output, xy, xy_grid_type=xy_grid_type)

max_stars_repo_name: rowhit/laygo

max_issues_repo_name: rowhit/laygo

max_forks_repo_name: rowhit/laygo


text:
"""
Copyright 2019 Anqi Fu, Junzi Zhang

This file is part of A2DR.

A2DR is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

A2DR is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with A2DR. If not, see <http://www.gnu.org/licenses/>.
"""

import numpy as np
import scipy as sp
import numpy.linalg as LA
import copy
import time
import scipy.sparse.linalg
import matplotlib
matplotlib.use('TKAgg')
import matplotlib.pyplot as plt

from cvxpy import *
from scipy import sparse
from scipy.optimize import nnls
from sklearn.datasets import make_sparse_spd_matrix

from a2dr import a2dr
from a2dr.proximal import *
from a2dr.tests.base_test import BaseTest

class TestPaper(BaseTest):
    """Unit tests for A2DR paper experiments."""

    def setUp(self):
        np.random.seed(1)
        self.eps_rel = 1e-8   # specify these in all examples?
        self.eps_abs = 1e-6
        self.MAX_ITER = 1000

    def test_coupled_qp(self):
        # Problem data.
        L = 4     # number of blocks
        s = 10    # number of coupling constraints
        ql = 30   # variable dimension of each subproblem QP
        pl = 50   # constraint dimension of each subproblem QP

        G_list = [np.random.randn(s,ql) for l in range(L)]
        F_list = [np.random.randn(pl,ql) for l in range(L)]
        c_list = [np.random.randn(ql) for l in range(L)]
        z_tld_list = [np.random.randn(ql) for l in range(L)]
        d_list = [F_list[l].dot(z_tld_list[l])+0.1 for l in range(L)]

        G = np.hstack(G_list)
        z_tld = np.hstack(z_tld_list)
        h = G.dot(z_tld)
        H_list = [np.random.randn(ql,ql) for l in range(L)]
        Q_list = [H_list[l].T.dot(H_list[l]) for l in range(L)]

        # Convert problem to standard form.
        def prox_qp_wrapper(l, Q_list, c_list, F_list, d_list):
            return lambda v, t: prox_qp(v, t, Q_list[l], c_list[l], F_list[l], d_list[l])
        # Use "map" method to avoid implicit overriding, which would make all the proximal operators the same
        prox_list = list(map(lambda l: prox_qp_wrapper(l, Q_list, c_list, F_list, d_list), range(L)))
        A_list = G_list
        b = h

        # Solve with DRS.
        drs_result = a2dr(prox_list, A_list, b, anderson=False, precond=True, max_iter=self.MAX_ITER)
        #drs_result = a2dr(prox_list, A_list, b, anderson=True, precond=True, max_iter=self.MAX_ITER, ada_reg=False)
        #drs_result = a2dr(prox_list, A_list, b, anderson=True, precond=True, max_iter=self.MAX_ITER, ada_reg=False, lam_accel=0)
        #drs_result = a2dr(prox_list, A_list, b, anderson=True, precond=True, max_iter=self.MAX_ITER, ada_reg=False, lam_accel=1e-12)
        print('DRS finished.')

        # Solve with A2DR.
        a2dr_result = a2dr(prox_list, A_list, b, anderson=True, precond=True, max_iter=self.MAX_ITER)
        #a2dr_result = a2dr(prox_list, A_list, b, anderson=True, precond=True, max_iter=self.MAX_ITER, lam_accel=1e-12)
        print('A2DR finished.')
        self.compare_total(drs_result, a2dr_result)

        # Check solution correctness.
        a2dr_z = a2dr_result['x_vals']
        a2dr_obj = np.sum([a2dr_z[l].dot(Q_list[l]).dot(a2dr_z[l])
                           + c_list[l].dot(a2dr_z[l]) for l in range(L)])
        a2dr_constr_vio = [np.linalg.norm(np.maximum(F_list[l].dot(a2dr_z[l])-d_list[l],0))**2
                                  for l in range(L)]
        a2dr_constr_vio += [np.linalg.norm(G.dot(np.hstack(a2dr_z))-h)**2]
        a2dr_constr_vio_val = np.sqrt(np.sum(a2dr_constr_vio))
        print('objective value of A2DR = {}'.format(a2dr_obj))
        print('constraint violation of A2DR = {}'.format(a2dr_constr_vio_val))

if __name__ == '__main__':
    tests = TestPaper()
    tests.setUp()
    tests.test_coupled_qp()

reason: import numpy,import scipy,from scipy,from cvxpy


text:
import numpy as np

"""Default values of carbon cycle parameters.

r0      : pre-industrial time-integrated airborne fraction of CO2 (yr)
rc      : change in time-integrated airborne fraction with CO2 (yr/GtC)
rt      : change in time-integrated airborne fraction with temperature (yr/K)
iirf_h  : time horizon for time integrated airborne fraction (yr)
iirf_max: maximum allowed value of iirf (yr)
a       : partition coefficient of carbon boxes
tau     : present-day decay time constants of CO2 (yr)
"""

r0       = 35.0
rc       = 0.019
rt       = 4.165
iirf_h   = 100.0
iirf_max = 97.0
a        = np.array([0.2173,0.2240,0.2824,0.2763])
tau      = np.array([1000000,394.4,36.54,4.304])

max_stars_repo_name: markperri/FAIR

max_issues_repo_name: juliaeis/FAIR

max_forks_repo_name: juliaeis/FAIR


text:
import numpy as np
import pandas as pd

class Agent:
    def __init__(self,
                 env,
                 data_path=None,
                 gamma=0.9,
                 learning_rate=0.1,
                 epsilon=.1):  # 初始化参数奖励衰减系数0.9，学习率0.1

        self.gamma = gamma
        self.learning_rate = learning_rate
        self.epsilon = epsilon
        self.time_count = env.time_count
        self.action_n = env.action_space
        self.x_count = env.x_count
        self.y_count = env.y_count

        if data_path is None:
            self.qshape = (self.x_count, self.y_count, self.time_count,
                           self.x_count, self.y_count, self.action_n)
            self.q = np.zeros(self.qshape)
            # 边界初始化
            for des_x in range(self.x_count):
                for des_y in range(self.y_count):
                    for t in range(self.time_count):
                        for x in range(self.x_count):
                            self.q[des_x, des_y, t, x, 0,
                                   0] = np.finfo(np.float32).min
                            self.q[des_x, des_y, t, x, 0,
                                   1] = np.finfo(np.float32).min
                            self.q[des_x, des_y, t, x, 0,
                                   7] = np.finfo(np.float32).min
                            self.q[des_x, des_y, t, x, self.y_count - 1,
                                   3] = np.finfo(np.float32).min
                            self.q[des_x, des_y, t, x, self.y_count - 1,
                                   4] = np.finfo(np.float32).min
                            self.q[des_x, des_y, t, x, self.y_count - 1,
                                   5] = np.finfo(np.float32).min

                        for y in range(self.y_count):
                            self.q[des_x, des_y, t, 0, y,
                                   5] = np.finfo(np.float32).min
                            self.q[des_x, des_y, t, 0, y,
                                   6] = np.finfo(np.float32).min
                            self.q[des_x, des_y, t, 0, y,
                                   7] = np.finfo(np.float32).min
                            self.q[des_x, des_y, t, self.x_count - 1, y,
                                   1] = np.finfo(np.float32).min
                            self.q[des_x, des_y, t, self.x_count - 1, y,
                                   2] = np.finfo(np.float32).min
                            self.q[des_x, des_y, t, self.x_count - 1, y,
                                   3] = np.finfo(np.float32).min
        else:
            self.q = np.load(data_path)

    def _obs2txy(self, obs):
        t = int(obs['time']) % 24
        x = obs['position'][0]
        y = obs['position'][1]
        return t, x, y

    def decide(self, obs, des_x, des_y):  # 智能体决策，epsilon贪心决策，输入参数状态
        t, x, y = self._obs2txy(obs)
        rand = np.random.uniform(0, 1)
        if rand > self.epsilon:
            # 需要洗牌，防止选idxmax时一直为最大值中最小的索引
            state_action = pd.Series(data=np.array(self.q[des_x, des_y, t, x,
                                                          y, :]))
            state_action = state_action.reindex(
                np.random.permutation(state_action.index))
            action = state_action.idxmax()
        else:
            action = np.random.randint(self.action_n)
        return action

    def learnQ(self, obs, action, reward, next_obs, done, des_x,
               des_y):  # 智能体学习，Q-learning
        t, x, y = self._obs2txy(obs)
        nt, nx, ny = self._obs2txy(next_obs)
        # print("此轮更新行动"+str(action))
        u = reward + self.gamma * self.q[des_x, des_y, nt, nx,
                                         ny].max() * (1. - done)
        td_error = u - self.q[des_x, des_y, t, x, y, action]
        self.q[des_x, des_y, t, x, y,
               action] += self.learning_rate * td_error  # 更新Q表

'''
    def learnS(self, obs, action, reward, next_obs,
               done):  # 智能体学习，SARSA-learning
        state = self._obs2state(obs)
        next_state = self._obs2state(next_obs)

        v = (self.q[des_x, des_y, next_state].sum() * self.epsilon +
             self.q[des_x, des_y, next_state].max() * (1. - self.epsilon)
             )  # 计算（next_state,next_action)Q值期望
        u = reward + self.gamma * v * (1. - done)
        td_error = u - self.q[des_x, des_y, state, action]
        self.q[des_x, des_y, state,
               action] += self.learning_rate * td_error  # 更新Q表
'''


text:
import re
import csv
import nltk
import pickle
import numpy as np
from collections import Counter
from nltk.util import trigrams, bigrams
from nltk.stem import SnowballStemmer
from nltk.sentiment import SentimentIntensityAnalyzer
from sklearn.feature_extraction.text import TfidfVectorizer
from merinjei_classification.preprocess.decorators import not_none
from merinjei_classification.preprocess.PreprocessData import PreprocessData
from merinjei_classification.preprocess.AutoCorrect import AutoCorrect
from merinjei_classification.preprocess.Lexicon import Lexicon

class PreprocessHateData(PreprocessData):

    def __init__(self, sub_directories: list, file_names: list, main_dir='data'):
        super().__init__(sub_directories, file_names, main_dir)
        self.vader_analyzer = SentimentIntensityAnalyzer()
        self.label_indx = 5
        self.txt_indx = 6
        self.corpus = None
        self.pos_features = None
        self.ngram_features = None
        self.other_features = None
        self.ngram_vectorizer = None
        self.pos_vectorizer = None
        self.lexicon = None

    def get_labels(self):
        files = self.open_files(self.paths)
        csv_readers = []
        self.corpus = []
        labels = []
        for file in files:
            csv_readers.append(csv.reader(file))

        for reader in csv_readers:
            skip_first = True
            for row in reader:
                if skip_first:
                    skip_first = False
                    continue
                label = int(row[self.label_indx])
                labels.append(label)
        return np.array(labels)

    def generate_corpus_get_labels(self):
        files = self.open_files(self.paths)
        csv_readers = []
        self.corpus = []
        labels = []
        for file in files:
            csv_readers.append(csv.reader(file))

        for reader in csv_readers:
            skip_first = True
            for row in reader:
                if skip_first:
                    skip_first = False
                    continue
                label = 1 if int(row[self.label_indx]) == 2 else 0
                labels.append(label)

                tweet = row[self.txt_indx].lower()
                self.corpus.append(tweet)

        self.close_files(files)
        return np.array(labels)

    def init_dataset(self):
        labels = self.generate_corpus_get_labels()
        additional_features = self.init_other_features()
        pos_data = self.init_pos_tags_ds()
        ngram_data = self.init_ngrams_datset()
        lexicon = self.init_lexicon_data()
        print(labels.shape)
        print(additional_features.shape)
        print(pos_data.shape)
        print(ngram_data.shape)
        print(lexicon.shape, np.sum(lexicon))
        self.dataset = np.concatenate([ngram_data, pos_data,
                                       additional_features, lexicon], axis=1)
        print("InitDS:", self.dataset.shape)
        return (self.dataset, labels)

    @not_none('corpus')
    def init_pos_tags_ds(self):
        dataset = []
        pos_vectorizer = TfidfVectorizer(
                tokenizer=None,
                lowercase=False,
                preprocessor=None,
                ngram_range=(1, 3),
                stop_words=None,
                use_idf=True,
                smooth_idf=False,
                norm=None,
                decode_error='replace',
                max_features=5000,
                min_df=5,
                max_df=0.75
        )

        for tweet in self.corpus:
            tweet = self.replace_mentions_urls(tweet)
            tokens = nltk.word_tokenize(tweet)
            tokens_tagged = nltk.pos_tag(tokens)
            # Taking only the part of speech (Word, PartOfSpeech)
            pos_tags = [pos[1] for pos in tokens_tagged if re.match('\w+',
                                                                    pos[1])]
            curr_row = ' '.join(pos_tags)
            dataset.append(curr_row)

        dataset = pos_vectorizer.fit_transform(dataset).toarray()
        vocab = {v:i for i, v in enumerate(pos_vectorizer.get_feature_names())}
        self.pos_features = vocab
        self.pos_vectorizer = pos_vectorizer
        return dataset

    def get_other_features(self, tweet):
        tweet = self.replace_mentions_urls(tweet, "URL", "MENTION", "HASHTAG")

        url_count = tweet.count("URL")
        mention_count = tweet.count('MENTION')
        hashtag_count = tweet.count("HASHTAG")

        number_of_words = len(re.split(r'[a-z]+', tweet))
        number_of_terms = len(tweet.split())
        number_of_unique_terms = len(set(tweet.split()))
        number_of_syllables = len(re.split(r'[aeiouy]', tweet))
        number_of_chars = len(re.split(r'[a-z]', tweet))
        number_of_chars_total = len(re.split(r'[a-z]|\W', tweet))

        avrg_syllables = number_of_syllables / number_of_words
        # Flesch–Kincaid grade level
        fkra_score = (0.39 * number_of_words) \
                    + ( 11.8 * number_of_syllables / number_of_words) - 15.59

        # Flesch reading ease
        fre_score = 206.835 - (1.015 * number_of_words)\
                    - (84.6 * number_of_syllables / number_of_words)

        tweet.replace('URL', '')
        tweet.replace('MENTION', '')
        tweet.replace('HASHTAG', '')
        polarity = self.vader_analyzer.polarity_scores(tweet)

        return [url_count, mention_count, hashtag_count, number_of_words,\
                number_of_terms, number_of_unique_terms,number_of_syllables,\
                number_of_chars, number_of_chars_total, avrg_syllables,\
                fkra_score, fre_score,\
                polarity['neg'], polarity['neu'], polarity['pos'],
                polarity['compound']]

    @not_none('corpus')
    def init_other_features(self):
        dataset = []
        for tweet in self.corpus:
            dataset.append(self.get_other_features(tweet))

        self.other_features = ['URLCOUNT', 'MENTIONCOUNT', 'HASHTAGCOUNT',
                               'WORDSCOUNT', 'TERMSCOUNT', 'UNIQUETERMSCOUNT',
                               'SYLLABLESCOUNT', 'CHARSCOUNT',
                               'TOTALCHARSCOUNT', 'AVRGSYLLABLESCOUNT', 'FKRA',
                               'FRE', 'POLARITYNEG', 'POLARITYNEU',
                               'POLARITYPOS', 'POLARITYCOMP']

        return np.array(dataset)

    @not_none('corpus')
    def init_ngrams_datset(self, pattern=r"\W+"):
        dataset = []

        ngram_vectorizer = TfidfVectorizer(use_idf=True,
                                           tokenizer=PreprocessHateData.tokenize,
                                           stop_words=nltk.corpus.stopwords.words('english'),
                                           decode_error='replace',
                                           min_df=5,
                                           max_df=0.75,
                                           max_features=10000,
                                           smooth_idf=False,
                                           norm=None,
                                           ngram_range=(1, 3),
                                           token_pattern=r'\b\w+\b')

        for tweet in self.corpus:
            tweet = self.replace_mentions_urls(tweet)
            # Adding the reduced sentence to the dataset(corpus)
            dataset.append(tweet)

        dataset = ngram_vectorizer.fit_transform(dataset).toarray()

        vocab = self.ngram_scores = {v:i for i, v in enumerate(ngram_vectorizer.get_feature_names())}
        self.ngram_features = {i:ngram_vectorizer.idf_[i] for i in vocab.values()}
        self.ngram_vectorizer = ngram_vectorizer

        return dataset

    @not_none('corpus')
    def init_lexicon_data(self):
        lx = Lexicon([''], ['refined_ngram_dict.csv'],
                     'merinjei_classification/data')
        lx.init_lexicon()
        self.lexicon = lx.get_lexicon()
        dataset = []

        for tweet in self.corpus:
            row = [0 for _ in range(len(self.lexicon))]
            tweet_ngrams = tweet.split() + list(trigrams(tweet.split()))\
                           + list(bigrams(tweet.split()))
            for ngram in tweet_ngrams:
                ngram = ' '.join(ngram)
                if ngram in self.lexicon:
                    # Add it to the dataset on its correct position
                    indx = list(self.lexicon.keys()).index(ngram)
                    row[indx] = self.lexicon[ngram]
            dataset.append(row)
        return np.array(dataset)

    def init_features(self):
        self.features = {
            'ngram_scores': self.ngram_scores,
            'ngram_features': self.ngram_features,
            'pos_features': self.pos_features,
            'other_features': self.other_features,
            'lexicon': self.lexicon,
            'ngram_vectorizer': self.ngram_vectorizer,
            'pos_vectorizer': self.pos_vectorizer
        }
        return self.features

    def replace_mentions_urls(self, tweet, replace_url='', replace_mention='', replace_hashtag=''):
        url_regex = re.compile(
            r'(?:http|ftp)s?://'
            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'
            r'localhost|'
            r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'
            r'(?::\d+)?'
            r'(?:/?|[/?]\S+)', re.IGNORECASE)

        mentions_regex = re.compile(r'@\w+')
        hashtag_regex = re.compile(r'#\w+')
        tweet = replace_url.join(re.split(url_regex, tweet))
        tweet = replace_mention.join(re.split(mentions_regex, tweet))
        tweet = replace_hashtag.join(re.split(hashtag_regex, tweet))
        tweet = ' '.join(re.split(r'\s+', tweet))

        return tweet

    @staticmethod
    def tokenize(tweet: str) -> list:
        stemmer = SnowballStemmer("english")
        tokens = tweet.split()
        tokens = [stemmer.stem(w) for w in tokens]
        return tokens


text:
from absl import app, flags, logging
from absl.flags import FLAGS

import tensorflow as tf
import numpy as np
from yolov3_tf2.models import (
    YoloV3, YoloV3Tiny, YoloLoss,
    yolo_anchors, yolo_anchor_masks,
    yolo_tiny_anchors, yolo_tiny_anchor_masks,
    convert_yolo_output
)
from yolov3_tf2.utils import freeze_all, create_detections, create_annotations, clear_directory
from yolov3_tf2.metrics import average_precisions, calculate_map
import yolov3_tf2.dataset as dataset
from yolov3_tf2.checkpoint_handler import BestEpochCheckpoint

flags.DEFINE_string('dataset', '', 'path to dataset')
flags.DEFINE_string('val_dataset', '', 'path to validation dataset')
flags.DEFINE_boolean('tiny', False, 'yolov3 or yolov3-tiny')
flags.DEFINE_string('weights', './checkpoints/yolov3.tf',
                    'path to weights file')
flags.DEFINE_string('classes', './data/coco.names', 'path to classes file')
flags.DEFINE_enum('transfer', 'none',
                  ['none', 'darknet', 'no_output', 'frozen', 'fine_tune'],
                  'none: Training from scratch, '
                  'darknet: Transfer darknet, '
                  'no_output: Transfer all but output, '
                  'frozen: Transfer and freeze all, '
                  'fine_tune: Transfer all and freeze darknet only')
flags.DEFINE_integer('size', 416, 'image size')
flags.DEFINE_integer('epochs', 2, 'number of epochs')
flags.DEFINE_integer('batch_size', 8, 'batch size')
flags.DEFINE_float('learning_rate', 1e-3, 'learning rate')
flags.DEFINE_boolean('augmentation', False, 'Enable image augmentation during training')
flags.DEFINE_integer('num_classes', 80, 'number of classes in the model')
flags.DEFINE_integer('weights_num_classes', None, 'specify num class for `weights` file if different, '
                                                  'useful in transfer learning with different number of classes')

def main(_argv):
    physical_devices = tf.config.experimental.list_physical_devices('GPU')
    for physical_device in physical_devices:
        tf.config.experimental.set_memory_growth(physical_device, True)

    if FLAGS.tiny:
        model = YoloV3Tiny(FLAGS.size, training=True,
                           classes=FLAGS.num_classes)
        anchors = yolo_tiny_anchors
        anchor_masks = yolo_tiny_anchor_masks
    else:
        model = YoloV3(FLAGS.size, training=True, classes=FLAGS.num_classes)
        anchors = yolo_anchors
        anchor_masks = yolo_anchor_masks

    train_dataset = dataset.load_fake_dataset()
    if FLAGS.dataset:
        train_dataset = dataset.load_tf_record(tfrecord=FLAGS.dataset, mode=tf.estimator.ModeKeys.TRAIN,
                                               class_file=FLAGS.classes, anchors=anchors,
                                               anchor_masks=anchor_masks, batch_size=FLAGS.batch_size,
                                               max_detections=FLAGS.yolo_max_boxes, size=FLAGS.size,
                                               augmentation=FLAGS.augmentation)

    val_dataset = dataset.load_fake_dataset()
    if FLAGS.val_dataset:
        val_dataset = dataset.load_tf_record(tfrecord=FLAGS.val_dataset, mode=tf.estimator.ModeKeys.EVAL,
                                             class_file=FLAGS.classes, anchors=anchors,
                                             anchor_masks=anchor_masks, batch_size=FLAGS.batch_size,
                                             max_detections=FLAGS.yolo_max_boxes, size=FLAGS.size,
                                             augmentation=False)

    # Configure the model for transfer learning
    if FLAGS.transfer == 'none':
        pass  # Nothing to do
    elif FLAGS.transfer in ['darknet', 'no_output']:
        # Darknet transfer is a special case that works
        # with incompatible number of classes

        # reset top layers
        if FLAGS.tiny:
            model_pretrained = YoloV3Tiny(
                FLAGS.size, training=True, classes=FLAGS.weights_num_classes or FLAGS.num_classes)
        else:
            model_pretrained = YoloV3(
                FLAGS.size, training=True, classes=FLAGS.weights_num_classes or FLAGS.num_classes)
        model_pretrained.load_weights(FLAGS.weights)

        if FLAGS.transfer == 'darknet':
            model.get_layer('yolo_darknet').set_weights(
                model_pretrained.get_layer('yolo_darknet').get_weights())
            freeze_all(model.get_layer('yolo_darknet'))

        elif FLAGS.transfer == 'no_output':
            for l in model.layers:
                if not l.name.startswith('yolo_output'):
                    l.set_weights(model_pretrained.get_layer(
                        l.name).get_weights())
                    freeze_all(l)

    else:
        # All other transfer require matching classes
        model.load_weights(FLAGS.weights)
        if FLAGS.transfer == 'fine_tune':
            # freeze darknet and fine tune other layers
            darknet = model.get_layer('yolo_darknet')
            freeze_all(darknet)
        elif FLAGS.transfer == 'frozen':
            # freeze everything
            freeze_all(model)

    optimizer = tf.keras.optimizers.Adam(lr=FLAGS.learning_rate)
    loss = [YoloLoss(anchors[mask], classes=FLAGS.num_classes)
            for mask in anchor_masks]

    train_log_dir = './logs/train'
    val_log_dir = './logs/valid'
    clear_directory(train_log_dir, clear_subdirectories=True)
    clear_directory(val_log_dir, clear_subdirectories=True)
    train_summary_writer = tf.summary.create_file_writer(train_log_dir)
    val_summary_writer = tf.summary.create_file_writer(val_log_dir)

    # Define checkpoint handler: track macro mAP
    ckpt_handler = BestEpochCheckpoint(model, './checkpoints/', 10,
                                                          min_delta=0.005, mode='max')

    # Eager mode is great for debugging
    # Non eager graph mode is recommended for real training
    avg_loss = tf.keras.metrics.Mean('loss', dtype=tf.float32)
    avg_val_loss = tf.keras.metrics.Mean('val_loss', dtype=tf.float32)

    # Training
    for epoch in range(1, FLAGS.epochs + 1):
        for batch, (images, labels) in enumerate(train_dataset):
            images, filenames = images
            with tf.GradientTape() as tape:
                outputs = model(images, training=True)
                regularization_loss = tf.reduce_sum(model.losses)
                pred_loss = []
                for output, label, loss_fn in zip(outputs, labels, loss):
                    pred_loss.append(loss_fn(label, output))
                total_loss = tf.reduce_sum(pred_loss) + regularization_loss

            grads = tape.gradient(total_loss, model.trainable_variables)
            optimizer.apply_gradients(
                zip(grads, model.trainable_variables))

            logging.info("{}_train_{}, {}, {}".format(
                epoch, batch, total_loss.numpy(),
                list(map(lambda x: np.sum(x.numpy()), pred_loss))))
            avg_loss.update_state(total_loss)

        with train_summary_writer.as_default():
            tf.summary.scalar('Avg_loss', avg_loss.result(), step=epoch)

        all_annotations = []
        all_detections = []

        for batch, (images, labels, int_labels) in enumerate(val_dataset):
            images, filenames = images
            outputs = model(images)
            boxes, scores, classes, valid_detections = convert_yolo_output(outputs[0], outputs[1], outputs[2],
                                                                           anchors=anchors,
                                                                           anchor_masks=anchor_masks,
                                                                           num_classes=FLAGS.num_classes,
                                                                           max_boxes=FLAGS.yolo_max_boxes,
                                                                           iou_threshold=FLAGS.yolo_iou_threshold,
                                                                           score_threshold=FLAGS.yolo_score_threshold
                                                                           )
            all_annotations = create_annotations(all_annotations, int_labels, FLAGS.num_classes)
            all_detections = create_detections(all_detections, boxes, scores, classes, valid_detections,
                                               FLAGS.num_classes)
            regularization_loss = tf.reduce_sum(model.losses)
            pred_loss = []
            for output, label, loss_fn in zip(outputs, labels, loss):
                pred_loss.append(loss_fn(label, output))
            total_loss = tf.reduce_sum(pred_loss) + regularization_loss

            logging.info("{}_val_{}, {}, {}".format(
                epoch, batch, total_loss.numpy(),
                list(map(lambda x: np.sum(x.numpy()), pred_loss))))
            avg_val_loss.update_state(total_loss)

        ap_val = average_precisions(all_detections, all_annotations, FLAGS.num_classes,
                                    FLAGS.yolo_iou_threshold)

        micro_map, macro_map = calculate_map(ap_val)
        logging.info("{}, train: {}, val: {}, micro_map: {}, macro_map:{}".format(
            epoch,
            avg_loss.result().numpy(),
            avg_val_loss.result().numpy(),
            micro_map,
            macro_map))
        print('micro_map: ', micro_map)
        print('macro_map: ', micro_map)

        with val_summary_writer.as_default():
            tf.summary.scalar('Avg_loss', avg_val_loss.result(), step=epoch)
            tf.summary.scalar('Micro_mAP', micro_map, step=epoch)
            tf.summary.scalar('Macro_mAP', macro_map, step=epoch)

        ckpt_handler.on_epoch_end(epoch=epoch, current_monitor=macro_map)

        avg_loss.reset_states()
        avg_val_loss.reset_states()

if __name__ == '__main__':
    try:
        app.run(main)
    except SystemExit:
        pass


text:
#!/usr/bin/python

import optparse
from matplotlib import pyplot as plt
import numpy as np
import sys
import math

parser = optparse.OptionParser(description='Graph combined electron data', usage='usage: %prog filename')
(options, args) = parser.parse_args()

'''Fail if command line arguments are invalid, see `README.md`'''
try:
	filename = args[0]
except IndexError:
	print 'No file specified.'
	sys.exit()

ref = []
times = []
distances = []
values = []
modeIsLinear = False
with open(filename) as file:
	for line in file.readlines():
		if modeIsLinear:
			if line.startswith('0000'):
				ref = [float(i) for i in line.split(',')[1:]]
			else:
				try:
					line = [float(i) for i in line.split(',')]
				except ValueError:
					break
				times.append(line[0])
				values.append(line[1:])
		else:
			if line.startswith('spherical'):
				continue
			elif line.startswith('linear'):
				modeIsLinear = True
				distances = [float(i) for i in line.split(',')[1:]]

# values looks like
# [    z: 0.217, 0.434, ...
#   ...
# 	4000: [... ,],
# 	4200: [... ,],
# 	...
# ]

# figure out where the Al sheet ends
i = 0
al = []
for v in ref:
	if v/max(ref) > 0.95:
		al.append(i)
	i += 1
firstAl = min(al)
lastAl = max(al)

res = []
for z in distances:
	if distances.index(z) > 0:
		res.append(z - distances[distances.index(z)-1])
res = sum(res)/len(res)

# arbitrarily avoid integrating within n a_B of the Al sheet
n = 10
startIndex = firstAl-int(n/res)
endIndex = lastAl+int(n/res)
print filename
print distances[startIndex], distances[endIndex]

seeBack = [sum(ref[:startIndex])]
seeForward = [sum(ref[endIndex:])]
seeCombined = [seeBack[0] + seeForward[0]]
for v in values:
	seeBack.append(sum(v[:startIndex]))
	seeForward.append(sum(v[endIndex:]))
	seeCombined.append(sum(v[:startIndex]) + sum(v[endIndex:]))

times = [0]+times

# plt.scatter(times, seeBack, color='#0000ff')
# plt.scatter(times, seeForward, color='#ff0000')
plt.scatter(times, seeCombined, color='#a000a0')
# plt.plot(times, seeBack, label='before sheet', color='#0000ff')
# plt.plot(times, seeForward, label='after sheet', color='#ff0000')
# plt.plot(times, seeCombined, label='combined', color='#a000a0')
plt.plot(times, seeCombined, color='#a000a0', label='max: %.2e'%(max(seeCombined)))
plt.axis([0, max(times), 0, 4.0]) # max(seeCombined)*1.1
plt.xlabel('time')
plt.ylabel('total electrons %i a_B outside the Al sheet'%(n))
plt.title(filename.split('.')[0])
# plt.title('Secondary Electron Emission')
plt.legend(loc='best')
# plt.show()
plt.savefig(filename.split('.')[0]+'.png')
plt.clf()

max_stars_repo_name: quells/ElectronDensityIntegration

max_issues_repo_name: quells/ElectronDensityIntegration

max_forks_repo_name: quells/ElectronDensityIntegration