text:
###############################################################################
### TIMELINE SHOWING QC COMMENTS
###############################################################################

###  Description
# The function 'qc_comment_timeline' creates a timeline showing when the
# various qc comments occur for a given parameter.

#------------------------------------------------------------------------------
### Import packages
import matplotlib.pyplot as plt
import os
import numpy as np
import string
import pandas as pd
import math

#------------------------------------------------------------------------------
### Set variables

# Figure sizes
FIG_WIDTH = 9
SUBPLOT_HEIGHT = 1.3

# Title fontsize:
TITLE_FONTSIZE = 9

# Specify the plot marker color
MARKER_COL = '#F5793A'

#------------------------------------------------------------------------------
### Functions

def qc_comment_timeline(parameter, vocab, df, output_dir):

	# Create list of unique QC comments for the given parameter (remove nan,
	# and split if more than one comment in a row)
	qc_comment_header = vocab[parameter]['col_header_name'] + ' QC Comment'
	unique_comments_raw = np.unique(list(df[qc_comment_header]))
	unique_comments_noNan = [x for x in unique_comments_raw if x != 'nan']
	comment_list = []
	for comment in unique_comments_noNan:
		if ';' in comment:
			splitted = comment.split(';')
			for split in splitted:
				comment_list.append(split.lstrip())
		else:
			comment_list.append(comment)
	comment_list = np.unique(comment_list)

	# Set up the figure, with one subplot per QC comment
	n_plots = len(comment_list)
	fig_height = SUBPLOT_HEIGHT * n_plots
	fig, axs = plt.subplots(nrows=n_plots, ncols=1, sharex=True, sharey=True,
		figsize=(FIG_WIDTH,fig_height))

	# Adjust height between subplots to make space for subplot titles
	plt.subplots_adjust(hspace=0.5)

	# Create a dummy value to use in subplot then an error occur without any
	# parameter value (e.g. for errors related to missing values). This dummy
	# value needs to be within the subplot y ranges (subplots share an y-axis),
	# so we can use the parameter mean value for all rows with a QC comment.
	df_comment = df[df[qc_comment_header].notnull()]
	dummy_value = np.nanmean(df_comment[vocab[parameter]['col_header_name']])

	# Add subplots to figure in a loop
	subplot_count = 0
	for comment in comment_list:

		# Set the axes for the plot
		ax = axs[subplot_count]

		# Extract data frame rows where QC comment field contains the 'comment'
		df_edit = df[df[qc_comment_header].str.contains(comment, na=False)]

		# Add subplot
		ax.plot(df_edit['Date/Time'],
			df_edit[vocab[parameter]['col_header_name']], marker='v',
			linestyle='None', c=MARKER_COL)

		# The subplot above will not show when errors related to missing values
		# occur, since there is no value to plot. Therefore, plot the dummy
		# value instead for such missing data errors. Create a list contaning
		# the dummy value in locations where the parameter value is missing in
		# the data frame - otherwise set this list value to None. Combine the
		# 'Date/Time' and dummy list into a dummy dataframe and plot it in the
		# current subplot.
		dummy_list = []
		value_list = list(df_edit[vocab[parameter]['col_header_name']])
		for value in value_list:
			if math.isnan(value):
				dummy_list.append(dummy_value)
			else:
				dummy_list.append(None)
		dummy_df = pd.DataFrame()
		dummy_df['Date/Time']  = list(df_edit['Date/Time'])
		dummy_df['dummy_values'] = dummy_list
		ax.plot(dummy_df['Date/Time'], dummy_df['dummy_values'],
			marker='v', linestyle='None', c='blue')

		# Add grid and subplot title
		ax.grid(True)
		title = '{letter})     \'{comment}\''.format(
			letter=string.ascii_lowercase[subplot_count], comment=comment)
		ax.set_title(title, loc='left', fontsize=TITLE_FONTSIZE,
			fontweight='bold')

		# Increase subplot counter
		subplot_count += 1
		if subplot_count == n_plots:
			break

	# Add y and x label
	fig.autofmt_xdate()
	fig.text(0.05, 0.5, vocab[parameter]['fig_label_name_python'],
		verticalalignment='center', rotation='vertical')

	# Save figure to file and close figure
	filename = parameter + '_qc_comment_timeline.png'
	filepath = os.path.join(output_dir, filename)
	plt.savefig(filepath, bbox_inches='tight')
	plt.close()

max_stars_repo_name: BjerknesClimateDataCentre/KPI

max_issues_repo_name: BjerknesClimateDataCentre/KPI


text:
import tensorflow as tf
import numpy as np

from PIL import Image
from PIL import ImageDraw

def age_gender_postprocessing(endnodes, device_pre_post_layers, **kwargs):
    age_predictions, gender_predictions = endnodes

    is_male = gender_predictions >= 0.6

    # Softmax is done outside of net due to HW issue when vector_sze % 8 != 0
    age_predictions = tf.nn.softmax(age_predictions)

    # Find 2 highest ranking ages per image
    indices = tf.argsort(age_predictions, direction='DESCENDING')[:, :2]
    batch_indices = tf.cast(tf.reshape(tf.range(0, tf.size(indices) / 2, 0.5), (tf.shape(indices)[0], 2)), tf.int32)
    indices_for_gather = tf.stack([batch_indices, indices], axis=-1)

    top_age_probs = tf.gather_nd(age_predictions, indices_for_gather)

    top_age_probs_sum = tf.reshape(tf.reduce_sum(top_age_probs, axis=1), (-1, 1))
    # Weighted average of age probability
    norm_preds = top_age_probs / top_age_probs_sum

    # Multiply age (index) by its weights average
    res_age = tf.reduce_sum(tf.cast(indices, tf.float32) * norm_preds, axis=1)
    res_age += 1
    return {'age': res_age, 'is_male': is_male}

def visualize_age_gender_result(logits, img, **kwargs):
    gender = 'Male' if logits['is_male'][0] else 'Female'
    img_orig = Image.fromarray(img[0])
    ImageDraw.Draw(img_orig).text((0, 0), 'Age: {:.2f}\t Gender: {}'.format(logits['age'][0], gender), (255, 0, 0))
    return np.array(img_orig, np.uint8)

ext: py

reason: import numpy


text:
import numpy as np
import matplotlib.pyplot as plt

from pytf import FilterBank
from ..algorithm.pac_ import (pad, mrpad, polar, pac_mvl, pac_hr, pac_mi)
from ..algorithm.blob_ import (detect_blob)
from ..utilities.parallel import (Parallel, ParallelDummy)
from ..viz.pac_plots import (_plot_comodulogram)
# Authors : David C.C. Lu <davidlu89@gmail.com>
#
# License : BSD (3-clause)

func = dict([
    ('mi', mrpad),
    ('hr', mrpad),
    ('mvl', polar)
])
pacfunc = dict([
    ('mi', pac_mi),
    ('hr', pac_hr),
    ('mvl', pac_mvl)
])
class PhaseAmplitudeCoupling(object):
    """ This class is for computing phase-amplitude coupling (PAC).

    Parameters:
    -----------
    nch: int
        The number of channels of the processing signal.

    nsamp: int
        The sample size of the processing signal.

    freq_phase: ndarray
        An array of frequency bands for phase signals.

    freq_amp: ndarray
        An array of frequency bands for amplitude signals.

    sample_rate: int
        Sampling of the input signal.

    nprocs: int
        Number of processes to enable for multiprocessing.

    pac: str
        The PAC method specified. Default is 'mi'.

    compression_factor:

    kwargs:

    """
    def __init__(self, nch=1, nsamp=2**11, binsize=2**14, freq_phase=None, freq_amp=None, sample_rate=None, \
                    nprocs=1, pac='mi', mprocs=False, compressed=True, **kwargs):

        # Pre-defined parameters
        _comp_threshold = 15
        _overlap_factor = 0.5

        # Phase-Amplitude Coupling Properties
        self._method = pac

        self._freq_phase = freq_phase if isinstance(freq_phase, np.ndarray) else np.asarray(freq_phase)
        self._freq_amp = freq_amp if isinstance(freq_amp, np.ndarray) else np.asarray(freq_amp)
        self._sample_rate = sample_rate
        self.fpsize = self.freq_phase.shape[0]
        self.fasize = self.freq_amp.shape[0]

        # Signal Properties
        _comp_ratio = int(np.floor(self.sample_rate / self.freq_amp[-1,-1] / 2))
        self._compression_factor = _comp_ratio if _comp_ratio < _comp_threshold else _comp_threshold
        self._compression_factor = self._compression_factor if compressed else 1

        self._nch = nch
        self._nsamp = nsamp
        self._nsamp_ = self.nsamp // self.compression_factor

        # Overlap-Window Parameters
        self._binsize = binsize
        self._nwin = int((self._nsamp / self._binsize) / _overlap_factor + 1)

        # Initialize filter bank parameters
        if self.freq_phase is not None or self.freq_amp is not None:
            self._order = self._binsize // 4
            self._los = FilterBank(
                    nch = self.nch,
                    nsamp = self.nsamp,
                    binsize = self._binsize,
                    order = self._order,
                    freq_bands = self.freq_phase,
                    sample_rate = self.sample_rate,
                    decimate_by = self.compression_factor, hilbert=True, nprocs=1
                )

            self._his = FilterBank(
                    nch = self.nch,
                    nsamp = self.nsamp,
                    binsize = self._binsize,
                    order = self._order,
                    freq_bands = self.freq_amp,
                    sample_rate = self.sample_rate,
                    decimate_by = self.compression_factor, hilbert=True, nprocs=1
                )

        # Initialize PAC
        self._nprocs = nprocs
        self._mprocs = True if self.nprocs > 1 else mprocs
        self._pfunc = Parallel(
                        self.get_pac, nprocs=self.nprocs, axis=1, flag=0,
                        ins_shape = [(self.nch, self.fpsize, self.nsamp_),
                                     (self.nch, self.fasize, self.nsamp_)],
                        out_shape = (self.nch, self.fpsize, self.fasize),
                        ins_dtype = [np.float32, np.float32],
                        out_dtype = np.float32,
                        method = self.method,
                        **kwargs
                    ) if self.mprocs else ParallelDummy(self.get_pac, method=self.method, **kwargs)

    def kill(self, opt=None): # kill the multiprocess
        """ Killing all the multiprocessing processes.
        """
        self._pfunc.kill(opt=opt)

    def comodulogram(self, x=None, x_lo=None, x_hi=None, **kwargs):
        """ Compute vectorized PAC. i.e., the computed PAC has the shape of (nch, nlo, nhi).

        Parameters:
        -----------
        x: ndarray (nch, nsamp)
            Raw input signal.

        x_lo: ndarray (nch, nfreq, nsamp)
            Lowpass filtered input signal.

        x_hi: ndarray (nch, nfreq, nsamp)
            Highpass filtered input signal.

        kwargs: various
            The key-word arguments for 'mrpad' or 'polar'. See '.algorithm.pac_.py'

        Return:
        -------
        The computed PAC.
        """
        self._xlo = x_lo if x_lo is not None else self.los.analysis(x, window='hanning')
        self._xhi = x_hi if x_hi is not None else self.his.analysis(x, window='hanning')

        self._comod = self._pfunc.result(np.angle(self._xlo), np.abs(self._xhi))
        return self._comod

    def plot_comodulogram(self, ch=None, axs=None, figsize=None, cbar=False,
                                title=None, label=False, xlabel=False, ylabel=False,
                                fontsize={'ticks': 15, 'axis': 15, 'title': 20}, **kwargs):
        """
        Plot comodulogram.

        Parameters:
        -----------
        ch: int
            Select the channel for plotting the comodulogram. Default: Select all channels.

        axs: ndarray
            Provide the Matplotlib Axes class to plot on. Default: Create a new figure and axes.

        figsize: tuple
            Specify the figure size.

        label: bool
            Label the plots. Default is False.

        Returns:
        --------
        The matplotlib figure object.
        """
        _comod = self._comod[ch,:,:][np.newaxis,:,:] if ch is not None else self._comod

        nch, nlo, nhi = _comod.shape

        # Build Figures
        figsize = (4 * nch, 5) if figsize is None else figsize
        if axs is None:
            fig, axs = plt.subplots(1, nch, figsize=figsize)
        else:
            fig = axs[0].figure

        self._axs = np.array(axs).ravel()
        self._fig = fig

        if not isinstance(title, list):
            title = list([title]) * len(self._axs)

        xlabel = True if label else xlabel
        ylabel = True if label else ylabel

        for (i,), ax in np.ndenumerate(self._axs):
            cbar_ = True if i==len(self._axs)-1 and cbar else False
            ylabel_ = True if i==0 and ylabel else False
            _plot_comodulogram(_comod[i,:,:], axs=ax, cbar=cbar_, title=title[i],\
                                xlabel=xlabel, ylabel=ylabel_, xaxis=self._los.center_freqs.squeeze(), yaxis=self._his.center_freqs.squeeze(),\
                                vmin=np.min(_comod), vmax=np.max(_comod), fontsize=fontsize, **kwargs)
        return self._fig

    def plot_pad(self, ch=None, axs=None, figsize=None, colors=None, nbins=10):
        """
        Plot PAD.
        TODO: Need more work on efficiency, as well as various plotting types.

        Parameters:
        -----------
        ch: int
            Select the channel for plotting the comodulogram. Default: Select all channels.

        axs: ndarray
            Provide the Matplotlib Axes class to plot on. Default: Create a new figure and axes.

        figsize: tuple
            Specify the figure size.

        colors:
            Specify the color of the bar.

        kwargs

        Returns:
        --------
        The matplotlib figure object.
        """
        _lo = self._xlo[ch,:,:][np.newaxis,:,:] if ch is not None else self._xlo
        _hi = self._xhi[ch,:,:][np.newaxis,:,:] if ch is not None else self._xhi

        xlo = _lo.mean(axis=1)[:,np.newaxis,:]
        xhi = _hi.mean(axis=1)[:,np.newaxis,:]
        pd = self._pfunc.result(np.angle(xlo), np.abs(xhi), nbins=nbins)

        nch, _ = pd.shape
        bin_centers = np.linspace(-np.pi, np.pi-np.pi/50, nbins+1) + np.pi/10

        if axs is None:
            fig, axs = plt.subplots(1, nch, figsize=figsize)
        else:
            fig = axs[0].figure

        self._axs = np.array(axs).ravel()
        self._fig = fig
        for (i,), ax in np.ndenumerate(self._axs):
            ax.bar(bin_centers[:-1], pd[i,:], width=.5)

        return self._fig

    @staticmethod
    def find_pac_blob(comod, threshold=.5, vmin=0, vmax=1):
        return detect_blob(comod, threshold=threshold, vmin=vmin, vmax=vmax)

    @staticmethod
    def get_pac(ang, amp, method='mi', **kwargs):
        """
        Compute PAC from phases and amplitudes of the signal.

        Parameters:
        -----------
        ang: ndarray
            The instantaneous phases of the given signal.

        amp: ndarray
            The instantaneous amplitudes of the given signal.

        method: str
            The PAC method to used. Default: 'mi'.

        kwargs: various
            The key-word arguments for 'mrpad' or 'polar'. See '.algorithm.pac_.py'

        Returns:
        --------
        The computed PAC/comodulogram
        """
        return pacfunc[method](func[method](ang, amp, **kwargs))

    # ===================================
    # Define Setters Parameters
    # ===================================
    @property
    def los(self):
        return self._los

    @property
    def his(self):
        return self._his

    @property
    def freq_phase(self):
        return self._freq_phase

    @property
    def freq_amp(self):
        return self._freq_amp

    @property
    def sample_rate(self):
        return self._sample_rate

    @property
    def method(self):
        return self._method

    @property
    def nprocs(self):
        return self._nprocs

    @property
    def unit(self):
        return self._unit

    @property
    def mprocs(self):
        return self._mprocs

    @property
    def nch(self):
        return self._nch

    @property
    def nsamp(self):
        return self._nsamp

    @property
    def nsamp_(self):
        return self._nsamp_

    @property
    def compression_factor(self):
        return self._compression_factor

max_issues_repo_name: SachsLab/mspacman

max_forks_repo_name: SachsLab/mspacman


text:
# This program is the "fixed" version of broken_matrix_ker.py

# This is to be used for an exercise where this code is translated to
# a pure CUDA-C version.

import pycuda.autoinit
import pycuda.driver as drv
from pycuda import gpuarray
from pycuda.compiler import SourceModule
import numpy as np

ker = SourceModule('''
// row-column dot-product for matrix multiplication
__device__ float rowcol_dot(float *matrix_a, float *matrix_b, int row, int col, int N)
{
	float val = 0;

	for (int k=0; k < N; k++)
	{
        val += matrix_a[ row*N + k ] * matrix_b[ col + k*N];
	}

	return(val);

}

// matrix multiplication kernel that is parallelized over row/column tuples.
__global__ void matrix_mult_ker(float * matrix_a, float * matrix_b, float * output_matrix, int N)
{

    int row = blockIdx.x*blockDim.x + threadIdx.x;
    int col = blockIdx.y*blockDim.y + threadIdx.y;

	output_matrix[col + row*N] = rowcol_dot(matrix_a, matrix_b, row, col, N);

}
''')

matrix_ker = ker.get_function('matrix_mult_ker')

test_a = np.float32([range(1,5)] * 4)
test_b = np.float32([range(14,10, -1)]*4 )

output_mat = np.matmul(test_a, test_b)

test_a_gpu = gpuarray.to_gpu(test_a)
test_b_gpu = gpuarray.to_gpu(test_b)
output_mat_gpu = gpuarray.empty_like(test_a_gpu)

matrix_ker(test_a_gpu, test_b_gpu, output_mat_gpu, np.int32(4), block=(2,2,1), grid=(2,2,1))

assert(np.allclose(output_mat_gpu.get(), output_mat) )

reason: import numpy,import pycuda,from pycuda


text:
# from dxl.learn.function.crop import random_crop_offset, random_crop, boundary_crop, align_crop, shape_as_list
import tensorflow as tf
# import numpy as np
import itertools
# import tables
from typing import Iterable
# import math

# class DataIterFactory:
#     class _DataGen(object):
#         def __init__(self, data_gen):
#             self.data_gen = data_gen
#
#         def __call__(self):
#             return next(self.data_gen)
#
#     def _gen(self):
#         for i in itertools.count(0):
#             try:
#                 if self.x_data[i - self.batch_size:i].shape[0] == self.batch_size:
#                     yield (self.x_data[i], self.y_data[i])
#                 else:
#                     break
#             except IndexError:
#                 print("Run out of data...")
#
#     def __init__(self, x_data, y_data, batch_size, num_epochs=1):
#         self.x_data = x_data
#         self.y_data = y_data
#         self.batch_size = batch_size
#         self.num_epochs = num_epochs
#         gen = self._gen
#         data_gen = self._DataGen(data_gen=gen)
#         self.next_batch = (tf.data.Dataset
#                            .from_generator(generator=data_gen,
#                                            output_types=(tf.int64, tf.int64),  #TODO get data type out of here
#                                            output_shapes=(self.x_data_shape,
#                                                           self.y_data_shape))
#                            .batch(self.batch_size)
#                            .repeat(self.num_epochs)
#                            .make_one_shot_iterator()
#                            .get_next())
#
#     @property
#     def x(self):
#         return tf.reshape(self.next_batch[0], [self.batch_size, 256, 256, 3])
#
#     @property
#     def y(self):
#         return tf.reshape(self.next_batch[1], [self.batch_size, 256, 256, 3])
#
#     @property
#     def x_data_shape(self):
#         return [self.x_data.shape[1], self.x_data.shape[2], self.x_data.shape[3]]
#
#     @property
#     def y_data_shape(self):
#         return [self.y_data.shape[1], self.y_data.shape[2], self.y_data.shape[3]]

    # def __call__(self):
    #     gen = self._gen
    #     data_gen = self._DataGen(data_gen=gen)
    #
    #     return
    #
    #     # iter_train = data_set_train.make_one_shot_iterator()
    #     # return iter_train

# class DataGen:
#     """
#     A data generator for phantom and sinogram dataset.
#     g = DataGen(file)
#
#     with tf.Session() as sess:
#         sess.run(g.next_batch)
#     """
#     def __init__(self, source:Iterable[tables.array.Array], batch_size=32, buffer_size=1):
#         self.source = source
#         self.batch_size = batch_size
#         self.next_batch = (
#             tf.data.Dataset
#             .from_generator(self._gen,
#                             (tf.int64, tf.int64),
#                             (tf.TensorShape(self.output_shape),
#                              tf.TensorShape(self.output_shape))
#                             .shuffle(buffer_size=buffer_size)
#                             .batch(batch_size)
#                             .make_one_shot_iterator()
#                             .get_next()
#
#     def _gen(self):
#         for i in itertools.count(0):
#             try:
#                 tmp = []
#                 for s in self.source[i]:
#                     tmp.append(s)
#                 yield tmp
#
#
#                 #
#                 #
#                 # yield (self._file.root.phantom[i][0],
#                 #        self._file.root.phantom[i][1],
#                 #        self._file.root.phantom[i][2],
#                 #        self._file.root.phantom[i][3])
#             except IndexError:
#                 break
#
#     @property
#     def source_shape(self):
#         return self.source[0].shape[1:]
#
#     @property
#     def output_shape(self):
#         tmp = list(self.source_shape)
#         return tmp.insert(0, self.batch_size)
#
#     @property
#     def x(self):
#         return tf.reshape(self.next_batch[0], self.output_shape)
#
#     @property
#     def y(self):
#         return tf.reshape(self.next_batch[1], self.output_shape)

# def show():
#     pass
#
#
# def psnr(inference, label, pix_max=255.0, idx=0):
#     mse = np.mean((rescale_single(inference[idx]) - rescale_single(label[idx])) ** 2)
#     if mse == 0:
#         return 100
#     return 20 * math.log10(pix_max / math.sqrt(mse))
#
#
# def rescale_single(inputs, bin_size=255):
#     window = inputs.max() - inputs.min()
#     scale_rate = bin_size / window
#     return inputs*scale_rate - inputs.min()*scale_rate
#
#
# def rescale_batch(images):
#     return np.array([rescale_single(img) for img in images])

max_stars_repo_name: Tsinglung-Tseng/hotpot

max_issues_repo_name: Tsinglung-Tseng/hotpot

max_forks_repo_name: Tsinglung-Tseng/hotpot


text:
import numpy as np
import pandas as pd
import xarray as xr

from sklearn.base import BaseEstimator,TransformerMixin

from .utils import events_to_xr_dim

class EpochTraceReducer(BaseEstimator,TransformerMixin):
    """Take event-aligned samples of traces from a population of neurons.

    Traces are sampled relative to the event time. There is no enforced
    constraint that the times of events or sample_times relative to the events
    need to align to trace sample times. Rather, samples are interpolated from
    the values in the traces DataFrame.

    Parameters
    ----------
    traces : pandas DataFrame with 'time' as the index and neuron IDs in columns
        The traces that will be sampled from when the transform method is called
    func : function
        Function that will be applied to trace samples within epochs

    Notes
    -----

    This estimator is stateless (besides constructor parameters), the
    fit method does nothing but is useful when used in a pipeline.
    """
    def __init__(self, traces, func):
        self.traces = traces
        self.func = func

    def fit(self, X, y=None):
        """Do nothing and return the estimator unchanged

        This method is here to implement the scikit-learn API and work in
        scikit-learn pipelines.

        Parameters
        ----------
        X : array-like

        Returns
        -------
        self

        """

        return self

    def transform(self, X):
        """Reduce traces around each event

        Parameters
        ----------
        X : pandas.DataFrame with a columns ['time','duration']

        Returns
        -------
        Xt : xarray.DataArray with columns ['event','neuron']
        """

        # define a local function that will extract traces around each event
        def extractor(ev):
            window = ev['time'], ev['time'] + ev['duration']
            mask = (
                (self.traces.index >= ev['time'])
                & (self.traces.index < (ev['time'] + ev['duration']))
                )
            return (
                self.traces[mask]
                .apply(self.func,axis=0)
                .to_xarray()
                .rename({'index':'neuron'})
            )

        # do the extraction
        tensor = [extractor(ev) for _,ev in X.iterrows()]
        concat_dim = events_to_xr_dim(X)

        # concatenate the DataArrays into a single DataArray
        return xr.concat(tensor,dim=concat_dim)

max_stars_repo_name: NileGraddis/neuroglia

max_issues_repo_name: j-friedrich/neuroglia

max_forks_repo_name: j-friedrich/neuroglia


text:
#!/usr/bin/env python
# $ -pe multicores 16
# $ -q mc_long
# $ -N xqml_sims
# $ -l h_rss=3500M
# $ -l sps=1
# $ -R y
# $ -j y
# $ -V
# $ -o /sps/planck/Users/tristram/Soft/Planck/NPIPE/output
# $ -e /sps/planck/Users/tristram/Soft/Planck/NPIPE/output
from numpy import *
import sys
import os

sys.path.insert(0, "/sps/planck/Users/tristram/Soft/LAL/libs/python")
sys.path.insert(0, "/sps/planck/Users/tristram/Soft/lib/python2.7/site-packages")
os.environ["OMP_NUM_THREADS"] = (
    os.environ["NSLOTS"] if os.environ.has_key("NSLOTS") else "16"
)

from healpy import *
from astropy.io import fits

# from planck import *
import xqml
import npipe

nside = 16
lmax = 45

bsimu = 0
esimu = 100

# datatype = "FFP10"; esimu=300
datatype = "NPIPE"
esimu = 100

galcut = 10

freq = [70, 100, 143, 217]

# beam
bl = npipe.cosbeam(nside)

clth = read_cl(
    "/sps/planck/Users/tristram/Models/planck_base_planck_2018_TTTEEElowllowE.fits"
)

if datatype == "FFP10":
    DBdir = (
        "/sps/planck/Users/tristram/Planck/NPIPE/Simu_%s_mask52_skymodel_ns512"
        % datatype
    )
    MAPdir = "%s/cleanmaps/ring" % (DBdir)
else:
    DBdir = (
        "/sps/planck/Users/tristram/Planck/NPIPE/Simu_%s_mask52_sync_ns512" % datatype
    )
    COVdir = "/sps/planck/Users/tristram/Planck/NPIPE/npipe6v19/lowres"
    MAPdir = "%s/cleanmaps" % (DBdir)

# CLdir  = "%s/xqml_%dpc" % (DBdir,galcut)
# os.system( 'mkdir '+CLdir)

nmaps = len(freq)
ntags = 6

# Masks
MASKFILE = (
    "/sps/planck/Users/tristram/Planck/Mask/lowl/mask_PR3_lowl_%dpc_ns512.fits"
    % (galcut)
)
mask = ud_grade(read_map(MASKFILE, verbose=False), nside_out=nside) > 0.5

print(MASKFILE)

# construct QML
print("Construct QML")
ellbins = arange(2, lmax + 2)
cross = xqml.xQML(
    mask, ellbins, clth, bell=bl, lmax=lmax, temp=False, polar=True, corr=False
)
deltaN = 1e-16

allcl = zeros((esimu - bsimu, len(freq), len(freq), 6, lmax + 1))
for f1 in range(len(freq)):
    N1 = (
        npipe.readcov(
            "%s/npipe6v19_ncm_ns0016_smoothed_%03d_bin.dat" % (COVdir, freq[f1]), mask
        )
        + identity(cross.npix * cross.nstokes) * deltaN
    )
    for f2 in range(f1, len(freq)):
        print("Compute xQML (%sx%s)" % (freq[f1], freq[f2]))
        N2 = (
            npipe.readcov(
                "%s/npipe6v19_ncm_ns0016_smoothed_%03d_bin.dat" % (COVdir, freq[f2]),
                mask,
            )
            + identity(cross.npix * cross.nstokes) * deltaN
        )
        cross.construct_esti(N1, N2)

        for n in range(bsimu, esimu):
            print("Simu %d" % n)

            if datatype == "NPIPE":
                simuname = "npipe6v19_map_mc_%04d" % n
                mapfile1 = "npipe6v19_%03d_map_%04d" % (freq[f1], n)
                mapfile2 = "npipe6v19_%03d_map_%04d" % (freq[f2], n)
            if datatype == "FFP10":
                simuname = "full_map_mc_%05d" % n
                mapfile1 = "%03d_%s" % (freq[f1], simuname)
                mapfile2 = "%03d_%s" % (freq[f2], simuname)

            m1 = npipe.smooth_and_degrade(
                "%s/%s_clean1.fits" % (MAPdir, mapfile1), bl, nside
            )
            m2 = npipe.smooth_and_degrade(
                "%s/%s_clean2.fits" % (MAPdir, mapfile2), bl, nside
            )
            clA = cross.get_spectra(m1, m2)

            m1 = npipe.smooth_and_degrade(
                "%s/%s_clean2.fits" % (MAPdir, mapfile1), bl, nside
            )
            m2 = npipe.smooth_and_degrade(
                "%s/%s_clean1.fits" % (MAPdir, mapfile2), bl, nside
            )
            clB = cross.get_spectra(m1, m2)

            allcl[n, f1, f2, 1:3, 2:] = (clA + clB) / 2.0
            allcl[n, f2, f1, 1:3, 2:] = (clA + clB) / 2.0

fits.writeto(
    "%s/allcl_npipe6v19_xqml_%dpc.fits" % (DBdir, galcut), allcl, overwrite=True
)

max_stars_repo_name: planck-npipe/toast-npipe

max_issues_repo_name: planck-npipe/toast-npipe

max_forks_repo_name: planck-npipe/toast-npipe

reason: from numpy,from astropy


text:
"""Phishing.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/126__574uNCBltWIvSvBxoB--lqW9YS2J
"""

from google.colab import drive
drive.mount('/content/drive')

# Commented out IPython magic to ensure Python compatibility.
# Importing necessary libraries

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
# %matplotlib inline

fishing = pd.read_csv("drive/MyDrive/Dataset/Phishing.csv")
fishing.head(5)

fishing = fishing.drop(['id'], axis = 1)

fishing.shape

fishing.isnull().sum()

# This dataset contains 48 features extracted from 5000 phishing webpages and 5000 legitimate webpages.

fishing.describe()

# Classes to predict from.

fishing['CLASS_LABEL'].value_counts()

# Finding more from the data

fishing_class = fishing.groupby('CLASS_LABEL')
fishing_class['NoHttps'].value_counts()

fishing_class['UrlLength'].mean()

fishing_class['NumPercent'].mean()

fishing_class['NumAmpersand'].mean()

fishing_class['NumHash'].mean()

fishing_class['IpAddress'].value_counts()

plt.figure(figsize=(30, 30))
sns.heatmap(fishing.corr(),annot=True,cmap='viridis',linewidths=.5)

fishing.columns

subset_fishing = fishing[['NumDots', 'PathLevel', 'NumDash', 'NumSensitiveWords', 'PctExtHyperlinks',
       'PctExtResourceUrls', 'InsecureForms', 'PctNullSelfRedirectHyperlinks', 'FrequentDomainNameMismatch',
       'SubmitInfoToEmail', 'IframeOrFrame', 'CLASS_LABEL']]

subset_fishing.head()

subset_fishing.shape

y = subset_fishing['CLASS_LABEL']
X = subset_fishing.drop(['CLASS_LABEL'], axis = 1)

from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.ensemble import RandomForestClassifier

Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, test_size=0.2, random_state = 42)

random_model = RandomForestClassifier(n_estimators=250, n_jobs = -1)

#Fit
random_model.fit(Xtrain, ytrain)

y_pred = random_model.predict(Xtest)

#Checking the accuracy
random_model_accuracy = round(random_model.score(Xtrain, ytrain)*100,2)
print(round(random_model_accuracy, 2), '%')

random_model_accuracy1 = round(random_model.score(Xtest, ytest)*100,2)
print(round(random_model_accuracy1, 2), '%')

# Save the trained model as a pickle string.
import pickle

saved_model = pickle.dump(random_model, open('drive/MyDrive/Dataset/Models/Phishing.pickle','wb'))


text:
#!/usr/bin/env python3

from pyrimidine import *
import numpy as np
from pyrimidine.benchmarks.special import *

# generate a knapsack problem randomly

n=10
_evaluate = schaffer

class _Individual(BaseEPIndividual):
    element_class = FloatChromosome, FloatChromosome

    def decode(self):
        return self.chromosomes[0]

    def _fitness(self):
        return - _evaluate(self.decode())

class _Population(EPPopulation, BasePopulation):
    element_class = _Individual
    default_size = 20

pop = _Population.random(sizes=(n, n))

stat={'Mean Fitness':'mean_fitness', 'Best Fitness': 'best_fitness'}
data = pop.evolve(stat=stat, n_iter=500, period=5, history=True)

import matplotlib.pyplot as plt
fig = plt.figure()
ax = fig.add_subplot(111)

data[['Mean Fitness', 'Best Fitness']].plot(ax=ax)
ax.set_xlabel('Generations * 5')
ax.set_ylabel('Fitness')
plt.show()

max_stars_repo_name: Freakwill/pyrimidine

max_issues_repo_name: Freakwill/pyrimidine

max_forks_repo_name: Freakwill/pyrimidine


text:
import os, sys
import cv2
import json
import numpy as np
from config import cfg, puts_debug, puts_info
import utility

class Data(object):
  '''Data
  dataset is a file which written in the format of:
  {
    "1": {
      "filename": "filename",
      "width": width,
      "height": height,
      "channel": channel,
      "object_num": num,
      "object_bbox": "xmin, ymin, xmax, ymax" * num (value normalized to [0, 1]),
      "object_label": "0, 1, 2, ..." (size is num)
    },
    "2": {
      "filename": "filename",
      ...
    }...
  }
  '''
  def __init__(self):
    self._provider = []
    self._groundtruth = []
    self._size = 0

  def load(self, data_filename = None):
    if None in [data_filename]:
      raise ValueError('data is empty')
    with open(data_filename) as fp:
      ctx = json.load(fp)
      fp.close()
      for key in ctx.keys():
        record, prop = ctx[key], {}
        image = cv2.imread(record['filename'])
        if image is None:
          continue
        # resize image to be continue
        num = record['object_num']
        labels = [float(val) for val in record['object_labels']]
        labels = utility.resize(int(num), labels, cfg.max_predictions)
        bboxes = [float(val) for val in record['object_bboxes']]
        bboxes = utility.resize(int(num), bboxes, cfg.max_predictions)
        puts_debug('{}\nval: \n{}, shape: {}\nval: \n{}, shape: {}'.format(num, labels, np.shape(labels), bboxes, np.shape(bboxes)))
        prop['raw_data'], prop['num'], prop['labels'], prop['bboxes'] = image / 255., int(num), labels, bboxes
        self._provider.append(prop)
        prop = np.concatenate([prop['labels'], prop['bboxes']], axis = 1)
        puts_debug('prop shape: {}'.format(np.shape(prop)))
        self._groundtruth.append(prop)
        self._size += 1

  def decode_and_fetch(self, batch_size = None):
    raw_data, labels, bboxes, groundtruth = [], [], [], []
    for i in range(batch_size):
      index = np.random.randint(0, self._size)
      raw_data.append(self._provider[index]['raw_data'])
      labels.append(self._provider[index]['labels'])
      bboxes.append(self._provider[index]['bboxes'])
      groundtruth.append(self._groundtruth[index])
    puts_debug('{}, {}, {}, {}'.format(np.shape(raw_data), np.shape(labels), np.shape(bboxes), np.shape(groundtruth)))
    return raw_data, labels, bboxes, groundtruth
    # return tf.convert_to_tensor(raw_data, tf.float32), tf.convert_to_tensor(labels, tf.float32), tf.convert_to_tensor(bboxes, tf.float32), tf.convert_to_tensor(groundtruth, tf.float32)

  def size(self):
    return self._size


text:
import json
from numpy import ndarray, array, concatenate, multiply
from thunder.images import fromtif, fromarray
from os import mkdir, makedirs
from os.path import join, exists
from glob import glob
from .metadata import load as load_metadata
from .data import load as load_data
from .data import reshape, split, smooth
from .metadata import merge

def load(path, metapath=None, engine=None):
    """
    Load raw mesoscope data and metadata.

    Parameters
    ----------
    path : str
        Location with both metadata (.json) and image files (.tif).

    metapath : str
        Optional location with metadata (.json) if different from path.

    engine : multiple
        A computational backend for parallelization can be None (for local compute)
        or a SparkContext (for Spark).
    """
    if metapath == None:
        metapath = path
    metadata = load_metadata(metapath)

    if metadata['logFramesPerFile'] == 1:
        data = load_data(path, engine=engine)
    elif not metadata['fastZ'] and metadata['logFramesPerFile'] == metadata['framesPerSlice']:
        data = load_data(path, engine=engine)
    else:
        data = load_data(path, nplanes=metadata['nplanes'], engine=engine)
    metadata['shape'] = data.shape
    return data, metadata

def convert(data, metadata):
    """
    Convert raw mesoscope data by reshaping and merging rois.

    Parameters
    ----------
    data : numpy array or images
        Raw image data as a numpy array or thunder images object.

    metadata : dict
        Dictionary with metadata.
    """
    if isinstance(data, ndarray):
        data = fromarray(data)
    nchannels = metadata['nchannels']
    nrois = metadata['nrois']
    nlines = metadata['nlines']
    order = metadata['order']
    if metadata['merge']:
        newdata = data.map(lambda x : reshape(x, nrois, nlines, order))
    else:
        newdata = data.map(lambda x : split(x, nrois, nlines))
    if not metadata['fastZ'] and metadata['logFramesPerFile'] == metadata['framesPerSlice']:
        newdata = fromarray(newdata.values.swapaxes(0,1))
    newmetadata = merge(metadata)
    newmetadata['shape'] = newdata.shape
    return newdata, newmetadata

def downsample(data, ds=None, dt=None):
    """
    Function for downsampling images in space and time.

    Parameters
    ----------
    data : numpy array or images
        Raw image data as a numpy array or thunder images object.

    ds : int or tuple that specifies downsampling in space

    dt : int that specifies downsampling in time
    """

    if not ds == None:
        data = data.median_filter(ds).subsample(ds)

    if not dt == None:
        data = data.map_as_series(lambda x: smooth(x, dt))

    return data

max_stars_repo_name: sofroniewn/Mesoscope

max_issues_repo_name: sofroniewn/Mesoscope

max_forks_repo_name: sofroniewn/Mesoscope

reason: from numpy


text:
# Taken and modified from:
# https://github.com/davidtvs/pytorch-lr-finder/torch_lr_finder/lr_finder.py

from __future__ import print_function, with_statement, division
import copy
import os
import torch
import pandas as pd
import numpy as np

from tqdm import tqdm
from torch.optim.lr_scheduler import _LRScheduler

from ..train.trainer import Step
from ..data import cudaify

global MPL_AVAIL

try:
    import matplotlib.pyplot as plt
    MPL_AVAIL = True
except:
    print('matplotlib is unavailable !')
    MPL_AVAIL = False

class LRFinder(Step):
    """Learning rate range test.

    The learning rate range test increases the learning rate in a pre-training run
    between two boundaries in a linear or exponential manner. It provides valuable
    information on how well the network can be trained over a range of learning rates
    and what is the optimal learning rate.

    Cyclical Learning Rates for Training Neural Networks: https://arxiv.org/abs/1506.01186
    fastai/lr_find: https://github.com/fastai/fastai

    """

    def __init__(self,
                 loader,
                 model,
                 optimizer,
                 criterion,
                 save_checkpoint_dir,
                 logger,
                 gradient_accumulation=1,
                 mixup=None,
                 cutmix=None,
                 cuda=True):

        super(LRFinder, self).__init__(loader=loader)

        self.model = model
        self.optimizer = optimizer
        self.criterion = criterion
        self.history = {"lr": [], "loss": []}
        self.save_checkpoint_dir = save_checkpoint_dir
        self.logger = logger
        self.print = self.logger.info
        self.gradient_accumulation = gradient_accumulation
        self.mixup = mixup
        self.cutmix = cutmix
        self.cuda = True

    def find_lr(
        self,
        start_lr,
        end_lr,
        num_iter,
        save_fig,
        step_mode="exp",
        smooth_f=0.05,
        diverge_th=5,
    ):
        """Performs the learning rate range test.

        Arguments:
            train_loader (torch.utils.data.DataLoader): the training set data laoder.
            val_loader (torch.utils.data.DataLoader, optional): if `None` the range test
                will only use the training loss. When given a data loader, the model is
                evaluated after each iteration on that dataset and the evaluation loss
                is used. Note that in this mode the test takes significantly longer but
                generally produces more precise results. Default: None.
            end_lr (float, optional): the maximum learning rate to test. Default: 10.
            num_iter (int, optional): the number of iterations over which the test
                occurs. Default: 100.
            step_mode (str, optional): one of the available learning rate policies,
                linear or exponential ("linear", "exp"). Default: "exp".
            smooth_f (float, optional): the loss smoothing factor within the [0, 1[
                interval. Disabled if set to 0, otherwise the loss is smoothed using
                exponential smoothing. Default: 0.05.
            diverge_th (int, optional): the test is stopped when the loss surpasses the
                threshold:  diverge_th * best_loss. Default: 5.

        """
        # Reset test results
        self.history = {"lr": [], "loss": []}
        self.best_loss = None

        for idx, group in enumerate(self.optimizer.param_groups):
            self.optimizer.param_groups[idx]['lr'] = start_lr

        # Initialize the proper learning rate policy
        if step_mode.lower() == "exp":
            lr_schedule = ExponentialLR(self.optimizer, end_lr, num_iter)
        elif step_mode.lower() == "linear":
            lr_schedule = LinearLR(self.optimizer, end_lr, num_iter)
        else:
            raise ValueError("expected one of (exp, linear), got {}".format(step_mode))

        if smooth_f < 0 or smooth_f >= 1:
            raise ValueError("smooth_f is outside the range [0, 1[")

        # Create an iterator to get data batch by batch
        for iteration in tqdm(range(num_iter)):
            # Get a new set of inputs and labels
            self.train_step()
            loss = self.loss_tracker.losses[-1]

            # Update the learning rate
            lr_schedule.step()
            self.history["lr"].append(lr_schedule.get_lr()[0])

            # Track the best loss and smooth it if smooth_f is specified
            if iteration == 0:
                self.best_loss = loss
            else:
                if smooth_f > 0:
                    loss = smooth_f * loss + (1 - smooth_f) * self.history["loss"][-1]
                if loss < self.best_loss:
                    self.best_loss = loss

            # Check if the loss has diverged; if it has, stop the test
            self.history["loss"].append(loss)
            if loss > diverge_th * self.best_loss:
                self.print("Stopping early, the loss has diverged ...")
                break

        pd.DataFrame(self.history).to_csv(os.path.join(self.save_checkpoint_dir, 'lrfind.csv'))
        if MPL_AVAIL and save_fig: self.plot(os.path.join(self.save_checkpoint_dir, 'lrfind.png'))

        self.print("Learning rate search finished !")

    def plot(self, save_file, skip_start=10, skip_end=5, log_lr=True, show_lr=None):
        """Plots the learning rate range test.

        Arguments:
            skip_start (int, optional): number of batches to trim from the start.
                Default: 10.
            skip_end (int, optional): number of batches to trim from the start.
                Default: 5.
            log_lr (bool, optional): True to plot the learning rate in a logarithmic
                scale; otherwise, plotted in a linear scale. Default: True.
            show_lr (float, optional): is set, will add vertical line to visualize
                specified learning rate; Default: None
        """

        if skip_start < 0:
            raise ValueError("skip_start cannot be negative")
        if skip_end < 0:
            raise ValueError("skip_end cannot be negative")
        if show_lr is not None and not isinstance(show_lr, float):
            raise ValueError("show_lr must be float")

        # Get the data to plot from the history dictionary. Also, handle skip_end=0
        # properly so the behaviour is the expected
        lrs = self.history["lr"]
        losses = self.history["loss"]
        if skip_end == 0:
            lrs = lrs[skip_start:]
            losses = losses[skip_start:]
        else:
            lrs = lrs[skip_start:-skip_end]
            losses = losses[skip_start:-skip_end]

        # Plot loss as a function of the learning rate
        plt.plot(lrs, losses)
        if log_lr:
            plt.xscale("log")
        plt.xlabel("Learning Rate")
        plt.ylabel("Loss")
        plt.ylim(top=losses[0]*1.25)

        if show_lr is not None:
            plt.axvline(x=show_lr, color="red")
        plt.savefig(save_file)

class LinearLR(_LRScheduler):
    """Linearly increases the learning rate between two boundaries over a number of
    iterations.

    Arguments:
        optimizer (torch.optim.Optimizer): wrapped optimizer.
        end_lr (float, optional): the initial learning rate which is the lower
            boundary of the test. Default: 10.
        num_iter (int, optional): the number of iterations over which the test
            occurs. Default: 100.
        last_epoch (int): the index of last epoch. Default: -1.

    """

    def __init__(self, optimizer, end_lr, num_iter, last_epoch=-1):
        self.end_lr = end_lr
        self.num_iter = num_iter
        super(LinearLR, self).__init__(optimizer, last_epoch)

    def get_lr(self):
        curr_iter = self.last_epoch + 1
        r = curr_iter / self.num_iter
        return [base_lr + r * (self.end_lr - base_lr) for base_lr in self.base_lrs]

class ExponentialLR(_LRScheduler):
    """Exponentially increases the learning rate between two boundaries over a number of
    iterations.

    Arguments:
        optimizer (torch.optim.Optimizer): wrapped optimizer.
        end_lr (float, optional): the initial learning rate which is the lower
            boundary of the test. Default: 10.
        num_iter (int, optional): the number of iterations over which the test
            occurs. Default: 100.
        last_epoch (int): the index of last epoch. Default: -1.

    """

    def __init__(self, optimizer, end_lr, num_iter, last_epoch=-1):
        self.end_lr = end_lr
        self.num_iter = num_iter
        super(ExponentialLR, self).__init__(optimizer, last_epoch)

    def get_lr(self):
        curr_iter = self.last_epoch + 1
        r = curr_iter / self.num_iter
        return [base_lr * (self.end_lr / base_lr) ** r for base_lr in self.base_lrs]

class StateCacher(object):
    def __init__(self, in_memory, cache_dir=None):
        self.in_memory = in_memory
        self.cache_dir = cache_dir

        if self.cache_dir is None:
            import tempfile
            self.cache_dir = tempfile.gettempdir()
        else:
            if not os.path.isdir(self.cache_dir):
                raise ValueError('Given `cache_dir` is not a valid directory.')

        self.cached = {}

    def store(self, key, state_dict):
        if self.in_memory:
            self.cached.update({key: copy.deepcopy(state_dict)})
        else:
            fn = os.path.join(self.cache_dir, 'state_{}_{}.pt'.format(key, id(self)))
            self.cached.update({key: fn})
            torch.save(state_dict, fn)

    def retrieve(self, key):
        if key not in self.cached:
            raise KeyError('Target {} was not cached.'.format(key))

        if self.in_memory:
            return self.cached.get(key)
        else:
            fn = self.cached.get(key)
            if not os.path.exists(fn):
                raise RuntimeError('Failed to load state in {}. File does not exist anymore.'.format(fn))
            state_dict = torch.load(fn, map_location=lambda storage, location: storage)
            return state_dict

    def __del__(self):
        """Check whether there are unused cached files existing in `cache_dir` before
        this instance being destroyed."""
        if self.in_memory:
            return

        for k in self.cached:
            if os.path.exists(self.cached[k]):
                os.remove(self.cached[k])

max_stars_repo_name: jphdotam/DFDC

max_issues_repo_name: jphdotam/DFDC

max_forks_repo_name: jphdotam/DFDC


text:
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.lines as lines
from numpy import linalg as lin

import random
import sys

class SamplePoint:
    def __init__(self, point, label):
        self.x = np.array(point)
        self.y = label

    def tounit(self):
        self.x = np.multiply(self.x, 1/lin.norm(self.x))

def getweights(sample):
    astar = np.zeros(len(sample[0].x))
    point = exists(sample, astar)
    while point != None:
        tmp = SamplePoint(point.x, point.y)
        tmp.tounit()
        astar = np.add(astar, np.multiply(tmp.y, tmp.x))
        point = exists(sample, astar)

    return astar

def exists(sample, astar):
    for point in sample:
        coord = point.x
        label = point.y
        if sign(np.inner(astar, coord)) != label:
            return point
    return None

def classify(weights, point):
    return sign(np.inner(weights, point))

def sign(val):
    return 1 if val >= 0 else -1

def samplegen(num):
    points = []
    sep = np.random.rand(1, 2)
    for i in range(num):
        xp, yp = [random.uniform(-1, 1) for i in range(2)]
        vec = np.array([xp, yp])
        label = sign(sep.dot(vec))
        point = SamplePoint([xp, yp, 1], label)
        points.append(point)
    return points

sample = []

def readPoints(fname):
    fin = open(fname, 'r')
    for line in fin:
        row = line.split(',')
        point = []
        for i in range(0, len(row)-1):
            point.append(float(row[i].strip()))
        point.append(1)
        label = int(row[len(row)-1])
        sp = SamplePoint(point, label)
        sample.append(sp)

def main():
    global sample
    if (len(sys.argv) == 3):
        if (sys.argv[1] == '-nd'):
            readPoints(sys.argv[2])
            print getweights(sample)
            return
        elif (sys.argv[1] == '-d'):
            readPoints(sys.argv[2])
    elif (len(sys.argv) == 2):
        readPoints(sys.argv[1])

    fig = plt.figure(figsize=(5, 5))
    fig.canvas.mpl_connect('button_release_event', onclick)
    plt.interactive = True
    update()

def update():
    global sample
    plt.clf()
    plt.xlim(-1.5, 1.5)
    plt.ylim(-1.5, 1.5)
    if len(sample) >= 2:
        weight = getweights(sample)
        if weight[1] != 1:
            yInt1 = (-(weight[0]*-1.5) - weight[2])/weight[1]
            yInt2 = (-(weight[0]*1.5) - weight[2])/weight[1]
            ax = plt.gca()
            ax.add_line(lines.Line2D([-1.5, 1.5], [yInt1, yInt2]))

    for s in sample:

        plt.plot(s.x[0], s.x[1], 'bo' if s.y == 1 else 'ro')
    plt.draw()
    plt.show()

def onclick(event):
    global sample
    if event.button == 1:
        sample.append(SamplePoint([event.xdata, event.ydata, 1], 1))
    elif event.button == 3:
        sample.append(SamplePoint([event.xdata, event.ydata, 1], -1))
    update()

if __name__ == "__main__":
    main()

max_stars_repo_name: hunterdquant/PyCeptron

max_issues_repo_name: hunterdquant/PyCeptron

max_forks_repo_name: hunterdquant/PyCeptron

reason: import numpy,from numpy


text:
#%%
import numpy as np
from scipy import sparse
from scipy.linalg import block_diag

#%%
def sdp_ymat( lines, Ybus ):
    nbus = Ybus.shape[0]
    nline = len(lines)
    # busset = np.arange(0, nbus)
    # lineset = np.arange(0, nline)
    #%%

    def e(k): return np.eye(nbus)[:, k][np.newaxis]  # size of e(k):  (1, nbus)
    def Yk_small(k): return (e(k).T @ e(k)) @ Ybus

    def Yk(k): return (1/2) * \
        np.block([
            [np.real(Yk_small(k) + Yk_small(k).T), np.imag(Yk_small(k).T - Yk_small(k))],
            [np.imag(Yk_small(k) - Yk_small(k).T), np.real(Yk_small(k) + Yk_small(k).T)]
        ])

    def Yk_(k): return -(1/2) * \
        np.block([
            [np.imag(Yk_small(k) + Yk_small(k).T), np.real(Yk_small(k) - Yk_small(k).T)],
            [np.real(Yk_small(k).T - Yk_small(k)), np.imag(Yk_small(k) + Yk_small(k).T)]
        ])

    def Mk(k): return block_diag(e(k).T @ e(k), e(k).T @ e(k))
    # Real part of line admittance
    def gl(l): return np.real(1 / (lines[l].r+1j*lines[l].x))
    # Imaginary part of line admittance
    def bl(l): return np.imag(1 / (lines[l].r+1j*lines[l].x))

    def tau(l): return 1 if lines[l].tap == 0 else lines[l].tap
    def theta(l): return lines[l].shft
    def gbcosft(l): return gl(l)*np.cos(theta(l)) + bl(l)*np.cos(theta(l)+np.pi/2)
    def gbsinft(l): return gl(l)*np.sin(theta(l)) + bl(l)*np.sin(theta(l)+np.pi/2)
    def gbcostf(l): return gl(l)*np.cos(-theta(l)) + bl(l)*np.cos(-theta(l)+np.pi/2)
    def gbsintf(l): return gl(l)*np.sin(-theta(l)) + bl(l)*np.sin(-theta(l)+np.pi/2)

    #%%
    def Ylineft(l): return 0.5*(
        sparse.coo_matrix((
            [gl(l)/(tau(l)**2), -gbcosft(l)/tau(l), gbsinft(l)/tau(l),
                gl(l)/(tau(l)**2), -gbsinft(l)/tau(l), -gbcosft(l)/tau(l)],
            ([lines[l].fbus, lines[l].fbus, lines[l].fbus, lines[l].fbus +
            nbus, lines[l].fbus+nbus, lines[l].fbus+nbus],
            [lines[l].fbus, lines[l].tbus, lines[l].tbus+nbus,
            lines[l].fbus+nbus, lines[l].tbus, lines[l].tbus+nbus])
        ), shape = (2*nbus, 2*nbus))
        +
        sparse.coo_matrix((
            [gl(l)/(tau(l)**2), -gbcosft(l)/tau(l), gbsinft(l)/tau(l),
                gl(l)/(tau(l)**2), -gbsinft(l)/tau(l), -gbcosft(l)/tau(l)],
            ([lines[l].fbus, lines[l].fbus, lines[l].fbus, lines[l].fbus +
            nbus, lines[l].fbus+nbus, lines[l].fbus+nbus],
            [lines[l].fbus, lines[l].tbus, lines[l].tbus+nbus,
            lines[l].fbus+nbus, lines[l].tbus, lines[l].tbus+nbus])
        ), shape=(2*nbus, 2*nbus)).T
        )

    def Y_lineft(l): return 0.5*(
        sparse.coo_matrix((
            [-(bl(l)+lines[l].b/2)/(tau(l)**2), gbsinft(l)/tau(l), gbcosft(l)/tau(l), -
            (bl(l)+lines[l].b/2)/(tau(l)**2), -gbcosft(l)/tau(l), gbsinft(l)/tau(l)],
            ([lines[l].fbus, lines[l].fbus,  lines[l].fbus,  lines[l].fbus +
                nbus, lines[l].fbus+nbus, lines[l].fbus+nbus],
            [lines[l].fbus, lines[l].tbus,  lines[l].tbus+nbus,
                lines[l].fbus+nbus, lines[l].tbus,  lines[l].tbus+nbus])
        ), shape=(2*nbus, 2*nbus))
        +
        sparse.coo_matrix((
            [-(bl(l)+lines[l].b/2)/(tau(l)**2), gbsinft(l)/tau(l), gbcosft(l)/tau(l), -
            (bl(l)+lines[l].b/2)/(tau(l)**2), -gbcosft(l)/tau(l), gbsinft(l)/tau(l)],
            ([lines[l].fbus, lines[l].fbus, lines[l].fbus,  lines[l].fbus +
                nbus, lines[l].fbus+nbus, lines[l].fbus+nbus],
            [lines[l].fbus, lines[l].tbus, lines[l].tbus+nbus,
                lines[l].fbus+nbus, lines[l].tbus, lines[l].tbus+nbus])
        ), shape=(2*nbus, 2*nbus)).T
        )

    def Ylinetf(l): return 0.5*(
        sparse.coo_matrix((
            [-gbcostf(l)/tau(l),   -gbsintf(l)/tau(l),    gbsintf(l) /
            tau(l),  -gbcostf(l)/tau(l), gl(l), gl(l)],
            ([lines[l].fbus, lines[l].fbus,  lines[l].fbus+nbus,
            lines[l].fbus+nbus, lines[l].tbus, lines[l].tbus+nbus],
            [lines[l].tbus, lines[l].tbus+nbus, lines[l].tbus,
                lines[l].tbus+nbus, lines[l].tbus, lines[l].tbus+nbus])
        ), shape = (2*nbus, 2*nbus))
        +
        sparse.coo_matrix((
            [-gbcostf(l)/tau(l), -gbsintf(l)/tau(l), gbsintf(l) /
            tau(l), -gbcostf(l)/tau(l), gl(l), gl(l)],
            ([lines[l].fbus, lines[l].fbus, lines[l].fbus+nbus,
            lines[l].fbus+nbus, lines[l].tbus, lines[l].tbus+nbus],
            [lines[l].tbus, lines[l].tbus+nbus, lines[l].tbus,
                lines[l].tbus+nbus, lines[l].tbus, lines[l].tbus+nbus])
        ), shape = (2*nbus, 2*nbus)).T
        )

    def Y_linetf(l): return 0.5*(
        sparse.coo_matrix((
            [gbsintf(l)/tau(l),   -gbcostf(l)/tau(l),    gbcostf(l)/tau(l),
            gbsintf(l)/tau(l),     -(bl(l)+lines[l].b/2), -(bl(l)+lines[l].b/2)],
            ([lines[l].fbus, lines[l].fbus, lines[l].fbus+nbus,
            lines[l].fbus+nbus, lines[l].tbus, lines[l].tbus+nbus],
            [lines[l].tbus, lines[l].tbus+nbus, lines[l].tbus,
                lines[l].tbus+nbus, lines[l].tbus, lines[l].tbus+nbus])
        ), shape=(2*nbus, 2*nbus))
        +
        sparse.coo_matrix((
            [gbsintf(l)/tau(l),   -gbcostf(l)/tau(l),    gbcostf(l)/tau(l),
            gbsintf(l)/tau(l),     -(bl(l)+lines[l].b/2),  -(bl(l)+lines[l].b/2)],
            ([lines[l].fbus, lines[l].fbus, lines[l].fbus+nbus,
            lines[l].fbus+nbus, lines[l].tbus, lines[l].tbus+nbus],
            [lines[l].tbus, lines[l].tbus+nbus, lines[l].tbus,
                lines[l].tbus+nbus, lines[l].tbus, lines[l].tbus+nbus])
        ), shape=(2*nbus, 2*nbus)).T
        )

    def YL(l): return sparse.coo_matrix((
            [1, -1, 1, -1, -1, 1, -1, 1],
            ([lines[l].fbus, lines[l].fbus, lines[l].fbus+nbus, lines[l].fbus+nbus,
                lines[l].tbus, lines[l].tbus, lines[l].tbus+nbus, lines[l].tbus+nbus],
            [lines[l].fbus, lines[l].tbus, lines[l].fbus+nbus, lines[l].tbus+nbus,
                lines[l].fbus, lines[l].tbus, lines[l].fbus+nbus, lines[l].tbus+nbus])
        ), shape = (2*nbus, 2*nbus)) * lines[l].r * (gl(l)**2 + bl(l)**2)

    def YL_(l): return (sparse.coo_matrix((
            [1, -1, 1, -1, -1, 1, -1, 1],
            ([lines[l].fbus, lines[l].fbus, lines[l].fbus+nbus, lines[l].fbus+nbus,
                lines[l].tbus, lines[l].tbus, lines[l].tbus+nbus, lines[l].tbus+nbus],
            [lines[l].fbus, lines[l].tbus, lines[l].fbus+nbus, lines[l].tbus+nbus,
                lines[l].fbus, lines[l].tbus, lines[l].fbus+nbus, lines[l].tbus+nbus])
        ), shape = (2*nbus, 2*nbus)) * lines[l].x * (gl(l)**2 + bl(l)**2)
        -
        sparse.coo_matrix((
            [1, 1, 1, 1],
            ([lines[l].fbus, lines[l].fbus+nbus, lines[l].tbus, lines[l].tbus+nbus],
            [lines[l].fbus, lines[l].fbus+nbus, lines[l].tbus, lines[l].tbus+nbus])
        ), shape = (2*nbus, 2*nbus)) * lines[l].b / 2)

    return Yk, Yk_, Mk, Ylineft, Ylinetf, Y_lineft, Y_linetf, YL, YL_

max_stars_repo_name: JipKim/ACOPF

max_issues_repo_name: JipKim/ACOPF

max_forks_repo_name: JipKim/ACOPF

reason: import numpy,from scipy


text:
import time
import copy
import json
import logging
import random
import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable

from evaluate import evaluate_generator, save_generator_evaluate_metric
from utils.util import AverageMeter, save, accuracy, min_max_normalize, bn_calibration
from utils.optim import cal_hc_loss
from utils.countmacs import MAC_Counter

class Trainer:
    def __init__(self, g_optimizer, writer, device, accuracy_predictor, flops_table, CONFIG):
        self.top1 = AverageMeter()
        self.top5 = AverageMeter()
        self.losses = AverageMeter()
        self.hc_losses = AverageMeter()

        self.writer = writer
        self.device = device

        self.criterion = criterion
        self.g_optimizer = g_optimizer

        self.CONFIG = CONFIG

        self.epochs = self.CONFIG.epochs
        self.warmup_epochs = self.CONFIG.warmup_epochs
        self.search_epochs = self.CONFIG.search_epochs

        self.hardware_pool = [i for i in range(self.CONFIG.low_macs, self.CONFIG.high_macs, 5)]
        self.hardware_index = 0
        random.shuffle(self.hardware_pool)

        self.noise_weight = self.CONFIG.noise_weight

        # ================== OFA ====================
        self.accuracy_predictor = accuracy_predictor
        self.flops_table = flops_table

        self.backbone = self.calculate_one_hot(torch.randn(8*21)).cuda()

    def search_train_loop(self, generator):
        self.epochs = self.warmup_epochs + self.search_epochs
        # Training generator
        best_loss = 10000.0
        best_top1 = 0
        tau = 5
        for epoch in range(self.warmup_epochs, self.search_epochs):
            logging.info("Start to train for search epoch {}".format(epoch))
            logging.info("Tau: {}".format(tau))
            self._generator_training_step(generator, val_loader, epoch, tau, info_for_logger="_gen_train_step")

            # ================ Train ============================================
            for i in range():
                # Training generator
                arch_param, hardware_constraint = self.set_arch_param(generator, tau=tau)

                # ============== evaluation flops ===============================
                gen_flops = self.flops_table.predict_arch_param_efficiency(arch_param)
                hc_loss = cal_hc_loss(gen_flops.cuda(), hardware_constraint.item(), self.CONFIG.alpha, self.CONFIG.loss_penalty)
                # ===============================================================
                self.g_optimizer.zero_grad()

                # ============== predict top1 accuracy ==========================
                top1_avg = self.accuracy_predictor(arch_param)
                ce_loss = -1 * top1_avg
                # ===============================================================
                loss = ce_loss + hc_loss
                logging.info("HC loss : {}".format(hc_loss))
                loss.backward()

                self.g_optimizer.step()
                self.g_optimizer.zero_grad()
            # ====================================================================

            # ============== Valid ===============================================
            hardware_constraint, arch_param = self._get_arch_param(generator, hardware_constraint, valid=True)
            arch_param = self.calculate_one_hot(arch_param)
            arch_param, hardware_constraint = self.set_arch_param(generator, model, hardware_constraint=hardware_constraint, arch_param=arch_param)
            # ============== evaluation flops ===============================
            gen_flops = self.flops_table.predict_arch_param_efficiency(arch_param)

            hc_loss = cal_hc_loss(gen_flops.cuda(), hardware_constraint.item(), self.CONFIG.alpha, self.CONFIG.loss_penalty)
            # ===============================================================

            # ============== predict top1 accuracy ==========================
            top1_avg = self.accuracy_predictor(arch_param)
            logger.info("Valid : Top-1 avg : {}".format(top1_avg))
            # ===============================================================

            # ====================================================================

            # ============== Evaluate ============================================
            total_loss = 0
            evaluate_metric = {"gen_flops":[], "true_flops":[]}
            for flops in range(self.CONFIG.low_macs, self.CONFIG.high_macs, 10):
                hardware_constraint = torch.tensor(flops, dtpye=torch.float32)
                hardware_constraint = hardware_constraint.view(-1, 1)
                hardware_constraint = hardware_constraint.to(self.device)

                normalize_hardware_constraint = min_max_normalize(self.CONFIG.high_macs, self.CONFIG.low_macs, hardware_constraint)

                noise = torch.randn(*self.backbone.shape)
                noise = noise.to(device)
                noise *= 0

                arch_param = generator(self.backbone, normalize_hardware_constraint, noise)
                # ============== evaluation flops ===============================
                gen_flops = self.flops_table.predict_arch_param_efficiency(arch_param)
                hc_loss = cal_hc_loss(gen_flops.cuda(), hardware_constraint.item(), self.CONFIG.alpha, self.CONFIG.loss_penalty)
                # ===============================================================

                evaluate_metric["gen_flops"].append(gen_flops)
                evaluate_metric["true_flops"].append(flops)

                total_loss += hc_loss.item()
            kendall_tau, _ = stats.kendalltau(evaluate_metric["gen_flops"], evaluate_metric["true_flops"])
            # ====================================================================

            logging.info("Total loss : {}".format(total_loss))
            if best_loss > total_loss:
                logging.info("Best loss by now: {} Tau : {}.Save model".format(total_loss, kendall_tau))
                best_loss = total_loss
                save_generator_evaluate_metric(evaluate_metric, self.CONFIG.path_to_generator_eval)
                save(generator, self.g_optimizer, self.CONFIG.path_to_save_generator)
            if top1_avg > best_top1 and total_loss < 0.4:
                logging.info("Best top1-avg by now: {}.Save model".format(top1_avg))
                best_top1 = top1_avg
                save(generator, self.g_optimizer, self.CONFIG.path_to_best_avg_generator)
            save(generator, self.g_optimizer, "./logs/generator/{}.pth".format(total_loss))

            tau *= self.CONFIG.tau_decay
            self.noise_weight = self.noise_weight * self.CONFIG.noise_decay if self.noise_weight > 0.0001 else 0
            logging.info("Noise weight : {}".format(self.noise_weight))
        logging.info("Best loss: {}".format(best_loss))
        save(generator, self.g_optimizer, self.CONFIG.path_to_fianl_generator)

    def _get_arch_param(self, generator, hardware_constraint=None, valid=False):
        # ====================== Strict fair sample
        if hardware_constraint is None:
            hardware_constraint = torch.tensor(self.hardware_pool[self.hardware_index]+random.random()-0.5, dtype=torch.float32).view(-1, 1)
            #hardware_constraint = torch.tensor(self.hardware_pool[self.hardware_index], dtype=torch.float32).view(-1, 1)
            self.hardware_index += 1
            if self.hardware_index == len(self.hardware_pool):
                self.hardware_index = 0
                random.shuffle(self.hardware_pool)
        else:
            hardware_constraint = torch.tensor(hardware_constraint, dtype=torch.float32).view(-1, 1)
        # ======================

        hardware_constraint = hardware_constraint.to(self.device)
        logging.info("Target macs : {}".format(hardware_constraint.item()))

        normalize_hardware_constraint = min_max_normalize(self.CONFIG.high_macs, self.CONFIG.low_macs, hardware_constraint)

        noise = torch.randn(*self.backbone.shape)
        noise = noise.to(self.device)
        noise *= self.noise_weight

        arch_param = generator(self.backbone, normalize_hardware_constraint, noise)

        return hardware_constraint, arch_param

    def set_arch_param(self, generator, hardware_constraint=None, arch_param=None, tau=None):
        """Sample the sub-network from supernet by arch_param(generate from generator or user specific)
        """
        if tau is not None:
            hardware_constraint, arch_param = self._get_arch_param(generator)
            arch_param = self.calculate_block_probability(arch_param, tau)

        arch_param = arch_param.to(self.device)
        return arch_param, hardware_constraint

    def calculate_block_probability(self, arch_param, tau):
        arch_param = arch_param.view(21, 8)
        p_arch_param = torch.zeros_like(arch_param)

        for i in range(20):
            p_arch_param[i, :4] = F.gumbel_softmax(arch_param[i, :4], tau=tau)
            p_arch_param[i, 4:] = F.gumbel_softmax(arch_param[i, 4:], tau=tau)

        p_arch_param[-1, :5] = F.gumbel_softmax(arch_param[-1, :5], tau=tau)
        return p_arch_param

    def calculate_one_hot(self, arch_param):
        arch_param = arch_param.view(21, 8)
        p_arch_param = torch.zeros_like(arch_param)

        for i in range(20):
            p_arch_param[i, arch_param[i, :4].argmax()] = 1
            p_arch_param[i, arch_param[i, 4:].argmax()] = 1

        p_arch_param[-1, arch_param[-1, :5].argmax()] = 1
        return p_arch_param


text:
import numpy as np
import os, sys
import argparse

parser = argparse.ArgumentParser(description='Reconstruct some image from a trained model.')
parser.add_argument('--arch', default=None, required=True, type=str, help='Vision model.')
parser.add_argument('--data', default=None, required=True, type=str, help='Vision dataset.')
parser.add_argument('--alpha', default=0.7, type=float, help='Privacy score weight')

opt = parser.parse_args()
def normalize(score):
    range = np.max(score)-np.min(score)
    score = (score-np.min(score))/range
    return score

if __name__ == '__main__':
    search_root = 'search/data_{}_arch_{}/'.format(opt.data, opt.arch)
    acc_root = 'accuracy/data_{}_arch_{}/'.format(opt.data, opt.arch)
    maxpath, maxval = None, -sys.maxsize
    minpath, minval = None, sys.maxsize

    pri_scores_list = list()
    pri_vars_list = list()
    results = list()
    for path in os.listdir(acc_root):
        pathname = os.path.join(search_root, path)
        pri_score_list = np.load(pathname).tolist()
        pri_mean = np.mean(pri_score_list)
        pri_var = np.var(pri_score_list)

        pri_scores_list.append(pri_mean)
        pri_vars_list.append(pri_var)

        pathname = os.path.join(acc_root, path)
        acc_score_list = np.load(pathname).tolist()
        acc_mean = np.mean(acc_score_list)
        if path == '.npy':
            print(pri_mean, acc_mean)
        results.append((path, pri_mean, acc_mean))

    pri_scores_list, pri_vars_list = normalize(pri_scores_list), normalize(pri_vars_list)
    pri_score_weight_sum = opt.alpha*pri_scores_list + (1-opt.alpha)*pri_vars_list

    indices = np.argsort(pri_score_weight_sum)
    for i, idx in enumerate(indices):
        if results[idx][2] >= −85:
            print(f'Policy:{results[idx][0]}, Privacy Score: {pri_score_weight_sum[idx]}, Accuracy Score: {results[idx][2]}')

max_stars_repo_name: LuckMonkeys/ATSPrivacy

max_issues_repo_name: LuckMonkeys/ATSPrivacy

max_forks_repo_name: LuckMonkeys/ATSPrivacy