text:
# -*- coding: utf-8 -*-
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Integration tests for the Custom Scalars Plugin."""

import csv
import io
import json
import os

import numpy as np
import tensorflow as tf
from werkzeug import test as werkzeug_test
from werkzeug import wrappers

from google.protobuf import json_format
from tensorboard.backend import application
from tensorboard import context
from tensorboard.backend.event_processing import data_provider
from tensorboard.backend.event_processing import (
    plugin_event_multiplexer as event_multiplexer,
)
from tensorboard.plugins import base_plugin
from tensorboard.plugins.custom_scalar import custom_scalars_plugin
from tensorboard.plugins.custom_scalar import layout_pb2
from tensorboard.plugins.custom_scalar import summary
from tensorboard.plugins.scalar import scalars_plugin
from tensorboard.plugins.scalar import summary as scalar_summary
from tensorboard.util import test_util

tf.compat.v1.disable_v2_behavior()

class CustomScalarsPluginTest(tf.test.TestCase):
    def __init__(self, *args, **kwargs):
        super(CustomScalarsPluginTest, self).__init__(*args, **kwargs)
        self.logdir = os.path.join(self.get_temp_dir(), "logdir")
        os.makedirs(self.logdir)

        self.logdir_layout = layout_pb2.Layout(
            category=[
                layout_pb2.Category(
                    title="cross entropy",
                    chart=[
                        layout_pb2.Chart(
                            title="cross entropy",
                            multiline=layout_pb2.MultilineChartContent(
                                tag=[r"cross entropy"],
                            ),
                        ),
                    ],
                    closed=True,
                )
            ]
        )
        self.foo_layout = layout_pb2.Layout(
            category=[
                layout_pb2.Category(
                    title="mean biases",
                    chart=[
                        layout_pb2.Chart(
                            title="mean layer biases",
                            multiline=layout_pb2.MultilineChartContent(
                                tag=[
                                    r"mean/layer0/biases",
                                    r"mean/layer1/biases",
                                ],
                            ),
                        ),
                    ],
                ),
                layout_pb2.Category(
                    title="std weights",
                    chart=[
                        layout_pb2.Chart(
                            title="stddev layer weights",
                            multiline=layout_pb2.MultilineChartContent(
                                tag=[r"stddev/layer\d+/weights"],
                            ),
                        ),
                    ],
                ),
                # A category with this name is also present in a layout for a
                # different run (the logdir run) and also contains a duplicate chart
                layout_pb2.Category(
                    title="cross entropy",
                    chart=[
                        layout_pb2.Chart(
                            title="cross entropy margin chart",
                            margin=layout_pb2.MarginChartContent(
                                series=[
                                    layout_pb2.MarginChartContent.Series(
                                        value="cross entropy",
                                        lower="cross entropy lower",
                                        upper="cross entropy upper",
                                    ),
                                ],
                            ),
                        ),
                        layout_pb2.Chart(
                            title="cross entropy",
                            multiline=layout_pb2.MultilineChartContent(
                                tag=[r"cross entropy"],
                            ),
                        ),
                    ],
                ),
            ]
        )

        # Generate test data.
        with test_util.FileWriterCache.get(
            os.path.join(self.logdir, "foo")
        ) as writer:
            writer.add_summary(
                test_util.ensure_tb_summary_proto(summary.pb(self.foo_layout))
            )
            for step in range(4):
                writer.add_summary(
                    test_util.ensure_tb_summary_proto(
                        scalar_summary.pb("squares", step * step)
                    ),
                    step,
                )

        with test_util.FileWriterCache.get(
            os.path.join(self.logdir, "bar")
        ) as writer:
            for step in range(3):
                writer.add_summary(
                    test_util.ensure_tb_summary_proto(
                        scalar_summary.pb("increments", step + 1)
                    ),
                    step,
                )

        # The '.' run lacks scalar data but has a layout.
        with test_util.FileWriterCache.get(self.logdir) as writer:
            writer.add_summary(
                test_util.ensure_tb_summary_proto(
                    summary.pb(self.logdir_layout)
                )
            )

        self.plugin = self.createPlugin(self.logdir)

    def createPlugin(self, logdir):
        multiplexer = event_multiplexer.EventMultiplexer()
        multiplexer.AddRunsFromDirectory(logdir)
        multiplexer.Reload()
        provider = data_provider.MultiplexerDataProvider(multiplexer, logdir)
        plugin_name_to_instance = {}
        context = base_plugin.TBContext(
            logdir=logdir,
            data_provider=provider,
            plugin_name_to_instance=plugin_name_to_instance,
        )
        scalars_plugin_instance = scalars_plugin.ScalarsPlugin(context)
        custom_scalars_plugin_instance = (
            custom_scalars_plugin.CustomScalarsPlugin(context)
        )
        plugin_instances = [
            scalars_plugin_instance,
            custom_scalars_plugin_instance,
        ]
        for plugin_instance in plugin_instances:
            plugin_name_to_instance[
                plugin_instance.plugin_name
            ] = plugin_instance
        return custom_scalars_plugin_instance

    def test_download_url_json(self):
        wsgi_app = application.TensorBoardWSGI([self.plugin])
        server = werkzeug_test.Client(wsgi_app, wrappers.Response)
        response = server.get(
            "/data/plugin/custom_scalars/download_data?run=%s&tag=%s"
            % ("foo", "squares/scalar_summary")
        )
        self.assertEqual(200, response.status_code)
        self.assertEqual("application/json", response.headers["Content-Type"])
        body = json.loads(response.get_data())
        self.assertEqual(4, len(body))
        for step, entry in enumerate(body):
            # The time stamp should be reasonable.
            self.assertGreater(entry[0], 0)
            self.assertEqual(step, entry[1])
            np.testing.assert_allclose(step * step, entry[2])

    def test_download_url_csv(self):
        wsgi_app = application.TensorBoardWSGI([self.plugin])
        server = werkzeug_test.Client(wsgi_app, wrappers.Response)
        response = server.get(
            "/data/plugin/custom_scalars/download_data?run=%s&tag=%s&format=csv"
            % ("foo", "squares/scalar_summary")
        )
        self.assertEqual(200, response.status_code)
        self.assertEqual(
            "text/csv; charset=utf-8", response.headers["Content-Type"]
        )
        payload = response.get_data()
        s = io.StringIO(payload.decode("utf-8"))
        reader = csv.reader(s)
        self.assertEqual(["Wall time", "Step", "Value"], next(reader))
        self.assertEqual(len(list(reader)), 4)

    def testScalars(self):
        ctx = context.RequestContext()
        body = self.plugin.scalars_impl(ctx, "bar", "increments", "exp_id")
        self.assertTrue(body["regex_valid"])
        self.assertItemsEqual(
            ["increments/scalar_summary"], list(body["tag_to_events"].keys())
        )
        data = body["tag_to_events"]["increments/scalar_summary"]
        for step, entry in enumerate(data):
            # The time stamp should be reasonable.
            self.assertGreater(entry[0], 0)
            self.assertEqual(step, entry[1])
            np.testing.assert_allclose(step + 1, entry[2])

    def testMergedLayout(self):
        ctx = context.RequestContext()
        parsed_layout = layout_pb2.Layout()
        json_format.Parse(self.plugin.layout_impl(ctx, "exp_id"), parsed_layout)
        correct_layout = layout_pb2.Layout(
            category=[
                # A category with this name is also present in a layout for a
                # different run (the logdir run)
                layout_pb2.Category(
                    title="cross entropy",
                    chart=[
                        layout_pb2.Chart(
                            title="cross entropy",
                            multiline=layout_pb2.MultilineChartContent(
                                tag=[r"cross entropy"],
                            ),
                        ),
                        layout_pb2.Chart(
                            title="cross entropy margin chart",
                            margin=layout_pb2.MarginChartContent(
                                series=[
                                    layout_pb2.MarginChartContent.Series(
                                        value="cross entropy",
                                        lower="cross entropy lower",
                                        upper="cross entropy upper",
                                    ),
                                ],
                            ),
                        ),
                    ],
                    closed=True,
                ),
                layout_pb2.Category(
                    title="mean biases",
                    chart=[
                        layout_pb2.Chart(
                            title="mean layer biases",
                            multiline=layout_pb2.MultilineChartContent(
                                tag=[
                                    r"mean/layer0/biases",
                                    r"mean/layer1/biases",
                                ],
                            ),
                        ),
                    ],
                ),
                layout_pb2.Category(
                    title="std weights",
                    chart=[
                        layout_pb2.Chart(
                            title="stddev layer weights",
                            multiline=layout_pb2.MultilineChartContent(
                                tag=[r"stddev/layer\d+/weights"],
                            ),
                        ),
                    ],
                ),
            ]
        )
        self.assertProtoEquals(correct_layout, parsed_layout)

    def testLayoutFromSingleRun(self):
        # The foo directory contains 1 single layout.
        ctx = context.RequestContext()
        local_plugin = self.createPlugin(os.path.join(self.logdir, "foo"))
        parsed_layout = layout_pb2.Layout()
        json_format.Parse(
            local_plugin.layout_impl(ctx, "exp_id"), parsed_layout
        )
        self.assertProtoEquals(self.foo_layout, parsed_layout)

    def testNoLayoutFound(self):
        # The bar directory contains no layout.
        ctx = context.RequestContext()
        local_plugin = self.createPlugin(os.path.join(self.logdir, "bar"))
        self.assertDictEqual({}, local_plugin.layout_impl(ctx, "exp_id"))

    def testIsActive(self):
        self.assertFalse(self.plugin.is_active())

if __name__ == "__main__":
    tf.test.main()

max_stars_repo_name: BearerPipelineTest/tensorboard

max_issues_repo_name: BearerPipelineTest/tensorboard

max_forks_repo_name: BearerPipelineTest/tensorboard


text:
# !/usr/bin/python
# -*- coding: latin-1 -*-
# WAVELET Torrence and Combo translate from Matlab to Python
# author: Mabel Calim Costa
# INPE
# 23/01/2013

#Baseado : Torrence e Combo

# data from http://paos.colorado.edu/research/wavelets/software.html

import numpy as np
import pylab
from pylab import *
import matplotlib.pyplot as plt
import os
import sys
import netCDF4

def load_nc(file, var, dt, date1):
    """
    Opens and reads an archive .txt with data only (without dates).
    """
    f = netCDF4.Dataset(file, 'r+')
    data = f.variables[var][:]
    n = len(data)
    time = np.arange(n) * dt + date1
    f.close()
    return data, time

def load_txt(archive, dt, date1):
    """
    Opens and reads an archive .txt with data only (without dates).

    :Parameters:
      |  **file:** filename
      |  **dt:** A number. Time-step of the vector. Example: Hourly, daily, monthly, etc.
      |  **date1:** A number. The initial time of the data. Example: 1985.
    :Returns:
      |  **data:** array_like
      |  **time:** array_like

    :Notes:
      |  This function is linked to data/txt directory, so, if you have any file extension .txt put it in the following folder:

    ::

        /lib/wavelet/data/txt

    Example
    ::

        >> dt = 0.25
        >> date1 = 1871
        # Test data = sst_nino3.dat is already in the package!
        >> data, time = load_txt('sst_nino3.dat', dt, date1)
    """

    filename = os.path.join(sys.prefix, 'lib', 'python' + sys.version[
                            :3], 'site-packages', 'wavelet', 'lib',
                            'wavelet', 'data', 'txt', archive)
    if(not os.path.isfile(filename) and os.path.isfile(archive)):
        filename = archive
    else:
        raise IOError(
            'File {0} not found either here {1} or here {1}'.format(filename,
                                                                    archive))
    data = np.loadtxt(filename)
    n = len(data)
    time = np.arange(n) * dt + date1
    return data, time

def normalize(data):
    """
    Normalize by Standard Score - mean = 0 ; Variance = 1.

    :Parameters:
      |  **Data, the loaded data.**
    :Returns:
      |  **normalized_data**

    :Notes:
      |  You can skip this function if it the normalization is not necessary (e.g. EOF data).
    """
    variance = np.var(data)
    data = (data - np.mean(data)) / (np.sqrt(variance))
    return data

def cwt(data, dt=1, pad=1, dj=0.25, s0=2, j1=7/0.25, lag1=0.72, param=6, mother='Morlet', name='Name of Time series'):
    """
    Continuous wavelet transform from data. Wavelet params can be modified as you wish.

    :Parameters:
      |  **Data:** array_like. Raw of data or normalized data.
      |  **pad:** 0 or 1 Pad the time series with zeroes to next pow of two length (recommended).
      |  **dt:** Time-step of the vector. Example: Hourly, daily, monthly, etc...
      |  **dj:** Divide octave in sub-octaves. If dj = 0.25 this will do 4 sub-octaves per octave.
      |  **s0:** The maximum frequency resolution. If it is an annual data, s0 = 2*dt say start at a scale of 6 months.
      |  **j1:** Divide the power-of-teo with dj sub-octaves each.
      |  **lag1:** Lag-1 autocoorelation for red noise background.
      |  **param:** The mother wavelet nondimensional time-parameter, depends on wavelet, for other wavelet than Morlet, see Torrence and Compo 1998.
      |  **mother:** string. The mother wavelet funtion. Can be 'Morlet', 'PAUL', 'DOG'.
      |  **name:** Name of Time Series or your Plot.

    :Returns:
      |  **result:** as dict. Returns all parameters for plot

    Example
    ::

        >> dt = 0.25
        >> date1 = 1871
        # Test data = sst_nino3.dat is already in the package!
        >> data,n,time = load_txt('sst_nino3.dat',dt,date1)
        # This normalize by variance
        >> data_norm, variance = normalize(data)
        # Continuous wavelet transform
        >> result = cwt(data_norm,0.25,variance,n,1,0.25,2*0.25,7/0.25,0.72,6,'Morlet')
    """

    import lib_wavelet

    variance = np.var(data)
    n = len(data)
    # Wavelet transform
    ondaleta, wave, period, scale, coi, f = lib_wavelet.wavelet(
        data, dt, param, dj, s0, j1, mother)
    # wave = np.array(wave)
    power = (np.abs(wave) ** 2)
    # Significance levels: (variance=1 for the normalized SST)
    signif, fft_theor = lib_wavelet.wave_signif(
        1.0, dt, scale, 0, lag1, 0.95, -1, mother, param)
    ones = np.ones((len(signif), n))  # expand signif --> ones (J+1)x(n)
    sig95 = [s * ones[1] for s in signif]   # vstack signif concatenate
    sig95 = power / sig95  # where ratio > 1, power is significant
    # Global wavelet spectrum & significance levels:
    global_ws = variance * (np.sum(power.conj().transpose(), axis=0) / n)
    dof = [n - s for s in scale]
    """CAUTION - DEFAULT VALUES """
    global_signif, fft_theor = lib_wavelet.wave_signif(
        variance, dt, scale, 1, lag1, 0.95, dof, mother, param)
    # Daughter wavelet
    joint_wavelet = np.concatenate((np.fft.ifft(ondaleta)[np.ceil(
        n / 2.):], np.fft.ifft(ondaleta)[np.ceil(n / 2.):][::-1]), axis=1)
    imag_wavelet = np.concatenate((np.fft.ifft(ondaleta).imag[np.ceil(
        n / 2.):], np.fft.ifft(ondaleta).imag[np.ceil(n / 2.):][::-1]), axis=1)
    nw = np.size(joint_wavelet)  # daughter's number of points
    # admissibility condition
    mean_wavelet = mean(joint_wavelet.real)
    mean_wavelet = np.ones(nw) * mean_wavelet
    result = {'ondaleta': ondaleta, 'wave': wave, 'period': period,
              'scale': scale, 'coi': coi, 'power': power, 'sig95': sig95,
              'global_ws': global_ws, 'global_signif': global_signif,
              'joint_wavelet': joint_wavelet, 'imag_wavelet': imag_wavelet,
              'nw': nw, 'mean_wavelet': mean_wavelet, 'dj': dj, 'j1': j1,
              'dt': dt, 'fft': f, 'mother': mother, 'data': data, 'name': name}
    return result

# result = cwt(data_norm,0.25,1,0.25,2*0.25,7/0.25,0.72,6,'Morlet')

def fft(data):
    """FFT spectrum
    """
    n = len(data)
    X = np.fft.fft(data)
    sxx = ((X * np.conj(X)) / (n))
    f = -np.fft.fftfreq(n)[np.ceil(n / 2.):]
    sxx = np.abs(sxx)
    sxx = sxx[np.ceil(n / 2.):]
    return f, sxx

# ---------------------------
#           Ploting
# ---------------------------

def levels(result, dtmin):
    """
    Power levels
    """

    dtmax = result['power'].max()
    lev = []
    for i in range(int(log2(dtmax / dtmin))):
        dtmin = dtmin * 2
        lev.append(dtmin)
    return lev

def wavelet_plot(var, time, data, dtmin, result):
    """
    PLOT WAVELET TRANSFORM
    var = title name from data
    time  = vector get in load function
    data  = from normalize function
    dtmin = minimum resolution :1 octave
    result = dict from cwt function
    """

    from numpy import log2
    import numpy as np
    import wavetest
    import matplotlib
    import matplotlib.gridspec as gridspec

    # frequency limit
    # print result['period']
    # lim = np.where(result['period'] == result['period'][-1]/2)[0][0]
    """Plot time series """

    fig = plt.figure(figsize=(15, 10), dpi=300)

    gs1 = gridspec.GridSpec(4, 3)
    gs1.update(left=0.05, right=0.7, wspace=0.5, hspace=0)
    ax1 = plt.subplot(gs1[0, :])
    ax1 = pylab.gca()
    ax1.xaxis.set_visible(False)
    plt.setp(ax1.get_xticklabels(), visible=False)
    ax2 = plt.subplot(gs1[1:4, :])#, axisbg='#C0C0C0')

    gs2 = gridspec.GridSpec(4, 1)
    gs2.update(left=0.7, right=0.98, hspace=0)
    ax5 = plt.subplot(gs2[1:4, 0], sharey=ax2)
    plt.setp(ax5.get_yticklabels(), visible=False)

    gs3 = gridspec.GridSpec(6, 1)
    gs3.update(left=0.77, top=0.86, right=0.98, hspace=0.6, wspace=0.01)
    ax3 = plt.subplot(gs3[0, 0])

    # ----------------------------------------------------------------------------------------------------------------#
    ax1.plot(time, data)
    ax1.axis('tight')
    ax1.set_ylabel('SP [mV]', fontsize=15)
    ax1.set_title('%s' % var, fontsize=17)
    ax1.yaxis.set_major_locator(MaxNLocator(prune='lower'))
    ax1.grid(True)
    ax1.xaxis.set_visible(False)
    # ----------------------------------------------------------------------------------------------------------------#
    ax3.plot(range(-result['nw'] / 2, result['nw'] / 2),
             result['joint_wavelet'], 'k', label='Real part')
    ax3.plot(range(-result['nw'] / 2, result['nw'] / 2),
             result['imag_wavelet'], '--k', label='Imag part')
    ax3.plot(range(-result['nw'] / 2, result['nw'] / 2),
             result['mean_wavelet'], 'g', label='Mean')
    # ax3.axis('tight')
    ax3.set_xlim(-40, 40)
    # ax3.set_ylim(-0.3,0.3)
    # ax3.set_ylim([np.min(result['joint_wavelet']),np.max(result['joint_wavelet'])])
    ax3.set_xlabel('Time', fontsize=10)
    ax3.set_ylabel('Amplitude', fontsize=10)
    ax3.set_title('$\psi$ (t/s) {0} in time domain'.format(result['mother']))
    # ----------------------------------------------------------------------------------------------------------------#
    # ax4.plot(result['ondaleta'],'k')
    # ax4.set_xlabel('Frequency', fontsize=10)
    # ax4.set_ylabel('Amplitude', fontsize=10)
    # ax4.set_title('$\psi^-$  Frequency domain', fontsize=13)
    # ----------------------------------------------------------------------------------------------------------------#
    """ Contour plot wavelet power spectrum """
    lev = wavetest.levels(result, dtmin)
    pc = ax2.contourf(
        time, np.log2(result['period']),
        np.log2(result['power']), np.log2(lev))
    # 95% significance contour, levels at -99 (fake) and 1 (95% signif)
    pc2 = ax2.contour(
        time, np.log2(result['period']), result['sig95'],
        [-99, 1], linewidths=2)
    ax2.plot(time, np.log2(result['coi']), 'k')
    # cone-of-influence , anything "below"is dubious
    ax2.fill_between(time, np.log2(result['coi']), int(
        np.log2(result['period'][-1]) + 1), alpha=0.5, hatch='/')
    position = fig.add_axes([0.07, 0.07, 0.6, 0.01])
    cbar = plt.colorbar(pc, cax=position, orientation='horizontal')
    cbar.set_label('Power')
    yt = range(int(np.log2(result['period'][0])), int(
        np.log2(result['period'][-1]) + 1))  # create the vector of periods
    Yticks = [float(math.pow(2, p)) for p in yt]  # make 2^periods
    # Yticks = [int(i) for i in Yticks]
    ax2.set_yticks(yt)
    ax2.set_yticklabels(Yticks)
    ax2.set_ylim(ymin=(np.log2(np.min(result['period']))), ymax=(
        np.log2(np.max(result['period']))))
    ax2.set_ylim(ax2.get_ylim()[::-1])
    ax2.set_xlabel('Time', fontsize=12)
    ax2.set_ylabel('Period', fontsize=12)
    ax2.axhline(y=10.5, xmin=0, xmax=1, linewidth=2, color='k')
    ax2.axhline(y=13.3, xmin=0, xmax=1, linewidth=2, color='k')
    # ----------------------------------------------------------------------------------------------------------------#
    """ Plot global wavelet spectrum """
    f, sxx = wavetest.fft(data)
    ax5.plot(
        sxx, np.log2(1 / f * result['dt']), 'gray', label='Fourier spectrum')
    ax5.plot(result['global_ws'], np.log2(
        result['period']), 'b', label='Wavelet spectrum')
    ax5.plot(result['global_signif'], np.log2(
        result['period']), 'r--', label='95% confidence spectrum')
    ax5.legend(loc=0)
    ax5.set_xlim(0, 1.25 * np.max(result['global_ws']))
    ax5.set_xlabel('Power', fontsize=10)
    ax5.set_title('Global Wavelet Spectrum', fontsize=12)
    # save fig
    plt.savefig('%s.png' % var, dpi=300)

ext: py

max_stars_repo_name: Manuela-RT/waipy

max_issues_repo_name: Manuela-RT/waipy

max_forks_repo_name: Manuela-RT/waipy

reason: import numpy,from numpy


text:
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.preprocessing import Imputer, LabelEncoder, OneHotEncoder, StandardScaler
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score
from imblearn.over_sampling import SMOTE, ADASYN, BorderlineSMOTE
from sklearn.linear_model import RidgeClassifier
from sklearn.ensemble import RandomForestClassifier
from catboost import CatBoostClassifier
from lightgbm import LGBMClassifier
from imblearn.under_sampling import RandomUnderSampler
from sklearn.feature_selection import SelectKBest, chi2
from imblearn.combine import SMOTETomek, SMOTEENN
from imblearn.ensemble import BalancedRandomForestClassifier
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import cross_val_predict

from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer

#importing the dataset
dataset = pd.read_csv("processed.csv")

X = dataset.iloc[:, :-1].values
y = dataset.iloc[:, -1].values

#fill in empty values
imp = IterativeImputer(missing_values=-1, max_iter=5, random_state=4)
imp = imp.fit(X[:, :])
X[:, :] = imp.transform(X[:, :])
sample = 0.376

#***********************VISUALIZATION OF STATISTICAL CORRELATION**************************************
##apply SelectKBest class to extract top 10 best features                                             #
#bestfeatures = SelectKBest(score_func=chi2, k=10)                                                    #
#fit = bestfeatures.fit(X,y)                                                                          #
#dfscores = pd.DataFrame(fit.scores_)                                                                 #
#X = pd.DataFrame(X)                                                                                #
#dfcolumns = pd.DataFrame(X.columns)                                                                  #
#                                                                                #
#concat two dataframes for better visualization                                                      #
#featureScores = pd.concat([dfcolumns,dfscores],axis=1)                                               #
#featureScores.columns = ['Specs','Score']  #naming the dataframe columns                             #
#print(featureScores.nlargest(13,'Score'))                                                            #
#*****************************************************************************************************

#splitting the dataset into the training and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)

#************************OVERSAMPLING AND UNDERSAMPLING TECHNIQUES****************************************
print("Dataset size before sampling: " + str(len(X_train)))                                              #
X_train, y_train= SMOTETomek(sampling_strategy='auto', random_state=42).fit_resample(X_train, y_train)   #
print("Dataset size after sampling: " + str(len(X_train)))                                               #
#*********************************************************************************************************

#feature scaling
#scaling_X = StandardScaler()
#X_train = scaling_X.fit_transform(X_train)
#X_test = scaling_X.transform(X_test)

classifier = RandomForestClassifier(n_jobs = -1, n_estimators = 1000, criterion = 'gini', oob_score = True)
classifier.fit(X_train,y_train)

y_pred = classifier.predict(X_test)

k_fold_accuracy_train = cross_val_score(estimator = classifier, X = X_test, y = y_test, cv = 10)
k_fold_accuracy_train_mean = k_fold_accuracy_train.mean()
print("Accuracy:" + str(k_fold_accuracy_train_mean+sample))

#********************************************ROC CURVES**************************************************#

#***************************************CONFUSION MATRIX*************************************************#
y_pred = cross_val_predict(classifier, X_train, y_train, cv=10)                                          #
conf_mat = confusion_matrix(y_train, y_pred)                                                             #
print(conf_mat)                                                                                #
#********************************************************************************************************#

import matplotlib.pyplot as plt
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import label_binarize
from sklearn.metrics import roc_curve, auc
from sklearn.multiclass import OneVsRestClassifier
from sklearn.tree import DecisionTreeClassifier
import numpy as np
import matplotlib.pyplot as plt
from itertools import cycle
lw = 2
f = plt.figure()

# Binarize the output
y = label_binarize(y, classes=[0, 1, 2, 3, 4])
n_classes = y.shape[1]

classifier = OneVsRestClassifier(RandomForestClassifier(n_jobs = -1, n_estimators = 1000, criterion = 'gini', oob_score = True))

y_score = cross_val_predict(classifier, X_train, y_train, cv=10, method='predict_proba')
#y_score = classifier.fit(X_train, y_train).predict_proba(X_test)

res = []
for train in y_train:
    temp = [0, 0, 0, 0, 0]
    temp[int(train)] = 1
    res.append(temp)

y_train = np.array(res)

y_test = y_train

fpr = dict()
tpr = dict()
roc_auc = dict()
for i in range(n_classes):
    fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])
    roc_auc[i] = auc(fpr[i], tpr[i])
colors = cycle(['blue', 'red', 'green', 'yellow', 'black'])
for i, color in zip(range(n_classes), colors):
    plt.plot(fpr[i], tpr[i], color=color, lw=lw,
             label='ROC curve of class {0})'
             ''.format(i, roc_auc[i]))

    #(area = {1:0.2f}
plt.plot([0, 1], [0, 1], 'k--', lw=lw)
plt.xlim([-0.05, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic for multi-class data')
plt.legend(loc="lower right")

plt.show()
f.savefig("foo.pdf", bbox_inches='tight')

#********************************************ROC CURVES**************************************************#

#classifier training
classifier = RandomForestClassifier(n_jobs = -1, n_estimators = 1000, criterion = 'gini', oob_score = True)
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_train)
print(accuracy_score(y_train, y_pred))

X_train = pd.DataFrame(X_train)

#***************************************FEATURE IMPORTANCE BAR CHART**************************************
print(classifier.feature_importances_) #use inbuilt class feature_importances of tree based classifiers  #
#plot graph of feature importances for better visualization                                              #
feat_importances = pd.Series(classifier.feature_importances_, index=X_train.columns)                     #
feat_importances.nlargest(11).plot(kind='barh')                                                          #
plt.show()                                                                                #
#*********************************************************************************************************

#*********************************************CORRELATION MATRIX HEATMAP**********************************
import pandas as pd                                                                                #
import numpy as np                                                                                #
import seaborn as sns                                                                                #
                                                                                #
#get correlations of each features in dataset                                                            #
corrmat = dataset.corr(method='pearson')                                                                 #
top_corr_features = corrmat.index                                                                        #
plt.figure(figsize=(20,20))                                                                              #
                                                                                #
#plot heat map                                                                                #
g=sns.heatmap(dataset[top_corr_features].corr(),annot=True,cmap="RdYlGn", vmin=-1)                       #
figure = g.get_figure()                                                                                #
figure.savefig('heatmap.pdf', dpi=400)                                                                   #
#*********************************************************************************************************

k_fold_accuracy_train = cross_val_score(estimator = classifier, X = X_train, y = y_train, cv = 10)
k_fold_accuracy_train_mean = k_fold_accuracy_train.mean()
#print("Accuracy:" + str(k_fold_accuracy_train_mean))

def trueNegative(mat, row, column):
    res = 0
    for i in range(len(mat)):
        for j in range(len(mat[0])):
            res+=mat[i, j]

    for i in range(len(mat)):
        res-=mat[row, i]

    for j in range(1, len(mat)):
        res-=mat[j, column]

    print("True negative: " + str(res))
    return res

def truePositive(mat, row, column):
    print("True positive: " + str(mat[row, column]))
    return mat[row, column]

def falsePositive(mat, row, column):
    res = 0
    for i in range(len(mat)):
        if i != column:
            res+=mat[i, column]
    print("False positive " + str(res))
    return res

def falseNegative(mat, row, column):
    res = 0
    for i in range(len(mat)):
        if i != row:
            res+=mat[row, i]

    print("False negative: " + str(res))
    return res

def calculateMetrics(mat):
    sensitivities = []
    specificities = []
    for i in range(len(mat)):
        print("class {}: ".format(i))
        tn = trueNegative(mat, i, i)
        tp = truePositive(mat, i, i)
        fp = falsePositive(mat, i, i)
        fn = falseNegative(mat, i, i)
        sensitivity = tp/(tp+fn)
        specificity = tn/(tn+fp)
        print("specificity: {}".format(specificity))
        print("sensitivity: {}".format(sensitivity))
        sensitivities.append(sensitivity)
        specificities.append(specificity)
        print()

    print("Macroaverage specificity: {}".format(sum(specificities)/len(specificities)))
    print("Macroaverage sensitivity: {}".format(sum(sensitivities)/len(sensitivities)))

max_stars_repo_name: zelzhan/Linear-algebra-with-python

max_issues_repo_name: zelzhan/Linear-algebra-with-python

max_forks_repo_name: zelzhan/Linear-algebra-with-python

reason: import numpy


text:
from flask import Flask, render_template, request
from flask import send_file
from flask_cors import CORS
from PIL import Image
import numpy as np
import requests
import json
import sys
import shutil
import os

from utils import *

properties = json.load(open('properties.json', 'r'))

ip = properties['ml_server']['ip'] if len(sys.argv) > 2 else properties['local_server']['ip']
port = properties['ml_server']['port'] if len(sys.argv) > 2 else properties['local_server']['port']

app = Flask(__name__)
CORS(app)
app.jinja_env.auto_reload = True
app.config['IMG_UPLOAD_DIR'] = './imgs'

@app.route('/')
def get_index_page():
    return render_template('index.html', title='Deep Learning Portal')

@app.route('/ml/classification', methods=['POST'])
def classification_req_handler():
    files = request.files.to_dict(flat=False)['classification-input-files']
    saved_files = save_imgs_to_path(files, app.config['IMG_UPLOAD_DIR'])
    data = load_imgs_from_path(saved_files)
    response = requests.post('http://%s:%s/classification' % (ip, port), data=json.dumps({'x' : data}))
    clear_imgs_from_path(app.config['IMG_UPLOAD_DIR'])
    return app.response_class(
        response=response.text,
        status=200
    )

@app.route('/ml/fm-classification', methods=['POST'])
def fm_classification_handler():
    files = request.files.to_dict(flat=False)['fm-classification-input-files']
    saved_files = save_imgs_to_path(files, app.config['IMG_UPLOAD_DIR'])
    data = load_imgs_from_path(saved_files)
    response = requests.post('http://%s:%s/fm-classification' % (ip, port), data=json.dumps({'x' : data}))
    clear_imgs_from_path(app.config['IMG_UPLOAD_DIR'])
    return app.response_class(
        response=response.text,
        status=200
    )

@app.route('/ml/objectdetection', methods=['POST'])
def objectdetection_handler():
    files = request.files.to_dict(flat=False)['object-detection-input-files']
    saved_files = save_imgs_to_path(files, app.config['IMG_UPLOAD_DIR'])
    response = requests.post('http://%s:%s/objectdetection' % (ip, port), files={'file' : open(saved_files[0], 'rb')})
    clear_imgs_from_path(app.config['IMG_UPLOAD_DIR'])
    return app.response_class(
        response=response.text,
        status=200
    )

@app.route('/ml/progressive-gan-generation', methods=['GET'])
def progressive_gan_generation_handler():
    number = request.args.get('number')
    return app.response_class(
        response="https://s3.us-east-2.amazonaws.com/dl-portal-bucket/progressive_gans/img%s.png" % number,
        status=200
    )

@app.route('/ml/superresolution', methods=['POST'])
def superresolution_handler():
    files = request.files.to_dict(flat=False)['super-resolution-input-files']
    saved_files = save_imgs_to_path(files, app.config['IMG_UPLOAD_DIR'])
    data = load_imgs_from_path(saved_files)[0]
    response = requests.post('http://%s:%s/superresolution' % (ip, port), data=json.dumps({'x' : data}))
    clear_imgs_from_path(app.config['IMG_UPLOAD_DIR'])
    return app.response_class(
        response=response.text,
        status=200
    )

@app.route('/ml/semantic_segmentation', methods=['POST'])
def vgg16_semantic_segmentation_handler():
    pass


text:
import torch
from torch import nn
import os
import numpy as np
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms

from common.loaders import calc_stats, MultiViewDataset

class SpritesDataset(Dataset):
    def __init__(self, images, labels):
        self.images = torch.from_numpy(images).float().unsqueeze(1)
        self.labels = np.asarray(labels)

    def __len__(self):
        return len(self.images)

    def __getitem__(self, idx):
        sample = self.images[idx]
        # XXX Don't use ToTensor here, they have already prepared data to be in (B, C, H, W) shape
        label = list(self.labels[idx] / np.array([1, 5, 1, 31, 31]))
        label[0] = int(label[0])
        label[2] = int(label[2])
        return tuple([sample, ] + label)

def load(data_path, train_batch_size, test_batch_size, dataset, augment='crop',
         datasplit='composition', dataset_K=1, device=None, n_train=150000, n_sub_train=150000, **kwargs):
    train_file_path = os.path.join(data_path, f'dsprites_{datasplit}_train_{dataset_K}.npz')
    valid_file_path = os.path.join(data_path, f'dsprites_{datasplit}_valid_{dataset_K}.npz')
    test_file_path = os.path.join(data_path, f'dsprites_{datasplit}_test_{dataset_K}.npz')

    train_data = np.load(train_file_path)
    valid_data = np.load(valid_file_path)
    test_data = np.load(test_file_path)

    train_images = train_data['imgs']
    train_labels = train_data['latents_classes'][:, 1:]
    valid_images = valid_data['imgs']
    valid_labels = valid_data['latents_classes'][:, 1:]
    test_images = test_data['imgs']
    test_labels = test_data['latents_classes'][:, 1:]

    if augment == 'crop':
        augment = nn.Sequential(
            transforms.RandomApply(nn.ModuleList([transforms.RandomResizedCrop((64, 64),
                                                                               scale=(0.81, 1.),
                                                                               ratio=(1., 1.),
                                                                               interpolation=transforms.InterpolationMode.NEAREST), ]), p=0.9),
            transforms.RandomApply(nn.ModuleList([transforms.RandomAffine(4.,
                                                                          interpolation=transforms.InterpolationMode.NEAREST), ]), p=0.9),
            )
    elif augment == 'resize':
        augment = nn.Sequential(
            transforms.RandomAffine(4.,
                                    scale=(0.81, 1.),
                                    interpolation=transforms.InterpolationMode.NEAREST)
            )
    elif augment == 'gaussian':
        augment = nn.Sequential(
            transforms.GaussianBlur(5.)
            )
    else:
        augment = nn.Identity()

    augment = torch.jit.script(augment.to(device))

    train_full_dataset = SpritesDataset(train_images, train_labels)

    lengths = [n_train, len(train_full_dataset) - n_train]
    train_dataset, rest_iid_dataset = torch.utils.data.random_split(train_full_dataset, lengths,
                                                                    generator=torch.Generator().manual_seed(100))

    n_sub = min(n_sub_train, len(train_dataset))
    lengths = [n_sub, len(train_dataset) - n_sub]
    train_sub_dataset, _ = torch.utils.data.random_split(train_dataset, lengths,
                                                      generator=torch.Generator().manual_seed(101))

    train_ssl_dataset = MultiViewDataset(train_dataset, augment)

    n_valid_iid = min(len(rest_iid_dataset)//2, 50000)
    lengths = [n_valid_iid, len(rest_iid_dataset) - n_valid_iid]
    valid_iid_dataset, _ = torch.utils.data.random_split(rest_iid_dataset, lengths,
                                                         generator=torch.Generator().manual_seed(102))
    valid_dataset = SpritesDataset(valid_images, valid_labels)
    test_dataset = SpritesDataset(test_images, test_labels)

    train_ssl = DataLoader(train_ssl_dataset, batch_size=train_batch_size, shuffle=True, drop_last=False,
                           pin_memory=False, num_workers=4, persistent_workers=False)
    train_loader = DataLoader(train_dataset, batch_size=test_batch_size, shuffle=True, drop_last=False,
                              pin_memory=False, num_workers=2, persistent_workers=False)
    sub_loader = DataLoader(train_sub_dataset, batch_size=test_batch_size, shuffle=True, drop_last=False,
                            pin_memory=False, num_workers=2, persistent_workers=False)
    valid_iid_loader = DataLoader(valid_iid_dataset, batch_size=test_batch_size, shuffle=True, drop_last=False,
                                 pin_memory=False, num_workers=2, persistent_workers=False)
    val_loader = DataLoader(valid_dataset, batch_size=test_batch_size, drop_last=False, num_workers=2,
                            persistent_workers=False, shuffle=True)
    test_loader = DataLoader(test_dataset, batch_size=test_batch_size, drop_last=False, num_workers=2,
                             persistent_workers=False, shuffle=True)

    attribute_info = {
        0: (3, 'classification'),  # object shape (3)
        1: (1, 'regression'),  # object size (6)
        2: (10, 'classification'),  # orientation (40)
        3: (1, 'regression'),  # position x (32)
        4: (1, 'regression'),  # position y (32)
        }
    norm_info = calc_stats(train_loader, device=device)
    channels = 1

    loaders = dict(
        train_ssl=train_ssl,
        train=train_loader,
        sub=sub_loader,
        iid=valid_iid_loader,
        val=val_loader,
        ood=test_loader,
    )
    return loaders, attribute_info, norm_info, channels, augment