text:
import numpy as np

def load_csv(filename, sep=';', skiprows=0):
    return np.loadtxt(open(filename, "rb"), delimiter=sep, skiprows=skiprows)

def load_cross_section(filename):
    data_cs = load_csv(filename)
    return data_cs[:,0], data_cs[:,1]

max_stars_repo_name: luihabl/gitgm

max_issues_repo_name: luihabl/gitgm

max_forks_repo_name: luihabl/gitgm


text:
# -*- coding: utf-8 -*-
#~ from __future__ import (unicode_literals, print_function, division, absolute_import)

import numpy as np

import matplotlib.cm
import matplotlib.colors

from .myqt import QT
import pyqtgraph as pg
from pyqtgraph.util.mutex import Mutex

from .base import ViewerBase, BaseMultiChannelViewer,  Base_MultiChannel_ParamController
from .datasource import FrameGrabber, MultiVideoFileSource
from .tools import create_plot_grid

import threading

default_params = [
    {'name': 'nb_column', 'type': 'int', 'value': 4},
    ]

default_by_channel_params = [
    {'name': 'visible', 'type': 'bool', 'value': True},
    ]

#~ class QFrameGrabber(QT.QObject, FrameGrabber):
    #~ frame_ready = QT.pyqtSignal(int, object)
    #~ def on_request_frame(self, video_index, target_frame):
        #~ if self.video_index!=video_index:
            #~ return
        #~ frame = self.get_frame(target_frame)
        #~ if not frame:
            #~ return
        #~ self.frame_ready.emit(self.video_index, frame)

class QFrameGrabber(QT.QObject):
    frame_ready = QT.pyqtSignal(int, object)

    def __init__(self, frame_grabber, video_index, parent=None):
        QT.QObject.__init__(self, parent)

        self.fg = frame_grabber
        self.video_index = video_index

        self.mutex = Mutex()
        self.request_list = []
        self._last_frame = None

    #~ def on_request_frame(self, video_index, target_frame):
        #~ if self.video_index!=video_index:
            #~ return
        #~ frame = self.fg.get_frame(target_frame)
        #~ if frame is None:
            #~ return
        #~ self.frame_ready.emit(self.video_index, frame)

    @property
    def last_frame(self):
        with self.mutex:
            return self._last_frame

    #~ @property
    #~ def active_frame(self):
        #~ return self.fg.active_frame

    #~ @active_frame.setter
    #~ def active_frame(self, value):
        #~ self.fg.active_frame = value

    #~ def queue_request_frame(self, target_frame):
        #~ print('queue_request_frame', threading.get_ident())
        #~ with self.mutex:
            #~ self.request_list.append(target_frame)
            #~ if len(self.request_list)>1:
                #~ self.request_frame.emit()

    def on_request_frame(self, video_index):
        if self.video_index!=video_index:
            return

        #~ print('on_request_frame', threading.get_ident())

        with self.mutex:
            #~ print('len(self.request_list)', len(self.request_list))
            if len(self.request_list)==0:
                return
            target_frame = self.request_list[-1]
            self.request_list = []

            if target_frame == self._last_frame:
                return

        frame = self.fg.get_frame(target_frame)
        with self.mutex:
            self._last_frame = target_frame
            #~ print('new self._last_frame', self._last_frame)
        #~ self.fg.active_frame = target_frame
        self.frame_ready.emit(self.video_index, frame)

class VideoViewer_ParamController(Base_MultiChannel_ParamController):
    pass

#~ class VideoViewer(ViewerBase):
class VideoViewer(BaseMultiChannelViewer):

    _default_params = default_params
    _default_by_channel_params = default_by_channel_params

    _ControllerClass = VideoViewer_ParamController

    request_frame = QT.pyqtSignal(int)

    def __init__(self, **kargs):
        BaseMultiChannelViewer.__init__(self, **kargs)

        self.make_params()
        self.make_param_controller()
        self.set_layout()

        self.qframe_grabbers = []
        self.threads = []
        #~ self.actual_frames = []
        for i, video_filename in enumerate(self.source.video_filenames):
            fg = QFrameGrabber(self.source.frame_grabbers[i], i)
            self.qframe_grabbers.append(fg)
            #~ fg.set_file(video_filename)
            #~ fg.video_index = i
            fg.frame_ready.connect(self.update_frame)

            thread = QT.QThread(parent=self)
            fg.moveToThread(thread)
            thread.start()
            self.threads.append(thread)

            self.request_frame.connect(self.qframe_grabbers[i].on_request_frame)

    @classmethod
    def from_filenames(cls, video_filenames, video_times, name):
        source = MultiVideoFileSource(video_filenames, video_times=video_times)
        view = cls(source=source, name=name)
        return view

    def closeEvent(self, event):
        for i, thread in enumerate(self.threads):
            thread.quit()
            thread.wait()
        event.accept()

    def set_layout(self):
        self.mainlayout = QT.QVBoxLayout()
        self.setLayout(self.mainlayout)

        self.graphiclayout = pg.GraphicsLayoutWidget()
        self.mainlayout.addWidget(self.graphiclayout)
        self.create_grid()

    def on_param_change(self):
        self.create_grid()
        self.refresh()

    def create_grid(self):
        visible_channels = self.params_controller.visible_channels
        self.plots = create_plot_grid(self.graphiclayout, self.params['nb_column'], visible_channels,
                     ViewBoxClass=pg.ViewBox, vb_params={'lockAspect':True})

        for plot in self.plots:
            plot.showAxis('left', False)
            plot.showAxis('bottom', False)

        self.images = []
        for c in range(self.source.nb_channel):
            if visible_channels[c]:
                image = pg.ImageItem()
                self.plots[c].addItem(image)
                self.images.append(image)
            else:
                self.images.append(None)

    def refresh(self):
        #~ print('videoviewer.refresh', self.t)
        visible_channels = self.params_controller.visible_channels

        #~ print()
        #~ print('refresh t=', self.t)
        for c in range(self.source.nb_channel):
            if visible_channels[c]:
                frame_index = self.source.time_to_frame_index(c, self.t)
                #~ print( 'c', c, 'frame_index', frame_index, 'self.qframe_grabbers[c].last_frame', self.qframe_grabbers[c].last_frame)

                #~ if self.qframe_grabbers[c].active_frame != frame_index:
                #~ print('self.qframe_grabbers[c].last_frame != frame_index', self.qframe_grabbers[c].last_frame != frame_index)
                if self.qframe_grabbers[c].last_frame != frame_index:

                    #~ self.qframe_grabbers[c].active_frame = frame_index
                    #~ self.request_frame.emit(c, frame_index)

                    #~ self.qframe_grabbers[c].queue_request_frame(frame_index)
                    #~ print('enque frame', threading.get_ident(), 'frame_index', frame_index)
                    with self.qframe_grabbers[c].mutex:

                        self.qframe_grabbers[c].request_list.append(frame_index)
                        if len(self.qframe_grabbers[c].request_list)>=1:
                            self.request_frame.emit(c)
                            #~ print('EMIT!!!!!')

    def update_frame(self, video_index, frame):
        #~ print('update_frame', video_index, frame)

        if frame is None:
            self.images[video_index].clear()
        else:
            #TODO : find better solution!!!! to avoid copy
            try:
                # PyAV >= 0.5.3
                img = frame.to_ndarray(format='rgb24')
            except AttributeError:
                # PyAV < 0.5.3
                img = frame.to_nd_array(format='rgb24')
            img = img.swapaxes(0,1)[:,::-1,:]
            #~ print(img.shape, img.dtype)
            self.images[video_index].setImage(img)

        #~ rgba = frame.reformat(frame.width, frame.height, "rgb24", 'itu709')
        #print rgba.to_image().save("test.png")
        # could use the buffer interface here instead, some versions of PyQt don't support it for some reason
        # need to track down which version they added support for it
        #~ bytearray(rgba.planes[0])
        #~ bytesPerPixel  =3
        #~ img = QT.QImage(bytearray(rgba.planes[0]), rgba.width, rgba.height, rgba.width * bytesPerPixel, QT.QImage.Format_RGB888)
        #~ self.images[video_index].setImage(img)

        #img = QtGui.QImage(rgba.planes[0], rgba.width, rgba.height, QtGui.QImage.Format_RGB888)

                #~ if self.actual_frames[i] != new_frame:

                #~ frame = self.source.get_frame(t=t,chan=c)
                #~ self.images[c].setImage(frame)
            #~ else:
                #~ pass

ext: py

max_stars_repo_name: rdarie/ephyviewer

max_issues_repo_name: rdarie/ephyviewer

max_forks_repo_name: rdarie/ephyviewer

reason: import numpy


text:
# -*- coding: utf-8 -*-
import pandas as pd
import numpy as np
import h5py
from pkg_resources import resource_filename

#%%
def _load_dataset(filename, partition='complete', data_type='pandas', **kwargs):
    """
    Load an example dataset from deepsurvk.datasets.
    All datasets correspond to the ones used originally in [#katzman0]_.

    Parameters
    ----------
    filename: string
        Name of the dataset (e.g., 'whas.h5')

    partition: string
        Partition of the data to load.

        Possible values are:
            * ``complete`` - The whole dataset (default)
            * ``training`` or ``train`` - Training partition as used in the original DeepSurv
            * ``testing`` or ``test`` - Testing partition as used in the original DeepSurv

    data_type: string
        Data type of the data.

        Possible values are:
            * ``pandas`` or ``pd`` or ``dataframe`` or ``df``- pandas DataFrame (default)
            * ``numpy`` or ``np`` - NumPy array

        .. note::
            NumPy is supported as an option, but DeepSurvK is built
            with pandas in mind.

    Returns
    -------
    tuple of pandas DataFrames
        * X - Features
        * Y - Target variable
        * E - Event variable

    References
    ----------
    .. [#katzman0] Katzman, Jared L., et al. "DeepSurv: personalized treatment recommender system using a Cox proportional hazards deep neural network." BMC medical research methodology 18.1 (2018): 24.
    """

    filename_ = resource_filename('deepsurvk', 'datasets/data/' + filename)

    # Read training data.
    with h5py.File(filename_, 'r') as f:
        X_train = f['train']['x'][()]
        E_train = f['train']['e'][()]
        Y_train = f['train']['t'][()].reshape(-1, 1)

    # Read testing data.
    with h5py.File(filename_, 'r') as f:
        X_test = f['test']['x'][()]
        E_test = f['test']['e'][()]
        Y_test = f['test']['t'][()].reshape(-1, 1)

    # Define data partitions.
    if partition == 'training' or partition == 'train':
        X = X_train
        Y = Y_train
        E = E_train
    elif partition == 'testing' or partition == 'test':
        X = X_test
        Y = Y_test
        E = E_test
    elif partition == 'complete':
        X = np.concatenate((X_train, X_test), axis=0)
        Y = np.concatenate((Y_train, Y_test), axis=0)
        E = np.concatenate((E_train, E_test), axis=0)
    else:
        raise ValueError('Invalid partition.')

    # If NumPy, do nothing (already in that format).
    if (data_type == 'numpy') or (data_type=='np') or (data_type=='array'):
        pass

    # Otherwise, convert to pandas DataFrame.
    elif (data_type == 'pandas') or (data_type=='pd') or (data_type=='dataframe') or (data_type=='df'):
        X, Y, E = _numpy_to_pandas(X, Y, E, filename)

    else:
        raise ValueError("Invalid data type.")

    return X, Y, E

#%%
def _numpy_to_pandas(X, Y, E, filename):
    """
    Convert a data NumPy array to pandas DataFrame.

    Parameters
    ----------
    X: NumPy array
        Feature matrix
    Y: NumPy array
        Output array
    E: NumPy array
        Event array
    filename: string
        Name of the dataset (e.g., 'whas.h5')

    Returns
    -------
    tuple
        Tuple with df_X, df_Y, and df_E
    """
    # Get columns names and data types.
    column_names = _get_column_names(filename)
    data_types = _get_data_types(filename)

    # Create pandas DataFrames
    df_X = pd.DataFrame(data=X, columns=column_names)
    df_X = df_X.astype(data_types)

    df_Y = pd.DataFrame(data=Y, columns=['T'], dtype=np.float64)
    df_E = pd.DataFrame(data=E, columns=['E'], dtype=np.int32)

    return df_X, df_Y, df_E

#%%
def _get_column_names(filename):
    """
    Get the data column names (needed to create a pandas DataFrame).

    Parameters
    ----------
    filename: string
        Name of the dataset (e.g., 'whas.h5')

    Returns
    -------
    column_names: list
        Column names corresponding to the data.
    """
    # Remove file name extension.
    filename = filename.split('.h5')[0]

    # Define column names for each dataset.
    if filename=='metabric':
        column_names = ['gene1', 'gene2', 'gene3', 'gene4', 'radio', 'horm', 'chemo', 'er_status', 'age']
    elif filename=='rgbsg':
        column_names = ['horm_treatment', 'grade', 'menopause' ,'age' ,'n_positive_nodes', 'progesterone' ,'estrogen']
    elif filename=='simulated_gaussian':
        column_names = ['x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7', 'x8', 'x9', 'x10']
    elif filename=='simulated_linear':
        column_names = ['x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7', 'x8', 'x9', 'x10']
    elif filename=='simulated_treatment':
        column_names = ['x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7', 'x8', 'x9', 'x10', 'treatment']
    elif filename=='support':
        column_names = ['age', 'sex', 'race', 'n_comorbidities', 'diabetes', 'dementia', 'cancer', 'blood_pressure', 'heart_rate', 'respiration_rate', 'temperature', 'white_blood_cell', 'serum_sodium', 'serum_creatinine']
    elif filename=='whas':
        column_names = ['shock', 'age', 'sex', 'bmi', 'chf', 'recurrent']

    return column_names

#%%
def _get_data_types(filename):
    """
    Get the right data types of the columns (needed to create a pandas DataFrame).

    Parameters
    ----------
    filename: string
        Name of the dataset (e.g., 'whas.h5')

    Returns
    -------
    data_types: dictionary
        Each key corresponds to the column name.
        Each value corresponds to the data type.
    """
    # Remove file name extension.
    filename = filename.split('.h5')[0]

    # Assign data type for each column for each dataset.
    if filename=='metabric':
        data_types_ = ['float64', 'float64', 'float64', 'float64', 'int32', 'int32', 'int32', 'int32', 'float64']
    elif filename=='rgbsg':
        data_types_ = ['int32', 'int32', 'int32', 'int32' ,'int32', 'int32', 'int32']
    elif filename=='simulated_gaussian':
        data_types_ = ['float64', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64']
    elif filename=='simulated_linear':
        data_types_ = ['float64', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64']
    elif filename=='simulated_treatment':
        data_types_ = ['float64', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', 'int32']
    elif filename=='support':
        data_types_ = ['float64', 'int32', 'int32', 'int32', 'int32', 'int32', 'int32', 'int32', 'int32', 'int32', 'float64', 'int32', 'float64', 'float64']
    elif filename=='whas':
        data_types_ = ['int32', 'int32', 'int32', 'float64', 'int32', 'int32']

    # Create dictionary.
    keys = _get_column_names(filename)
    data_types = dict(zip(keys, data_types_))

    return data_types

#%%
def load_metabric(partition='complete', **kwargs):
    """
    Data from the Molecular Taxonomy of Breast Cancer International
    Consortium (METABRIC), which uses gene and protein expression
    profiles to determine new breast cancer subgroups

    It consists of clinical features of 1980 patients, of which 57.72%
    have an observed death due to breast cancer with a median survival time
    of 116 months. However, the file only contains data of 1904 patients.

    For more information, see [#curtis]_ as well as the accompanying README.

    Parameters
    ----------
    partition: string
        Partition of the data to load.

        Possible values are:
            * ``complete`` - The whole dataset (default)
            * ``training`` or ``train`` - Training partition as used in the original DeepSurv
            * ``testing`` or ``test`` - Testing partition as used in the original DeepSurv

    data_type: string
        Data type of the data.

        Possible values are:
            * ``pandas`` or ``pd`` or ``dataframe`` or ``df``- pandas DataFrame (default)
            * ``numpy`` or ``np`` - NumPy array

        .. note::
            NumPy is supported as an option, but DeepSurvK is built
            with pandas in mind.

    Returns
    -------
    tuple of pandas DataFrames
        * X - Features
        * Y - Target variable
        * E - Event variable

    References
    ----------
    .. [#curtis] Curtis, Christina, et al. "The genomic and transcriptomic architecture of 2,000 breast tumours reveals novel subgroups." Nature 486.7403 (2012): 346-352.
    """
    return _load_dataset('metabric.h5', partition=partition, **kwargs)

#%%
def load_rgbsg(partition='complete', **kwargs):
    """
    The training partition belongs to the Rotterdam tumor bank dataset [#foekens]_.
    It contains records of 1546 patients with node-positive breast cancer.
    Nearly 90% of the patients have an observed death time.

    The testing partitiong belongs to the German Breast Cancer Study
    Group (GBSG) [#schumacher]_. It contains records for 686 patients (of which 56 % are
    censored) in a randomized clinical trial that studied the effects of
    chemotherapy and hormone treatment on survival rate.

    For more information, see [#foekens]_ and [#schumacher]_, as well as the accompanying README.

    Parameters
    ----------
    partition: string
        Partition of the data to load.

        Possible values are:
            * ``complete`` - The whole dataset (default)
            * ``training`` or ``train`` - Training partition as used in the original DeepSurv
            * ``testing`` or ``test`` - Testing partition as used in the original DeepSurv

    data_type: string
        Data type of the data.

        Possible values are:
            * ``pandas`` or ``pd`` or ``dataframe`` or ``df``- pandas DataFrame (default)
            * ``numpy`` or ``np`` - NumPy array

        .. note::
            NumPy is supported as an option, but DeepSurvK is built
            with pandas in mind.

    Returns
    -------
    tuple of pandas DataFrames
        * X - Features
        * Y - Target variable
        * E - Event variable

    References
    ----------
    .. [#foekens] Foekens, John A., et al. "The urokinase system of plasminogen activation and prognosis in 2780 breast cancer patients." Cancer research 60.3 (2000): 636-643.

    .. [#schumacher] Schumacher, M., et al. "Randomized 2 x 2 trial evaluating hormonal treatment and the duration of chemotherapy in node-positive breast cancer patients. German Breast Cancer Study Group." Journal of Clinical Oncology 12.10 (1994): 2086-2093.
    """
    return _load_dataset('rgbsg.h5', partition=partition, **kwargs)

#%%
def load_simulated_gaussian(partition='complete', **kwargs):
    """
    Synthetic data with a Gaussian (non-linear) log-risk function.

    For more information, see [#katzman1]_ as well as the accompanying README.

    Parameters
    ----------
    partition: string
        Partition of the data to load.

        Possible values are:
            * ``complete`` - The whole dataset (default)
            * ``training`` or ``train`` - Training partition as used in the original DeepSurv
            * ``testing`` or ``test`` - Testing partition as used in the original DeepSurv

    data_type: string
        Data type of the data.

        Possible values are:
            * ``pandas`` or ``pd`` or ``dataframe`` or ``df``- pandas DataFrame (default)
            * ``numpy`` or ``np`` - NumPy array

        .. note::
            NumPy is supported as an option, but DeepSurvK is built
            with pandas in mind.

    Returns
    -------
    tuple of pandas DataFrames
        * X - Features
        * Y - Target variable
        * E - Event variable

    References
    ----------
    .. [#katzman1] Katzman, Jared L., et al. "DeepSurv: personalized treatment recommender system using a Cox proportional hazards deep neural network." BMC medical research methodology 18.1 (2018): 24.
    """
    return _load_dataset('simulated_gaussian.h5', partition=partition, **kwargs)

#%%
def load_simulated_linear(partition='complete', **kwargs):
    """
    Synthetic data with a linear log-risk function.

    For more information, see [#katzman2]_ as well as the accompanying README.

    Parameters
    ----------
    partition: string
        Partition of the data to load.

        Possible values are:
            * ``complete`` - The whole dataset (default)
            * ``training`` or ``train`` - Training partition as used in the original DeepSurv
            * ``testing`` or ``test`` - Testing partition as used in the original DeepSurv

    data_type: string
        Data type of the data.

        Possible values are:
            * ``pandas`` or ``pd`` or ``dataframe`` or ``df``- pandas DataFrame (default)
            * ``numpy`` or ``np`` - NumPy array

        .. note::
            NumPy is supported as an option, but DeepSurvK is built
            with pandas in mind.

    Returns
    -------
    tuple of pandas DataFrames
        * X - Features
        * Y - Target variable
        * E - Event variable

    References
    ----------
    .. [#katzman2] Katzman, Jared L., et al. "DeepSurv: personalized treatment recommender system using a Cox proportional hazards deep neural network." BMC medical research methodology 18.1 (2018): 24.
    """
    return _load_dataset('simulated_linear.h5', partition=partition, **kwargs)

#%%
def load_simulated_treatment(partition='complete', **kwargs):
    """
    Synthetic data similar to the simulated_gaussian one, with an additional
    column representing treatment.

    For more information, see [#katzman3]_ as well as the accompanying README.

    Parameters
    ----------
    partition: string
        Partition of the data to load.

        Possible values are:
            * ``complete`` - The whole dataset (default)
            * ``training`` or ``train`` - Training partition as used in the original DeepSurv
            * ``testing`` or ``test`` - Testing partition as used in the original DeepSurv

    data_type: string
        Data type of the data.

        Possible values are:
            * ``pandas`` or ``pd`` or ``dataframe`` or ``df``- pandas DataFrame (default)
            * ``numpy`` or ``np`` - NumPy array

        .. note::
            NumPy is supported as an option, but DeepSurvK is built
            with pandas in mind.

    Returns
    -------
    tuple of pandas DataFrames
        * X - Features
        * Y - Target variable
        * E - Event variable

    References
    ----------
    .. [#katzman3] Katzman, Jared L., et al. "DeepSurv: personalized treatment recommender system using a Cox proportional hazards deep neural network." BMC medical research methodology 18.1 (2018): 24.
    """
    return _load_dataset('simulated_treatment.h5', partition=partition, **kwargs)

#%%
def load_support(partition='complete', **kwargs):
    """
    Data from the Study to Understand Prognoses Preferences Outcomes
    and Risks of Treatment (SUPPORT), which studied the survival time of
    seriously ill hospitalized adults.

    Originally, it consists of 14 clinical features of 9105 patients.
    However, patients with missing features were dropped, leaving a total
    of 8873 patients.

    For more information, see [#knaus]_ as well as the accompanying README.

    Parameters
    ----------
    partition: string
        Partition of the data to load.

        Possible values are:
            * ``complete`` - The whole dataset (default)
            * ``training`` or ``train`` - Training partition as used in the original DeepSurv
            * ``testing`` or ``test`` - Testing partition as used in the original DeepSurv

    data_type: string
        Data type of the data.

        Possible values are:
            * ``pandas`` or ``pd`` or ``dataframe`` or ``df``- pandas DataFrame (default)
            * ``numpy`` or ``np`` - NumPy array

        .. note::
            NumPy is supported as an option, but DeepSurvK is built
            with pandas in mind.

    Returns
    -------
    tuple of pandas DataFrames
        * X - Features
        * Y - Target variable
        * E - Event variable

    References
    ----------
    .. [#knaus] Knaus, William A., et al. "The SUPPORT prognostic model: Objective estimates of survival for seriously ill hospitalized adults." Annals of internal medicine 122.3 (1995): 191-203.
    """
    return _load_dataset('support.h5', partition=partition, **kwargs)

#%%
def load_whas(partition='complete', **kwargs):
    """
    Data from the Worcester Heart Attack Study (WHAS), which investigates
    the effects of a patient's factors on acute myocardial infraction (MI)
    survival.

    It consists of 1638 observations and 5 features: age, sex,
    body-mass-index (BMI), left heart failure complications (CHF), and order
    of MI (MIORD).

    For more information, see [#hosmer]_ as well as the accompanying README.

    Parameters
    ----------
    partition: string
        Partition of the data to load.

        Possible values are:
            * ``complete`` - The whole dataset (default)
            * ``training`` or ``train`` - Training partition as used in the original DeepSurv
            * ``testing`` or ``test`` - Testing partition as used in the original DeepSurv

    data_type: string
        Data type of the data.

        Possible values are:
            * ``pandas`` or ``pd`` or ``dataframe`` or ``df``- pandas DataFrame (default)
            * ``numpy`` or ``np`` - NumPy array

        .. note::
            NumPy is supported as an option, but DeepSurvK is built
            with pandas in mind.

    Returns
    -------
    tuple of pandas DataFrames
        * X - Features
        * Y - Target variable
        * E - Event variable

    References
    ----------
    .. [#hosmer] Hosmer Jr, David W., Stanley Lemeshow, and Susanne May. Applied survival analysis: regression modeling of time-to-event data. Vol. 618. John Wiley & Sons, 2011.
    """
    return _load_dataset('whas.h5', partition=partition, **kwargs)

max_stars_repo_name: arturomoncadatorres/deepsurvk

max_issues_repo_name: arturomoncadatorres/deepsurvk

max_forks_repo_name: arturomoncadatorres/deepsurvk


text:
"""
Seismic: Synthetic convolutional seismogram for a simple two layer velocity
model
"""
import numpy as np
import matplotlib.pyplot as plt
from fatiando.seismic import conv
from fatiando.vis import mpl
# model parameters
n_samples, n_traces = [600, 20]
rock_grid = 1500.*np.ones((n_samples, n_traces))
rock_grid[300:, :] = 2500.
# synthetic calculation
vel_l = conv.depth_2_time(rock_grid, rock_grid, dt=2.e-3, dz=1.)
rho_l = np.ones(np.shape(vel_l))
rc = conv.reflectivity(vel_l, rho_l)
synt = conv.convolutional_model(rc, 30., conv.rickerwave, dt=2.e-3)
# plot input model
plt.figure()
plt.subplot(3, 1, 1)
plt.ylabel('Depth (m)')
plt.title("Depth Vp model", fontsize=13, family='sans-serif', weight='bold')
plt.imshow(rock_grid, extent=[0, n_traces, n_samples, 0],
           cmap=mpl.pyplot.cm.bwr, aspect='auto', origin='upper')
# plot resulted seismogram using wiggle
plt.subplot(3, 1, 2)
mpl.seismic_wiggle(synt, dt=2.e-3)
mpl.seismic_image(synt, dt=2.e-3, cmap=mpl.pyplot.cm.jet, aspect='auto')
plt.ylabel('time (seconds)')
plt.title("Convolutional seismogram", fontsize=13, family='sans-serif',
          weight='bold')

# plot resulted seismogram using wiggle over Vp model
plt.subplot(3, 1, 3)
mpl.seismic_image(vel_l, dt=2.e-3, cmap=mpl.pyplot.cm.jet, aspect='auto')
mpl.seismic_wiggle(synt, dt=2.e-3)
plt.ylabel('time (seconds)')
plt.title("Convolutional seismogram over Vp model", fontsize=13,
          family='sans-serif', weight='bold')
plt.show()


text:
import tensorflow as tf
import tensorflow_addons as tfa
import numpy as np

def get_initializer(range: float = 0.02) -> tf.initializers.TruncatedNormal:
    return tf.initializers.TruncatedNormal(stddev=range)

def shape_list(x):
    static = x.shape.as_list()
    dynamic = tf.shape(x)
    return [dynamic[i] if s is None else s for i, s in enumerate(static)]

def create_attention_mask(x):
    x = tf.reduce_any(x != 0, axis=-1)
    x = x[:, tf.newaxis, :, tf.newaxis]
    x = tf.cast(x, tf.float32)
    return (1 - x) * -1e9

class Conv1D(tf.keras.layers.Layer):
    def __init__(self, nf, nx, **kwargs):
        super().__init__(**kwargs)
        self.nf = nf
        self.nx = nx

    def build(self, input_shape):
        self.weight = self.add_weight(
            'weight',
            shape=[self.nx, self.nf],
            initializer=get_initializer(0.02),
        )
        self.bias = self.add_weight(
            'bias',
            shape=[1, self.nf],
            initializer=tf.zeros_initializer(),
        )

    def call(self, x):
        bz, sl = shape_list(x)[:2]

        x = tf.reshape(x, [-1, self.nx])
        x = tf.matmul(x, self.weight) + self.bias
        x = tf.reshape(x, [bz, sl, self.nf])

        return x

class Attention(tf.keras.layers.Layer):
    def __init__(self, n_days, n_features, n_heads, **kwargs):
        super().__init__(**kwargs)

        assert n_features % n_heads == 0
        self.n_days = n_days
        self.n_features = n_features
        self.n_heads = n_heads

        self.c_attn = Conv1D(n_features * 3, n_features, name='c_attn')
        self.c_proj = Conv1D(n_features, n_features, name='c_proj')
        self.attn_dropout = tf.keras.layers.Dropout(0.1)
        self.resid_dropout = tf.keras.layers.Dropout(0.1)

    @staticmethod
    def causal_attention_mask(nd, ns, dtype):
        """1's in the lower triangle, counting from the lower right corner.
        Same as tf.matrix_band_part(tf.ones([nd, ns]), -1, ns-nd), but doesn't produce garbage on TPUs.
        """
        i = tf.range(nd)[:, None]
        j = tf.range(ns)
        m = i >= j - ns + nd
        return tf.cast(m, dtype)

    def _attn(self, q, k, v, mask, training=False):
        # q, k, p, v have shape [batch, heads, days, head_features]
        w = tf.matmul(q, k, transpose_a=True)
        dk = tf.cast(shape_list(k)[-1], tf.float32)  # scale attention_scores
        w = w / tf.math.sqrt(dk)

        # w has shape [batch, heads, dst_sequence, src_sequence]
        # where information flows from src to dst.
        _, _, nd, ns = shape_list(w)
        b = self.causal_attention_mask(nd, ns, dtype=w.dtype)
        b = b[tf.newaxis, tf.newaxis, :, :]
        w = w * b - 1e4 * (1 - b)

        # Apply attention mask and softmax the scores
        w = tf.matmul(w, v, transpose_b=True)
        w = tf.divide(w, tf.math.square(dk))
        w = tf.transpose(w, perm=(0, 1, 3, 2))
        w = tf.add(w, mask)
        w = tf.nn.softmax(w, axis=-2)
        w = self.attn_dropout(w, training=training)

        # Apply attention weights
        v = tf.multiply(v, w)

        # return outputs and attn weights
        return v, w

    def merge_heads(self, x):
        x = tf.transpose(x, [0, 2, 1, 3])
        x_shape = shape_list(x)
        new_x_shape = x_shape[:-2] + [x_shape[-2] * x_shape[-1]]
        return tf.reshape(x, new_x_shape)

    def split_heads(self, x):
        x_shape = shape_list(x)
        new_x_shape = x_shape[:-1] + \
            [self.n_heads, x_shape[-1] // self.n_heads]
        x = tf.reshape(x, new_x_shape)
        # (batch, head, seq_length, head_features)
        return tf.transpose(x, (0, 2, 1, 3))

    def call(self, x, mask, training=False):
        x = self.c_attn(x)
        q, k, v = tf.split(x, 3, axis=2)
        q = self.split_heads(q)
        k = self.split_heads(k)
        v = self.split_heads(v)

        a, w = self._attn(q, k, v, mask, training=training)
        a, w = self.merge_heads(a), self.merge_heads(w)
        a = self.c_proj(a)
        a = self.resid_dropout(a, training=training)

        return a, w

class MLP(tf.keras.layers.Layer):
    def __init__(self, n_state, n_features, **kwargs):
        super().__init__(**kwargs)
        nx = n_features
        self.c_fc = Conv1D(n_state, nx, name="c_fc")
        self.c_proj = Conv1D(nx, n_state, name="c_proj")
        self.dropout = tf.keras.layers.Dropout(0.1)

    def call(self, x, training=False):
        h = tfa.activations.gelu(self.c_fc(x))
        h2 = self.c_proj(h)
        h2 = self.dropout(h2, training=training)
        return h2

class Block(tf.keras.layers.Layer):
    def __init__(self, n_days, n_features, n_heads, **kwargs):
        super().__init__(**kwargs)

        self.ln_1 = tf.keras.layers.LayerNormalization(
            epsilon=1e-5,
            name='ln_1',
        )
        self.attn = Attention(n_days, n_features, n_heads, name='attn')
        self.ln_2 = tf.keras.layers.LayerNormalization(
            epsilon=1e-5,
            name='ln_2',
        )
        self.mlp = MLP(4 * n_features, n_features, name='mlp')

    def call(self, x, mask, training=False):
        a = self.ln_1(x)
        a, w = self.attn(a, mask, training=training)
        x = x + a

        m = self.ln_2(x)
        m = self.mlp(m, training=training)
        x = x + m

        return x, w

class Mimic3Gpt2(tf.keras.Model):
    def __init__(self, n_features, n_heads, n_days=14, n_layers=1, **kwargs):
        super().__init__(**kwargs)

        self.n_days = n_days
        self.n_features = n_features
        self.n_layers = n_layers

        # positional embedding layer
        self.wpe = tf.keras.layers.Embedding(
            n_days,
            n_features,
            embeddings_initializer=get_initializer(0.02),
            name='wpe',
        )
        self.drop = tf.keras.layers.Dropout(0.1)

        # transformer-decoder block layer
        self.blocks = [
            Block(n_days, n_features, n_heads, name=f'block_{i}')
            for i in range(n_layers)
        ]
        self.ln_f = tf.keras.layers.LayerNormalization(
            epsilon=1e-5,
            name='ln_f',
        )

        # final projection layer
        self.proj = tf.keras.layers.Dense(
            1,
            activation='sigmoid',
            kernel_initializer=get_initializer(0.02),
            name='proj'
        )
        self.proj_drop = tf.keras.layers.Dropout(0.1)

    def call(self, x, training=False):
        # sanity check
        _, n_days, n_features = shape_list(x)
        assert n_days == self.n_days, "x's shape should be [batch_size, n_days, n_features]"
        assert n_features == self.n_features, "x's shape should be [batch_size, n_days, n_features]"

        # create attention mask so that the model won't
        # treat padding days as inputs
        attn_mask = create_attention_mask(x)

        # add positional encodings
        position_ids = tf.range(0, n_days, dtype=tf.int32)[tf.newaxis, :]
        x = x + self.wpe(position_ids)
        x = self.drop(x, training=training)

        # feed x to n decoder blocks
        w = tf.zeros(shape_list(x))
        for block in self.blocks:
            x, wb = block(x, attn_mask, training=training)
            w = tf.add(w, wb)

        # feed to final normalization layer
        x = self.ln_f(x)

        # project linearly to get probabilities for each day
        x = self.proj(x)
        x = self.proj_drop(x, training=training)

        return x, tf.divide(w, self.n_layers)

max_stars_repo_name: miggymigz/mimic-lstm

max_issues_repo_name: miggymigz/mimic-lstm

max_forks_repo_name: miggymigz/mimic-lstm


text:
import pandas as pd
import numpy as np

from new_set_points import new_set_points

test_data = pd.read_csv('test_data.csv')

next_week_data = pd.read_csv('next_week_test_data.csv')

test_points = new_set_points(next_week_data, test_data)

test_points.to_csv('weekly_set_points.csv')


text:
"""util functions for visualization

"""

import argparse
import time
import csv
import yaml
import os
import logging
from pathlib import Path

import numpy as np
from tqdm import tqdm

from tensorboardX import SummaryWriter
import cv2
import matplotlib.pyplot as plt

def plot_imgs(imgs, titles=None, cmap='brg', ylabel='', normalize=False, ax=None, dpi=100):
    n = len(imgs)
    if not isinstance(cmap, list):
        cmap = [cmap]*n
    if ax is None:
        fig, ax = plt.subplots(1, n, figsize=(6*n, 6), dpi=dpi)
        if n == 1:
            ax = [ax]
    else:
        if not isinstance(ax, list):
            ax = [ax]
        assert len(ax) == len(imgs)
    for i in range(n):
        if imgs[i].shape[-1] == 3:
            imgs[i] = imgs[i][..., ::-1]  # BGR to RGB
        ax[i].imshow(imgs[i], cmap=plt.get_cmap(cmap[i]),
                     vmin=None if normalize else 0,
                     vmax=None if normalize else 1)
        if titles:
            ax[i].set_title(titles[i])
        ax[i].get_yaxis().set_ticks([])
        ax[i].get_xaxis().set_ticks([])
        for spine in ax[i].spines.values():  # remove frame
            spine.set_visible(False)
    ax[0].set_ylabel(ylabel)
    plt.tight_layout()

# from utils.draw import img_overlap
def img_overlap(img_r, img_g, img_gray):  # img_b repeat
    img = np.concatenate((img_gray, img_gray, img_gray), axis=0)
    img[0, :, :] += img_r[0, :, :]
    img[1, :, :] += img_g[0, :, :]
    img[img > 1] = 1
    img[img < 0] = 0
    return img

def draw_keypoints(img, corners, color=(0, 255, 0), radius=3, s=3):
    '''

    :param img:
        image:
        numpy [H, W]
    :param corners:
        Points
        numpy [N, 2]
    :param color:
    :param radius:
    :param s:
    :return:
        overlaying image
        numpy [H, W]
    '''
    img = np.repeat(cv2.resize(img, None, fx=s, fy=s)[..., np.newaxis], 3, -1)
    for c in np.stack(corners).T:
        # cv2.circle(img, tuple(s * np.flip(c, 0)), radius, color, thickness=-1)
        cv2.circle(img, tuple((s * c[:2]).astype(int)), radius, color, thickness=-1)
    return img

# def draw_keypoints(img, corners, color=(0, 255, 0), radius=3, s=3):
#     '''

#     :param img:
#         np (H, W)
#     :param corners:
#         np (3, N)
#     :param color:
#     :param radius:
#     :param s:
#     :return:
#     '''
#     img = np.repeat(cv2.resize(img, None, fx=s, fy=s)[..., np.newaxis], 3, -1)
#     for c in np.stack(corners).T:
#         # cv2.circle(img, tuple(s * np.flip(c, 0)), radius, color, thickness=-1)
#         cv2.circle(img, tuple((s*c[:2]).astype(int)), radius, color, thickness=-1)
#     return img

def draw_matches(rgb1, rgb2, match_pairs, lw = 0.5, color='g', if_fig=True,
                filename='matches.png', show=False):
    '''

    :param rgb1:
        image1
        numpy (H, W)
    :param rgb2:
        image2
        numpy (H, W)
    :param match_pairs:
        numpy (keypoiny1 x, keypoint1 y, keypoint2 x, keypoint 2 y)
    :return:
        None
    '''
    from matplotlib import pyplot as plt

    h1, w1 = rgb1.shape[:2]
    h2, w2 = rgb2.shape[:2]
    canvas = np.zeros((max(h1, h2), w1 + w2, 3), dtype=rgb1.dtype)
    canvas[:h1, :w1] = rgb1[:,:,np.newaxis]
    canvas[:h2, w1:] = rgb2[:,:,np.newaxis]
    # fig = plt.figure(frameon=False)
    if if_fig:
        fig = plt.figure(figsize=(15,5))
    plt.axis("off")
    plt.imshow(canvas, zorder=1)

    xs = match_pairs[:, [0, 2]]
    xs[:, 1] += w1
    ys = match_pairs[:, [1, 3]]

    alpha = 1
    sf = 5
    # lw = 0.5
    # markersize = 1
    markersize = 2

    plt.plot(
        xs.T, ys.T,
        alpha=alpha,
        linestyle="-",
        linewidth=lw,
        aa=False,
        marker='o',
        markersize=markersize,
        fillstyle='none',
        color=color,
        zorder=2,
        # color=[0.0, 0.8, 0.0],
    );
    plt.tight_layout()
    if filename is not None:
        plt.savefig(filename, dpi=300, bbox_inches='tight')
    print('#Matches = {}'.format(len(match_pairs)))
    if show:
        plt.show()

# from utils.draw import draw_matches_cv
def draw_matches_cv(data):
    keypoints1 = [cv2.KeyPoint(p[1], p[0], 1) for p in data['keypoints1']]
    keypoints2 = [cv2.KeyPoint(p[1], p[0], 1) for p in data['keypoints2']]
    inliers = data['inliers'].astype(bool)
    matches = np.array(data['matches'])[inliers].tolist()
    def to3dim(img):
        if img.ndim == 2:
            img = img[:, :, np.newaxis]
        return img
    img1 = to3dim(data['image1'])
    img2 = to3dim(data['image2'])
    img1 = np.concatenate([img1, img1, img1], axis=2)
    img2 = np.concatenate([img2, img2, img2], axis=2)
    return cv2.drawMatches(img1, keypoints1, img2, keypoints2, matches,
                           None, matchColor=(0,255,0), singlePointColor=(0, 0, 255))

def drawBox(points, img, offset=np.array([0,0]), color=(0,255,0)):
#     print("origin", points)
    offset = offset[::-1]
    points = points + offset
    points = points.astype(int)
    for i in range(len(points)):
        img = img + cv2.line(np.zeros_like(img),tuple(points[-1+i]), tuple(points[i]), color,5)
    return img

max_stars_repo_name: Dai-z/pytorch-superpoint

max_issues_repo_name: Dai-z/pytorch-superpoint

max_forks_repo_name: Dai-z/pytorch-superpoint


text:
import numpy as np
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import pickle
from combined_thresh import combined_thresh
from perspective_transform import perspective_transform
from Line import Line
from line_fit import line_fit, tune_fit, final_viz, calc_curve, calc_vehicle_offset, viz2
from moviepy.editor import VideoFileClip

# Global variables (just to make the moviepy video annotation work)
with open('calibrate_camera.p', 'rb') as f:
	save_dict = pickle.load(f)
mtx = save_dict['mtx']
dist = save_dict['dist']
window_size = 5  # how many frames for line smoothing
left_line = Line(n=window_size)
right_line = Line(n=window_size)
detected = False  # did the fast line fit detect the lines?
left_curve, right_curve = 0., 0.  # radius of curvature for left and right lanes
left_lane_inds, right_lane_inds = None, None  # for calculating curvature
frameCount = 0
retLast = {}

# MoviePy video annotation will call this function
def annotate_image(img_in):
	"""
	Annotate the input image with lane line markings
	Returns annotated image
	"""
	global mtx, dist, left_line, right_line, detected, frameCount, retLast
	global left_curve, right_curve, left_lane_inds, right_lane_inds

	frameCount += 1
	src = np.float32(
		[[200, 720],
		 [1100, 720],
		 [520, 500],
		 [760, 500]])

	x = [src[0, 0], src[1, 0], src[3, 0], src[2, 0], src[0, 0]]
	y = [src[0, 1], src[1, 1], src[3, 1], src[2, 1], src[0, 1]]

	# Undistort, threshold, perspective transform
	undist = cv2.undistort(img_in, mtx, dist, None, mtx)
	img, abs_bin, mag_bin, dir_bin, hls_bin = combined_thresh(undist)
	binary_warped, binary_unwarped, m, m_inv = perspective_transform(img)

	# Perform polynomial fit
	if not detected:
		# Slow line fit
		ret = line_fit(binary_warped)
		# if detect no lanes, use last result instead.
		if len(ret) == 0:
			ret = retLast
		left_fit = ret['left_fit']
		right_fit = ret['right_fit']
		nonzerox = ret['nonzerox']
		nonzeroy = ret['nonzeroy']
		out_img = ret['out_img']
		left_lane_inds = ret['left_lane_inds']
		right_lane_inds = ret['right_lane_inds']
		histogram = ret['histo']

		# Get moving average of line fit coefficients
		left_fit = left_line.add_fit(left_fit)
		right_fit = right_line.add_fit(right_fit)

		# Calculate curvature
		left_curve, right_curve = calc_curve(left_lane_inds, right_lane_inds, nonzerox, nonzeroy)

		detected = True  # slow line fit always detects the line

	else:  # implies detected == True
		# Fast line fit
		left_fit = left_line.get_fit()
		right_fit = right_line.get_fit()
		ret = tune_fit(binary_warped, left_fit, right_fit)
		left_fit = ret['left_fit']
		right_fit = ret['right_fit']
		nonzerox = ret['nonzerox']
		nonzeroy = ret['nonzeroy']
		left_lane_inds = ret['left_lane_inds']
		right_lane_inds = ret['right_lane_inds']

		# Only make updates if we detected lines in current frame
		if ret is not None:
			left_fit = ret['left_fit']
			right_fit = ret['right_fit']
			nonzerox = ret['nonzerox']
			nonzeroy = ret['nonzeroy']
			left_lane_inds = ret['left_lane_inds']
			right_lane_inds = ret['right_lane_inds']

			left_fit = left_line.add_fit(left_fit)
			right_fit = right_line.add_fit(right_fit)
			left_curve, right_curve = calc_curve(left_lane_inds, right_lane_inds, nonzerox, nonzeroy)
		else:
			detected = False

	vehicle_offset = calc_vehicle_offset(undist, left_fit, right_fit)

	# Perform final visualization on top of original undistorted image
	result = final_viz(undist, left_fit, right_fit, m_inv, left_curve, right_curve, vehicle_offset)

	retLast = ret

	save_viz2 = './output_images/polyfit_test%d.jpg' % (frameCount)

	viz2(binary_warped, ret, save_viz2)

	save_warped = './output_images/warped_test%d.jpg' % (frameCount)
	plt.imshow(binary_warped, cmap='gray', vmin=0, vmax=1)
	if save_warped is None:
		plt.show()
	else:
		plt.savefig(save_warped)
	plt.gcf().clear()

	save_binary = './output_images/binary_test%d.jpg' % (frameCount)
	plt.imshow(img,	cmap='gray', vmin=0, vmax=1)
	if save_binary is None:
		plt.show()
	else:
		plt.savefig(save_binary)
	plt.gcf().clear()

	if frameCount > 0:
		fig = plt.gcf()
		fig.set_size_inches(16.5, 8.5)
		plt.subplot(2, 3, 1)
		plt.imshow(undist)
		# plt.plot(undist)
		plt.plot(x, y)
		plt.title('undist')
		plt.subplot(2, 3, 2)
		plt.imshow(hls_bin, cmap='gray', vmin=0, vmax=1)
		plt.title('hls_bin')
		plt.subplot(2, 3, 3)
		plt.imshow(abs_bin, cmap='gray', vmin=0, vmax=1)
		plt.title('abs_bin')
		plt.subplot(2, 3, 4)
		plt.imshow(img, cmap='gray', vmin=0, vmax=1)
		plt.title('img')
		plt.subplot(2, 3, 5)
		plt.imshow(out_img)
		plt.title('out_img')
		plt.subplot(2, 3, 6)
		plt.imshow(result, cmap='gray', vmin=0, vmax=1)
		plt.title('result')

		save_result = 'D:/code/github_code/CarND-Advanced-Lane-Lines-P4/output_images/result-test%d.jpg' % (frameCount)
		if save_result is None:
			plt.show()
		else:
			plt.savefig(save_result)
		plt.gcf().clear()

	return result

def annotate_video(input_file, output_file):
	""" Given input_file video, save annotated video to output_file """
	video = VideoFileClip(input_file)
	annotated_video = video.fl_image(annotate_image)
	annotated_video.write_videofile(output_file, audio=False)

if __name__ == '__main__':
	# Annotate the video
	# annotate_video('challenge_video.mp4', 'challenge_video_out.mp4')

	# Show example annotated image on screen for sanity check
	for i in range (1, 7):

		img_file = 'test_images/test%d.jpg' % (i)
		img = mpimg.imread(img_file)
		result = annotate_image(img)
		plt.imshow(result)
		save_file = 'D:/code/github_code/CarND-Advanced-Lane-Lines-P4/output_images/test%d.jpg' % (i)
		if save_file is None:
			plt.show()
		else:
			plt.savefig(save_file)
		plt.gcf().clear()


text:
import numpy as np
import tensorflow as tf

from DeepSparseCoding.tf1x.ops.init_ops import L2NormalizedTruncatedNormalInitializer
from DeepSparseCoding.tf1x.utils.trainable_variable_dict import TrainableVariableDict

class AeModule(object):
  def __init__(self, data_tensor, layer_types, enc_channels, dec_channels, patch_size,
    conv_strides, w_decay_mult, w_norm_mult, act_funcs, dropout, tie_dec_weights,
    w_init_type, variable_scope="ae"):
    """
    Autoencoder module
    Inputs:
      data_tensor
      enc_channels [list of ints] the number of output channels per encoder layer
        Last entry is the number of latent units
      dec_channels [list of ints] the number of output channels per decoder layer
        Last entry must be the number of input pixels for FC layers and channels for CONV layers
      w_decay_mult: tradeoff multiplier for weight decay loss
      w_norm_mult: tradeoff multiplier for weight norm loss (asks weight norm to == 1)
      act_funcs: activation functions
      dropout: specifies the keep probability or None
      conv: if True, do convolution
      conv_strides: list of strides for convolution [batch, y, x, channels]
      patch_size: number of (y, x) inputs for convolutional patches
      w_init_type: [str] which w_init to use, options are 'normal', 'xavier', or 'l2_normed'
      variable_scope: specifies the variable_scope for the module
    Outputs:
      dictionary
    """
    self.conv_strides = conv_strides
    self.variable_scope = variable_scope
    self.trainable_variables = TrainableVariableDict()
    self.data_tensor = data_tensor
    self.enc_channels = enc_channels
    self.dec_channels = dec_channels
    self.patch_size_y = [int(size[0]) for size in patch_size]
    self.patch_size_x = [int(size[1]) for size in patch_size]
    self.dropout = dropout
    self.w_decay_mult = w_decay_mult
    self.w_norm_mult = w_norm_mult
    self.act_funcs = act_funcs
    self.num_enc_layers = len(self.enc_channels)
    self.num_dec_layers = len(self.dec_channels)
    self.tie_dec_weights = tie_dec_weights
    self.enc_layer_types = layer_types[:self.num_enc_layers]
    self.dec_layer_types = layer_types[self.num_enc_layers:]
    self.layer_types = [self.enc_layer_types, self.dec_layer_types]
    self.num_enc_conv_layers = self.enc_layer_types.count("conv")
    self.num_dec_conv_layers = self.dec_layer_types.count("conv")
    self.num_conv_layers = self.num_enc_conv_layers + self.num_dec_conv_layers
    self.num_enc_fc_layers = self.enc_layer_types.count("fc")
    self.num_dec_fc_layers = self.dec_layer_types.count("fc")
    self.num_fc_layers = self.num_enc_fc_layers + self.num_dec_fc_layers
    self.num_layers = self.num_enc_layers + self.num_dec_layers
    data_ndim = len(data_tensor.get_shape().as_list())
    self.all_strides = [] # Full list of strides, including FC layers
    for enc_conv_id in range(self.num_enc_conv_layers):
      self.all_strides.append(self.conv_strides[enc_conv_id])
    for enc_fc_id in range(self.num_enc_fc_layers):
      self.all_strides.append(None)
    for dec_fc_id in range(self.num_dec_fc_layers):
      self.all_strides.append(None)
    for dec_conv_id in range(self.num_dec_conv_layers):
      self.all_strides.append(self.conv_strides[self.num_enc_conv_layers + dec_conv_id])
    if data_ndim == 2:
      self.batch_size, self.num_pixels = self.data_tensor.get_shape()
    else:
      self.batch_size, self.num_pixels_y, self.num_pixels_x, self.num_channels = \
        self.data_tensor.get_shape()
      self.num_pixels = self.num_pixels_y * self.num_pixels_x * self.num_channels
    self.w_init_type = w_init_type
    # Parameter checks
    if self.enc_layer_types[0] == "conv":
      assert data_ndim == 4, (
        "Module requires data_tensor to have shape" +
        " [batch, num_pixels_y, num_pixels_x, num_features] if first layer is conv")
    else:
      assert data_ndim == 2, (
        "Module requires data_tensor to have shape [batch, num_pixels]")
    if(self.tie_dec_weights):
      assert self.num_enc_layers == self.num_dec_layers, (
        "num_enc_layers must equal num_dec_layers, but are %g and %g"%(
        self.num_enc_layers, self.num_dec_layers))
    if self.num_enc_conv_layers > 0 and self.num_enc_fc_layers > 0:
      assert np.all("conv" in self.enc_layer_types[:self.num_enc_conv_layers]), \
        ("Encoder conv layers must come before fc layers")
    if self.num_dec_conv_layers > 0 and self.num_dec_fc_layers > 0:
      assert np.all("fc" in self.dec_layer_types[:self.num_dec_fc_layers]), \
        ("Decoder fc layers must come before conv layers")
    assert self.num_enc_layers == len(self.enc_layer_types), \
      ("The number of encoder channels must match the number of encoder layer types")
    assert self.num_dec_layers == len(self.dec_layer_types), \
      ("The number of decoder channels must match the number of decoder layer types")
    assert all([layer_type in ["conv", "fc"] for layer_type in layer_types]), \
      ("All layer_types must be conv or fc")
    assert len(self.patch_size_y) == self.num_conv_layers, \
      ("patch_size_y must be a list of size " + str(self.num_conv_layers))
    assert len(self.patch_size_x) == self.num_conv_layers, \
      ("patch_size_x must be a list of size " + str(self.num_conv_layers))
    assert len(self.conv_strides) == self.num_conv_layers, \
      ("conv_strides must be a list of size " + str(self.num_conv_layers))
    assert len(self.act_funcs) == self.num_layers, \
      ("act_funcs parameter must be a list of size " + str(self.num_layers))
    self.build_graph()

  def compute_weight_norm_loss(self):
    with tf.compat.v1.variable_scope("w_norm"):
      w_norm_list = []
      for w in self.w_list:
        reduc_axis = np.arange(1, len(w.get_shape().as_list()))
        w_norm = tf.reduce_sum(input_tensor=tf.square(1 - tf.reduce_sum(input_tensor=tf.square(w), axis=reduc_axis)))
        w_norm_list.append(w_norm)
      norm_loss = tf.multiply(0.5 * self.w_norm_mult, tf.add_n(w_norm_list))
    return norm_loss

  def compute_weight_decay_loss(self):
    with tf.compat.v1.variable_scope("unsupervised"):
      w_decay_list = [tf.reduce_sum(input_tensor=tf.square(w)) for w in self.w_list]
      decay_loss = tf.multiply(0.5*self.w_decay_mult, tf.add_n(w_decay_list))
    return decay_loss

  def compute_recon_loss(self, reconstruction):
    with tf.compat.v1.variable_scope("unsupervised"):
      # If the encoder and decoder are different types (conv vs fc) then there may be a shape mismatch
      recon_shape = reconstruction.get_shape()
      data_shape = self.data_tensor.get_shape()
      if(recon_shape.ndims != data_shape.ndims):
        if(np.prod(recon_shape.as_list()[1:]) == np.prod(data_shape.as_list()[1:])):
          reconstruction = tf.reshape(reconstruction, tf.shape(input=self.data_tensor))
        else:
          assert False, ("Reconstructiion and input must have the same size")
      reduc_dim = list(range(1, len(reconstruction.shape)))# We want to avg over batch
      recon_loss = 0.5 * tf.reduce_mean(
        input_tensor=tf.reduce_sum(input_tensor=tf.square(tf.subtract(reconstruction, self.data_tensor)),
        axis=reduc_dim), name="recon_loss")
    return recon_loss

  def compute_total_loss(self):
    with tf.compat.v1.variable_scope("loss") as scope:
      self.loss_dict = {"recon_loss":self.compute_recon_loss(self.reconstruction),
        "weight_decay_loss":self.compute_weight_decay_loss(),
        "weight_norm_loss":self.compute_weight_norm_loss()}
      self.total_loss = tf.add_n([loss for loss in self.loss_dict.values()], name="total_loss")

  def flatten_feature_map(self, feature_map):
    """
    Flatten input tensor from [batch, y, x, f] to [batch, y*x*f]
    """
    map_shape = feature_map.get_shape()
    if(map_shape.ndims == 4):
      (batch, y, x, f) = map_shape
      prev_input_features = int(y * x * f)
      resh_map  = tf.reshape(feature_map, [-1, prev_input_features])
    elif(map_shape.ndims == 2):
      resh_map = feature_map
    else:
      assert False, ("Input feature_map has incorrect ndims")
    return resh_map

  def get_dec_shapes(self, input_shape):
    # The following assumes decoder fc->conv operation mirrors encoder conv->fc
    conv_output_length = tf.python.keras.utils.conv_utils.conv_output_length
    in_y, in_x, in_f = input_shape[1:]
    dec_conv_strides = self.conv_strides[:-self.num_dec_conv_layers]
    filter_size_y = self.patch_size_y[:-self.num_dec_conv_layers]
    filter_size_x = self.patch_size_x[:-self.num_dec_conv_layers]
    dec_channels = self.dec_channels[:self.num_dec_conv_layers][::-1]
    last_enc_conv_channels = self.enc_channels[self.num_enc_conv_layers-1]
    dec_channels[-1] = last_enc_conv_channels
    layer_shapes = [[int(in_y), int(in_x), int(in_f)]]
    for layer_id in range(self.num_dec_conv_layers):
      out_y = conv_output_length(
        input_length=layer_shapes[layer_id][0],
        filter_size=filter_size_y[layer_id],
        padding="same",
        stride=dec_conv_strides[layer_id][1])
      out_x = conv_output_length(
        input_length=layer_shapes[layer_id][1],
        filter_size=filter_size_x[layer_id],
        padding="same",
        stride=dec_conv_strides[layer_id][2])
      layer_shapes.append([int(out_y), int(out_x), int(dec_channels[layer_id])])
    return layer_shapes[::-1]

  def compute_pre_activation(self, layer_id, input_tensor, w, b, conv, decode):
    if conv:
      strides = self.all_strides[layer_id]
      if decode:
        height_const = tf.shape(input=input_tensor)[1] % strides[1]
        out_height = (tf.shape(input=input_tensor)[1] * strides[1]) - height_const
        width_const = tf.shape(input=input_tensor)[2] % strides[2]
        out_width = (tf.shape(input=input_tensor)[2] * strides[2]) - width_const
        out_shape = tf.stack([tf.shape(input=input_tensor)[0], # Batch
          out_height, # Height
          out_width, # Width
          tf.shape(input=w)[2]]) # Channels
        pre_act = tf.add(tf.nn.conv2d_transpose(input_tensor, w, out_shape, strides,
          padding="SAME"), b)
      else:
        pre_act = tf.add(tf.nn.conv2d(input=input_tensor, filters=w, strides=strides, padding="SAME"), b)
    else:
      pre_act = tf.add(tf.matmul(input_tensor, w), b)
    return pre_act

  def layer_maker(self, layer_id, input_tensor, activation_function, w_shape,
    keep_prob=1.0, conv=False, decode=False, tie_dec_weights=False, name_suffix=""):
    """
    Make layer that does act(u*w+b) where * is a dot product or convolution
    Example case for w_read_id logic:
      layer_id: [0 1 2 3 4] [5 6 7 8 9]
                              10-6  10-7  10-8 10-9  10-10
      weight_id: [0 1 2 3 4] [ 4     3     2     1     0 ]
      num_layers: 10
      weight_id = num_layers - (layer_id + 1)
    """
    with tf.compat.v1.variable_scope("layer"+str(layer_id), reuse=tf.compat.v1.AUTO_REUSE) as scope:
      if tie_dec_weights:
        w_read_id = self.num_layers - (layer_id+1)
      else:
        w_read_id = layer_id
      name_prefix = "conv_" if conv else "fc_"
      w_name = name_prefix+"w_"+str(w_read_id)+name_suffix
      if self.w_init_type.lower() == "normal":
        w = tf.compat.v1.get_variable(name=w_name, shape=w_shape, dtype=tf.float32,
          initializer=self.w_normal_init, trainable=True)
      elif self.w_init_type.lower() == "xavier":
        w = tf.compat.v1.get_variable(name=w_name, shape=w_shape, dtype=tf.float32,
          initializer=self.w_xavier_init, trainable=True)
      elif self.w_init_type.lower() == "l2_normed":
        if decode:
          w = tf.compat.v1.get_variable(name=w_name, shape=w_shape, dtype=tf.float32,
            initializer=self.w_normed_dec_init, trainable=True)
        else:
          w = tf.compat.v1.get_variable(name=w_name, shape=w_shape, dtype=tf.float32,
            initializer=self.w_normed_enc_init, trainable=True)
      else:
        assert False, ("w_init_type parameter must be 'normal', 'xavier', or 'l2_normed', not %s"%(
          self.w_init_type))
      b_name = name_prefix+"b_"+str(layer_id)+name_suffix
      if conv and decode:
        b_shape = w_shape[-2]
      else:
        b_shape = w_shape[-1]
      b = tf.compat.v1.get_variable(name=b_name, shape=b_shape,
        dtype=tf.float32, initializer=self.b_init, trainable=True)
      pre_act = self.compute_pre_activation(layer_id, input_tensor, w, b, conv, decode)
      output_tensor = activation_function(pre_act)
      output_tensor = tf.nn.dropout(output_tensor, rate=1-keep_prob)
    return output_tensor, w, b

  def build_encoder(self, input_tensor, activation_functions):
    enc_u_list = [input_tensor]
    enc_w_list = []
    enc_b_list = []
    prev_input_features = input_tensor.get_shape().as_list()[-1]
    # Make conv layers first
    for layer_id in range(self.num_enc_conv_layers):
      w_shape = [self.patch_size_y[layer_id], self.patch_size_x[layer_id],
        int(prev_input_features), int(self.enc_channels[layer_id])]
      u_out, w, b = self.layer_maker(layer_id, enc_u_list[layer_id],
        activation_functions[layer_id], w_shape, keep_prob=self.dropout[layer_id],
        conv=True, decode=False, tie_dec_weights=self.tie_dec_weights)
      enc_u_list.append(u_out)
      enc_w_list.append(w)
      enc_b_list.append(b)
      prev_input_features = int(self.enc_channels[layer_id])
    # Make fc layers second
    for enc_fc_layer_id in range(self.num_enc_fc_layers):
      layer_id = enc_fc_layer_id + self.num_enc_conv_layers
      if enc_fc_layer_id == 0: # Input needs to be reshaped to [batch, num_units] for FC layers
        in_tensor = self.flatten_feature_map(enc_u_list[-1])
        prev_input_features = in_tensor.get_shape().as_list()[1]
      else:
        in_tensor = enc_u_list[layer_id]
      w_shape = [int(prev_input_features), int(self.enc_channels[layer_id])]
      u_out, w, b = self.layer_maker(layer_id, in_tensor, activation_functions[layer_id],
        w_shape, keep_prob=self.dropout[layer_id], conv=False, decode=False,
        tie_dec_weights=self.tie_dec_weights)
      enc_u_list.append(u_out)
      enc_w_list.append(w)
      enc_b_list.append(b)
      prev_input_features = int(self.enc_channels[layer_id])
    return enc_u_list, enc_w_list, enc_b_list

  def build_decoder(self, input_tensor, activation_functions):
    dec_u_list = [input_tensor]
    dec_w_list = []
    dec_b_list = []
    # Build FC layers first
    for dec_layer_id in range(self.num_dec_fc_layers):
      layer_id = self.num_enc_layers + dec_layer_id
      input_shape = dec_u_list[dec_layer_id].get_shape()
      if input_shape.ndims == 4: # if final enc layer was conv then flatten
        in_tensor = self.flatten_feature_map(dec_u_list[dec_layer_id])
      else: # final enc layer was fc
        in_tensor = dec_u_list[dec_layer_id]
      if dec_layer_id == self.num_dec_fc_layers - 1 and self.num_dec_conv_layers > 0:
        # If there are decoder conv layers, then
        # the last decoder FC layer needs to output a vector of the correct length
        # correct_length = feature_map_y * feature_map_x * feature_map_f
        # where feature_map_f = self.dec_channels[dec_layer_id]
        conv_layer_shapes = self.get_dec_shapes(self.data_tensor.get_shape())
        out_channels = np.prod(conv_layer_shapes[0])
      else:
        out_channels = self.dec_channels[dec_layer_id]
      w_shape = [in_tensor.get_shape()[-1], out_channels]
      u_out, w, b = self.layer_maker(layer_id, in_tensor,
        activation_functions[dec_layer_id], w_shape, keep_prob=self.dropout[layer_id],
        conv=False, decode=True, tie_dec_weights=self.tie_dec_weights)
      dec_u_list.append(u_out)
      dec_w_list.append(w)
      dec_b_list.append(b)
    # Build conv layers second
    for dec_conv_layer_id in range(self.num_dec_conv_layers):
      dec_layer_id = self.num_dec_fc_layers + dec_conv_layer_id
      layer_id = self.num_enc_layers + dec_layer_id
      input_shape = dec_u_list[dec_layer_id].get_shape()
      if input_shape.ndims == 4: # prev layer was conv
        (batch, y, x, f) = input_shape
        in_tensor = dec_u_list[dec_layer_id]
        w_shape = [
          self.patch_size_y[self.num_enc_conv_layers + dec_conv_layer_id],
          self.patch_size_x[self.num_enc_conv_layers + dec_conv_layer_id],
          self.dec_channels[dec_layer_id],
          f]
      else: # prev layer was fc
        conv_layer_shapes = self.get_dec_shapes(self.data_tensor.get_shape())
        new_shape = [-1] + conv_layer_shapes[dec_conv_layer_id]
        in_tensor = tf.reshape(dec_u_list[dec_layer_id], new_shape)
        w_shape = [
          self.patch_size_y[self.num_enc_conv_layers + dec_conv_layer_id],
          self.patch_size_x[self.num_enc_conv_layers + dec_conv_layer_id],
          self.dec_channels[dec_layer_id],
          new_shape[-1]]
      u_out, w, b = self.layer_maker(layer_id, in_tensor, activation_functions[dec_layer_id],
        w_shape, keep_prob=self.dropout[layer_id], conv=True, decode=True,
        tie_dec_weights=self.tie_dec_weights)
      dec_u_list.append(u_out)
      dec_w_list.append(w)
      dec_b_list.append(b)
    return dec_u_list, dec_w_list, dec_b_list

  def build_graph(self):
    with tf.compat.v1.variable_scope(self.variable_scope) as scope:
      with tf.compat.v1.variable_scope("weight_inits") as scope:
        self.w_normal_init = tf.compat.v1.initializers.truncated_normal(mean=0.0, stddev=0.001)
        self.w_xavier_init = tf.compat.v1.keras.initializers.VarianceScaling(scale=1.0, mode="fan_avg", distribution=("uniform" if False else "truncated_normal"))
        self.w_normed_enc_init = L2NormalizedTruncatedNormalInitializer(mean=0.0, stddev=0.001,
          axis=0, epsilon=1e-12, dtype=tf.float32) #TODO: Fix axis to be general to conv layers
        self.w_normed_dec_init = L2NormalizedTruncatedNormalInitializer(mean=0.0, stddev=0.001,
          axis=-1, epsilon=1e-12, dtype=tf.float32)
        self.b_init = tf.compat.v1.initializers.constant(1e-4)
      self.u_list = [self.data_tensor]
      self.w_list = []
      self.b_list = []
      enc_u_list, enc_w_list, enc_b_list = self.build_encoder(self.u_list[0],
        self.act_funcs[:self.num_enc_layers])
      self.u_list += enc_u_list[1:] # build_encoder() will place self.u_list[0] as enc_u_list[0]
      self.w_list += enc_w_list
      self.b_list += enc_b_list
      with tf.compat.v1.variable_scope("inference") as scope:
        self.a = tf.identity(enc_u_list[-1], name="activity")
      dec_u_list, dec_w_list, dec_b_list = self.build_decoder(self.a,
        self.act_funcs[self.num_enc_layers:])
      self.u_list += dec_u_list[1:] # build_decoder() will place self.u_list[-1] as dec_u_list[0]
      if not self.tie_dec_weights:
        self.w_list += dec_w_list
      self.b_list += dec_b_list
      with tf.compat.v1.variable_scope("norm_weights") as scope:
        w_enc_norm_dim = list(range(len(self.w_list[0].get_shape().as_list())-1))
        self.norm_enc_w = self.w_list[0].assign(tf.nn.l2_normalize(self.w_list[0],
          axis=w_enc_norm_dim, epsilon=1e-8, name="row_l2_norm"))
        self.norm_dec_w = self.w_list[-1].assign(tf.nn.l2_normalize(self.w_list[-1],
          axis=-1, epsilon=1e-8, name="col_l2_norm"))
        self.norm_w = tf.group(self.norm_enc_w, self.norm_dec_w, name="l2_norm_weights")
      for w,b in zip(self.w_list, self.b_list):
        self.trainable_variables[w.name] = w
        self.trainable_variables[b.name] = b
      with tf.compat.v1.variable_scope("output") as scope:
        self.reconstruction = tf.identity(self.u_list[-1], name="reconstruction")
      self.compute_total_loss()

max_stars_repo_name: dpaiton/DeepSparseCoding

max_issues_repo_name: dpaiton/DeepSparseCoding

max_forks_repo_name: dpaiton/DeepSparseCoding


text:
"""Sparse ops implementation using tf primitives.
"""

import numpy as np
from collections import namedtuple
import tensorflow as tf

from artifice.log import logger  # noqa: unused
from artifice import utils

def _compute_bcount(size, bstride):
  return [utils.divup(size[0], bstride[0]),
          utils.divup(size[1], bstride[1])]

def _compute_input_padding(size, bcount, bsize, boffset, bstride):
  """Computes the padding for the operation.

  :param size: `[SZH, SZW]` list-like of ints, size of image
  :param bcount: `[BCH, BCW]` list of ints
  :param bsize:
  :param boffset:
  :param bstride:
  :returns: `pad_h, pad_w` for _pad_inputs function, possibly negative.
  :rtype:

  """
  pad_h = [boffset[0],
           boffset[0] + bstride[0] * bcount[0] + bsize[0] - size[0]]
  pad_w = [boffset[1],
           boffset[1] + bstride[1] * bcount[1] + bsize[1] - size[1]]
  return pad_h, pad_w

def _pad_inputs(mask, bcount, bsize, boffset, bstride):
  """Pad the inputs for then

  :param mask: 4D tensor containing the inputs.
  :param bcount:
  :param bsize:
  :param boffset:
  :param bstride:
  :returns: Padded (or cropped) mask
  :rtype: tf.Tensor

  """
  pad_h, pad_w = _compute_input_padding(
    mask.shape[1:3], bcount, bsize, boffset, bstride)

  if pad_h[0] < 0:
    mask = mask[:, -pad_h[0]:, :, :]
    pad_h[0] = 0
  if pad_h[1] < 0:
    mask = mask[:, :-pad_h[1], :, :]
    pad_h[1] = 0
  if pad_w[0] < 0:
    mask = mask[:, :, -pad_w[0]:, :]
    pad_w[0] = 0
  if pad_w[1] < 0:
    mask = mask[:, :, :-pad_w[1], :]
    pad_w[1] = 0

  pad_n = pad_c = [0, 0]
  return tf.pad(mask, [pad_n, pad_h, pad_w, pad_c])

def _compute_upsample_offsets(bsize):
  """Compute the offsets for blocks with bsize.

  Assumes that the given coordinate is at the top left of the block.

  So for example, if the block size were [3, 4], the returned offsets would be:
```
  [[[0], [1], [2], [3]],
   [[1], [2], [3], [4]],
   [[2], [3], [4], [5]]]
```
  which has shape [1, 3, 4, 1]

  :param bsize: `[BSZH, BSZW]` size of the blocks.
  :returns: [1, bsize[0], bsize[1], 1] array of offsets to upsample a set of
  block_indices.
  :rtype: tf.Tensor

  """
  offsets = np.array([[[0, i, j] for j in range(bsize[1])]
                      for i in range(bsize[0])], np.int32)
  # todo: fix this so it isn't crazy
  offsets = tf.constant(offsets, tf.int32)
  offsets = tf.expand_dims(offsets, 0)
  return offsets

def _upsample_block_indices(active_block_indices, bsize, boffset, bstride):
  """Upsamples the indices to have all indices in a rectangle.

  :param active_block_indices: [M,3] Tensor. Corresponds to top left coordinate
  after offset and scaling.
  :param bsize: block size
  :param boffset:
  :param bstride:
  :returns: [M, bsize[0], bsize[1], 3] locations of all pixels in the blocks.
  :rtype:

  """
  ops = []
  logger.debug(f"bsize: {bsize}")
  logger.debug(f"bstride: {bstride}")
  # ops.append(tf.print(active_block_indices, summarize=-1))
  offset = tf.constant([0, boffset[0], boffset[1]], dtype=tf.int32)
  scale = tf.constant([1, bstride[0], bstride[1]], dtype=tf.int32)
  indices = tf.cast(active_block_indices, tf.int32) + offset
  indices *= scale                                       # [M, 3]
  indices = tf.expand_dims(indices, 1)
  indices = tf.expand_dims(indices, 2)  # [M, 1, 1, 3]
  upsample_offsets = _compute_upsample_offsets(
    bsize)  # [1, bsize[0], bsize[1], 3]
  logger.debug(f"indices: {indices.shape}")
  logger.debug(f"upsample_offsets: {upsample_offsets.shape}")
  # ops.append(tf.print(indices, summarize=-1))
  # ops.append(tf.print(upsample_offsets, summarize=-1))
  with tf.control_dependencies(ops):
    indices += upsample_offsets  # [M, bsize[0], bsize[1], 3]

  return indices

"""
TensorFlow primitive implementations.
"""

def reduce_mask(mask,
                bcount, *,
                bsize,
                boffset,
                bstride,
                tol=0.5,
                avgpool=False):
  """Reduce the mask to namedtuple `(bin_counts, active_block_indices)`, indices.

  :param mask:
  :param block_count:
  :param bsize:
  :param boffset:
  :param bstride:
  :param tol:
  :param avgpool:
  :returns:
  :rtype:

  """
  logger.debug(f"mask: {mask.shape}")
  mask = _pad_inputs(mask, bcount, bsize, boffset, bstride)
  logger.debug(f"padded mask: {mask.shape}")
  mask = tf.nn.pool(
    mask,
    window_shape=bsize,
    pooling_type='AVG' if avgpool else 'MAX',
    padding='SAME',
    strides=bstride)
  mask = tf.squeeze(mask, axis=3)
  active_block_indices = tf.where(mask > tf.constant(tol, mask.dtype))
  active_block_indices = tf.cast(active_block_indices, tf.int32)
  bin_counts = tf.shape(active_block_indices)[0]
  Indices = namedtuple('Indices', ['active_block_indices', 'bin_counts'])
  return Indices(active_block_indices, bin_counts)

def gather(
        inputs,
        bin_counts,
        active_block_indices, *,
        bsize,
        boffset,
        bstride):
  """FIXME! briefly describe function

  :param inputs:
  :param bin_counts: number of blocks?
  :param active_block_indices:
  :param bsize:
  :param boffset:
  :param bstride:
  :returns:
  :rtype:

  """
  logger.debug(f"inputs: {inputs.shape}")
  size = inputs.shape[1:3]
  bcount = _compute_bcount(size, bstride)
  inputs = _pad_inputs(inputs, bcount, bsize, boffset, bstride)

  logger.debug(f"padded inputs: {inputs.shape}")
  indices = _upsample_block_indices(
    active_block_indices,
    bsize,
    boffset,
    bstride)
  ops = []
  # ops.append(tf.print(indices, summarize=-1))
  logger.debug(f"gather indices: {indices.shape}")
  with tf.control_dependencies(ops):
    blocks = tf.gather_nd(inputs, indices)  # todo: fix index error
  blocks = tf.reshape(
    blocks, [bin_counts, bsize[0], bsize[1], tf.shape(inputs)[3]])
  return blocks

def scatter(
        blocks,
        bin_counts,                 # pylint: disable=unused-argument
        active_block_indices,
        outputs, *,
        bsize,
        boffset,
        bstride,
        add=False):
  """Scatter the blocks back onto outputs.

  Note that currently this only uses `outputs.shape` to scatter onto a tensor
  of zeros.

  In tf >= 1.14, the functions tf.tensor_scatter_nd_update and
  tf.tensor_scatter_nd_add would overcome this barrier.

  :param blocks: [M, bsize[0], bsize[1], C]
  :param bin_counts:
  :param active_block_indices:
  :param outputs: [N, H, W, C]
  :param bsize:
  :param boffset:
  :param bstride:
  :returns:
  :rtype:

  """
  size = outputs.shape[1:3]
  bcount = _compute_bcount(size, bstride)
  outputs = _pad_inputs(outputs, bcount, bsize, boffset, bstride)

  indices = _upsample_block_indices(
    active_block_indices,
    bsize,
    boffset,
    bstride)                    # [M, bsize[0], bsize[1], 3]

  if add:
    raise NotImplementedError
  else:
    outputs = tf.case(
      [(tf.equal(tf.shape(blocks)[0], tf.constant(0, tf.int32)),
        (lambda: outputs))],
      default=lambda: tf.scatter_nd(indices, blocks, tf.shape(outputs)))

  return outputs[:, :size[0], :size[1], :]

def scatter_var(
        blocks,
        bin_counts,                 # pylint: disable=unused-argument
        active_block_indices,
        outputs, *,
        bsize,
        boffset,
        bstride,
        add=False):

  raise NotImplementedError("no gradient for sparse_lib.scatter_var")

  indices = _upsample_block_indices(
    active_block_indices,
    bsize,
    boffset,
    bstride)                    # [M, bsize[0], bsize[1], 3]

  if add:
    outputs = tf.scatter_nd_add(outputs, indices, blocks)
  else:
    outputs = tf.scatter_nd_update(outputs, indices, blocks)

  return outputs

max_stars_repo_name: zhuokaizhao/artifice

max_issues_repo_name: zhuokaizhao/artifice

max_forks_repo_name: zhuokaizhao/artifice


text:
# https://machinelearnings.co/text-classification-using-neural-networks-f5cd7b8765c6

# use natural language toolkit
import nltk
from nltk.stem.lancaster import LancasterStemmer
import os
import json
import datetime
import json
nltk.download('punkt')

stemmer = LancasterStemmer()

training_data = []

with open('data/examples_violence.json') as json_file:
    data = json.load(json_file)
    for text in data:
        training_data.append({"class":"violencia machista", "sentence":text})
        print(training_data)

with open('data/examples_no_violence.json') as json_file:
    data = json.load(json_file)
    for text in data:
        training_data.append({"class":"no violencia machista", "sentence":text})
        print(training_data)

# tokeninzation
words = []
classes = []
documents = []
ignore_words = ['?']
# loop through each sentence in our training data
for pattern in training_data:
    # tokenize each word in the sentence
    w = nltk.word_tokenize(pattern['sentence'])
    # add to our words list
    words.extend(w)
    # add to documents in our corpus
    documents.append((w, pattern['class']))
    # add to our classes list
    if pattern['class'] not in classes:
        classes.append(pattern['class'])

# stem and lower each word and remove duplicates
words = [stemmer.stem(w.lower()) for w in words if w not in ignore_words]
words = list(set(words))

# remove duplicates
classes = list(set(classes))

print (len(documents), "documents")
print (len(classes), "classes", classes)
print (len(words), "unique stemmed words", words)

# create our training data
training = []
output = []
# create an empty array for our output
output_empty = [0] * len(classes)

# training set, bag of words for each sentence
for doc in documents:
    # initialize our bag of words
    bag = []
    # list of tokenized words for the pattern
    pattern_words = doc[0]
    # stem each word
    pattern_words = [stemmer.stem(word.lower()) for word in pattern_words]
    # create our bag of words array
    for w in words:
        bag.append(1) if w in pattern_words else bag.append(0)

    training.append(bag)
    # output is a '0' for each tag and '1' for current tag
    output_row = list(output_empty)
    output_row[classes.index(doc[1])] = 1
    output.append(output_row)

# sample training/output
i = 0
w = documents[i][0]
print ([stemmer.stem(word.lower()) for word in w])
print (training[i])
print (output[i])

import numpy as np
import time

# compute sigmoid nonlinearity
def sigmoid(x):
    output = 1/(1+np.exp(-x))
    return output

# convert output of sigmoid function to its derivative
def sigmoid_output_to_derivative(output):
    return output*(1-output)

def clean_up_sentence(sentence):
    # tokenize the pattern
    sentence_words = nltk.word_tokenize(sentence)
    # stem each word
    sentence_words = [stemmer.stem(word.lower()) for word in sentence_words]
    return sentence_words

# return bag of words array: 0 or 1 for each word in the bag that exists in the sentence
def bow(sentence, words, show_details=False):
    # tokenize the pattern
    sentence_words = clean_up_sentence(sentence)
    # bag of words
    bag = [0]*len(words)
    for s in sentence_words:
        for i,w in enumerate(words):
            if w == s:
                bag[i] = 1
                if show_details:
                    print ("found in bag: %s" % w)

    return(np.array(bag))

def think(sentence, show_details=False):
    x = bow(sentence.lower(), words, show_details)
    if show_details:
        print ("sentence:", sentence, "\n bow:", x)
    # input layer is our bag of words
    l0 = x
    # matrix multiplication of input and hidden layer
    l1 = sigmoid(np.dot(l0, synapse_0))
    # output layer
    l2 = sigmoid(np.dot(l1, synapse_1))
    return l2

def train(X, y, hidden_neurons=10, alpha=1, epochs=50000, dropout=False, dropout_percent=0.5):

    print("Training with %s neurons, alpha:%s, dropout:%s %s" % (hidden_neurons, str(alpha), dropout, dropout_percent if dropout else '') )
    print ("Input matrix: %sx%s    Output matrix: %sx%s" % (len(X),len(X[0]),1, len(classes)) )
    np.random.seed(1)

    last_mean_error = 1
    # randomly initialize our weights with mean 0
    synapse_0 = 2*np.random.random((len(X[0]), hidden_neurons)) - 1
    synapse_1 = 2*np.random.random((hidden_neurons, len(classes))) - 1

    prev_synapse_0_weight_update = np.zeros_like(synapse_0)
    prev_synapse_1_weight_update = np.zeros_like(synapse_1)

    synapse_0_direction_count = np.zeros_like(synapse_0)
    synapse_1_direction_count = np.zeros_like(synapse_1)

    for j in iter(range(epochs+1)):

        # Feed forward through layers 0, 1, and 2
        layer_0 = X
        layer_1 = sigmoid(np.dot(layer_0, synapse_0))

        if(dropout):
            layer_1 *= np.random.binomial([np.ones((len(X),hidden_neurons))],1-dropout_percent)[0] * (1.0/(1-dropout_percent))

        layer_2 = sigmoid(np.dot(layer_1, synapse_1))

        # how much did we miss the target value?
        layer_2_error = y - layer_2

        if (j% 10000) == 0 and j > 5000:
            # if this 10k iteration's error is greater than the last iteration, break out
            if np.mean(np.abs(layer_2_error)) < last_mean_error:
                print ("delta after "+str(j)+" iterations:" + str(np.mean(np.abs(layer_2_error))) )
                last_mean_error = np.mean(np.abs(layer_2_error))
            else:
                print ("break:", np.mean(np.abs(layer_2_error)), ">", last_mean_error )
                break

        # in what direction is the target value?
        # were we really sure? if so, don't change too much.
        layer_2_delta = layer_2_error * sigmoid_output_to_derivative(layer_2)

        # how much did each l1 value contribute to the l2 error (according to the weights)?
        layer_1_error = layer_2_delta.dot(synapse_1.T)

        # in what direction is the target l1?
        # were we really sure? if so, don't change too much.
        layer_1_delta = layer_1_error * sigmoid_output_to_derivative(layer_1)

        synapse_1_weight_update = (layer_1.T.dot(layer_2_delta))
        synapse_0_weight_update = (layer_0.T.dot(layer_1_delta))

        if(j > 0):
            synapse_0_direction_count += np.abs(((synapse_0_weight_update > 0)+0) - ((prev_synapse_0_weight_update > 0) + 0))
            synapse_1_direction_count += np.abs(((synapse_1_weight_update > 0)+0) - ((prev_synapse_1_weight_update > 0) + 0))

        synapse_1 += alpha * synapse_1_weight_update
        synapse_0 += alpha * synapse_0_weight_update

        prev_synapse_0_weight_update = synapse_0_weight_update
        prev_synapse_1_weight_update = synapse_1_weight_update

    now = datetime.datetime.now()

    # persist synapses
    synapse = {'synapse0': synapse_0.tolist(), 'synapse1': synapse_1.tolist(),
            'datetime': now.strftime("%Y-%m-%d %H:%M"),
            'words': words,
            'classes': classes
            }
    synapse_file = "synapses.json"

    with open(synapse_file, 'w') as outfile:
        json.dump(synapse, outfile, indent=4, sort_keys=True)
    print ("saved synapses to:", synapse_file)

X = np.array(training)
y = np.array(output)

start_time = time.time()

train(X, y, hidden_neurons=20, alpha=0.1, epochs=100000, dropout=False, dropout_percent=0.2)

elapsed_time = time.time() - start_time
print ("processing time:", elapsed_time, "seconds")

test_new_no_violence="La encomiable resistencia de Denis Shapovalov se transforma en una masa viscosa de la que no es fácil despegarse, pero el desagradable efecto se agota y llega el gran momento cuando el joven estrella una derecha en la red: 6-3 y 7-6(7), en 1h 55m. Entonces, Rafael Nadal cae a plomo al suelo y la Caja Mágica entra en erupción. Ocho años después, España reconquista la Copa Davis y eleva la sexta ensaladera de su historia, porque antes Roberto Bautista ha logrado una emotiva victoria en la apertura de la final contra Canadá. Se añade el festejo a los de 2000, 2004, 2008, 2009 y 2011, y Suecia queda ahora a un solo trofeo en el historial.  MÁS INFORMACIÓN España conquista la Davis de NadalCOLUMNA | 'El abrazo a Bautista', por TONI NADAL España conquista la Davis de NadalFOTOGALERÍA Las escenas de la final España conquista la Davis de NadalEmoción y trasnoche para un giro total  Del Sant Jordi a la Caja Mágica, de aquel éxito de hace 19 años a este último en 2019, hay infinidad de capítulos reservados, nombres propios y escenas, pero existe un eje vertebrador llamado Nadal. No estuvo presente el balear en la primera ensaladera, porque entonces aún estaba desarrollándose el chico que se convertiría en mito y que ese día fue el abanderado, mientras el fotograma final correspondía a Juan Carlos Ferrero; sin embargo, el hilo conductor del éxito en las dos últimas décadas tiene origen en Manacor. Preside él, imponente, el último laurel, el primero en el rompedor diseño propuesto por Gerard Piqué y su equipo. Se cacareaba desde hace meses el cansino sobrenombre de la Davis de Piqué, y al final la realidad del torneo ha derivado en otro: la Davis de Nadal.  A contrapié, la majestuosa actuación del mallorquín en la última semana –ocho puntos de los once obtenidos– contradice de alguna manera el espíritu de fondo de la Copa Davis, competición que acostumbra a premiar la coralidad. Pero esta vez, más nunca, ha sido un triunfo de autor. Ha ido España casi siempre a remolque y él ha ido empujando la nave un día sí y otro también. De principio a fin, desdoblándose. Resolvió los cruces contra Rusia y Argentina, despachó a Croacia y sentenció a Gran Bretaña en el territorio de dobles que tan bien manejan los británicos. Y en la final, más Nadal, presente en cuatro de las cinco rúbricas de las ensaladeras previas, aunque no compitió en la final de Mar del Plata (2008).  Debutó con 17 años en Brno, cuando Jordi Arrese le dio la alternativa para medirse al checo Jiri Novak, y ese mismo año ya dejó huella en la competición al derrotar en la final de La Cartuja al estadounidense Andy Roddick siendo todavía un rookie. A partir de ahí, un largo vuelo que comprende 23 eliminatorias, con un saldo final de 29 victorias y una sola derrota en los individuales, y 8-4 en dobles. Ha vivido España a su compás, en función de que le respetase su cuerpo y su calendario le permitiera aumentar en mayor medida la implicación; también sujeta a la circunstancia institucional, porque la década gloriosa dio paso luego a un periodo sombrío (en ocasiones esperpéntico) de inestabilidad. ADVERTISING  inRead invented by Teads   Contra una torre de Babel Corren ahora buenos tiempos, la federación intenta relanzar la base y el presente sonríe con la inauguración triunfal del nuevo formato. Un éxito que llevará para siempre el sello de Nadal, respaldado por el otro protagonista, Bautista, autor de dos puntos; el restante fue obra de Feliciano López y Marcel Granollers. Aunque se contuviera por dentro, en ese brinco del castellonense hacia el banquillo viajaban un millón de emociones. Proclive a relatos épicos y heroicidades, la Davis regaló este domingo otro de esos capítulos inolvidables, porque la competición le había reservado un emotivo episodio en la jornada final.  Después de haber perdido a su padre hace solo tres días y de haber abandonado la concentración tras jugar dos partidos –derrota en la apertura, contra Andrey Rublev, y triunfo ante el croata Nikola Mektic el segundo día–, Bautista cogió el coche y regresó a Madrid el sábado, y al día siguiente quiso saltar a la pista. Sergi Bruguera le alineó, en su condición de número dos del equipo, y firmó el primer punto de la final para España al vencer a Auger-Aliassime por 7-6(3) y 6-3, en 1h 38m.  Pese a la desgracia familiar, el número nueve del mundo hizo toda una demostración de aplomo, decidiendo jugar y abrir la serie frente a la multiétnica Canadá; esta, una verdadera torre de Babel. Sus componentes mezclan orígenees ruso, israelí (Shapovalov), checo (Pospisil) y togolés, en el caso de Aliassime. El joven, de 19 años, es uno de los tenistas a los que se observa con lupa, porque tiene muchos ingredientes para hacerse notar en los próximos años. Este año alcanzó las semifinales de Miami y hace cuatro se proclamó campeón júnior de la Davis, precisamente en la Caja Mágica.  Una atmósfera emotiva Aquejado de un esguince de tobillo, todavía no había intervenido en la competición, pero su técnico apostó por él y no le salió la jugada. Sí, en cambio, a Bruguera. Introdujo a Bautista y este logró una de las victorias más importantes de su carrera. En una atmósfera emotiva, el castellonense (31 años) sacó del maletín su buen oficio y desarmó al rival, que cedió en su apuesta a todo o nada, con 45 errores no forzados. Le pesó al canadiense la pérdida del tie break en el primer parcial, y en el segundo Bautista administró con madurez las distancias. Quebró para 3-0 y pese a la réplica (3-2) cerró con autoridad el duelo.  Entonces alzó los brazos, dedicó el punto a su padre y después del abrazo con el capitán se abalanzó hacia el resto del equipo. A base de corazón, situó a España a un solo punto de la sexta ensaladera y a continuación remató el trabajo el de casi siempre, Nadal, artífice fundamental del reencuentro con la felicidad. A la tercera bola de partido y después de levantar una de set en contra, el número uno decidió. Suma 29 victorias y una sola derrota en los partidos individuales, y ostenta además el récord absoluto de 31 triunfos consecutivos incluyendo los dobles. Redondea así una temporada fabulosa, con dos grandes (Roland Garros y el US Open), otros dos títulos (Roma y Montreal) y el número uno.  Y estaba escrito, con todas las letras: esta Davis era de él.  LAS SEIS ENSALADERAS DE ESPAÑA 2000. España, 3 - Australia, 1. Palau Sant Jordi (Barcelona). Costa, Ferrero, Corretja, Balcells.  2004. Estados Unidos, 2 - España, 3. Estadio La Cartuja (Sevilla). Moyá, Nadal, Ferrero, Robredo.  2008. Argentina, 1 - España, 3. Mar del Plata (Argentina). Nadal, Ferrer, Feliciano López, Verdasco.  2009. República Checa, 0 - España, 5. Palau Sant Jordi (Barcelona). Nadal, Ferrer, Feliciano López, Verdasco.  2011. Argentina, 1 - España, 3. Estadio La Cartuja (Sevilla). Nadal, Ferrer, Feliciano López, Verdasco.  2019. España, 2 - Canadá, 0. Caja Mágica (Madrid). Nadal, Bautista, Carreño, Feliciano López, Granollers."
test_new_violence="Una activista mexicana defensora de los indígenas ha sido asesinada este jueves en el Estado mexicano de Sonora. Raquel Padilla Ramos, historiadora y antropóloga murió debido a las “heridas de arma blanca” que le infligió su pareja, un hombre de 55 años, en una casa en un pequeño poblado llamado Ures, según ha anunciado la Fiscalía estatal. La policía local detuvo al agresor en el mismo momento en que cometía el crimen, sobre las cuatro de la tarde. La Procuraduría local ha abierto una investigación por feminicidio contra el hombre. La víctima, de 53 años, trabajaba como investigadora para el Instituto Nacional de Antropología e Historia mexicano (INAH).  MÁS INFORMACIÓN Asesinada por su pareja una defensora de los indígenas en el Estado mexicano de SonoraLiberados los dos activistas indígenas secuestrados en Guerrero La violencia provoca casi nueve millones de desplazados en México desde 2011 México pone en marcha un programa para buscar a 40.000 desaparecidos  De perfil académico y feminista, las redes sociales de Padilla Ramos se han vuelto este viernes una especie de bitácora rememorativa. En el muro de Facebook la activista aún se condolía por el asesinato de una mujer el pasado octubre en Playa del Carmen, en el Estado de Quintana Roo: “Tenía 25 años y le robaron la vida”, escribió. “A usted también se la robaron, descanse en paz”, lamentan ahora los que conocieron a Ramos Padilla. Entremezcladas con estas publicaciones, la activista también compartía con regularidad fotos y anécdotas junto a sus hijos y su pareja.  Días antes a su muerte, la activista había dejado numerosas reflexiones en sus redes sociales sobre la matanza en Bavispe, en la que el pasado lunes seis niños y tres mujeres de la familia LeBarón fueron brutalmente asesinados. “La masacre de mujeres y niños es lo importante”, compartió en su cuenta de Twitter el comentario de la periodista mexicana Lydia Cacho. Desde que trascendió la noticia en este poblado a 250 kilómetros al norte de Ures, la académica había ofrecido varias explicaciones sobre la historia del lugar. “Bavispe fue un pueblo de los indios ópatas, refundado como pueblo de misión por la Compañía de Jesús”, había señalado.  Padilla Ramos tenía un importante recorrido en el mundo académico. Había cursado una licenciatura en Ciencias Antropológicas en la Universidad Autónoma de Yucatán, una maestría en Ciencias Antropológicas y un doctorado en Estudios Mesoamericanos en la Universidad de Hamburgo, en Alemania. El secretario de Cultura de Ciudad de México, Alfonso Suárez, ha sido uno de los primeros en sumarse a las condolencias este viernes. “Condenamos la violencia y exigimos justicia”, ha escrito en su cuenta de Twitter. El INAH también lamentó  la sensible pérdida por el fallecimiento  de la “destacada escritora e intelectual”, a quién calificó como “una luchadora incansable por los derechos territoriales y culturales de los pueblos indígenas”. El comunicado, sin embargo, levantó críticas por referirse al hecho como fallecimiento y no como feminicidio.  Sonora es uno de los Estados mexicanos con peores cifras en agresiones machistas en México. Solo entre enero y julio de este año, 24 mujeres fueron asesinadas por sus parejas en esa entidad. Unos números que lo convierten en el quinto más violento para las mujeres. El detenido por el homicidio de Ramos Padilla fue señalado por varios testigos como el autor del crimen y enfrentará un proceso por feminicidio."
classes=["violencia machista", "no violencia machista"]

# probability threshold
ERROR_THRESHOLD = 0.2
# load our calculated synapse values
synapse_file = 'synapses.json'
with open(synapse_file) as data_file:
    synapse = json.load(data_file)
    synapse_0 = np.asarray(synapse['synapse0'])
    synapse_1 = np.asarray(synapse['synapse1'])

def classify(sentence, show_details=False):
    results = think(sentence, show_details)

    results = [[i,r] for i,r in enumerate(results) if r>ERROR_THRESHOLD ]
    results.sort(key=lambda x: x[1], reverse=True)
    return_results =[[classes[r[0]],r[1]] for r in results]
    print ("%s \n classification: %s" % (sentence, return_results))
    return return_results

classify(test_new_violence)
classify(test_new_no_violence)

print()
classify(test_new_violence, show_details=True)


text:
import os
import sys
import numpy as np
import faiss
import rocketqa

def build_index(encoder_conf, index_file_name, title_list, para_list):

    dual_encoder = rocketqa.load_model(**encoder_conf)
    para_embs = dual_encoder.encode_para(para=para_list, title=title_list)
    para_embs = np.array(list(para_embs))

    indexer = faiss.IndexFlatIP(768)
    indexer.add(para_embs.astype('float32'))
    faiss.write_index(indexer, index_file_name)

if __name__ == '__main__':
    if len(sys.argv) != 4:
        print ("USAGE: ")
        print ("      python3 index.py ${language} ${data_file} ${index_file}")
        print ("--For Example:")
        print ("      python3 index.py zh ../data/dureader.para test.index")
        exit()

    language = sys.argv[1]
    data_file = sys.argv[2]
    index_file = sys.argv[3]
    if language == 'zh':
        model = 'zh_dureader_de_v2'
    elif language == 'en':
        model = 'v1_marco_de'
    else:
        print ("illegal language, only [zh] and [en] is supported", file=sys.stderr)
        exit()

    para_list = []
    title_list = []
    for line in open(data_file):
        t, p = line.strip().split('\t')
        para_list.append(p)
        title_list.append(t)

    de_conf = {
            "model": model,
            "use_cuda": True,
            "device_id": 0,
            "batch_size": 32
    }
    build_index(de_conf, index_file, title_list, para_list)

max_stars_repo_name: sfwydyc/rocketqa

max_issues_repo_name: sfwydyc/rocketqa

max_forks_repo_name: sfwydyc/rocketqa


text:
# --------------
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# code starts here
df = pd.read_csv(path)
#print(df.info())
p_a = ((df['fico'] > 700).sum())/len(df)
print(p_a)
p_b = ((df['purpose'] == 'debt_consolidation').sum())/len(df)
print(p_b)
df1 = df[df['purpose']== 'debt_consolidation']
p_a_b = df1[df1['fico'].astype(float) >700].shape[0]/df1.shape[0]
print(p_a_b)
result = p_a_b == p_a
print(result)
# code ends here

# --------------
# code starts here
prob_lp = (df['paid.back.loan'] == 'Yes').sum()/len(df)
print(prob_lp)
prob_cs = (df['credit.policy'] == 'Yes').sum()/len(df)
print(prob_cs)
new_df = df[df['paid.back.loan'] == 'Yes']
prob_pd_cs = (new_df['credit.policy'] == 'Yes').sum()/len(new_df)
print(prob_pd_cs)
bayes = (prob_pd_cs*prob_lp)/prob_cs
print(bayes)
# code ends here

# --------------
# code starts here
plt.bar(df['purpose'],df['purpose'].index)
df1 = df[df['paid.back.loan'] == 'No']
df1
plt.bar(df1['purpose'],df1['purpose'].index)
# code ends here

# --------------
# code starts here
inst_median = df['installment'].median()
print(inst_median)
inst_mean = df['installment'].mean()
print(inst_mean)
plt.hist(df['installment'])
plt.hist(df['log.annual.inc'])
# code ends here