text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2018 Birkbeck College. All rights reserved.
#
# Licensed under the MIT license. See file LICENSE for details.
#
# Author: Cosmin Stamate

from __future__ import division
from __future__ import unicode_literals
from __future__ import print_function
from __future__ import absolute_import
from builtins import super
from builtins import int
from builtins import range
from future import standard_library
standard_library.install_aliases()
import sys
import traceback

import numpy as np
import pandas as pd
import json

from scipy import interpolate, signal, fft
from scipy.fftpack import rfft
from pywt import wavedec

from pdkit.processor import Processor
from pdkit.gait_time_series import GaitTimeSeries

from scipy.integrate import cumtrapz
import matplotlib.pylab as plt
import matplotlib.patches as mpatches

from pdkit.utils import (load_data,
                        numerical_integration,
                        autocorrelation,
                        peakdet,
                        compute_interpeak,
                        butter_lowpass_filter,
                        crossings_nonzero_pos2neg,
                        autocorrelate,
                        get_signal_peaks_and_prominences,
                        BellmanKSegment)

class GaitProcessor(Processor):
    """
        This is the main Gait Processor class. Once the data is loaded it will be
        accessible at data_frame, where it looks like:
        data_frame.x, data_frame.y, data_frame.z: x, y, z components of the acceleration
        data_frame.index is the datetime-like index

        These values are recommended by the author of the pilot study :cite:`g-Kassavetis2015, g-BachlinPRMHGT10`

        :param sampling_frequency: (optional) the sampling frequency in Hz (100.0 default)
        :type sampling_frequency: float
        :param cutoff_frequency: (optional) the cutoff frequency in Hz (2.0 default)
        :type cutoff_frequency: float
        :param filter_order: (optional) filter order (2 default)
        :type filter_order: int
        :param window: (optional) The size of the window used to traverse the time series (256 default)
        :type window: int
        :param lower_frequency: (optional) lower frequency in Hz (2.0 default)
        :type lower_frequency: float
        :param upper_frequency: (optional) upper frequency in Hz (10.0 default)
        :type upper_frequency: float
        :param loco_band: The ratio of the energy in the locomotion band, measured in Hz ([0.5, 3] default)
        :type loco_band: list
        :param freeze_band: The ration of energy in the freeze band, measured in Hz ([3, 8] default)
        :type freeze_band: list
        :param step_size: The average step size in centimeters (50.0 default).
        :type step_size: float
        :param delta: A point is considered a maximum peak if it has the maximal value, and was preceded (to the left) by a value lower by delta (0.5 default).
        :type delta: float
        :param duration: The duration in the the same time unit as the time series (None default).
        :type duration: float
        :param distance: The total walking distance in meters (None default).
        :type distance: float

        :Examples:

        >>> import pdkit
        >>> gp = pdkit.GaitProcessor()
        >>> ts = pdkit.GaitTimeSeries().load(path_to_data)
        >>> rts = gp.resample(ts)
    """

    def __init__(self,
                 sampling_frequency=100.0,
                 cutoff_frequency=2.0,
                 filter_order=2,
                 window=256,
                 lower_frequency=2.0,
                 upper_frequency=10.0,
                 loco_band=[0.5, 3],
                 freeze_band=[3, 8],
                 step_size=50.0,
                 delta=0.5,
                 stride_fraction=1.0/8.0,
                 duration=None,
                 distance=None
                 ):

        super().__init__(sampling_frequency,
                         cutoff_frequency,
                         filter_order,
                         window,
                         lower_frequency,
                         upper_frequency)

        self.delta = delta
        self.step_size = step_size
        self.loco_band = loco_band
        self.freeze_band = freeze_band
        self.stride_fraction = stride_fraction
        self.duration=duration
        self.distance=distance

    def freeze_of_gait(self, x):
        """
            This method assess freeze of gait following :cite:`g-BachlinPRMHGT10`.

            :param x: The time series to assess freeze of gait on. This could be x, y, z or mag_sum_acc.
            :type x: pandas.Series
            :return freeze_time: What times do freeze of gait events occur. [measured in time (h:m:s)]
            :rtype freeze_time: numpy.ndarray
            :return freeze_indexe: Freeze Index is defined as the power in the “freeze” band [3–8 Hz] divided by the power in the “locomotor” band [0.5–3 Hz] [3]. [measured in Hz]
            :rtype freeze_indexe: numpy.ndarray
            :return list locomotor_freeze_index: Locomotor freeze index is the power in the “freeze” band [3–8 Hz] added to power in the “locomotor” band [0.5–3 Hz]. [measured in Hz]
            :rtype locomotor_freeze_index: numpy.ndarray
        """

        data = self.resample_signal(x).values

        f_res = self.sampling_frequency / self.window

        f_nr_LBs = int(self.loco_band[0] / f_res)
        f_nr_LBe = int(self.loco_band[1] / f_res)
        f_nr_FBs = int(self.freeze_band[0] / f_res)
        f_nr_FBe = int(self.freeze_band[1] / f_res)

        jPos = self.window + 1
        i = 0

        time = []
        sumLocoFreeze = []
        freezeIndex = []

        while jPos < len(data):

            jStart = jPos - self.window
            time.append(jPos)

            y = data[int(jStart):int(jPos)]
            y = y - np.mean(y)

            Y = np.fft.fft(y, int(self.window))
            Pyy = abs(Y*Y) / self.window

            areaLocoBand = numerical_integration( Pyy[f_nr_LBs-1 : f_nr_LBe], self.sampling_frequency )
            areaFreezeBand = numerical_integration( Pyy[f_nr_FBs-1 : f_nr_FBe], self.sampling_frequency )

            sumLocoFreeze.append(areaFreezeBand + areaLocoBand)

            freezeIndex.append(areaFreezeBand / areaLocoBand)

            jPos = jPos + self.step_size
            i = i + 1

        freeze_time = np.asarray(time, dtype=np.int32)
        freeze_index = np.asarray(freezeIndex, dtype=np.float32)
        locomotor_freeze_index = np.asarray(sumLocoFreeze, dtype=np.float32)

        return freeze_time, freeze_index, locomotor_freeze_index

    def frequency_of_peaks(self, x, start_offset=100, end_offset=100):
        """
            This method assess the frequency of the peaks on any given 1-dimensional time series.

            :param x: The time series to assess freeze of gait on. This could be x, y, z or mag_sum_acc.
            :type x: pandas.Series
            :param start_offset: Signal to leave out (of calculations) from the begining of the time series (100 default).
            :type start_offset: int
            :param end_offset: Signal to leave out (from calculations) from the end of the time series (100 default).
            :type end_offset: int
            :return frequency_of_peaks: The frequency of peaks on the provided time series [measured in Hz].
            :rtype frequency_of_peaks: float
        """

        peaks_data = x[start_offset: -end_offset].values
        maxtab, mintab = peakdet(peaks_data, self.delta)

        x = np.mean(peaks_data[maxtab[1:,0].astype(int)] - peaks_data[maxtab[:-1,0].astype(int)])

        frequency_of_peaks = abs(1/x)

        return frequency_of_peaks

    def speed_of_gait(self, x, wavelet_type='db3', wavelet_level=6):
        """
            This method assess the speed of gait following :cite:`g-MartinSB11`.

            It extracts the gait speed from the energies of the approximation coefficients of wavelet functions.
            Prefferably you should use the magnitude of x, y and z (mag_acc_sum) here, as the time series.

            :param x: The time series to assess freeze of gait on. This could be x, y, z or mag_sum_acc.
            :type x: pandas.Series
            :param wavelet_type: The type of wavelet to use. See https://pywavelets.readthedocs.io/en/latest/ref/wavelets.html for a full list ('db3' default).
            :type wavelet_type: str
            :param wavelet_level: The number of cycles the used wavelet should have. See https://pywavelets.readthedocs.io/en/latest/ref/wavelets.html for a fill list (6 default).
            :type wavelet_level: int
            :return: The speed of gait [measured in meters/second].
            :rtype: float
        """

        coeffs = wavedec(x.values, wavelet=wavelet_type, level=wavelet_level)

        energy = [sum(coeffs[wavelet_level - i]**2) / len(coeffs[wavelet_level - i]) for i in range(wavelet_level)]

        WEd1 = energy[0] / (5 * np.sqrt(2))
        WEd2 = energy[1] / (4 * np.sqrt(2))
        WEd3 = energy[2] / (3 * np.sqrt(2))
        WEd4 = energy[3] / (2 * np.sqrt(2))
        WEd5 = energy[4] / np.sqrt(2)
        WEd6 = energy[5] / np.sqrt(2)

        gait_speed = 0.5 * np.sqrt(WEd1+(WEd2/2)+(WEd3/3)+(WEd4/4)+(WEd5/5)+(WEd6/6))

        return gait_speed

    def walk_regularity_symmetry(self, data_frame):
        """
            This method extracts the step and stride regularity and also walk symmetry.

            :param data_frame: The data frame. It should have x, y, and z columns.
            :type data_frame: pandas.DataFrame
            :return step_regularity: Regularity of steps on [x, y, z] coordinates, defined as the consistency of the step-to-step pattern.
            :rtype step_regularity: numpy.ndarray
            :return stride_regularity: Regularity of stride on [x, y, z] coordinates, defined as the consistency of the stride-to-stride pattern.
            :rtype stride_regularity: numpy.ndarray
            :return walk_symmetry: Symmetry of walk on [x, y, z] coordinates, defined as the difference between step and stride regularity.
            :rtype walk_symmetry: numpy.ndarray
        """

        def _symmetry(v):
            maxtab, _ = peakdet(v, self.delta)
            return maxtab[1][1], maxtab[2][1]

        step_regularity_x, stride_regularity_x = _symmetry(autocorrelation(data_frame.x))
        step_regularity_y, stride_regularity_y = _symmetry(autocorrelation(data_frame.y))
        step_regularity_z, stride_regularity_z = _symmetry(autocorrelation(data_frame.z))

        symmetry_x = step_regularity_x - stride_regularity_x
        symmetry_y = step_regularity_y - stride_regularity_y
        symmetry_z = step_regularity_z - stride_regularity_z

        step_regularity = np.array([step_regularity_x, step_regularity_y, step_regularity_z])
        stride_regularity = np.array([stride_regularity_x, stride_regularity_y, stride_regularity_z])
        walk_symmetry = np.array([symmetry_x, symmetry_y, symmetry_z])

        return step_regularity, stride_regularity, walk_symmetry

    def walk_direction_preheel(self, data_frame):
        """
            Estimate local walk (not cardinal) direction with pre-heel strike phase.

            Inspired by Nirupam Roy's B.E. thesis: "WalkCompass: Finding Walking Direction Leveraging Smartphone's Inertial Sensors"

            :param data_frame: The data frame. It should have x, y, and z columns.
            :type data_frame: pandas.DataFrame
            :return: Unit vector of local walk (not cardinal) direction.
            :rtype: numpy.ndarray
        """

        # Sum of absolute values across accelerometer axes:
        data = data_frame.x.abs() + data_frame.y.abs() + data_frame.z.abs()

        # Find maximum peaks of smoothed data:
        dummy, ipeaks_smooth = self.heel_strikes(data)
        data = data.values

        # Compute number of samples between peaks using the real part of the FFT:
        interpeak = compute_interpeak(data, self.sampling_frequency)
        decel = np.int(np.round(self.stride_fraction * interpeak))

        # Find maximum peaks close to maximum peaks of smoothed data:
        ipeaks = []
        for ipeak_smooth in ipeaks_smooth:
            ipeak = np.argmax(data[ipeak_smooth - decel:ipeak_smooth + decel])
            ipeak += ipeak_smooth - decel
            ipeaks.append(ipeak)

        # Compute the average vector for each deceleration phase:
        vectors = []
        for ipeak in ipeaks:
            decel_vectors = np.asarray([[data_frame.x[i], data_frame.y[i], data_frame.z[i]]
                                        for i in range(ipeak - decel, ipeak)])
            vectors.append(np.mean(decel_vectors, axis=0))

        # Compute the average deceleration vector and take the opposite direction:
        direction = -1 * np.mean(vectors, axis=0)

        # Return the unit vector in this direction:
        direction /= np.sqrt(direction.dot(direction))

        return direction

    def heel_strikes(self, x):
        """ Estimate heel strike times between sign changes in accelerometer data.

            :param x: The time series to assess freeze of gait on. This could be x, y, z or mag_sum_acc.
            :type x: pandas.Series
            :return strikes: Heel strike timings measured in seconds.
            :rtype striles: numpy.ndarray
            :return strikes_idx: Heel strike timing indices of the time series.
            :rtype strikes_idx: numpy.ndarray
        """

        # Demean data:
        data = x.values
        data -= data.mean()

        # TODO: fix this
        # Low-pass filter the AP accelerometer data by the 4th order zero lag
        # Butterworth filter whose cut frequency is set to 5 Hz:
        filtered = butter_lowpass_filter(data, self.sampling_frequency, self.cutoff_frequency, self.filter_order)

        # Find transitional positions where AP accelerometer changes from
        # positive to negative.
        transitions = crossings_nonzero_pos2neg(filtered)

        # Find the peaks of AP acceleration preceding the transitional positions,
        # and greater than the product of a threshold and the maximum value of
        # the AP acceleration:
        strike_indices_smooth = []
        filter_threshold = np.abs(self.delta * np.max(filtered))
        for i in range(1, np.size(transitions)):
            segment = range(transitions[i-1], transitions[i])
            imax = np.argmax(filtered[segment])
            if filtered[segment[imax]] > filter_threshold:
                strike_indices_smooth.append(segment[imax])

        # Compute number of samples between peaks using the real part of the FFT:
        interpeak = compute_interpeak(data, self.sampling_frequency)
        decel = np.int(interpeak / 2)

        # Find maximum peaks close to maximum peaks of smoothed data:
        strikes_idx = []
        for ismooth in strike_indices_smooth:
            istrike = np.argmax(data[ismooth - decel:ismooth + decel])
            istrike = istrike + ismooth - decel
            strikes_idx.append(istrike)

        strikes = np.asarray(strikes_idx)
        strikes -= strikes[0]
        strikes = strikes / self.sampling_frequency

        return strikes, np.array(strikes_idx)

    def gait_regularity_symmetry(self, x, average_step_duration='autodetect', average_stride_duration='autodetect', unbias=1, normalize=2):
        """
            Compute step and stride regularity and symmetry from accelerometer data with the help of steps and strides.

            :param x: The time series to assess freeze of gait on. This could be x, y, z or mag_sum_acc.
            :type x: pandas.Series
            :param average_step_duration: Average duration of each step using the same time unit as the time series. If this is set to 'autodetect' it will infer this from the time series.
            :type average_step_duration: float
            :param average_stride_duration: Average duration of each stride using the same time unit as the time series. If this is set to 'autodetect' it will infer this from the time series.
            :type average_stride_duration: float
            :param unbias: Unbiased autocorrelation: divide by range (unbias=1) or by weighted range (unbias=2).
            :type unbias: int
            :param int normalize: Normalize: divide by 1st coefficient (normalize=1) or by maximum abs. value (normalize=2).
            :type normalize: int
            :return step_regularity: Step regularity measure along axis.
            :rtype step_regularity: float
            :return stride_regularity: Stride regularity measure along axis.
            :rtype stride_regularity: float
            :return symmetry: Symmetry measure along axis.
            :rtype symmetry: float
        """
        if (average_step_duration=='autodetect') or (average_stride_duration=='autodetect'):

            strikes, _ = self.heel_strikes(x)

            step_durations = []
            for i in range(1, np.size(strikes)):
                step_durations.append(strikes[i] - strikes[i-1])

            average_step_duration = np.mean(step_durations)

            number_of_steps = np.size(strikes)

            strides1 = strikes[0::2]
            strides2 = strikes[1::2]

            stride_durations1 = []
            for i in range(1, np.size(strides1)):
                stride_durations1.append(strides1[i] - strides1[i-1])

            stride_durations2 = []
            for i in range(1, np.size(strides2)):
                stride_durations2.append(strides2[i] - strides2[i-1])

            strides = [strides1, strides2]
            stride_durations = [stride_durations1, stride_durations2]

            average_stride_duration = np.mean((np.mean(stride_durations1),
                                        np.mean(stride_durations2)))

            return self.gait_regularity_symmetry(x, average_step_duration, average_stride_duration)

        else:
            coefficients, _ = autocorrelate(x, unbias=1, normalize=2)

            step_period = np.int(np.round(1 / average_step_duration))
            stride_period = np.int(np.round(1 / average_stride_duration))

            step_regularity = coefficients[step_period]
            stride_regularity = coefficients[stride_period]
            symmetry = np.abs(stride_regularity - step_regularity)

            return step_regularity, stride_regularity, symmetry

    def gait(self, x):
        """
            Extract gait features from estimated heel strikes and accelerometer data.

            :param x: The time series to assess freeze of gait on. This could be x, y, z or mag_sum_acc.
            :type x: pandas.Series
            :return number_of_steps: Estimated number of steps based on heel strikes [number of steps].
            :rtype number_of_steps: int
            :return velocity: Velocity (if distance is provided) [meters/second].
            :rtype velocity: float
            :return avg_step_length: Average step length (if distance is provided) [meters].
            :rtype avg_step_length: float
            :return avg_stride_length: Average stride length (if distance is provided) [meters].
            :rtyoe avg_stride_length: float
            :return cadence: Number of steps divided by duration [steps/second].
            :rtype cadence: float
            :return array step_durations: Step duration [seconds].
            :rtype step_durations: np.ndarray
            :return float avg_step_duration: Average step duration [seconds].
            :rtype avg_step_duration: float
            :return float sd_step_durations: Standard deviation of step durations [seconds].
            :rtype sd_step_durations: np.ndarray
            :return list strides: Stride timings for each side [seconds].
            :rtype strides: numpy.ndarray
            :return float avg_number_of_strides: Estimated number of strides based on alternating heel strikes [number of strides].
            :rtype avg_number_of_strides: float
            :return list stride_durations: Estimated stride durations [seconds].
            :rtype stride_durations: numpy.ndarray
            :return float avg_stride_duration: Average stride duration [seconds].
            :rtype avg_stride_duration: float
            :return float sd_step_durations: Standard deviation of stride durations [seconds].
            :rtype sd_step_duration: float
            :return float step_regularity: Measure of step regularity along axis [percentage consistency of the step-to-step pattern].
            :rtype step_regularity: float
            :return float stride_regularity: Measure of stride regularity along axis [percentage consistency of the stride-to-stride pattern].
            :rtype stride_regularity: float
            :return float symmetry: Measure of gait symmetry along axis [difference between step and stride regularity].
            :rtype symmetry: float
        """

        data = x

        strikes, _ = self.heel_strikes(data)

        step_durations = []
        for i in range(1, np.size(strikes)):
            step_durations.append(strikes[i] - strikes[i-1])

        avg_step_duration = np.mean(step_durations)
        sd_step_durations = np.std(step_durations)

        number_of_steps = np.size(strikes)

        strides1 = strikes[0::2]
        strides2 = strikes[1::2]

        stride_durations1 = []
        for i in range(1, np.size(strides1)):
            stride_durations1.append(strides1[i] - strides1[i-1])

        stride_durations2 = []
        for i in range(1, np.size(strides2)):
            stride_durations2.append(strides2[i] - strides2[i-1])

        strides = [strides1, strides2]
        stride_durations = [stride_durations1, stride_durations2]

        avg_number_of_strides = np.mean([np.size(strides1), np.size(strides2)])
        avg_stride_duration = np.mean((np.mean(stride_durations1),
                                    np.mean(stride_durations2)))
        sd_stride_durations = np.mean((np.std(stride_durations1),
                                    np.std(stride_durations2)))

        step_period = np.int(np.round(1 / avg_step_duration))
        stride_period = np.int(np.round(1 / avg_stride_duration))

        step_regularity, stride_regularity, symmetry = self.gait_regularity_symmetry(data,
                                                                                average_step_duration=avg_step_duration,
                                                                                average_stride_duration=avg_stride_duration)

        cadence = None
        if self.duration:
            cadence = number_of_steps / self.duration

        velocity = None
        avg_step_length = None
        avg_stride_length = None

        if self.distance:
            velocity = self.distance / self.duration
            avg_step_length = number_of_steps / self.distance
            avg_stride_length = avg_number_of_strides / self.distance

        return [number_of_steps, cadence,
               velocity,
               avg_step_length,
               avg_stride_length,
               step_durations,
               avg_step_duration,
               sd_step_durations,
               strides,
               stride_durations,
               avg_number_of_strides,
               avg_stride_duration,
               sd_stride_durations,
               step_regularity,
               stride_regularity,
               symmetry]

    def separate_into_sections(self, data_frame, labels_col='anno', labels_to_keep=[1,2], min_labels_in_sequence=100):
        """ Helper function to separate a time series into multiple sections based on a labeled column.

            :param data_frame: The data frame. It should have x, y, and z columns.
            :type data_frame: pandas.DataFrame
            :param labels_col: The column which has the labels we would like to separate the data_frame on on ('anno' default).
            :type labels_col: str
            :param labels_to_keep: The unique labele ids of the labels which we would like to keep, out of all the labels in the labels_col ([1, 2] default).
            :type labels_to_keep: list
            :param min_labels_in_sequence: The minimum number of samples which can make up a section (100 default).
            :type min_labels_in_sequence: int

            :return: A list of DataFrames, segmented accordingly.
            :rtype: list
        """
        sections = [[]]

        mask = data_frame[labels_col].apply(lambda x: x in labels_to_keep)

        for i,m in enumerate(mask):
            if m:
                sections[-1].append(i)

            if not m and len(sections[-1]) > min_labels_in_sequence:
                sections.append([])

        sections.pop()

        sections = [self.rebuild_indexes(data_frame.iloc[s]) for s in sections]

        return sections

    def bellman_segmentation(self, x, states):
        """
            Divide a univariate time-series, data_frame, into states contiguous segments, using Bellman k-segmentation algorithm on the peak prominences of the data.

            :param x: The time series to assess freeze of gait on. This could be x, y, z or mag_sum_acc.
            :type x: pandas.Series
            :param states: Number of contigous segments.
            :type states: int
            :return peaks: The peaks in our data_frame.
            :rtype peaks: list
            :return prominences: Peaks prominences.
            :rtype prominences: list
            :return bellman_idx: The indices of the segments.
            :rtype bellman_idx: list
        """
        peaks, prominences = get_signal_peaks_and_prominences(x)
        bellman_idx = BellmanKSegment(prominences, states)

        return peaks, prominences, bellman_idx

    def sklearn_segmentation(self, x, cluster_fn):
        """
            Divide a univariate time-series, data_frame, into states contiguous segments, using sk-learn clustering algorithms on the peak prominences of the data.

            :param x: The time series to assess freeze of gait on. This could be x, y, z or mag_sum_acc.
            :type x: pandas.Series
            :param cluster_fn: Any unsupervised learning algorithm from the sklearn library. It needs to have the `fit_predict` method.
            :param cluster_fn: sklearn.aglorithm
            :return peaks: The peaks in our data_frame.
            :rtype peaks: list
            :return prominences: Peaks prominences.
            :rtype prominences: list
            :return sklearn_idx: The indices of the segments.
            :rtype sklearn_idx: list
        """
        peaks, prominences = get_signal_peaks_and_prominences(x)

        # sklearn fix: reshape to (-1, 1)
        sklearn_idx = cluster_fn.fit_predict(prominences.reshape(-1, 1))

        return peaks, prominences, sklearn_idx

    def add_manual_segmentation_to_data_frame(self, data_frame, segmentation_dictionary):
        """
            Utility method to store manual segmentation of gait time series.

            :param data_frame: The data frame. It should have x, y, and z columns.
            :type data_frame: pandas.DataFrame
            :param segmentation_dictionary: A dictionary of the form {'signal_type': [(from, to), (from, to)], ..., 'signal_type': [(from, to), (from, to)]}. The from and to can either be of type numpy.datetime64 or int, depending on how you are segmenting the time series.
            :type segmentation_dictionary: dict
            :return: The data_frame with a new column named 'segmentation'.
            :rtype: pandas.DataFrame
        """

        # add some checks to see if dictionary is in the right format!

        data_frame['segmentation'] = 'unknown'

        for i, (k, v) in enumerate(segmentation_dictionary.items()):
            for start, end in v:
                if type(start) != np.datetime64:
                    if start < 0: start = 0
                    if end > data_frame.size: end = data_frame.size

                    start = data_frame.index.values[start]
                    end = data_frame.index.values[end]

                data_frame.loc[start: end, 'segmentation'] = k

        return data_frame

    def plot_segmentation_dictionary(self, x, segmentation_dictionary, figsize=(10, 5)):
        """
            Utility method used to visualize how the segmentation dictionary interacts with the time series.

            :param data_frame: The data frame. It should have x, y, and z columns.
            :type data_frame: pandas.DataFrame
            :param segmentation_dictionary: A dictionary of the form {'signal_type': [(from, to), (from, to)], ..., 'signal_type': [(from, to), (from, to)]}.
            :type segmentation_dictionary: dict
            :param figsize: The size of the figure where we will plot the segmentation on top of the provided time series ((10, 5) default).
            :type figsize: tuple
        """
        data = x

        fig, ax = plt.subplots()
        fig.set_size_inches(figsize[0], figsize[1])

        # fix this!!
        colors = 'bgrcmykwbgrcmykwbgrcmykw'

        data.plot(ax=ax)

        for i, (k, v) in enumerate(segmentation_dictionary.items()):
            for start, end in v:
                if type(start) != np.datetime64:
                    start = data.index.values[start]
                    end = data.index.values[end]

                plt.axvspan(start, end, color=colors[i], alpha=0.5)

        legend = [mpatches.Patch(color=colors[i], label="{}".format(k)) for i, k in enumerate(segmentation_dictionary.keys())]

        plt.legend(handles=legend)

        plt.show()

    def plot_segmentation_data_frame(self, segmented_data_frame, axis='mag_sum_acc', figsize=(10, 5)):
        """
            Utility method used to visualize how the segmentation dictionary interacts with the time series.

            :param segmented_data_frame: The segmented data frame. It should have x, y, z and segmentation columns.
            :type segmented_data_frame: pandas.DataFrame
            :param axis: The axis which we want to plot. We can choose from x, y, z and mag_sum_acc ('mag_acc_sum' default).
            :type axis: str
            :param figsize: The size of the figure where we will plot the segmentation on top of the provided time series ((10, 5) default).
            :type figsize: tuple
        """
        # fix this!!
        colors = 'bgrcmykwbgrcmykwbgrcmykw'

        keys = np.unique(segmented_data_frame['segmentation'])

        fig, ax = plt.subplots()
        fig.set_size_inches(figsize[0], figsize[1])

        segmented_data_frame[axis].plot(ax=ax)

        for i, k in enumerate(keys):
            patch = segmented_data_frame['segmentation'].loc[segmented_data_frame['segmentation'] == k].index
            for p in patch:
                ax.axvline(p, color=colors[i], alpha=0.1)

        legend = [mpatches.Patch(color=colors[i], label="{}".format(k)) for i, k in enumerate(keys)]

        plt.legend(handles=legend)

        plt.show()

max_stars_repo_name: gkroussos/pdkit

max_issues_repo_name: gkroussos/pdkit

max_forks_repo_name: gkroussos/pdkit


text:
"""
Conjugate Gradient method with preconditioning for solving sparse
linear systems.

@var TOLERANCE: iterate in CG until the error is less than this value
@var MAX_ITER: maximum number of CG iterations
"""

import numpy as np
import sparse
import time

TOLERANCE = 1E-10
MAX_ITER = 1000

def conjugate_gradient(A, b, prec=None):
    """Solve the system of linear equations using (preconditioned)
    Conjugate Gradient method.

    @arg A: I{m} times I{n} sparse matrix
    @type A: L{SparseMatrix}
    @arg b: right-hand side vector
    @type b: C{numpy.ndarray} of shape (I{m},1)
    @return: solution vector
    @rtype: C{numpy.ndarray} of shape (I{n},1)
    @arg prec: preconditioner function (or any callable) that returns I{z} for given I{r}
    @type prec: 1-argument Python callable
    """

    t1 = time.clock()
    x = np.zeros((A.n, 1))
    r = b - A * x
    it = 0

    while np.sqrt(np.sum(r ** 2)) > TOLERANCE and it < MAX_ITER:

        if prec:
            z = prec(r)
        else:
            z = r.copy()

        rt = r.transpose()
        rtr = 0
        for j in range(len(r)):
            rtr += rt[0, j] * z[j, 0]

        if it == 0:
            p = z
        else:
            beta = rtr / rtr_last
            p = z + beta * p_last

        Ap = A * p
        pt = p.transpose()
        ptAp = 0

        for j in range(len(Ap)):
            ptAp += pt[0, j] * Ap[j, 0]

        alfa = rtr / ptAp
        x_next = x + alfa * p
        r_next = r - alfa * Ap

        x = x_next.copy()
        r = r_next.copy()

        rtr_last = rtr.copy()
        p_last = p.copy()
        it += 1

    t2 = time.clock()
    print "CG number of iterations: %d" % it
    print "CG error: %.15f" % np.sqrt(np.sum(r ** 2))
    print "CG time: %.15f" % (t2 - t1)

    return x

ext: py

reason: import numpy


text:
# Copyright (C) 2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

import numpy as np
import pytest

import openvino as ov
import openvino.opset8 as ops
from openvino.impl import Function, Output, Type
from openvino.utils.decorators import custom_preprocess_function
from openvino import Core
from tests.runtime import get_runtime
from openvino.preprocess import PrePostProcessor, InputInfo, PreProcessSteps, InputTensorInfo, \
    OutputTensorInfo, OutputNetworkInfo, InputNetworkInfo, ColorFormat, OutputInfo, \
    PostProcessSteps, ResizeAlgorithm

def test_ngraph_preprocess_mean():
    shape = [2, 2]
    parameter_a = ops.parameter(shape, dtype=np.float32, name="A")
    model = parameter_a
    function = Function(model, [parameter_a], "TestFunction")

    function = PrePostProcessor(function)\
        .input(InputInfo()
               .preprocess(PreProcessSteps()
                           .mean(1.)
                           )
               )\
        .build()

    input_data = np.array([[1, 2], [3, 4]]).astype(np.float32)
    expected_output = np.array([[0, 1], [2, 3]]).astype(np.float32)

    runtime = get_runtime()
    computation = runtime.computation(function)
    output = computation(input_data)
    assert np.equal(output, expected_output).all()

def test_ngraph_preprocess_mean_vector():
    shape = [2, 2]
    parameter_a = ops.parameter(shape, dtype=np.float32, name="A")
    model = parameter_a
    function = Function(model, [parameter_a], "TestFunction")
    layout = ov.Layout("NCHW")

    function = PrePostProcessor(function)\
        .input(InputInfo()
               .tensor(InputTensorInfo().set_layout(layout))
               .preprocess(PreProcessSteps()
                           .mean([1., 2.])
                           )
               )\
        .build()

    input_data = np.array([[1, 2], [3, 4]]).astype(np.float32)
    expected_output = np.array([[0, 0], [2, 2]]).astype(np.float32)

    runtime = get_runtime()
    computation = runtime.computation(function)
    output = computation(input_data)
    assert np.equal(output, expected_output).all()

def test_ngraph_preprocess_scale_vector():
    shape = [2, 2]
    parameter_a = ops.parameter(shape, dtype=np.float32, name="A")
    model = parameter_a
    function = Function(model, [parameter_a], "TestFunction")
    layout = ov.Layout("NCHW")

    function = PrePostProcessor(function)\
        .input(InputInfo()
               .tensor(InputTensorInfo().set_layout(layout))
               .preprocess(PreProcessSteps()
                           .scale([0.5, 2.])
                           )
               )\
        .build()

    input_data = np.array([[1, 2], [3, 4]]).astype(np.float32)
    expected_output = np.array([[2, 1], [6, 2]]).astype(np.float32)

    runtime = get_runtime()
    computation = runtime.computation(function)
    output = computation(input_data)
    assert np.equal(output, expected_output).all()

def test_ngraph_preprocess_mean_scale_convert():
    shape = [2, 2]
    param1 = ops.parameter(shape, dtype=np.int32, name="A")
    param2 = ops.parameter(shape, dtype=np.int32, name="B")
    function = Function([param1, param2], [param1, param2], "TestFunction")

    @custom_preprocess_function
    def custom_preprocess(output: Output):
        return ops.abs(output)

    function = PrePostProcessor(function) \
        .input(InputInfo(1)
               .tensor(InputTensorInfo()
                       .set_element_type(Type.i32))
               .preprocess(PreProcessSteps()
                           .convert_element_type(Type.f32)
                           .mean(1.)
                           .scale(2.)
                           )
               ) \
        .input(InputInfo(0)
               .preprocess(PreProcessSteps()
                           .convert_element_type(Type.f32)
                           .mean(1.)
                           .custom(custom_preprocess)
                           )
               ) \
        .build()

    input_data1 = np.array([[0, 1], [2, -2]]).astype(np.int32)
    input_data2 = np.array([[1, 3], [5, 7]]).astype(np.int32)
    expected_output1 = np.array([[1, 0], [1, 3]]).astype(np.float32)
    expected_output2 = np.array([[0, 1], [2, 3]]).astype(np.float32)

    runtime = get_runtime()
    computation = runtime.computation(function)
    [output1, output2] = computation(input_data1, input_data2)
    assert np.equal(output1, expected_output1).all()
    assert np.equal(output2, expected_output2).all()

def test_ngraph_preprocess_output_postprocess():
    shape = [2, 2]
    parameter_a = ops.parameter(shape, dtype=np.int32, name="A")
    model = parameter_a
    function = Function(model, [parameter_a], "TestFunction")
    layout1 = ov.Layout("NCHW")
    layout2 = ov.Layout("NHWC")
    layout3 = [0, 1]

    @custom_preprocess_function
    def custom_postprocess(output: Output):
        return ops.abs(output)

    function = PrePostProcessor(function)\
        .input(InputInfo()
               .tensor(InputTensorInfo().set_layout(layout1))
               .preprocess(PreProcessSteps()
                           .convert_element_type(Type.f32)
                           .mean([1., 2.])
                           )
               ) \
        .output(OutputInfo().postprocess(PostProcessSteps()
                                         .convert_element_type(Type.f32)
                                         .convert_layout(layout2)
                                         .convert_layout(layout3)
                                         .custom(custom_postprocess))) \
        .build()

    input_data = np.array([[-1, -2], [-3, -4]]).astype(np.int32)
    expected_output = np.array([[2, 4], [4, 6]]).astype(np.float32)

    runtime = get_runtime()
    computation = runtime.computation(function)
    output = computation(input_data)
    assert np.equal(output, expected_output).all()

def test_ngraph_preprocess_spatial_static_shape():
    shape = [2, 2, 2]
    parameter_a = ops.parameter(shape, dtype=np.int32, name="A")
    model = parameter_a
    function = Function(model, [parameter_a], "TestFunction")
    layout = ov.Layout("CHW")

    color_format = ColorFormat.RGB
    function = PrePostProcessor(function)\
        .input(InputInfo()
               .tensor(InputTensorInfo()
                       .set_layout(layout)
                       .set_spatial_static_shape(2, 2)
                       .set_color_format(color_format, []))
               .preprocess(PreProcessSteps()
                           .convert_element_type(Type.f32)
                           .mean([1., 2])
                           )
               .network(InputNetworkInfo().set_layout(layout))
               ) \
        .output(OutputInfo()
                .tensor(OutputTensorInfo()
                        .set_layout(layout)
                        .set_element_type(Type.f32))
                .network(OutputNetworkInfo().set_layout(layout))) \
        .build()

    input_data = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]).astype(np.int32)
    expected_output = np.array([[[0, 1], [2, 3]], [[3, 4], [5, 6]]]).astype(np.float32)

    runtime = get_runtime()
    computation = runtime.computation(function)
    output = computation(input_data)
    assert np.equal(output, expected_output).all()

@pytest.mark.parametrize(
    "algorithm, color_format1, color_format2, is_failing",
    [(ResizeAlgorithm.RESIZE_LINEAR, ColorFormat.UNDEFINED, ColorFormat.BGR, True),
     (ResizeAlgorithm.RESIZE_LINEAR, ColorFormat.RGB, ColorFormat.NV12_SINGLE_PLANE, True),
     (ResizeAlgorithm.RESIZE_LINEAR, ColorFormat.RGB, ColorFormat.NV12_TWO_PLANES, True),
     (ResizeAlgorithm.RESIZE_CUBIC, ColorFormat.RGB, ColorFormat.UNDEFINED, True),
     (ResizeAlgorithm.RESIZE_CUBIC, ColorFormat.RGB, ColorFormat.BGR, False),
     (ResizeAlgorithm.RESIZE_CUBIC, ColorFormat.BGR, ColorFormat.RGB, False),
     (ResizeAlgorithm.RESIZE_NEAREST, ColorFormat.BGR, ColorFormat.NV12_SINGLE_PLANE, True),
     (ResizeAlgorithm.RESIZE_NEAREST, ColorFormat.BGR, ColorFormat.NV12_TWO_PLANES, True),
     (ResizeAlgorithm.RESIZE_NEAREST, ColorFormat.BGR, ColorFormat.UNDEFINED, True)])
def test_ngraph_preprocess_steps(algorithm, color_format1, color_format2, is_failing):
    shape = [1, 1, 3, 3]
    parameter_a = ops.parameter(shape, dtype=np.float32, name="A")
    model = parameter_a
    function = Function(model, [parameter_a], "TestFunction")
    layout1 = ov.Layout("NCWH")
    layout2 = ov.Layout("NCHW")
    custom_processor = PrePostProcessor(function)\
        .input(InputInfo()
               .tensor(InputTensorInfo()
                       .set_layout(layout1)
                       .set_color_format(color_format1, []))
               .preprocess(PreProcessSteps()
                           .mean(1.)
                           .resize(algorithm, 3, 3)
                           .convert_layout(layout2)
                           .convert_color(color_format2)))
    if is_failing:
        with pytest.raises(RuntimeError) as e:
            function = custom_processor.build()
        assert "is not convertible to" in str(e.value)
    else:
        function = custom_processor.build()
        input_data = np.array([[[[1, 2, 3], [4, 5, 6], [7, 8, 9]]]]).astype(np.float32)
        expected_output = np.array([[[[0, 3, 6], [1, 4, 7], [2, 5, 8]]]]).astype(np.float32)

        runtime = get_runtime()
        computation = runtime.computation(function)
        output = computation(input_data)
        assert np.equal(output, expected_output).all()

def test_ngraph_preprocess_postprocess_layout():
    shape = [1, 1, 3, 3]
    parameter_a = ops.parameter(shape, dtype=np.float32, name="A")
    model = parameter_a
    function = Function(model, [parameter_a], "TestFunction")
    layout1 = ov.Layout("NCWH")
    layout2 = ov.Layout("NCHW")

    function = PrePostProcessor(function)\
        .input(InputInfo()
               .tensor(InputTensorInfo()
                       .set_layout(layout1))
               .preprocess(PreProcessSteps()
                           .mean(1.)
                           .convert_layout(layout2)
                           .reverse_channels()
                           )
               ) \
        .output(OutputInfo()
                .postprocess(PostProcessSteps()
                             .convert_layout([0, 1, 2, 3]))) \
        .build()

    input_data = np.array([[[[1, 2, 3], [4, 5, 6], [7, 8, 9]]]]).astype(np.float32)
    expected_output = np.array([[[[0, 3, 6], [1, 4, 7], [2, 5, 8]]]]).astype(np.float32)

    runtime = get_runtime()
    computation = runtime.computation(function)
    output = computation(input_data)
    assert np.equal(output, expected_output).all()

def test_ngraph_preprocess_reverse_channels():
    shape = [1, 2, 2, 2]
    parameter_a = ops.parameter(shape, dtype=np.float32, name="A")
    model = parameter_a
    function = Function(model, [parameter_a], "TestFunction")
    layout1 = ov.Layout("NCWH")

    function = PrePostProcessor(function)\
        .input(InputInfo()
               .tensor(InputTensorInfo()
                       .set_layout(layout1))
               .preprocess(PreProcessSteps()
                           .mean(1.)
                           .reverse_channels()
                           )
               ) \
        .build()

    input_data = np.array([[[[1, 2], [3, 4]], [[5, 6], [7, 8]]]]).astype(np.float32)
    expected_output = np.array([[[[4, 5], [6, 7]], [[0, 1], [2, 3]]]]).astype(np.float32)

    runtime = get_runtime()
    computation = runtime.computation(function)
    output = computation(input_data)
    assert np.equal(output, expected_output).all()

def test_ngraph_preprocess_resize_algorithm():
    shape = [1, 1, 3, 3]
    parameter_a = ops.parameter(shape, dtype=np.float32, name="A")
    model = parameter_a
    function = Function(model, [parameter_a], "TestFunction")
    resize_alg = ResizeAlgorithm.RESIZE_CUBIC
    layout1 = ov.Layout("NCWH")

    function = PrePostProcessor(function)\
        .input(InputInfo()
               .tensor(InputTensorInfo()
                       .set_layout(layout1))
               .preprocess(PreProcessSteps()
                           .mean(1.)
                           .resize(resize_alg, 3, 3))
               )\
        .build()

    input_data = np.array([[[[1, 2, 3], [4, 5, 6], [7, 8, 9]]]]).astype(np.float32)
    expected_output = np.array([[[[0, 1, 2], [3, 4, 5], [6, 7, 8]]]]).astype(np.float32)

    runtime = get_runtime()
    computation = runtime.computation(function)
    output = computation(input_data)
    assert np.equal(output, expected_output).all()

def test_ngraph_preprocess_model():
    model = bytes(b"""<net name="add_model" version="10">
    <layers>
    <layer id="0" name="x" type="Parameter" version="opset1">
        <data element_type="i32" shape="2,2,2"/>
        <output>
            <port id="0" precision="FP32">
                <dim>2</dim>
                <dim>2</dim>
                <dim>2</dim>
            </port>
        </output>
    </layer>
    <layer id="1" name="y" type="Parameter" version="opset1">
        <data element_type="i32" shape="2,2,2"/>
        <output>
            <port id="0" precision="FP32">
                <dim>2</dim>
                <dim>2</dim>
                <dim>2</dim>
            </port>
        </output>
    </layer>
    <layer id="2" name="sum" type="Add" version="opset1">
        <input>
            <port id="0">
                <dim>2</dim>
                <dim>2</dim>
                <dim>2</dim>
            </port>
            <port id="1">
                <dim>2</dim>
                <dim>2</dim>
                <dim>2</dim>
            </port>
        </input>
        <output>
            <port id="2" precision="FP32">
                <dim>2</dim>
                <dim>2</dim>
                <dim>2</dim>
            </port>
        </output>
    </layer>
    <layer id="3" name="sum/sink_port_0" type="Result" version="opset1">
        <input>
            <port id="0">
                <dim>2</dim>
                <dim>2</dim>
                <dim>2</dim>
            </port>
        </input>
    </layer>
    </layers>
    <edges>
    <edge from-layer="0" from-port="0" to-layer="2" to-port="0"/>
    <edge from-layer="1" from-port="0" to-layer="2" to-port="1"/>
    <edge from-layer="2" from-port="2" to-layer="3" to-port="0"/>
    </edges>
</net>""")
    core = Core()
    function = core.read_model(model=model)

    @custom_preprocess_function
    def custom_preprocess(output: Output):
        return ops.abs(output)

    function = PrePostProcessor(function) \
        .input(InputInfo(1)
               .preprocess(PreProcessSteps()
                           .convert_element_type(Type.f32)
                           .scale(0.5)
                           )
               ) \
        .input(InputInfo(0)
               .preprocess(PreProcessSteps()
                           .convert_element_type(Type.f32)
                           .mean(5.))) \
        .output(OutputInfo(0)
                .postprocess(PostProcessSteps()
                             .custom(custom_preprocess))) \
        .build()

    input_data = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]).astype(np.float32)
    expected_output = np.array([[[2, 1], [4, 7]], [[10, 13], [16, 19]]]).astype(np.float32)

    runtime = get_runtime()
    computation = runtime.computation(function)
    output = computation(input_data, input_data)

    assert np.equal(output, expected_output).all()

max_stars_repo_name: tpwrules/openvino

max_issues_repo_name: tpwrules/openvino

max_forks_repo_name: evkotov/openvino


text:
import numpy
import torch

from .GoalsBuffer    import *

class AgentGoExplore():
    def __init__(self, envs, Model, config):
        self.envs = envs

        self.steps                  = config.steps
        self.batch_size             = config.batch_size

        self.training_epochs        = config.training_epochs
        self.envs_count             = config.envs_count

        self.buffer_size                = config.buffer_size
        self.buffer_add_threshold       = config.buffer_add_threshold
        self.buffer_downsample          = config.buffer_downsample

        self.state_shape    = self.envs.observation_space.shape
        self.actions_count  = self.envs.action_space.n

        self.goals_buffer   =  GoalsBuffer(self.buffer_size, self.state_shape, self.buffer_add_threshold, self.buffer_downsample, True)

        #self.model          = Model.Model(self.state_shape, self.actions_count, self.buffer_downsample)
        #self.optimizer      = torch.optim.Adam(self.model.parameters(), lr=config.learning_rate)

        #reset envs and fill initial state
        self.states = numpy.zeros((self.envs_count, ) + self.state_shape, dtype=numpy.float32)
        for e in range(self.envs_count):
            self.states[e] = self.envs.reset(e).copy()

        #all agents into explore mode - nothing stored in buffer yet, so explore some new states
        self.mode = numpy.zeros(self.envs_count, dtype=bool)

        self.goals_t                = torch.zeros((self.envs_count, ) + self.goals_buffer.goal_shape)

        self.buffer_actions_list    = [[] for _ in range(self.envs_count)]
        self.current_actions_list   = [[] for _ in range(self.envs_count)]
        self.actions_step           = numpy.zeros(self.envs_count, dtype=int)
        self.episode_ext_reward_sum = numpy.zeros(self.envs_count)
        self.episode_int_reward_sum = numpy.zeros(self.envs_count)

        self.enable_training()
        self.iterations                     = 0

        self.log_goals_count    = 0.0
        self.log_mode           = 0.0
        self.log_length         = 0.0

    def enable_training(self):
        self.enabled_training = True

    def disable_training(self):
        self.enabled_training = False

    def main(self):
        #state to tensor
        states_t            = torch.tensor(self.states, dtype=torch.float).detach()

        actions = self._sample_actions()

        for e in range(self.envs_count):
            self.current_actions_list[e].append(actions[e])

        #execute action
        states, rewards_ext, dones, infos = self.envs.step(actions)

        self.states = states.copy()

        self.episode_ext_reward_sum+= rewards_ext

        #TODO, use RND ?
        #self.episode_int_reward_sum+= rewards_int

        self.goals_buffer.add(states_t, self.episode_ext_reward_sum, self.episode_int_reward_sum, self.current_actions_list)

        #check where goal reached
        goals_reached = self.goals_buffer.goals_reached(states_t, self.goals_t)

        #switch mode to explore, if goal reached
        for e in range(self.envs_count):
            if goals_reached[e]:
                self.mode[e] = True

        for e in range(self.envs_count):
            if dones[e]:
                self.states[e]      = self.envs.reset(e).copy()

                #switch agent to GoMode
                self.mode[e]  = False

                #sample goal, and actions leading to goal
                goal, actions = self.goals_buffer.get_goal(1.0, 1.0, 0.001)

                self.goals_t[e]                 = goal.clone()
                self.buffer_actions_list[e]     = actions
                self.current_actions_list[e]    = []

                self.episode_ext_reward_sum[e]  = 0.0
                self.episode_int_reward_sum[e]  = 0.0

        avg_length = 0
        for e in range(self.envs_count):
            avg_length+= len(self.buffer_actions_list[e])
        avg_length = avg_length / self.envs_count

        #collect stats
        k = 0.02
        self.log_mode     = (1.0 - k)*self.log_mode + k*100.0*self.mode.mean()
        self.log_length   = (1.0 - k)*self.log_length + k*avg_length

        self.iterations+= 1
        return rewards_ext[0], dones[0], infos[0]

    def save(self, save_path):
        pass

    def load(self, load_path):
        pass

    def get_log(self):
        result = ""

        result+= str(round(self.goals_buffer.goals_ptr, 7)) + " "
        result+= str(round(self.log_mode, 7)) + " "
        result+= str(round(self.log_length, 7)) + " "

        return result

    def _sample_actions(self):
        actions = numpy.zeros(self.envs_count, dtype=int)
        for e in range(self.envs_count):
            #agent GO mode
            actions_list     = self.buffer_actions_list[e]
            action_step_idx  = self.actions_step[e]
            if self.mode[e] == False and action_step_idx < len(actions_list):
                actions[e] = actions_list[action_step_idx]
                self.actions_step[e]+= 1
            #agent Explore mode
            else:
                actions[e]      = numpy.random.randint(0, self.actions_count)

        return actions


text:
# -*- coding: utf-8 -*-

import numpy as np
import paddle

EPS = np.finfo(np.float32).eps

def compress_cIRM(mask, K=10, C=0.1):
    if paddle.is_tensor(mask):
        mask = -100 * (mask <= -100).astype(paddle.float32) + mask * (mask > -100).astype(paddle.float32)
        mask = K * (1 - paddle.exp(-C * mask)) / (1 + paddle.exp(-C * mask))
        return mask.astype(paddle.float32)
    else:
        mask = -100 * (mask <= -100) + mask * (mask > -100)
        mask = K * (1 - np.exp(-C * mask)) / (1 + np.exp(-C * mask))
        return mask.astype(np.float32)

def decompress_cIRM(mask, K=10, limit=9.999):
    if paddle.is_tensor(mask):
        mask = (
            limit * (mask >= limit).astype(paddle.float32)
            - limit * (mask <= -limit).astype(paddle.float32)
            + mask * (paddle.abs(mask) < limit).astype(paddle.float32)
        )
        mask = -K * paddle.log((K - mask) / (K + mask))
        return mask.astype(paddle.float32)
    else:
        mask = limit * (mask >= limit) - limit * (mask <= -limit) + mask * (np.abs(mask) < limit)
        mask = -K * np.log((K - mask) / (K + mask))
        return mask.astype(np.float32)

def get_cIRM(noisy_spec, clean_spec):
    if paddle.is_tensor(noisy_spec) and paddle.is_tensor(clean_spec):
        noisy_real = paddle.real(noisy_spec)
        noisy_imag = paddle.imag(noisy_spec)
        clean_real = paddle.real(clean_spec)
        clean_imag = paddle.imag(clean_spec)

        denominator = (noisy_real ** 2 + clean_imag ** 2) + EPS

        cIRM_real = (noisy_real * clean_real + noisy_imag * clean_imag) / denominator
        cIRM_imag = (noisy_real * clean_imag - noisy_imag * clean_real) / denominator
        cIRM = paddle.stack([cIRM_real, cIRM_imag], axis=-1)
    else:
        noisy_real = np.real(noisy_spec)
        noisy_imag = np.imag(noisy_spec)
        clean_real = np.real(clean_spec)
        clean_imag = np.imag(clean_spec)

        denominator = (noisy_real ** 2 + clean_imag ** 2) + EPS

        cIRM_real = (noisy_real * clean_real + noisy_imag * clean_imag) / denominator
        cIRM_imag = (noisy_real * clean_imag - noisy_imag * clean_real) / denominator
        cIRM = np.stack((cIRM_real, cIRM_imag), axis=-1)

    return compress_cIRM(cIRM)  # [B, F, T, 2]


text:
import cv2
import numpy as np

RED = (0, 0, 255)
BLUE = (255, 0, 0)
GREEN = (0, 255, 0)

class LaneDetector:
    def __init__(self):
        self.angle = 0
        self.previous_angles = []

    def __insert_previous_angle(self):
        max_size = 10
        if len(self.previous_angles) >= max_size:
            del self.previous_angles[0]
        self.previous_angles.append(self.angle)

    def detect_lane(self, image: np.ndarray):
        self.original_image_array = image.copy()
        self.image_array = image.copy()
        self.right = []
        self.left = []
        self.done = True

        self._blur_image()
        self._get_grayscale_image()
        self._detect_edges()
        self._get_roi_image()
        self._hough_line()
        if self.hough_lines is None:
           return False, self.angle  # not done
        self._get_lane_candidates()
        self._intersection_of_lanes()
        self._get_angle_error()

        add_term = 0.1  # TODO: consider previous angles
        if not self.right or not self.left:
            self.angle = np.mean(self.previous_angles)
            if not self.right:
                self.angle += add_term
            if not self.left:
                self.angle -= add_term
        self.__insert_previous_angle()
        return self.done, self.angle

    def _blur_image(self):
        kernel_size = 3
        self.image_array = cv2.GaussianBlur(
            self.image_array, (kernel_size, kernel_size), 0)

    def _get_grayscale_image(self):
        self.image_array = cv2.cvtColor(self.image_array, cv2.COLOR_BGR2GRAY)

    def _detect_edges(self):
        self.image_array = cv2.Canny(self.image_array,
                                     50,
                                     150,
                                     )

    def _get_roi_image(self):
        xsize = self.image_array.shape[1]
        ysize = self.image_array.shape[0]
        mask = np.zeros_like(self.image_array)
        dx_bottom = int(0 * xsize)
        dx_up = int(0.15 * xsize)
        dy_bottom = int(0.1 * ysize)
        dy_up = int(0.55 * ysize)
        vertices = np.array([[(xsize - dx_bottom, ysize - dy_bottom),
                              (xsize - dx_up, dy_up),
                              (dx_up, dy_up),
                              (dx_bottom, ysize - dy_bottom)]],
                            dtype=np.int32)

        if len(self.image_array.shape) > 2:
            channel_count = self.image_array.shape[2]
            ignore_mask_color = (255,) * channel_count
        else:
            ignore_mask_color = 255
        cv2.fillPoly(mask, vertices, ignore_mask_color)
        self.image_array = cv2.bitwise_and(self.image_array, mask)

    def _hough_line(self):
        self.hough_lines = cv2.HoughLinesP(self.image_array,
                                           rho=1,
                                           theta=np.pi/180,
                                           threshold=25,
                                           minLineLength=20,
                                           maxLineGap=20,
                                           )
        if self.hough_lines is None:
            return

    def __draw_lines(self, lines, color):
        for line in lines:
            for x1, y1, x2, y2 in line:
                if x1 == x2 or y1 == y2:
                    continue
                cv2.line(self.original_image_array,
                         (x1, y1), (x2, y2), color, 2)

    def _get_lane_candidates(self):
        self.r_x_points, self.r_y_points = [], []
        self.l_x_points, self.l_y_points = [], []
        horizontal_clip_slope = 0.5
        for x1, y1, x2, y2 in self.hough_lines[:, 0]:
            m = self.__slope(x1, y1, x2, y2)
            if m >= horizontal_clip_slope:
                self.right.append([[x1, y1, x2, y2]])
                self.r_x_points.append(x1)
                self.r_x_points.append(x2)
                self.r_y_points.append(y1)
                self.r_y_points.append(y2)

            elif m < -horizontal_clip_slope:
                self.left.append([[x1, y1, x2, y2]])
                self.l_x_points.append(x1)
                self.l_x_points.append(x2)
                self.l_y_points.append(y1)
                self.l_y_points.append(y2)

        self.__draw_lines(self.right, RED)
        self.__draw_lines(self.left, BLUE)

    def __slope(self, x1, y1, x2, y2):
        if x1 != x2:
            return (y1 - y2) / (x1 - x2)
        else:
            return np.inf

    def _intersection_of_lanes(self):
        self.r_m, r_c = self.__one_line_linear_regression(
            self.r_x_points, self.r_y_points)
        self.l_m, l_c = self.__one_line_linear_regression(
            self.l_x_points, self.l_y_points)
        a = np.array([[-self.r_m, 1], [-self.l_m, 1]])
        b = np.array([r_c, l_c])
        try:
            self.intersection_point = np.linalg.solve(a, b)
            self.intersection_point = tuple(int(i)
                                            for i in self.intersection_point)
        except:
            print("exception")
            self.intersection_point = (self.image_array.shape[1]/2, 0)
        if self.intersection_point[0] > 0 and self.intersection_point[1] > 0:
            cv2.circle(img=self.original_image_array,
                       center=self.intersection_point,
                       radius=1,
                       color=GREEN,
                       thickness=3,
                       )

    def __one_line_linear_regression(self, x_points, y_points):
        A = np.vstack([x_points, np.ones(len(x_points))]).T
        m, c = np.linalg.lstsq(A, y_points, rcond=None)[0]
        return m, c

    def _get_angle_error(self):
        xsize = self.image_array.shape[1]
        ysize = self.image_array.shape[0]
        # < 0 when the car have to go left
        dist_to_baseline = self.intersection_point[0] - xsize/2
        dist_to_bottom = ysize - self.intersection_point[1]
        self.angle = np.arctan(dist_to_baseline / dist_to_bottom)
        cv2.line(self.original_image_array, (int(xsize/2), 0),
                 (int(xsize/2), ysize), RED, 1)

if __name__ == "__main__":
    image = cv2.imread("default_controller/sample0_7.png", cv2.IMREAD_COLOR)
    detector = LaneDetector()
    print(detector.detect_lane(image))
    # conda install -c conda-forge opencv=4.1.0
    cv2.imshow('processed', detector.image_array)
    cv2.imshow('original', detector.original_image_array)
    cv2.waitKey(0)

    image = cv2.imread("default_controller/sample0_29.png", cv2.IMREAD_COLOR)
    detector = LaneDetector()
    print(detector.detect_lane(image))
    # conda install -c conda-forge opencv=4.1.0
    cv2.imshow('processed', detector.image_array)
    cv2.imshow('original', detector.original_image_array)
    cv2.waitKey(0)


text:
import logging
import os
import datetime
import torchvision.models as models
import math
import torch
import yaml
from easydict import EasyDict
import shutil
import numpy as np
import torch.nn.functional as F
import gco

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def adjust_learning_rate(initial_lr, optimizer, epoch, n_repeats):
    """Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""
    lr = initial_lr * (0.1**(epoch // int(math.ceil(30. / n_repeats))))
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

def fgsm(gradz, step_size):
    return step_size * torch.sign(gradz)

def to_onehot(inp, num_classes, device='cuda'):
    y_onehot = torch.zeros((inp.size(0), num_classes), dtype=torch.float32, device=device)

    y_onehot.scatter_(1, inp.unsqueeze(1), value=1.)

    return y_onehot

def random_initialize(n_input, n_output, height, width):
    return np.random.randint(0, n_input, (n_output, width, height))

def accuracy(output, target, topk=(1, )):
    """Computes the accuracy over the k top predictions for the specified values of k"""
    with torch.no_grad():
        maxk = max(topk)
        batch_size = target.size(0)

        _, pred = output.topk(maxk, 1, True, True)
        pred = pred.t()
        correct = pred.eq(target.view(1, -1).expand_as(pred))

        res = []
        for k in topk:
            correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True)
            res.append(correct_k.mul_(100.0 / batch_size))
        return res

def initiate_logger(rank, output_path, evaluate):
    if not os.path.isdir(os.path.join('output', output_path)) and rank == 0:
        os.makedirs(os.path.join('output', output_path))
    else:
        import time
        while not os.path.isdir(os.path.join('output', output_path)):
            time.sleep(1)

    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger()
    logger.addHandler(
        logging.FileHandler(
            os.path.join('output', output_path, 'eval.txt' if evaluate else 'log.txt'), 'w'))
    logger.info(pad_str(' LOGISTICS '))
    logger.info('Experiment Date: {}'.format(datetime.datetime.now().strftime('%Y-%m-%d %H:%M')))
    logger.info('Output Name: {}'.format(output_path))
    logger.info('User: {}'.format(os.getenv('USER')))
    return logger

def get_model_names():
    return sorted(
        name for name in models.__dict__
        if name.islower() and not name.startswith("__") and callable(models.__dict__[name]))

def pad_str(msg, total_len=70):
    rem_len = total_len - len(msg)
    return '*'*int(rem_len/2) + msg + '*'*int(rem_len/2)\

def parse_config_file(args):
    with open(args.config) as f:
        config = EasyDict(yaml.load(f))

    # Add args parameters to the dict
    for k, v in vars(args).items():
        config[k] = v

    # Add the output path
    config.output_name = '{:s}'.format(args.output_prefix)
    return config

def save_checkpoint(state, is_best, filepath):
    filename = os.path.join(filepath, 'checkpoint.pth.tar')
    # Save model
    torch.save(state, filename)
    # Save best model
    if is_best:
        shutil.copyfile(filename, os.path.join(filepath, 'model_best.pth.tar'))

def distance(z, dist_type='l2'):
    with torch.no_grad():
        diff = z.unsqueeze(1) - z.unsqueeze(0)
        if dist_type[:2] == 'l2':
            A_dist = (diff**2).sum(-1)
            if dist_type == 'l2':
                A_dist = torch.sqrt(A_dist)
            elif dist_type == 'l22':
                pass
        elif dist_type == 'l1':
            A_dist = diff.abs().sum(-1)
        elif dist_type == 'linf':
            A_dist = diff.abs().max(-1)[0]
        else:
            return None
    return A_dist

def mix_input(mask_onehot, input_sp, target_reweighted):
    n_output, height, width, n_input = mask_onehot.shape
    _, n_class = target_reweighted.shape
    mask_onehot_im = F.interpolate(mask_onehot.permute(0, 3, 1, 2),
                                   size=input_sp.shape[-1],
                                   mode='nearest')
    output = torch.sum(mask_onehot_im.unsqueeze(2) * input_sp.unsqueeze(0), dim=1)

    mask_target = torch.matmul(mask_onehot, target_reweighted)
    target = mask_target.reshape(n_output, height * width, n_class).sum(-2) / height / width

    return output, target

def graphcut_multi(cost, beta=1, algorithm='swap', n_label=0, add_idx=None):
    height, width, n_input = cost.shape

    unary = np.ascontiguousarray(cost)

    if add_idx is not None:
        add_idx = add_idx.astype(np.bool)
    pairwise = (np.ones(shape=(n_input, n_input), dtype=np.float32) -
                np.eye(n_input, dtype=np.float32))
    if n_label == 2:
        pairwise[-1, :-1][add_idx] = 0.25
        pairwise[:-1, -1][add_idx] = 0.25
    elif n_label == 3:
        pairwise[-3:, :-3][:, add_idx] = np.array([[0.25, 0.25, 1], [0.25, 1, 0.25],
                                                   [1, 0.25, 0.25]])
        pairwise[:-3, -3:][add_idx, :] = np.array([[0.25, 0.25, 1], [0.25, 1, 0.25],
                                                   [1, 0.25, 0.25]])

    cost_v = beta * np.ones(shape=[height - 1, width], dtype=np.float32)
    cost_h = beta * np.ones(shape=[height, width - 1], dtype=np.float32)

    mask_idx = gco.cut_grid_graph(unary, pairwise, cost_v, cost_h, algorithm='swap')

    return mask_idx

def mixup_match(out, target_reweighted, param_list, sc=None, A_dist=None, device='cuda'):
    mixup_alpha = param_list['mixup_alpha']
    lam_dist = param_list['lam_dist']
    m_part = 16
    m_block_num = param_list['m_block_num']
    m_beta = param_list['m_beta']
    m_gamma = 1
    m_eta = 0.05
    m_thres = param_list['thres']
    m_thres_type = 'hard'
    set_resolve = param_list['set_resolve']
    m_niter = 3

    with torch.no_grad():
        n_input = out.shape[0]
        n_output = n_input
        width = out.shape[-1]

        if A_dist is None:
            A_dist = torch.eye(n_input, device=device)
        A_base = torch.eye(m_part, device=device)

        block_size = width // m_block_num
        sc = F.avg_pool2d(sc, block_size)
        sc_norm = sc / sc.view(n_input, -1).sum(1).view(n_input, 1, 1)
        cost_matrix = -sc_norm

        out_list = []
        target_part_list = []

        for i in range(n_output // m_part):
            A_dist_part = A_dist[i * m_part:(i + 1) * m_part, i * m_part:(i + 1) * m_part]
            A_dist_part = A_dist_part / torch.sum(A_dist_part) * m_part
            A = (1 - lam_dist) * A_base + lam_dist * A_dist_part

            cost_matrix_part = cost_matrix[i * m_part:(i + 1) * m_part]
            mask_onehot = get_onehot_matrix(cost_matrix_part,
                                            A,
                                            m_part,
                                            beta=m_beta,
                                            gamma=m_gamma,
                                            eta=m_eta,
                                            thres=m_thres,
                                            thres_type=m_thres_type,
                                            set_resolve=set_resolve,
                                            niter=m_niter,
                                            device=device)
            output_part, target_part = mix_input(mask_onehot, out[i * m_part:(i + 1) * m_part],
                                                 target_reweighted[i * m_part:(i + 1) * m_part])

            out_list.append(output_part)
            target_part_list.append(target_part)

        out = torch.cat(out_list, dim=0)
        target_reweighted = torch.cat(target_part_list, dim=0)

    return out, target_reweighted

def get_onehot_matrix(cost_matrix,
                      A,
                      n_output,
                      idx=None,
                      beta=1,
                      gamma=1,
                      eta=0.,
                      thres=0.,
                      thres_type='hard',
                      set_resolve=False,
                      niter=2,
                      print_log=False,
                      device='cuda'):
    n_input, height, width = cost_matrix.shape
    thres = thres * height * width
    beta = beta / height / width
    gamma = gamma / height / width
    eta = eta / height / width

    add_cost = None

    # prior
    lam = torch.ones(n_input, device=device)
    alpha = torch.distributions.dirichlet.Dirichlet(lam).sample().reshape(n_input, 1, 1)
    cost_matrix -= eta * torch.log(alpha + 1e-8)

    with torch.no_grad():
        # Init
        if idx is None:
            mask_idx = torch.tensor(random_initialize(n_input, n_output, height, width),
                                    device=device)
            mask_onehot = to_onehot(mask_idx.reshape(-1), n_input,
                                    device=device).reshape([n_output, height, width, n_input])

        loss_prev = obj_fn(cost_matrix, mask_onehot, beta, gamma)
        penalty = to_onehot(mask_idx.reshape(-1), n_input, device=device).sum(0).reshape(-1, 1, 1)

        # main loop
        for iter_idx in range(niter):
            for i in range(n_output):
                label_count = mask_onehot[i].reshape([height * width, n_input]).sum(0)
                penalty -= label_count.reshape(-1, 1, 1)
                if thres_type == 'hard':
                    modular_penalty = (2 * gamma * (
                        (A @ penalty.squeeze() > thres).float() * A @ penalty.squeeze())).reshape(
                            -1, 1, 1)
                elif thres_type == 'soft':
                    modular_penalty = (2 * gamma * ((A @ penalty.squeeze() > thres).float() *
                                                    (A @ penalty.squeeze() - thres))).reshape(
                                                        -1, 1, 1)
                else:
                    raise AssertionError("wrong threshold type!")

                if add_cost is not None:
                    cost_penalty = (cost_matrix + modular_penalty +
                                    gamma * add_cost[i].reshape(-1, 1, 1)).permute(1, 2, 0)
                else:
                    cost_penalty = (cost_matrix + modular_penalty).permute(1, 2, 0)
                mask_onehot[i] = graphcut_wrapper(cost_penalty, label_count, n_input, height, width,
                                                  beta, device, iter_idx)
                penalty += mask_onehot[i].reshape([height * width,
                                                   n_input]).sum(0).reshape(-1, 1, 1)

            if iter_idx == niter - 2 and set_resolve:
                assigned_label_total = (mask_onehot.reshape(n_output, -1, n_input).sum(1) >
                                        0).float()
                add_cost = resolve_label(assigned_label_total, device=device)

            loss = obj_fn(cost_matrix, mask_onehot, beta, gamma)
            if (loss_prev - loss).abs() / loss.abs() < 1e-6:
                break
            loss_prev = loss

    return mask_onehot

def resolve_label(assigned_label_total, device='cuda'):
    n_output, n_input = assigned_label_total.shape
    add_cost = torch.zeros_like(assigned_label_total)

    dist = torch.min(
        (assigned_label_total.unsqueeze(1) - assigned_label_total.unsqueeze(0)).abs().sum(-1),
        torch.tensor(1.0, device=device))
    coincide = torch.triu(1. - dist, diagonal=1)

    for i1, i2 in coincide.nonzero():
        nonzeros = assigned_label_total[i1].nonzero()
        if len(nonzeros) == 1:
            continue
        else:
            add_cost[i1][nonzeros[0]] = 1.
            add_cost[i2][nonzeros[1]] = 1.

    return add_cost

def graphcut_wrapper(cost_penalty, label_count, n_input, height, width, beta, device, iter_idx=0):
    '''Wrapper of graphcut_multi performing efficient extension to multi-label'''
    assigned_label = (label_count > 0)
    if iter_idx > 0:
        n_label = int(assigned_label.float().sum())
    else:
        n_label = 0

    if n_label == 2:
        cost_add = cost_penalty[:, :, assigned_label].mean(-1, keepdim=True) - 5e-4
        cost_penalty = torch.cat([cost_penalty, cost_add], dim=-1)
        unary = cost_penalty.cpu().numpy()

        mask_idx_np = graphcut_multi(unary,
                                     beta=beta,
                                     n_label=2,
                                     add_idx=assigned_label.cpu().numpy(),
                                     algorithm='swap')
        mask_idx_onehot = to_onehot(torch.tensor(mask_idx_np, device=device, dtype=torch.long),
                                    n_input + 1).reshape(height, width, n_input + 1)

        idx_matrix = torch.zeros([1, 1, n_input], device=device)
        idx_matrix[:, :, assigned_label] = 0.5
        mask_onehot_i = mask_idx_onehot[:, :, :n_input] + mask_idx_onehot[:, :,
                                                                          n_input:] * idx_matrix
    elif n_label >= 3:
        soft_label = torch.tensor([[0.5, 0.5, 0], [0.5, 0, 0.5], [0, 0.5, 0.5]], device=device)

        _, indices = torch.topk(label_count, k=3)
        assigned_label = torch.zeros_like(assigned_label)
        assigned_label[indices] = True

        cost_add = torch.matmul(cost_penalty[:, :, assigned_label], soft_label) - 5e-4
        cost_penalty = torch.cat([cost_penalty, cost_add], dim=-1)
        unary = cost_penalty.cpu().numpy()

        mask_idx_np = graphcut_multi(unary,
                                     beta=beta,
                                     n_label=3,
                                     add_idx=assigned_label.cpu().numpy(),
                                     algorithm='swap')
        mask_idx_onehot = to_onehot(torch.tensor(mask_idx_np, device=device, dtype=torch.long),
                                    n_input + 3).reshape(height, width, n_input + 3)

        idx_matrix = torch.zeros([3, n_input], device=device)
        idx_matrix[:, assigned_label] = soft_label
        mask_onehot_i = mask_idx_onehot[:, :, :n_input] + torch.matmul(
            mask_idx_onehot[:, :, n_input:], idx_matrix)
    else:
        unary = cost_penalty.cpu().numpy()
        mask_idx_np = graphcut_multi(unary, beta=beta, algorithm='swap')
        mask_onehot_i = to_onehot(torch.tensor(mask_idx_np, device=device, dtype=torch.long),
                                  n_input).reshape(height, width, n_input)

    return mask_onehot_i

def obj_fn(cost_matrix, mask_onehot, beta, gamma):
    n_output, height, width, n_input = mask_onehot.shape
    mask_idx_sum = mask_onehot.reshape(n_output, height * width, n_input).sum(1)

    loss = 0
    loss += torch.sum(cost_matrix.permute(1, 2, 0).unsqueeze(0) * mask_onehot)  # unary
    loss += beta / 2 * (
        (mask_onehot[:, :-1, :, :] - mask_onehot[:, 1:, :, :]).abs().sum() +
        (mask_onehot[:, :, :-1, :] - mask_onehot[:, :, 1:, :]).abs().sum())  # submodular
    loss += gamma * (torch.sum(mask_idx_sum.sum(0)**2) - torch.sum(mask_idx_sum**2))

    return loss

max_stars_repo_name: snu-mllab/Co-Mixup

max_issues_repo_name: snu-mllab/Co-Mixup

max_forks_repo_name: snu-mllab/Co-Mixup


text:
import random
import time
import tensorflow as tf
import numpy as np

from sklearn import preprocessing

def limit_loss(phs, prs, pts, nhs, nrs, nts, pos_margin, neg_margin, neg_param):
    print("limit_loss")
    with tf.name_scope('limit_loss_distance'):
        pos_distance = phs + prs - pts
        neg_distance = nhs + nrs - nts
    with tf.name_scope('limit_loss'):
        pos_score = tf.reduce_sum(tf.square(pos_distance), axis=1)
        neg_score = tf.reduce_sum(tf.square(neg_distance), axis=1)
        pos_loss = tf.reduce_sum(tf.maximum(pos_score - tf.constant(pos_margin), 0))
        neg_loss = tf.reduce_sum(tf.maximum(tf.constant(neg_margin) - neg_score, 0))
        loss = pos_loss + neg_param * neg_loss
    return loss

def xavier_init(mat_x, mat_y, name, is_l2=False):
    print("xavier_init")
    embeddings = tf.get_variable(name, shape=[mat_x, mat_y], initializer=tf.contrib.layers.xavier_initializer(uniform=False))
    if is_l2:
        return tf.nn.l2_normalize(embeddings, 1) if is_l2 else embeddings
    return embeddings

def embed_init(mat_x, mat_y, name, is_l2=False):
    print("embed_init")
    embeddings = tf.Variable(tf.truncated_normal([mat_x, mat_y]), name=name)
    if is_l2:
        return tf.nn.l2_normalize(embeddings, 1) if is_l2 else embeddings
    return embeddings

def random_unit_embeddings(dim1, dim2, name, is_l2=False):
    print("random_unit_embeddings")
    vectors = list()
    for i in range(dim1):
        vectors.append([random.gauss(0, 1) for j in range(dim2)])
    if is_l2:
        vectors = preprocessing.normalize(np.matrix(vectors))
    return tf.Variable(vectors, dtype=tf.float32, name=name)

def mul(tensor1, tensor2, session, num, sigmoid):
    t = time.time()
    if num < 20000:
        sim_mat = tf.matmul(tensor1, tensor2, transpose_b=True)
        if sigmoid:
            res = tf.sigmoid(sim_mat).eval(session=session)
        else:
            res = sim_mat.eval(session=session)
    else:
        res = np.matmul(tensor1.eval(session=session), tensor2.eval(session=session).T)
    print("mat mul costs: {:.3f}".format(time.time() - t))
    return res

def gelu(x):
    cdf = 0.5 * (1.0 + tf.tanh(
        (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3)))))
    return x * cdf

def get_activation(activation_string):
    act = activation_string.lower()
    if act == "linear":
        return None
    elif act == "relu":
        return tf.nn.relu
    elif act == "gelu":
        return gelu
    elif act == "tanh":
        return tf.tanh
    elif act == "sigmoid":
        return tf.sigmoid
    else:
        raise ValueError("Unsupported activation: %s" % act)

max_stars_repo_name: nju-websoft/TransEdge

max_issues_repo_name: nju-websoft/TransEdge

max_forks_repo_name: nju-websoft/TransEdge


text:
import numpy

class bernoulli():
    def pmf(x,p):
        """
        probabilty mass function (pmf)

        where x is a random variable and p is the probabilty
        of an event with a Bernoulli distribution
        """
        return (p**x)*((1-p)**(1-x))

    def mean(p):
        """
        mean (expected value E(x)) of a Bernoulli random variable x
        """
        return p

    def var(p):
        """
        variance of bernoulli random variable x
        """
        return p*(1-p)

    @classmethod
    def std(self, p):
        """
        standard deviation as the square root of variance
        """
        return self.var(p)**(1/2)

    def rvs(p,size):
        """
        generate random variates corresponding to a Bernoulli distribution

        size determines number of generated variates
        """
        rvs = numpy.array([])
        for i in range(0, size):
            if numpy.random.rand() <= p:
                a = 1
            else:
                a = 0
            rvs = numpy.append(rvs,a)
        return rvs

max_stars_repo_name: hossainsadman/bernoulli-distribution

max_issues_repo_name: hossainsadman/bernoulli-distribution

max_forks_repo_name: hossainsadman/bernoulli-distribution


text:
# Copyright 2018 Xanadu Quantum Technologies Inc.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" This scripts converts Tetris numpy images into a .png figure."""

import numpy as np
import matplotlib.pyplot as plt
import os

##################### set local directories ########
# Model name
model_string = 'tetris'

# Output folder
folder_locator = './outputs/'

# Locations of saved data and output figure
save_string = folder_locator + 'models/' + model_string + '/'

# Loading of images
images_out = np.load(save_string + 'images_out.npy')
images_out_big = np.load(save_string + 'images_out_big.npy')

num_labels = 7
plot_scale = 1

# Plotting of the final image.
fig_images, axs = plt.subplots(
    nrows=2, ncols=num_labels, figsize=(num_labels * plot_scale, 2 * plot_scale)
)

all_images = [images_out, images_out_big]
for i in range(2):
    for lable in range(num_labels):
        ax = axs[i][lable]
        ax.imshow(all_images[i][lable], cmap='gray')
        ax.axis('off')
        ax.set_xticklabels([])
        ax.set_yticklabels([])
plt.tight_layout()
fig_images.savefig(save_string + 'fig_images.png')

max_stars_repo_name: rickyHong/Quantum-neuralNet-repl

max_issues_repo_name: JairEscamilla/quantum-neural-networks

max_forks_repo_name: JairEscamilla/quantum-neural-networks


text:
# app4.py

import numpy as np
import pandas as pd
import altair as alt
import plotly.express as px
import plotly.graph_objs as go
import pickle as pkle
import os.path
import streamlit as st

def app():
    st.title('Writeup')

    st.write('We created several different visualizations of the data set from both a macro and micro lens by\
    first illustrated the general statistical distributions of the data scientist candidate population in terms\
    of city development index, training hour, experience, latest jobs, and education levels (in Analysis 1 Tab),\
    then drilling down to a gender and education level-specific breakdowns to see if there are any noticeable employment\
    trends among groups of different gender identities and academic achievements. The City Development Indices of the cities\
    that the candidates reside in are extremely left-skewed, with the median at 0.86 with an overwhelming majority of residents\
    residing in cities with well-established Infrastructure, Waste Management, Health, Education, and City Product, as defined by\
    the United Nations. The specific country/regional information wasn’t provided in the dataset, so the developers would refrain\
    from making potentially biased assumptions, but it’s interesting to note that there’s a spike at 0.62 with a moderately large\
    group of candidates residing in less developed cities. Our box plot distributions of CDI by Education Level show that Masters,\
    Graduates, and PhD’s are highly centered around cities with high CDI’s, while high school and primary school grads are scattered\
    towards lower CDI’s with slightly more outliers. We hypothesize that candidates in developing cities may have access to online/open-source\
    material that can supplement or replace formal training in DS, hence the supply in the job market.')

    st.write('60% of the dataset are graduate students, with 60% having Graduate Degrees, 22.8% with Undergraduate, 2.15% with PhDs\
    and 10.5% with high school degrees. The developers found this distribution quite jarringly different from the job market situation\
    in the U.S., where closely 50-60% of the data scientist job applicants hold Master’s degrees or higher, so we deemed it a factor highly\
    dependent on the region/continent, which is unknown. The years of experience by education level is as expected with PhD’s and Master’s\
    students having the upper bound of 20> years, followed by Undergraduate degree holders, then High School and Primary Schoolers. Since Data\
    Scientists used to be primarily PhD’s or academic scholars, it’s not surprising that those groups have more experiences than others.\
    The experience distribution by major was quite contrary to our hypothesis - that STEM degree holders will have more YoE on average - with \
    all disciplines having pretty much equivalent distributions.')

    st.write('We must note that our dataset’s intrinsically imbalanced in terms of the candidates’ experience and gender, with nearly ~40% of the\
    candidates having 20+ years of work experience as a Data Scientist. Another limitation that the dataset had was its ambiguity in certain\
    columns, including training hours, which the developers assumed as the # of hours formally trained as a data scientist apart from general\
    work experience. This information would have been more meaningful if it were the number of hours per week spent on training oneself as a\
    better Data Professional, but since it isn’t, the more relevant work experiences as a Data Scientist, the longer the training hours, hence\
    the apparent correlation between having relevant work experience and higher training hours.')

    st.write('Last New Job distribution was quite interesting, with 60% of the candidates only having worked at their current job for less than a year.\
    Given that DS’s are predominantly hired in tech companies or at tech functions, it’s not surprising that frequent job switches are common\
    and not necessarily frowned upon, compared to more traditional industries.')

    st.write('We include two gender-related distributions before deep-diving into it in Analysis 2, as the dataset has 1.5x more male than female candidates,\
    it was hard to recognize the data points that represent female data scientists in Distribution of Data Scientists by Experience and Last New Job.\
    In almost all cases, the number of male data scientists is much higher than the female data scientists, female data scientists points were covered.\
    This graph showed that the respondents who have worked for their previous company for 4 years tended to have longer working experience.\
    On the other hand, the ones who have shorter working experience have changed their positions or jobs more often.')

    st.write('To classify the data scientists by their gender and education level, we added two drop-down menus, so the users can easily select\
    the data that matches with a certain condition and only use it to create plots. Using these two options, we created three-bar plots\
    which show the distribution of data scientists by their enrolled university types, majors, and company types. The majority of the data\
    scientists in the given data set are not currently enrolled in university. However, most Ph.D.-level data scientists answered that they\
    are currently enrolled in university. Also, the proportion of data scientists who are currently attending university full-time was much\
    higher in the female data scientists group than in the male group.')

    st.write('In the major graph, as was expected, the majority of data scientists studied STEM majors regardless of gender,\
    and those who did not attend university are classified as ‘other’ in the major graph. The number of data scientists who studied Arts\
    during their undergrad was the lowest in this distribution graph.')

    st.write('Lastly, to find which type of companies hire data scientists the most, we drew a graph that shows the company type and size\
    that the respondents are currently working for. According to their answers, the majority works for small private firms regardless of gender.\
    However, when we selected only Ph.D.-level data scientists, the result was different. The proportion of respondents\
    who work for the public sector has increased.')