text:
"""Functions for identifying daytime"""
import numpy as np
import pandas as pd
from pvanalytics import util

def _rolling_by_minute(data, days, f):
    # apply `f` to a rolling window of length `days` at each minute of
    # the day.
    rolling = data.groupby(
        data.index.hour * 60 + data.index.minute
    ).rolling(
        min_periods=1,
        center=True,
        window=days
    )
    result = f(rolling).reset_index(0, drop=True)
    return result.sort_index()

def _run_lengths(series):
    # Count the number of equal values adjacent to each value.
    #
    # Examples
    # --------
    # >>> _run_lengths(pd.Series([True, True, True]))
    # 0    3
    # 1    3
    # 2    3
    #
    # >>> _run_lengths(
    # ...     pd.Series([True, False, False, True, True, False]
    # ... ))
    # 0    1
    # 1    2
    # 2    2
    # 3    2
    # 4    2
    # 5    1
    runs = (series != series.shift(1)).cumsum()
    return runs.groupby(runs).transform('count')

def _correct_if_invalid(series, invalid, correction_window):
    # For every time marked `invalid` replace the truth value in `series`
    # with the truth value at the same time in the majority of the
    # surrounding days. The number of surrounding days to examine is indicated
    # by `correction_window`.
    rolling_majority = _rolling_by_minute(
        series,
        days=correction_window,
        f=lambda x: x.sum() / x.count() > 0.5
    )
    return (~invalid & series) | (invalid & rolling_majority)

def _correct_midday_errors(night, minutes_per_value, hours_min,
                           correction_window):
    # identify periods of time that appear to switch from night to day
    # (or day to night) on too short a time scale to be reasonable.
    invalid = _run_lengths(night)*minutes_per_value <= hours_min*60
    return _correct_if_invalid(night, invalid, correction_window)

def _correct_edge_of_day_errors(night, minutes_per_value,
                                day_length_difference_max,
                                day_length_window, correction_window):
    # Identify day-time periods that are "too short" and replace
    # values with the majority truth-value for the same time in the
    # surrounding days.
    #
    # Because daylight savings shifts happen at night we cannot look
    # at night-length directly. Instead we look for too-short days and
    # flag the full day for correction. This may result in slightly
    # reduced accuracy for sunrise/sunset times on these days (even if
    # the day/night boundary at one end of the day - sunrise or sunset
    # - was correctly marked, it will be replaced with the rolling
    # median for that minute).
    day_length = night.groupby(night.cumsum()).transform(
        lambda x: len(x) * minutes_per_value
    )
    # remove night time values so they don't interfere with the median
    # day length.
    day_length.loc[night] = np.nan
    day_length_median = day_length.rolling(
        window=str(day_length_window) + 'D'
    ).median()
    # flag days that are more than 30 minutes shorter than the median
    short_days = day_length < (day_length_median - day_length_difference_max)
    invalid = short_days.groupby(short_days.index.date).transform(
        lambda day: any(day)
    )
    return _correct_if_invalid(night, invalid, correction_window)

def _filter_and_normalize(series, outliers):
    # filter a series by removing outliers and clamping the minimum to
    # 0. Then normalize the series by the maximum deviation.
    if outliers is not None:
        series.loc[outliers] = np.nan
    series.loc[series < 0] = 0
    return (series - series.min()) / (series.max() - series.min())

def _freqstr_to_minutes(freqstr):
    return util.freq_to_timedelta(freqstr).seconds / 60

def power_or_irradiance(series, outliers=None,
                        low_value_threshold=0.003,
                        low_median_threshold=0.0015,
                        low_diff_threshold=0.0005, median_days=7,
                        clipping=None, freq=None,
                        correction_window=31, hours_min=5,
                        day_length_difference_max=30,
                        day_length_window=14):
    """Return True for values that are during the day.

    After removing outliers and normalizing the data, a time is
    classified as night when two of the following three criteria are
    satisfied:

    - near-zero value
    - near-zero first-order derivative
    - near-zero rolling median at the same time over the surrounding
      week (see `median_days`)

    Mid-day times where power goes near zero or
    stops changing may be incorrectly classified as night. To correct
    these errors, night or day periods with duration that is too long or
    too short are identified, and times in these periods are re-classified
    to have the majority value at the same time on preceding and
    following days (as set by `correction_window`).

    Finally any values that are True in `clipping` are marked as day.

    Parameters
    ----------
    series : Series
        Time series of power or irradiance.
    outliers : Series, optional
        Boolean time series with True for values in `series` that are
        outliers.
    low_value_threshold : float, default 0.003
        Maximum normalized power or irradiance value for a time to be
        considered night.
    low_median_threshold : float, default 0.0015
        Maximum rolling median of power or irradiance for a time to be
        considered night.
    low_diff_threshold : float, default 0.0005
        Maximum derivative of normalized power or irradiance for a time
        to be considered night.
    median_days : int, default 7
        Number of days to use to calculate the rolling median at each
        minute. [days]
    clipping : Series, optional
        True when clipping indicated. Any values where clipping is
        indicated are automatically considered 'daytime'.
    freq : str, optional
        A pandas freqstr specifying the expected timestamp spacing for
        the series. If None, the frequency will be inferred from the index.
    correction_window : int, default 31
        Number of adjacent days to examine when correcting
        day/night classification errors. [days]
    hours_min : float, default 5
        Minimum number of hours in a contiguous period of day or
        night. A day/night period shorter than `hours_min` is
        flagged for error correction. [hours]
    day_length_difference_max : float, default 30
        Days with length that is `day_length_difference_max` minutes less
        than the median length of surrounding days are flagged for
        corrections.
    day_length_window : int, default 14
        The length of the rolling window used for calculating the
        median length of the day when correcting errors in the morning
        or afternoon. [days]

    Returns
    -------
    Series
        Boolean time series with True for times that are during the
        day.

    Notes
    -----

    ``NA`` values are treated like zeros.

    Derived from the PVFleets QA Analysis project.

    """
    series = series.fillna(value=0)
    series_norm = _filter_and_normalize(series, outliers).fillna(value=0)
    minutes_per_value = _freqstr_to_minutes(
        freq or pd.infer_freq(series.index)
    )
    first_order_diff = series_norm.diff() / minutes_per_value
    rolling_median = _rolling_by_minute(
        series_norm,
        days=median_days,
        f=pd.core.window.RollingGroupby.median
    )

    # Night-time if two of the following are satisfied:
    # - Near-zero value
    # - Near-zero first-order derivative
    # - Near-zero rolling median
    low_value = series_norm <= low_value_threshold
    low_median = rolling_median <= low_median_threshold
    low_diff = abs(first_order_diff) <= low_diff_threshold

    night = ((low_value & low_diff)
             | (low_value & low_median)
             | (low_diff & low_median))
    # Fix erroneous classifications (e.g. midday outages where power
    # goes to 0 and stays there for several hours, clipping classified
    # as night, and night-time periods that are too long)
    night_corrected_midday = _correct_midday_errors(
        night, minutes_per_value, hours_min, correction_window
    )
    night_corrected_clipping = ~((clipping or False)
                                 | (~night_corrected_midday))
    night_corrected_edges = _correct_edge_of_day_errors(
        night_corrected_clipping,
        minutes_per_value,
        day_length_difference_max,
        day_length_window,
        correction_window
    )
    return ~night_corrected_edges

max_stars_repo_name: kanderso-nrel/pvanalytics

max_issues_repo_name: kanderso-nrel/pvanalytics

max_forks_repo_name: kanderso-nrel/pvanalytics


text:
import glob
import cv2
import numpy as np
import time
import os.path
from sklearn import svm
from sklearn.preprocessing import StandardScaler
from skimage.feature import hog
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.model_selection import GridSearchCV
from sklearn.externals import joblib

def svm_classifier(accuracy=False):
    if os.path.isfile('svm.pkl'):
        clf = joblib.load('svm.pkl')
    else:
        vehicles = []
        non_vehicles = []
        non_v_files = glob.glob('../data/non-vehicles/*/*.png')
        v_files = glob.glob('../data/vehicles/*/*.png')

        print("Num vehicles: ", len(v_files))
        print("Num non-vehicles: ", len(non_v_files))
        t0=time.time()

        # X
        non_v_features = extract_features(non_v_files, cspace='YUV', orient=9, pix_per_cell=8, cell_per_block=2, hog_channel='ALL')
        v_features = extract_features(v_files, cspace='YUV', orient=9, pix_per_cell=8, cell_per_block=2, hog_channel='ALL')
        print(round(time.time()-t0, 2), 'Seconds to extract features...')
        X = np.vstack((v_features, non_v_features)).astype(np.float64)
        X_scaler = StandardScaler().fit(X)
        scaled_X = X_scaler.transform(X)
        np.savez("scaler.npz", mean=X_scaler.mean_, scale=X_scaler.scale_)

        # y
        y = np.hstack((np.ones(len(v_features)), np.zeros(len(non_v_features))))

        rand_state = np.random.randint(0, 100)
        X_train, X_test, y_train, y_test = train_test_split(scaled_X, y, test_size=0.2, random_state=rand_state)
        print(X_train.shape, y_train.shape)

        # SVM
        # C_range = np.logspace(-2, 10, 13)
        # gamma_range = np.logspace(-9, 3, 13)
        # param_grid = dict(gamma=gamma_range, C=C_range)
        # cv = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)
        # grid = GridSearchCV(svm.SVC(), param_grid=param_grid, cv=cv)
        # grid.fit(X_train[0:500], y_train[0:500])
        # print("The best parameters are %s with a score of %0.2f" % (grid.best_params_, grid.best_score_))

        print('Start training SVM')
        t0=time.time()
        clf = svm.LinearSVC(random_state=rand_state)
        # clf = svm.SVC() # RBF
        clf.fit(X_train, y_train)
        print(round(time.time()-t0, 2), 'Seconds to train SVC...')

        if accuracy:
            print('Test Accuracy of SVC = ', round(clf.score(X_test, y_test), 4))

        joblib.dump(clf, 'svm.pkl')
    return clf

def get_hog_features(img, orient, pix_per_cell, cell_per_block, vis=False, feature_vec=True):
    if vis == True:
        features, hog_image = hog(img, orientations=orient, pixels_per_cell=(pix_per_cell, pix_per_cell),
                                  cells_per_block=(cell_per_block, cell_per_block), transform_sqrt=False,
                                  visualise=True, feature_vector=False)
        return features, hog_image
    else:
        features = hog(img, orientations=orient, pixels_per_cell=(pix_per_cell, pix_per_cell),
                       cells_per_block=(cell_per_block, cell_per_block), transform_sqrt=False,
                       visualise=False, feature_vector=feature_vec)
        return features

def bin_spatial(img, size=(32, 32)):
    feature_image = np.copy(img)
    # Use cv2.resize().ravel() to create the feature vector
    features = cv2.resize(feature_image, size).ravel()
    # Return the feature vector
    return features

def color_hist(img, nbins=32, bins_range=(0, 256)):
    feature_image = np.copy(img)
    # Compute the histogram of the color channels separately
    channel1_hist = np.histogram(feature_image[:,:,0], bins=nbins, range=bins_range)
    channel2_hist = np.histogram(feature_image[:,:,1], bins=nbins, range=bins_range)
    channel3_hist = np.histogram(feature_image[:,:,2], bins=nbins, range=bins_range)
    # Concatenate the histograms into a single feature vector
    hist_features = np.concatenate((channel1_hist[0], channel2_hist[0], channel3_hist[0]))
    # Return the individual histograms, bin_centers and feature vector
    return hist_features

def transform_features(img, cspace='BGR', orient=9, pix_per_cell=8, cell_per_block=2, hog_channel=0, spatial_size=(16, 16)):
    features = []

    file_features = []
    if cspace != 'BGR':
        if cspace == 'HSV':
            feature_image = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
        elif cspace == 'LUV':
            feature_image = cv2.cvtColor(img, cv2.COLOR_BGR2LUV)
        elif cspace == 'HLS':
            feature_image = cv2.cvtColor(img, cv2.COLOR_BGR2HLS)
        elif cspace == 'YUV':
            feature_image = cv2.cvtColor(img, cv2.COLOR_BGR2YUV)
        elif cspace == 'YCrCb':
            feature_image = cv2.cvtColor(img, cv2.COLOR_BGR2YCrCb)
    else: feature_image = np.copy(img)

    if hog_channel == 'ALL':
        hog_features = []
        for channel in range(feature_image.shape[2]):
            hog_features.append(get_hog_features(feature_image[:,:,channel],
                                orient, pix_per_cell, cell_per_block,
                                vis=False, feature_vec=True))
        hog_features = np.ravel(hog_features)
    else:
        hog_features = get_hog_features(feature_image[:,:,hog_channel], orient,
                    pix_per_cell, cell_per_block, vis=False, feature_vec=True)

    file_features.append(hog_features)

    spatial_features = bin_spatial(feature_image, size=spatial_size)
    file_features.append(spatial_features)

    # color_features = color_hist(feature_image)
    # file_features.append(color_features)

    features.append(np.concatenate(file_features))

    return np.array(features)

def extract_features(imgs, cspace='BGR', orient=9, pix_per_cell=8, cell_per_block=2, hog_channel=0, spatial_size=(16, 16)):
    # Create a list to append feature vectors to
    features = []
    # Iterate through the list of images
    for file in imgs:
        file_features = []
        # Read in each one by one
        image = cv2.imread(file)
        if cspace != 'BGR':
            if cspace == 'HSV':
                feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
            elif cspace == 'LUV':
                feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2LUV)
            elif cspace == 'HLS':
                feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2HLS)
            elif cspace == 'YUV':
                feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2YUV)
            elif cspace == 'YCrCb':
                feature_image = cv2.cvtColor(image, cv2.COLOR_BGR2YCrCb)
        else: feature_image = np.copy(image)

        # Call get_hog_features() with vis=False, feature_vec=True
        if hog_channel == 'ALL':
            hog_features = []
            for channel in range(feature_image.shape[2]):
                hog_features.append(get_hog_features(feature_image[:,:,channel],
                                    orient, pix_per_cell, cell_per_block,
                                    vis=False, feature_vec=True))
            hog_features = np.ravel(hog_features)
        else:
            hog_features = get_hog_features(feature_image[:,:,hog_channel], orient,
                        pix_per_cell, cell_per_block, vis=False, feature_vec=True)
        # Append the new feature vector to the features list
        file_features.append(hog_features)

        spatial_features = bin_spatial(feature_image, size=spatial_size)
        file_features.append(spatial_features)

        # color_features = color_hist(feature_image)
        # file_features.append(color_features)

        features.append(np.concatenate(file_features))

    # Return list of feature vectors
    return np.array(features)

if __name__ == "__main__":
    svm_classifier(accuracy=True)

ext: py

max_stars_repo_name: shernshiou/CarND

max_issues_repo_name: shernshiou/CarND

max_forks_repo_name: shernshiou/CarND

reason: import numpy


text:
# Bite Size Bayes

Copyright 2020 Allen B. Downey

License: [Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0)](https://creativecommons.org/licenses/by-nc-sa/4.0/)

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

## The "Girl Named Florida" problem

In [The Drunkard's Walk](https://www.goodreads.com/book/show/2272880.The_Drunkard_s_Walk), Leonard Mlodinow presents "The Girl Named Florida Problem":

>"In a family with two children, what are the chances, if [at least] one of the children is a girl named Florida, that both children are girls?"

I added "at least" to Mlodinow's statement of the problem to avoid a subtle ambiguity (which I'll explain at the end).

To avoid some real-world complications, let's assume that this question takes place in an imaginary city called Statesville where:

* Every family has two children.

* 50% of children are male and 50% are female.

* All children are named after U.S. states, and all state names are chosen with equal probability.

* Genders and names within each family are chosen independently.

To answer Mlodinow's question, I'll create a DataFrame with one row for each family in Statesville and a column for the gender and name of each child.

Here's a list of genders and a [dictionary of state names](https://gist.github.com/tlancon/9794920a0c3a9990279de704f936050c):

gender = ['B', 'G']

us_states = {
    'Alabama': 'AL',
    'Alaska': 'AK',
    'Arizona': 'AZ',
    'Arkansas': 'AR',
    'California': 'CA',
    'Colorado': 'CO',
    'Connecticut': 'CT',
    'Delaware': 'DE',
#    'District of Columbia': 'DC',
    'Florida': 'FL',
    'Georgia': 'GA',
    'Hawaii': 'HI',
    'Idaho': 'ID',
    'Illinois': 'IL',
    'Indiana': 'IN',
    'Iowa': 'IA',
    'Kansas': 'KS',
    'Kentucky': 'KY',
    'Louisiana': 'LA',
    'Maine': 'ME',
    'Maryland': 'MD',
    'Massachusetts': 'MA',
    'Michigan': 'MI',
    'Minnesota': 'MN',
    'Mississippi': 'MS',
    'Missouri': 'MO',
    'Montana': 'MT',
    'Nebraska': 'NE',
    'Nevada': 'NV',
    'New Hampshire': 'NH',
    'New Jersey': 'NJ',
    'New Mexico': 'NM',
    'New York': 'NY',
    'North Carolina': 'NC',
    'North Dakota': 'ND',
    'Ohio': 'OH',
    'Oklahoma': 'OK',
    'Oregon': 'OR',
    'Pennsylvania': 'PA',
    'Rhode Island': 'RI',
    'South Carolina': 'SC',
    'South Dakota': 'SD',
    'Tennessee': 'TN',
    'Texas': 'TX',
    'Utah': 'UT',
    'Vermont': 'VT',
    'Virginia': 'VA',
    'Washington': 'WA',
    'West Virginia': 'WV',
    'Wisconsin': 'WI',
    'Wyoming': 'WY'
}

To enumerate all possible combinations of genders and names, I'll use `from_product`, which makes a Pandas MultiIndex.

names = ['gender1', 'name1', 'gender2', 'name2']

index = pd.MultiIndex.from_product([gender, us_states]*2,
                                   names=names)

Now I'll create a DataFrame with that index:

df = pd.DataFrame(index=index)
df.head()

It will be easier to work with if I reindex it so the levels in the MultiIndex become columns.

df = df.reset_index()
df.head()

This DataFrame contains one row for each family in Statesville; for example, the first row represents a family with two boys, both named Alabama.

As it turns out, there are 10,000 families in Statesville:

len(df)

## Probabilities

To compute probabilities, we'll use Boolean Series.  For example, the following Series is `True` for each family where the first child is a girl:

girl1 = (df['gender1']=='G')

The following function takes a Boolean Series and counts the number of `True` values, which is the probability that the condition is true.

def prob(A):
    """Computes the probability of a proposition, A.

    A: Boolean series

    returns: probability
    """
    assert isinstance(A, pd.Series)
    assert A.dtype == 'bool'

    return A.mean()

Not surprisingly, the probability is 50% that the first child is a girl.

prob(girl1)

And so is the probability that the second child is a girl.

girl2 = (df['gender2']=='G')
prob(girl2)

Mlodinow's question is a conditional probability: given that one of the children is a girl named Florida, what is the probability that both children are girls?

To compute conditional probabilities, I'll use this function, which takes two Boolean Series, `A` and `B`, and computes the conditional probability $P(A~\mathrm{given}~B)$.

def conditional(A, B):
    """Conditional probability of A given B.

    A: Boolean series
    B: Boolean series

    returns: probability
    """
    return prob(A[B])

For example, here's the probability that the second child is a girl, given that the first child is a girl.

conditional(girl2, girl1)

The result is 50%, which is the same as the unconditioned probability that the second child is a girl:

prob(girl2)

So that confirms that the genders of the two children are independent, which is one of my assumptions.

Now, Mlodinow's question asks about the probability that both children are girls, so let's compute that.

gg = (girl1 & girl2)
prob(gg)

In 25% of families, both children are girls.  And that should be no surprise: because they are independent, the probability of the conjunction is the product of the probabilities:

prob(girl1) * prob(girl2)

While we're at it, we can also compute the conditional probability of two girls, given that the first child is a girl.

conditional(gg, girl1)

That's what we should expect.  If we know the first child is a girl, and the probability is 50% that the second child is a girl, the probability of two girls is 50%.

## At least one girl

Before I answer Mlodinow's question, I'll warm up with a simpler version: given that at least one of the children is a girl, what is the probability that both are?

To compute the probability of "at least one girl" I will use the `|` operator, which computes the logical `OR` of the two Series:

at_least_one_girl = (girl1 | girl2)
prob(at_least_one_girl)

75% of the families in Statesville have at least one girl.

Now we can compute the conditional probability of two girls, given that the family has at least one girl.

conditional(gg, at_least_one_girl)

Of the families that have at least one girl, `1/3` have two girls.

If you have not thought about questions like this before, that result might surprise you.  The following figure might help:

In the top left, the gray square represents a family with two boys; in the lower right, the dark blue square represents a family with two girls.

The other two quadrants represent families with one girl, but note that there are two ways that can happen: the first child can be a girl or the second child can be a girl.

There are an equal number of families in each quadrant.

If we select families with at least one girl, we eliminate the gray square in the upper left.  Of the remaining three squares, one of them has two girls.

So if we know a family has at least one girl, the probability they have two girls is 33%.

## What's in a name?

So far, we have computed two conditional probabilities:

* Given that the first child is a girl, the probability is 50% that both children are girls.

* Given that at least one child is a girl, the probability is 33% that both children are girls.

Now we're ready to answer Mlodinow's question:

* Given that at least one child is a girl *named Florida*, what is the probability that both children are girls?

If your intuition is telling you that the name of the child can't possibly matter, brace yourself.

Here's the probability that the first child is a girl named Florida.

gf1 = girl1 & (df['name1']=='Florida')
prob(gf1)

And the probability that the second child is a girl named Florida.

gf2 = girl2 & (df['name2']=='Florida')
prob(gf2)

To compute the probability that at least one of the children is a girl named Florida, we can use the `|` operator again.

at_least_one_girl_named_florida = (gf1 | gf2)
prob(at_least_one_girl_named_florida)

We can double-check it by using the disjunction rule:

prob(gf1) + prob(gf2) - prob(gf1 & gf2)

So, the percentage of families with at least one girl named Florida is a little less than 2%.

Now, finally, here is the answer to Mlodinow's question:

conditional(gg, at_least_one_girl_named_florida)

That's right, the answer is about 49.7%.  To summarize:

* Given that the first child is a girl, the probability is 50% that both children are girls.

* Given that at least one child is a girl, the probability is 33% that both children are girls.

* Given that at least one child is a girl *named Florida*, the probability is 49.7% that both children are girls.

If your brain just exploded, I'm sorry.

Here's my best attempt to put your brain back together.

For each child, there are three possibilities: boy (B), girl not named Florida (G), and girl named Florida (GF), with these probabilities:

$P(B) = 1/2 $

$P(G) = 1/2 - x $

$P(GF) = x $

where $x$ is the percentage of people who are girls named Florida.

In families with two children, here are the possible combinations and their probabilities:

$P(B, B) = (1/2)(1/2)$

$P(B, G) = (1/2)(1/2-x)$

$P(B, GF) = (1/2)(x)$

$P(G, B) = (1/2-x)(1/2)$

$P(G, G) = (1/2-x)(1/2-x)$

$P(G, GF) = (1/2-x)(x)$

$P(GF, B) = (x)(1/2)$

$P(GF, G) = (x)(1/2-x)$

$P(GF, GF) = (x)(x)$

If we select only the families that have at least one girl named Florida, here are their probabilities:

$P(B, GF) = (1/2)(x)$

$P(G, GF) = (1/2-x)(x)$

$P(GF, B) = (x)(1/2)$

$P(GF, G) = (x)(1/2-x)$

$P(GF, GF) = (x)(x)$

Of those, if we select the families with two girls, here are their probabilities:

$P(G, GF) = (1/2-x)(x)$

$P(GF, G) = (x)(1/2-x)$

$P(GF, GF) = (x)(x)$

To get the conditional probability of two girls, given at least one girl named Florida, we can add up the last 3 probabilities and divide by the sum of the previous 5 probabilities.

With a little algebra, we get:

$P(\mathrm{two~girls} ~|~ \mathrm{at~least~one~girl~named~Florida}) = (1 - x) / (2 - x)$

As $x$ approaches $0$ the answer approaches $1/2$.

As $x$ approaches $1/2$, the answer approaches $1/3$.

Here's what all of that looks like graphically:

Here `B` a boy, `Gx` is a girl with some property `X`, and `G` is a girl who doesn't have that property.  If we select all families with at least one `Gx`, we get the five blue squares (light and dark).  Of those, the families with two girls are the three dark blue squares.

If property `X` is common, the ratio of dark blue to all blue approaches `1/3`.  If `X` is rare, the same ratio approaches `1/2`.

In the "Girl Named Florida" problem, `x` is 1/100, and we can compute the result:

x = 1/100
(1-x) / (2-x)

Which is what we got by counting all of the families in Statesville.

## Controversy

[I wrote about this problem in my blog in 2011](http://allendowney.blogspot.com/2011/11/girl-named-florida-solutions.html).  As you can see in the comments, my explanation was not met with universal acclaim.

One of the issues that came up is the challenge of stating the question unambiguously.  In this article, I rephrased Mlodinow's statement to clarify it.

But since we have come all this way, let me also answer a different version of the problem.

>Suppose you choose a house in Statesville at random and ring the doorbell.  A girl (who lives there) opens the door and you learn that her name is Florida.  What is the probability that the other child in this house is a girl?

In this version of the problem, the selection process is different.  Instead of selecting houses with at least one girl named Florida, you selected a house, then selected a child, and learned that her name is Florida.

Since the selection of the child was arbitrary, we can say without loss of generality that the child you met is the first child in the table.

In that case, the conditional probability of two girls is:

conditional(gg, gf1)

Which is the same as the conditional probability, given that the first child is a girl:

conditional(gg, girl1)

So in this version of the problem, the girl's name is irrelevant.

max_stars_repo_name: myamullaciencia/Bayesian-statistics

max_issues_repo_name: myamullaciencia/Bayesian-statistics

max_forks_repo_name: myamullaciencia/Bayesian-statistics


text:
import torch
import torch.nn as nn
from torch.optim import LBFGS

import hamiltorch
import numpy as np
import matplotlib.pyplot as plt
import pickle
from tqdm.auto import tqdm, trange

# from multiprocessing import Pool
from core import Inference

def init_random_inputs_w_bounds(np_lb, np_ub, N=1, torch_tensor=True, seed=None, device=torch.device('cpu')):
    dim = np_lb.size
    scale = (np_ub - np_lb).reshape([1,-1])

    rand_state = np.random.get_state()
    try:
        if seed is None:
            from datetime import datetime
            seed=int(datetime.utcnow().timestamp()*10000%(2**32))
        #
        np.random.seed(seed)
        uniform_noise = np.random.uniform(0,1,size=[N,dim])
    except:
        print('Errors occured when generating random noise...')
    finally:
        np.random.set_state(rand_state)

    X_init = uniform_noise*scale + np_lb

    if torch_tensor:
        X_init = torch.tensor(X_init, requires_grad=True, dtype=torch.float).to(device)

    return X_init

class HamilBayesOpt:
    def __init__(self, model, dataset, sampling):
        self.model = model
        self.dataset = dataset
        self.sampling = sampling
        self.device = model.device

        print('Initialize the HMC inference module...')
        self.inference = Inference.MFHMC(self.model, self.sampling, self.device)

    def fit(self, constraint):
        print('Evaluating the posterior samples given training data...')
        nXtr_list, nYtr_list = self.dataset.get_data(train=True, norm=True, torch_tensor=True, device=self.device)
        hmc_samples = self.inference.posterior(nXtr_list, nYtr_list, constraint)

        return hmc_samples

    def predict(self, hmc_samples):
        nXte_list, nYte_list = self.dataset.get_data(train=False, norm=True, torch_tensor=True, device=self.device)
        mf_nPred_list = self.inference.predict(nXte_list, hmc_samples)
        return mf_nPred_list

    def _functional_forward_(self, m, sample):

        params = self.inference._unflatten_net_params(sample, flat_tau=self.inference.flat_tau)

        def func_forward(nX):
            Ym = self.model.forward_by_params(nX, m, params['mf_weights'], params['mf_biases'])
            return Ym

        return func_forward

    def _eval_sample_f_star_(self, m, sample):
        n_np_lb, n_np_ub = self.dataset.get_scaled_bounds(m)
        #print(n_np_lb, n_np_ub)
        bounds = torch.tensor(np.vstack((n_np_lb, n_np_ub))).to(self.device)

        nXq = init_random_inputs_w_bounds(n_np_lb, n_np_ub, device=self.device)

        optimizer_f_star = LBFGS(
            [nXq], lr=1e-1, max_iter=20, max_eval=None,
            tolerance_grad=1e-8, tolerance_change=1e-12, history_size=100)

        objective = self._functional_forward_(m, sample)

        def closure():
            optimizer_f_star.zero_grad()

            loss = -torch.squeeze(objective(nXq))
            loss.backward(retain_graph=True)

            with torch.no_grad():
                for j, (lb, ub) in enumerate(zip(*bounds)):
                    nXq.data[..., j].clamp_(lb, ub) # need to do this on the data not X itself
                #
            #
            return loss
        #

        optimizer_f_star.step(closure)

        for j, (lb, ub) in enumerate(zip(*bounds)):
            nXq.data[..., j].clamp_(lb, ub) # need to do this on the data not X itself
        #

        nfstar = objective(nXq)

        return nfstar, nXq

    def fidelity_samples_extremes(self, m, samples):
        restart = 5

        samples_fstar = []
        samples_Xq = []

        for s in samples:

            fstar_buffer = []
            Xq_buffer = []

            for r in range(restart):
                nfstar, nXq = self._eval_sample_f_star_(m, s)
                fstar_buffer.append(nfstar)
                Xq_buffer.append(nXq)
            #

            # argmax the restart trials
            argi = torch.argmax(torch.tensor(fstar_buffer))

            samples_fstar.append(fstar_buffer[argi])
            samples_Xq.append(Xq_buffer[argi])
        #

        samples_nfstar = torch.stack(samples_fstar).squeeze()
        samples_nXq = torch.stack(samples_Xq)

        return samples_nfstar, samples_nXq

    def _functional_info_gain_(self, m, samples, Fstar):

        def info_gain_function(nX):

            pred_list = []
            for s in samples:
                params = self.inference._unflatten_net_params(s, flat_tau=self.inference.flat_tau)
                pred_s = self.model.forward_by_params(nX, m, params['mf_weights'], params['mf_biases'])
                pred_list.append(pred_s)
            #

            Ns = len(samples)
            pred_samples = torch.stack(pred_list).squeeze()

            H_pred_samples = 0.5*torch.log(2*np.pi*np.e*torch.square(torch.std(pred_samples)))
            H_Fstar = 0.5*torch.log(2*np.pi*np.e*torch.square(torch.std(Fstar)))

            joint = torch.cat([Fstar, pred_samples])
            H_joint = 0.5*torch.log(2*np.pi*np.e*torch.square(torch.std(joint)))

            gain = H_pred_samples + H_Fstar - H_joint

            return gain
        #

        return info_gain_function

    def _batch_functional_info_gain_(self, m, samples, Fstar, batch_queried):

        def info_gain_function(nX):

            pred_list = []
            for s in samples:
                params = self.inference._unflatten_net_params(s, flat_tau=self.inference.flat_tau)
                pred_s = self.model.forward_by_params(nX, m, params['mf_weights'], params['mf_biases'])
                pred_list.append(pred_s)
            #

            Ns = len(samples)
            pred_samples = torch.stack(pred_list).squeeze()

#             prev_batch = torch.stack(batch_queried).flatten()
            prev_batch = torch.stack(batch_queried).flatten().repeat(Ns)
            #print(prev_batch.shape)

            curr_batch = torch.cat([pred_samples, prev_batch])

            H_curr_batch = 0.5*torch.log(2*np.pi*np.e*torch.square(torch.std(curr_batch)))
            H_Fstar = 0.5*torch.log(2*np.pi*np.e*torch.square(torch.std(Fstar)))

            joint = torch.cat([Fstar, curr_batch])
            H_joint = 0.5*torch.log(2*np.pi*np.e*torch.square(torch.std(joint)))

            gain = H_curr_batch + H_Fstar - H_joint

            return gain

        return info_gain_function

    def _eval_info_gain_(self, m, samples, Fstar, batch_queried=[]):
        n_np_lb, n_np_ub = self.dataset.get_scaled_bounds(m)
        bounds = torch.tensor(np.vstack((n_np_lb, n_np_ub))).to(self.device)

        nXq = init_random_inputs_w_bounds(n_np_lb, n_np_ub, device=self.device)

        optimizer_f_star = LBFGS(
            [nXq], lr=1e-1, max_iter=20, max_eval=None,
            tolerance_grad=1e-8, tolerance_change=1e-12, history_size=100)

        if not batch_queried:
            #print('no batch found, SIGNLE acquisition function used.')
            objective = self._functional_info_gain_(m, samples, Fstar)
        else:
            #print('found previous batch, BATCH acquisition function used.')
            objective = self._batch_functional_info_gain_(m, samples, Fstar, batch_queried)
        #

        def closure():
            optimizer_f_star.zero_grad()

            loss = -torch.squeeze(objective(nXq))
            loss.backward(retain_graph=True)

            with torch.no_grad():
                for j, (lb, ub) in enumerate(zip(*bounds)):
                    nXq.data[..., j].clamp_(lb, ub) # need to do this on the data not X itself
                #
            #
            return loss
        #

        optimizer_f_star.step(closure)

        for j, (lb, ub) in enumerate(zip(*bounds)):
            nXq.data[..., j].clamp_(lb, ub) # need to do this on the data not X itself
        #

        eval_acq = objective(nXq)

        return eval_acq, nXq

    def fidelity_info_gain_by_samples(self, m, samples, Fstar, batch_queried=[]):

        restart = 5

        acq_buffer = []
        Xq_buffer = []

        for r in range(restart):
            eval_acq, nXq = self._eval_info_gain_(m, samples, Fstar, batch_queried)
            acq_buffer.append(eval_acq)
            Xq_buffer.append(nXq)
        #

        # argmax the restart trials
        argi = torch.argmax(torch.tensor(acq_buffer))
        #print(acq_buffer)

        arg_acq_eval = acq_buffer[argi]
        arg_nXq = Xq_buffer[argi]

        return arg_acq_eval, arg_nXq

    def info_gain_by_samples(self, samples, batch_queried=[]):

        evals_info_gain = []
        evals_nXq = []

        Fstar, _ = self.fidelity_samples_extremes(self.model.M-1, samples)

        for m in range(self.model.M):
            fidelity_info_gain, fidelity_nXq = self.fidelity_info_gain_by_samples(m, samples, Fstar, batch_queried)
            evals_info_gain.append(fidelity_info_gain)
            evals_nXq.append(fidelity_nXq)
        #
        return evals_info_gain, evals_nXq

    def info_gain_step(self, samples):

        evals_info_gain, evals_nXq = self.info_gain_by_samples(samples)

        penalty = torch.from_numpy(np.array(self.dataset.penalty)).float().to(self.device)

        reg_evals_info_gain = torch.stack(evals_info_gain)/penalty

        argm = torch.argmax(reg_evals_info_gain)
        argx = evals_nXq[argm]

        return argm, argx

    def fixed_fidelity_step(self, samples, fixed_fidelty):
        evals_info_gain, evals_nXq = self.info_gain_by_samples(samples)
        argm = fixed_fidelty
        argx = evals_nXq[argm]

        return argm, argx

    def ratio_batch_step(self, samples, batch_size):

        batch_argm = []
        batch_argx = []
        batch_queried = []
        batch_costs = []

        penalty = np.array(self.dataset.penalty)

        for b in range(batch_size):
            evals_info_gain, evals_nXq = self.info_gain_by_samples(samples, batch_queried)

            buffer_acquisition = []
            for m in range(self.model.M):
                eval_acquisition = evals_info_gain[m]/(sum(batch_costs)+penalty[m])
                buffer_acquisition.append(eval_acquisition)
            #

            argm = torch.argmax(torch.stack(buffer_acquisition))
            argx = evals_nXq[argm]
#             print(argm)
#             print(argx)
            batch_argm.append(argm)
            batch_argx.append(argx)

#             print(buffer_acquisition )
#             print(argm, argx)

            batch_costs.append(penalty[argm])
            nyq = self.dataset.query_n_sample(argx, argm, torch_tensor=True, device=self.device)

            batch_queried.append(nyq)
#             print(batch_costs)
#             print(batch_queried)

        #

        return batch_argm, batch_argx, batch_queried

    def ao_ratio_batch_step(self, samples, batch_size, alters):

        batch_argm = []
        batch_argx = []
        batch_queried = []
        batch_costs = []

        penalty = np.array(self.dataset.penalty)

        if self.model.M == 2:
            p_init = [0.8, 0.2]
        elif self.model.M == 3:
            p_init = [0.8, 0.15, 0.05]
        #

        np_batch_argm = np.random.choice(self.model.M, size=batch_size, p=p_init).tolist()
        batch_argm = [torch.tensor(x).to(self.device) for x in np_batch_argm]
        batch_argx = []
        batch_queried = []
        batch_costs = []

        for i in range(batch_size):
            argm = batch_argm[i]
            n_np_lb, n_np_ub = self.dataset.get_scaled_bounds(argm)
            nXq = init_random_inputs_w_bounds(n_np_lb, n_np_ub, device=self.device)
            nyq = self.dataset.query_n_sample(nXq,  argm, torch_tensor=True, device=self.device)
            batch_argx.append(nXq)
            batch_queried.append(nyq)
            batch_costs.append(penalty[argm])
        #

        for k in range(alters):

            for b in range(batch_size):
                fixed_batch_argm = batch_argm[:b]+batch_argm[b+1:]
                fixed_batch_argx = batch_argx[:b]+batch_argx[b+1:]
                fixed_batch_queried = batch_queried[:b]+batch_queried[b+1:]
                fixed_batch_costs = batch_costs[:b]+batch_costs[b+1:]

                evals_info_gain, evals_nXq = self.info_gain_by_samples(samples, fixed_batch_queried)

                buffer_acquisition = []
                for m in range(self.model.M):
                    eval_acquisition = evals_info_gain[m]/(sum(fixed_batch_costs)+penalty[m])
                    buffer_acquisition.append(eval_acquisition)
                #

                argm = torch.argmax(torch.stack(buffer_acquisition))
                argx = evals_nXq[argm]

                nyq = self.dataset.query_n_sample(argx, argm, torch_tensor=True, device=self.device)

                batch_argm = batch_argm[:b]+ [argm] + batch_argm[b+1:]
                batch_argx = batch_argx[:b]+ [argx] + batch_argx[b+1:]
                batch_queried = batch_queried[:b]+ [nyq] + batch_queried[b+1:]
                batch_costs = batch_costs[:b]+ [penalty[argm]] + batch_costs[b+1:]

            #
        #

        return batch_argm, batch_argx, batch_queried

    def pseudo_par_step(self, samples, n_threads):
        # n_threads
        Ns = len(samples)
        perm = np.random.permutation(Ns)
        ns_per_thread = int(Ns/n_threads)

        pool_samples = []
        for i in range(0, Ns, ns_per_thread):
            i_subset = perm[i:i+ns_per_thread].astype(int).tolist()
            subset_samples = []
            for j in i_subset:
                subset_samples.append(samples[j])
            #
            pool_samples.append(subset_samples)
        #

        pool_argm = []
        pool_argx = []

        for subset_samples in pool_samples:
            argm, argx = self.info_gain_step(subset_samples)
            print(argm, argx)
            pool_argm.append(argm)
            pool_argx.append(argx)
        #

        return pool_argm, pool_argx


text:
# coding=utf-8
# Copyright 2020 The HuggingFace Team All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Sparse Fine-tuning the library models for question answering.
"""
# You can also adapt this script on your own question answering task. Pointers for this are left as comments.
import copy
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as nn_functional
from transformers import AutoConfig, AutoModelForQuestionAnswering
from dataclasses import dataclass, field
from collections import defaultdict

from nn_pruning.model_structure import struct_from_config, ModelStructureNotFound

from .modules.masked_nn import (
    ChannelPruningModulePatcher,
    JointPruningModulePatcher,
    LinearPruningModulePatcher,
    LinearPruningArgs,
    MaskedLinear,
    MaskedLinearModelCompiler,
    GenericLinearPruningContextModule,
    head_mask,
)
from .modules.nonorm import Layer2NoNorm, NoNorm, NoNormCompiler, Layer2NoNormPatcher
from .modules.gelu2relu import GeLU2ReLUModelPatcher
from .inference_model_patcher import BertHeadsPruner

from nn_pruning.training_patcher import (
    LinearModelPatcher,
    PatcherContext,
    PatcherContextModule,
)
from pathlib import Path
import json

@dataclass
class SparseTrainingArguments:
    """
    Sparse training specific arguments
    """

    mask_scores_learning_rate: float = field(
        default=1e-2, metadata={"help": "The initial learning rate for mask_scores."}
    )

    dense_pruning_method: str = field(default="topK", metadata={"help": "Dense Layers pruning method."})

    attention_pruning_method: str = field(default="topK", metadata={"help": "Dense Layers pruning method."})

    ampere_pruning_method: str = field(
        default="disabled",
        metadata={"help": "Ampere sparse method ('disabled' for no ampere sparsity, topK, annealing, sigmoied_threshold, threshold)"},
    )

    attention_output_with_dense: bool = field(
        default=True,
        metadata={"help": "share the same pruning parameters for attention output and other dense matrices"},
    )

    bias_mask: bool = field(
        default=True,
        metadata={"help": "Apply the mask built on weight to the bias too (not doing so will impact somewhat ability to prune complete heads, as bias is then pruned while being nonzero)"},
    )

    mask_init: str = field(default="constant", metadata={"help": "Mask scores initialization method"})

    mask_scale: float = field(
        default=0.0,
        metadata={"help": "Parameter to use with mask_init."},
    )

    dense_block_rows: int = field(
        default=1,
        metadata={"help": "Block size in rows for dense layers."},
    )

    dense_block_cols: int = field(
        default=1,
        metadata={"help": "Block size in cols for dense layers."},
    )

    attention_block_rows: int = field(
        default=1,
        metadata={"help": "Block size in rows for attention."},
    )

    attention_block_cols: int = field(
        default=1,
        metadata={"help": "Block size in cols for attention."},
    )

    initial_threshold: float = field(
        default=1.0,
        metadata={"help": "Initial value of the threshold (for scheduling)."},
    )
    final_threshold: float = field(
        default=0.5,
        metadata={"help": "Final value of the threshold (for scheduling)."},
    )

    initial_warmup: float = field(
        default=1,
        metadata={
            "help": "Run `initial_warmup` * `warmup_steps` steps of threshold warmup during which threshold stays at its `initial_threshold` value (sparsity schedule)."
        },
    )
    final_warmup: float = field(
        default=2,
        metadata={
            "help": "Run `final_warmup` * `warmup_steps` steps of threshold cool-down during which threshold stays"
        },
    )

    initial_ampere_temperature: float = field(
        default=0.0,
        metadata={"help": "Initial value of the ampere temperature (for scheduling)."},
    )
    final_ampere_temperature: float = field(
        default=20.0,
        metadata={"help": "Final value of the ampere temperature (for scheduling)."},
    )

    regularization: str = field(
        default="disabled",
        metadata={"help": "Add L0 or L1 regularization to the mask scores."},
    )

    regularization_final_lambda: float = field(
        default=0.0,
        metadata={"help": "Regularization intensity (used in conjunction with `regularization`)."},
    )

    attention_lambda: float = field(
        default=1.0,
        metadata={"help": "Regularization intensity for attention (attention lambda will be regularization_lambda * attention_lambda)."},
    )

    dense_lambda: float = field(
        default=1.0,
        metadata={
            "help": "Regularization intensity for dense (attention lambda will be regularization_lambda * dense_lambda)."},
    )

    decoder_attention_lambda: float = field(
        default=None,
        metadata={"help": "Regularization intensity for decoder attention (attention lambda will be regularization_lambda * decoder_attention_lambda)."},
    )

    decoder_dense_lambda: float = field(
        default=None,
        metadata={
            "help": "Regularization intensity for decoder dense (attention lambda will be regularization_lambda * decoder_dense_lambda)."},
    )

    distil_teacher_name_or_path: str = field(
        default=None,
        metadata={"help": "Path to the already SQuAD fine-tuned teacher model. Only for distillation."},
    )

    distil_alpha_ce: float = field(
        default=0.5,
        metadata={"help": "Cross entropy loss linear weight. Only for distillation."},
    )

    distil_alpha_teacher: float = field(
        default=0.5,
        metadata={"help": "Distillation loss linear weight. Only for distillation."},
    )

    distil_temperature: float = field(
        default=2.0,
        metadata={"help": "Distillation temperature. Only for distillation."},
    )

    final_finetune: bool = field(
        default=False,
        metadata={
            "help": "Run a final fine tune pass on the network"
        },
    )

    layer_norm_patch: bool = field(
        default=False,
        metadata={
            "help": "Transform the LayerNorms in a MobileBert NoNorm"
        },
    )
    layer_norm_patch_steps: int = field(
        default=50000,
        metadata={
            "help": "Number of steps for the transition from LayerNorm to NoNorm"
        },
    )

    layer_norm_patch_start_delta: int = field(
        default=0.99,
        metadata={
            "help": "Starting smoothing factor for transition from LayerNorm to NoNorm (final is 1.0)."
        },
    )

    gelu_patch: bool = field(
        default=False,
        metadata={
            "help": "Transform the GeLU in ReLU"
        },
    )

    gelu_patch_steps: int = field(
        default=50000,
        metadata={
            "help": "Number of steps for the transition from GeLU to ReLU"
        },
    )

    linear_min_parameters: int = field(
        default=0.005,
        metadata={
            "help": "Minimum fraction of parameters which should be non zero when using ThresholdBinarizer"
        },
    )

    rewind_model_name_or_path: str = field(
        default=None,
        metadata={
           "help": "Model that will be used as a guide to prevent pruning of some attention heads while redoing fine-pruning."
        },
    )

    eval_with_current_patch_params: bool = field(
        default=False,
        metadata={
            "help": "Whether to keep the transition parameters used during training for eval. Only for Layer2NoNorm, GeLU2ReLU and pruning threshold."
        },
    )

    qat: bool = field(
        default=False,
        metadata={"help": "Whether to prepare the model for Quantization Aware Training"},
    )

    qconfig: str = field(
        default="default",
        metadata={"help": "The quantization scheme configuration to use for QAT"},
    )

    schedule_type: str = field(default="linear", metadata={"help": "LR Schedule type."})

    @classmethod
    def hybrid(cls, regularization_lambda):
        sparse_args = cls()
        sparse_args.dense_pruning_method = "sigmoied_threshold:1d_alt"
        sparse_args.attention_pruning_method = "sigmoied_threshold"
        sparse_args.attention_block_rows = 32
        sparse_args.attention_block_cols = 32
        sparse_args.attention_output_with_dense = False
        sparse_args.initial_threshold = 0.0
        sparse_args.final_threshold = 0.1

        sparse_args.regularization = "l1"
        sparse_args.regularization_final_lambda = regularization_lambda
        return sparse_args

class ModelPatchingCoordinator:

    def __init__(self, sparse_args, device, cache_dir, model_name_or_path, logit_names, teacher_constructor):
        # logit_names is ["start_logits", "end_logits"] for qa, ["logits"] for glue etc
        # teacher model is AutoModelForQuestionAnswering for qa, AutoModelForSequenceClassification for glue etc
        self.sparse_args = sparse_args
        self.patcher_context = PatcherContext()
        self.teacher_constructor = teacher_constructor
        self.device = device
        self.cache_dir = cache_dir
        self.teacher = None
        self.layer_head_mask = self.create_head_rewind_info(device, cache_dir)
        self.logit_names = logit_names
        self.model_name_or_path = model_name_or_path
        config = AutoConfig.from_pretrained(model_name_or_path, cache_dir=cache_dir)
        self.model_structure = struct_from_config(config.__class__)

    def parse_pruning_method(self, method):
        parts = method.split(":")
        if len(parts) == 2:
            return parts
        elif len(parts) == 1:
            return parts[0], "default"
        else:
            raise RuntimeError("Could not parse pruning method")

    def log(self):
        logs = {}
        for k, v in self.patcher_context.enumerate_context_data():
            logs[k] = v

        return logs

    def create_teacher(self):
        if self.teacher is not None:
            return self.teacher

        device = self.device
        cache_dir = self.cache_dir

        sparse_args = self.sparse_args

        if sparse_args.distil_teacher_name_or_path is not None:
            assert sparse_args.distil_alpha_ce > 0.0
            assert sparse_args.distil_alpha_ce + sparse_args.distil_alpha_teacher > 0.0

            model_config = AutoConfig.from_pretrained(sparse_args.distil_teacher_name_or_path, cache_dir=cache_dir)

            teacher = self.teacher_constructor.from_pretrained(
                sparse_args.distil_teacher_name_or_path,
                from_tf=bool(".ckpt" in sparse_args.distil_teacher_name_or_path),
                config=model_config,
                cache_dir=cache_dir,
            )
            teacher.to(device)
            self.teacher = teacher

        return self.teacher

    def create_head_rewind_info(self, device, cache_dir):
        if not hasattr(self.sparse_args, "rewind_model_name_or_path"):
            return None

        rewind_model_name_or_path = self.sparse_args.rewind_model_name_or_path
        if rewind_model_name_or_path is None:
            return None
        else:
            rewind_config = AutoConfig.from_pretrained(rewind_model_name_or_path, cache_dir=cache_dir)

            return head_mask(rewind_config, device)

    def schedule_threshold(
        self,
        step: int = -1,
        total_step: int = -1,
        warmup_steps: int = -1,
        training: bool = False,
        compile:bool = False,
    ):
        sparse_args = self.sparse_args

        initial_threshold = sparse_args.initial_threshold
        final_threshold = sparse_args.final_threshold
        initial_warmup = sparse_args.initial_warmup
        final_warmup = sparse_args.final_warmup
        final_lambda = sparse_args.regularization_final_lambda
        initial_ampere_temperature = sparse_args.initial_ampere_temperature
        final_ampere_temperature = sparse_args.final_ampere_temperature

        if not training:
            step -= 1

        eval_with_current_patch_params = (hasattr(sparse_args, "eval_with_current_patch_params") and sparse_args.eval_with_current_patch_params)
        use_scheduler = training or eval_with_current_patch_params

        if compile:
            if use_scheduler:
                base_path = Path(self.model_name_or_path)
                training_args = torch.load(str(base_path / "training_args.bin"))
                warmup_steps = training_args.warmup_steps

                with (base_path / "trainer_state.json").open() as f:
                    trainer_state = json.load(f)

                step = trainer_state["global_step"]
                total_step = trainer_state["max_steps"]

        if use_scheduler:
            if step <= initial_warmup * warmup_steps:
                mul_coeff = 1.0
                threshold = initial_threshold
                ampere_temperature = initial_ampere_temperature
            elif step > (total_step - final_warmup * warmup_steps):
                mul_coeff = 0.0
                threshold = final_threshold
                ampere_temperature = final_ampere_temperature
            else:
                spars_warmup_steps = initial_warmup * warmup_steps
                spars_schedu_steps = (final_warmup + initial_warmup) * warmup_steps
                mul_coeff = 1 - (step - spars_warmup_steps) / (total_step - spars_schedu_steps)
                threshold = final_threshold + (initial_threshold - final_threshold) * (mul_coeff ** 3)
                ampere_temperature = final_ampere_temperature + (
                    initial_ampere_temperature - final_ampere_temperature
                ) * (mul_coeff ** 3)
        else:
            mul_coeff = 0.0
            threshold = final_threshold
            ampere_temperature = final_ampere_temperature

        regu_lambda = final_lambda * threshold / final_threshold

        context_data = dict(
            threshold=threshold,
            regu_lambda=regu_lambda,
            ampere_temperature = ampere_temperature,
            progress = 1.0 - mul_coeff
        )

        def interp(a,b, interpf):
            return a * interpf + (1.0 - interpf) * b

        if hasattr(sparse_args, "layer_norm_patch") and sparse_args.layer_norm_patch:
            if use_scheduler:
                interpf = 0.0
                layer_norm_patch_steps = sparse_args.layer_norm_patch_steps
                if step < layer_norm_patch_steps:
                    interpf = 1.0 - (step / layer_norm_patch_steps)

                delta = interp(sparse_args.layer_norm_patch_start_delta, 1.0, interpf)
                mix = interpf

                context_data["layernorm_to_nonorm_delta"] = delta
                context_data["layernorm_to_nonorm_mix"] = mix
            else:
                context_data["layernorm_to_nonorm_delta"] = 1.0
                context_data["layernorm_to_nonorm_mix"] = 0.0

        if hasattr(sparse_args, "gelu_patch") and sparse_args.gelu_patch:
            if use_scheduler:
                interpf = 0.0
                gelu_patch_steps = sparse_args.gelu_patch_steps
                if step < gelu_patch_steps:
                    interpf = 1.0 - (step / gelu_patch_steps)

                context_data["gelu_to_relu_mix"] = interpf
            else:
                context_data["gelu_to_relu_mix"] = 0.0

        self.patcher_context.set_context_data_dict(context_data)

    def regularization_loss(self, model: nn.Module):
        # Return regularization, lambda, and information on the network sparsity
        mode = self.sparse_args.regularization

        info = {}

        regul_modes = ["l1", "l0"]
        if mode in regul_modes:
            threshold = self.patcher_context.get_context_data("threshold")

        for name, module in model.named_modules():
            module_regu = 0
            module_nnz_info = {"nnz":0, "numel":0}
            nummod = 1
            # if isinstance(module, nn.Linear): print("found")
            if mode not in regul_modes:
                # if isinstance(module, nn.Linear):
                #     weight = module.weight
                #     module_nnz_info["nnz"] = (weight != 0).sum()
                #     module_nnz_info["numel"] = weight.numel()
                # elif isinstance(module, MaskedLinear): # added - why wasn't this here before? sparsity only with regu?
                #     module_nnz_info = module.get_sparsity_info()
                #     nummod = 1
                if isinstance(module, MaskedLinear): # added - why wasn't this here before? sparsity only with regu?
                    module_nnz_info = module.get_sparsity_info()
                    nummod = 1
                else:
                    continue
            elif isinstance(module, GenericLinearPruningContextModule):
                module_regu = module.regularization(mode)
            elif isinstance(module, MaskedLinear):
                module_nnz_info = module.get_sparsity_info()
                # try:
                #     module_nnz_info = module.get_sparsity_info()
                # except:
                #     print("failure getting module_nnz_info", name)
                #     continue
                nummod = 1
            elif hasattr(module, "regularization"):
                module_regu = module.regularization()
                if hasattr(module, "get_sparsity_info"):
                    module_nnz_info = module.get_sparsity_info()
            else:
                continue

            key = "decoder_" if self.model_structure.is_decoder(name) else ""
            exclude_att_dense = not hasattr(self.sparse_args, "attention_output_with_dense") or self.sparse_args.attention_output_with_dense
            key += "attention" if self.model_structure.is_attention(name, exclude_att_dense=exclude_att_dense) else "dense"

            if key not in info:
                info[key] = defaultdict(float)

            key_info = info[key]
            key_info["regu"] += module_regu
            key_info["nummod"] += nummod

            for k,v in module_nnz_info.items():
                key_info[k] += float(v)

        if mode not in regul_modes:
            lamb = 0
            lambdas = {k: 0 for k in info.keys()}
        else:
            lamb = self.patcher_context.get_context_data("regu_lambda")
            lambdas = {}
            n = len(info)
            for k in info.keys():
                if k.endswith('attention'):
                    if k.startswith('decoder'):
                        if self.sparse_args.decoder_attention_lambda is None:
                            self.sparse_args.decoder_attention_lambda = self.sparse_args.attention_lambda
                        lambdas[k] = self.sparse_args.decoder_attention_lambda / n
                    else:
                        lambdas[k] = self.sparse_args.attention_lambda / n
                else:
                    if k.startswith('decoder'):
                        if self.sparse_args.decoder_dense_lambda is None:
                            self.sparse_args.decoder_dense_lambda = self.sparse_args.dense_lambda
                        lambdas[k] = self.sparse_args.decoder_dense_lambda / n
                    else:
                        lambdas[k] = self.sparse_args.dense_lambda / n

        info["total"] = defaultdict(float)

        for key, value in info.items():
            if key == "total":
                continue
            for k, v in value.items():
                if k == "numel" or "nnz" in k:
                    info["total"][k] += v

        for key, value in info.items():
            if value["numel"] != 0:
                # No patching (no pruning) -> no information on nnz -> dense linear layers
                value["nnz_perc"] = value["nnz"] / value["numel"]
            else:
                value["nnz_perc"] = 1.0
            for k in "nnz", "numel":
                if k in value:
                    del value[k]
            if key == "total":
                continue
            if value["nummod"] != 0:
                value["regu_loss"] = value["regu"] * lambdas[key] / value["nummod"]
                info["total"]["regu_loss"] += value["regu_loss"]
            for k in "regu", "nummod":
                if k in value:
                    del value[k]

        return info["total"]["regu_loss"], lamb, info

    def distil_loss_combine(self, ce_loss, model_inputs, model_outputs):
        sparse_args = self.sparse_args
        teacher = self.create_teacher()

        if teacher == None:
            return ce_loss, 0.0

        temperature = sparse_args.distil_temperature

        teacher_inputs_ = model_inputs.copy()
        if 'labels' in teacher_inputs_:
            del teacher_inputs_['labels']

        teacher_inputs = {}
        for k,v in teacher_inputs_.items():
            teacher_inputs[k] = v.detach().clone()

        with torch.no_grad():
            teacher_outputs = teacher(**teacher_inputs)

        loss_logits = 0
        for logit_name in self.logit_names:
            logits_stu = model_outputs[logit_name]
            logits_tea = teacher_outputs[logit_name].detach().clone()

            loss_logits_part = nn_functional.kl_div(
                input=nn_functional.log_softmax(logits_stu / temperature, dim=-1),
                target=nn_functional.softmax(logits_tea / temperature, dim=-1),
                reduction="batchmean",
            ) * (temperature ** 2)

            loss_logits = loss_logits + loss_logits_part

        loss_logits = loss_logits / len(self.logit_names)

        loss = sparse_args.distil_alpha_teacher * loss_logits + sparse_args.distil_alpha_ce * ce_loss

        return loss, loss_logits

    def create_optimizer_groups(self, model, args, sparse_args):
        # Prepare optimizer and schedule (linear warmup and decay)
        no_decay = ["bias", "LayerNorm.weight", "NoNorm.weight", "layer_norm.weight", "layernorm_embedding.weight",
                    "final_layer_norm.weight", "ln_1.weight", "ln_2.weight", "ln_f.weight"]
        mask_params = []
        no_decay_params = []
        decay_params = []

        for n, p in model.named_parameters():
            if not p.requires_grad:
                continue
            if "mask_score" in n:
                mask_params.append(p)
            elif any(nd in n for nd in no_decay):
                no_decay_params.append(p)
            else:
                decay_params.append(p)

        optimizer_grouped_parameters = [
            {
                "params": mask_params,
                "lr": sparse_args.mask_scores_learning_rate,
            },
            {
                "params": no_decay_params,
                "lr": args.learning_rate,
                "weight_decay": 0.0,
            },
            {
                "params": decay_params,
                "lr": args.learning_rate,
                "weight_decay": args.weight_decay,
            },
        ]

        return optimizer_grouped_parameters

    def patch_model(self, model, trial = None):
        layers_count = model.config.num_hidden_layers
        sparse_args = self.sparse_args

        device = model.device

        attention_pruning_method_parts = self.parse_pruning_method(sparse_args.attention_pruning_method)

        if hasattr(sparse_args, "bias_mask"):
            bias_mask = sparse_args.bias_mask
        else:
            bias_mask = False

        if hasattr(sparse_args, "linear_min_parameters"):
            linear_min_parameters = sparse_args.linear_min_parameters
        else:
            linear_min_parameters = 0.005

        patcher_context = self.patcher_context

        if attention_pruning_method_parts[0] != "disabled" or sparse_args.ampere_pruning_method != "disabled":
            args_attention = LinearPruningArgs(
                method=attention_pruning_method_parts[0],
                submethod=attention_pruning_method_parts[1],
                ampere_method=sparse_args.ampere_pruning_method,
                block_rows=sparse_args.attention_block_rows,
                block_cols=sparse_args.attention_block_cols,
                bias_mask=bias_mask,
                min_elements=linear_min_parameters,
            )

            args_attention_t = LinearPruningArgs(
                method=attention_pruning_method_parts[0],
                submethod=attention_pruning_method_parts[1],
                ampere_method=sparse_args.ampere_pruning_method,
                block_rows=sparse_args.attention_block_cols,
                block_cols=sparse_args.attention_block_rows,
                bias_mask=bias_mask,
                min_elements=linear_min_parameters,
            )

            if args_attention.submethod == "joint":
                p_attention = JointPruningModulePatcher(patcher_context, args_attention, model_structure=self.model_structure, suffix=".attention")
                p_attention_t = JointPruningModulePatcher(patcher_context, args_attention_t, model_structure=self.model_structure, suffix=".attention")
            else:
                p_attention = LinearPruningModulePatcher(patcher_context,
                                                         args_attention,
                                                         model_structure=self.model_structure,
                                                         row_additive_mask = self.layer_head_mask)
                p_attention_t = LinearPruningModulePatcher(patcher_context,
                                                           args_attention_t,
                                                           model_structure = self.model_structure,
                                                           col_additive_mask = self.layer_head_mask)
        else:
            p_attention = None
            p_attention_t = None

        dense_pruning_method_parts = self.parse_pruning_method(sparse_args.dense_pruning_method)

        if dense_pruning_method_parts[0] != "disabled" or sparse_args.ampere_pruning_method != "disabled":
            args_dense = LinearPruningArgs(
                method=dense_pruning_method_parts[0],
                submethod=dense_pruning_method_parts[1],
                ampere_method=sparse_args.ampere_pruning_method,
                block_rows=sparse_args.dense_block_rows,
                block_cols=sparse_args.dense_block_cols,
                bias_mask=bias_mask,
                min_elements=linear_min_parameters,
            )
            if args_dense.submethod.startswith("1d"):
                p_dense = ChannelPruningModulePatcher(
                    patcher_context, args_dense, model_structure=self.model_structure, suffix="dense"
                )
            else:
                p_dense = LinearPruningModulePatcher(patcher_context, args_dense, model_structure=self.model_structure)
        else:
            p_dense = None

        if not hasattr(sparse_args, "attention_output_with_dense") or sparse_args.attention_output_with_dense:
            p_att_dense = p_dense
        else:
            p_att_dense = p_attention_t

        module_patchers = dict(
            query=p_attention,
            key=p_attention,
            value=p_attention,
            att_dense=p_att_dense,
            encoder_decoder_query=p_attention,
            encoder_decoder_key=p_attention,
            encoder_decoder_value=p_attention,
            encoder_decoder_att_dense=p_att_dense,
            interm_dense=p_dense,
            output_dense=p_dense,
        )

        if hasattr(sparse_args, "layer_norm_patch"):
            layer_norm_patch = sparse_args.layer_norm_patch
        else:
            layer_norm_patch = False

        if hasattr(sparse_args, "gelu_patch"):
            gelu_patch = sparse_args.gelu_patch
        else:
            gelu_patch = False

        patcher = LinearModelPatcher(module_patchers, model_structure=self.model_structure)

        patcher.patch(model)
        model = model.to(device)  # TODO: change this by making sure the mask_scores are located at the right place.

        self.stats = {}
        self.stats["main"] = patcher.stats

        if layer_norm_patch:
            def schedule_callback():
                mix = self.patcher_context.get_context_data("layernorm_to_nonorm_mix")
                delta = self.patcher_context.get_context_data("layernorm_to_nonorm_delta")
                return dict(mix=mix, delta=delta)

            layer_norm_patcher = Layer2NoNormPatcher(schedule_callback=schedule_callback)
            layer_norm_patcher.patch(model)
            self.stats["layer_norm"] = layer_norm_patcher.stats

        if gelu_patch:
            def schedule_callback():
                mix = self.patcher_context.get_context_data("gelu_to_relu_mix")
                return dict(mix=mix)

            gelu_patcher = GeLU2ReLUModelPatcher(schedule_callback=schedule_callback)
            gelu_patcher.patch(model)
            self.stats["gelu"] = gelu_patcher.stats

        return patcher

    def compile_model(self, model):
        self.schedule_threshold(compile=True)
        compiler = MaskedLinearModelCompiler()
        compiler.patch(model)

        if hasattr(self.sparse_args, "layer_norm_patch") and self.sparse_args.layer_norm_patch:
            nnc = NoNormCompiler()
            nnc.patch(model)
            model.config.layer_norm_type = "no_norm"

        if hasattr(self.sparse_args, "gelu_patch") and self.sparse_args.gelu_patch:
            model.config.hidden_act = "relu"

        pruner = BertHeadsPruner(model)
        removed_heads, total_heads = pruner.run()
        return removed_heads, total_heads


text:
"""
Random processes exist everywhere. Roughly speaking, a random process is a
system of related random variables, usually indexed with respect to time t ≥ 0,
for a continuous random process, or by natural numbers n = 1, 2, ..., for a
discrete random process.

This module examine the Poisson process, which is a simple yet interesing
random process.
"""
import matplotlib.pyplot as plt
import numpy as np

from numpy.random import default_rng
from scipy.special import factorial

def probability(events, param, time=1):
    return ((param*time)**events/factorial(events))*np.exp(-param*time)

rng = default_rng(12345)

rate = 4.0
inter_arrival_times = rng.exponential(scale=1./rate, size=50)

arrivals = np.add.accumulate(inter_arrival_times)
count = np.arange(50)

fig1, ax1 = plt.subplots()
ax1.step(arrivals, count, where="post")
ax1.set_xlabel("Time")
ax1.set_ylabel("Number of arrivals")
ax1.set_title("Arrivals over time")

N = np.arange(15)
estimated_scale = np.mean(inter_arrival_times)
estimated_rate = 1.0/estimated_scale

fig2, ax2 = plt.subplots()
ax2.plot(N, probability(N, param=rate), "k", label="True distribution")
ax2.plot(N, probability(N, param=estimated_rate), "k--",
         label="Estimated distribution")
ax2.set_xlabel("Number of arrivals in 1 time unit")
ax2.set_ylabel("Probability")
ax2.set_title("Probability Distribution")

plt.show()

max_stars_repo_name: jeantardelli/math-with-python

max_issues_repo_name: jeantardelli/math-with-python

max_forks_repo_name: jeantardelli/math-with-python

reason: import numpy,from numpy,from scipy


text:
import numpy as np
import cv2
import tensorflow as tf
import tensorflow.keras as keras
from tensorflow.keras.layers import Dense,Dropout,Conv2D,Conv2DTranspose,\
ReLU,Softmax,Flatten,Reshape,UpSampling2D,Input,Activation,LayerNormalization
from tqdm import tqdm
import argparse
import os

#from AR9
def mkdiring(input):
    arr=input.split("/");input=""
    for inp in arr:
        if not os.path.exists(input+inp+"/"):os.mkdir(input+inp)
        input+=inp+"/"
    return input.rstrip("/")
def ffzk(input_dir):#Relative directory for all existing files
    imgname_array=[];input_dir=input_dir.strip("\"\'")
    for fd_path, _, sb_file in os.walk(input_dir):
        for fil in sb_file:imgname_array.append(fd_path.replace('\\','/') + '/' + fil)
    if os.path.isfile(input_dir):imgname_array.append(input_dir.replace('\\','/'))
    return imgname_array

def img2np(dir=[],img_len=0):
    img=[]
    for x in dir:
        try:img.append(cv2.imread(x))
        except:continue
        if img_len!=0:img[-1]=cv2.resize(img[-1],(img_len,img_len))
        elif img[-1].shape!=img[0].shape:img.pop(-1);continue#Leave only the same shape
        img[-1] = img[-1].astype(np.float32)/ 256
    return np.stack(img, axis=0)

def tf2img(tfs,dir="./",name="",epoch=0,ext=".png"):
    mkdiring(dir)
    if type(tfs)!=np.ndarray:tfs=tfs.numpy()
    tfs=(tfs*256).astype(np.uint8)
    for i in range(tfs.shape[0]):
        cv2.imwrite(os.path.join(dir,name+"_epoch-num_"+str(epoch)+"-"+str(i)+ext),tfs[i])

def tf_ini():#About GPU resources
    physical_devices = tf.config.experimental.list_physical_devices('GPU')
    for k in range(len(physical_devices)):
        tf.config.experimental.set_memory_growth(physical_devices[k], True)
    if len(physical_devices)==0:print("GPU_failed!")
    return len(physical_devices)

class c3c(keras.Model):
    def __init__(self,dim=4):#plz define used layers below...
        super().__init__()
        self.layer1=[Conv2D(dim,1,padding="same"),
                ]
        self.layer1_1=[Conv2D(dim,3,padding="same"),
                       Conv2D(dim,3,padding="same"),
                       Conv2D(dim,3,padding="same"),
                ]
        self.dim=dim
        return
    def call(self,mod):#plz add layers below...
        with tf.name_scope("c3c"):
            mod_1=mod
            for i in range(len(self.layer1)):mod=self.layer1[i](mod)
            for i in range(len(self.layer1_1)):mod_1=self.layer1_1[i](mod_1)
            mod=keras.layers.add([mod,mod_1])
            mod=Dropout(0.05)(mod)
            mod=Activation("relu")(mod)
        return mod

class m_gen(keras.Model):
    def __init__(self,trials=[]):#plz define used layers below...
        super().__init__()
        self.layer1=[Reshape((1,1,8)),
                     UpSampling2D(2),
                     c3c(12),
                     UpSampling2D(2),
                     c3c(16),
                     UpSampling2D(2),
                     c3c(24),
                     c3c(32),
                     UpSampling2D(2),
                     c3c(32),
                     c3c(48),
                     UpSampling2D(2),
                     c3c(48),
                     c3c(54),
                     c3c(64),
                     UpSampling2D(2),
                     c3c(64),
                     c3c(72),
                     c3c(84),
                     c3c(94),
                     Conv2D(3,1,padding="same",activation="sigmoid"),
                     ]
        return
    def call(self, mod,training=None):#plz add layers below...
        for i in range(len(self.layer1)):
            mod=self.layer1[i](mod)
        return mod

class m_dis(keras.Model):
    def __init__(self):#plz define used layers below...
        super().__init__()
        self.layer1=[c3c(16),
                     c3c(18),
                     c3c(24),
                     Conv2D(24,4,2,padding="same",activation="relu"),
                     LayerNormalization(),
                     c3c(28),
                     c3c(32),
                     c3c(36),
                     Conv2D(36,4,2,padding="same",activation="relu"),
                     LayerNormalization(),
                     c3c(42),
                     c3c(46),
                     c3c(48),
                     Conv2D(48,4,2,padding="same",activation="relu"),
                     LayerNormalization(),
                     c3c(54),
                     c3c(64),
                     Conv2D(64,4,2,padding="same",activation="relu"),
                     LayerNormalization(),
                     c3c(84),
                     c3c(84),
                     Conv2D(84,4,2,padding="same",activation="relu"),
                     c3c(84),
                     LayerNormalization(),
                     Dropout(0.05),
                     Flatten(),
                     Dense(84,activation="relu"),
                     Dense(64,activation="relu"),
                     LayerNormalization(),
                     Dropout(0.05),
                     Dense(32,activation="relu"),
                     Dense(1,activation="sigmoid"),
                ]
        return
    def call(self, mod,training=None):#plz add layers below...
        for i in range(len(self.layer1)):
            mod=self.layer1[i](mod)
        return mod

def baches(a,b):
    a.reshape((b,a.shape()))

class gan():
    def __init__(self,trials=[]):
        self.gen=m_gen()
        self.dis=m_dis()
    def pred(self,batch=4):
        return self.gen(np.random.rand(batch,8).astype(np.float32))
    def train(self,data=[],epoch=1000,batch=16,predbatch=8):
        #unco
        self.gen.build(input_shape=(batch,8))
        self.dis.build(input_shape=(batch,data.shape[1],data.shape[2],data.shape[3]))
        self.gen.summary()
        self.dis.summary()

        optimizer = keras.optimizers.SGD(0.003)
        self.gen.compile(optimizer = optimizer,
                          loss=keras.losses.binary_crossentropy,metrics=['accuracy'])
        self.dis.compile(optimizer = optimizer,
                          loss=keras.losses.binary_crossentropy,metrics=['accuracy'])

        try:self.gen.load_weights(os.path.join(args.outdir,"disw.hdf5"))
        except:print("\nCannot_use_savedata...")
        try:self.dis.load_weights(os.path.join(args.outdir,"genw.hdf5"))
        except:print("\nCannot_use_savedata...")

        ones=np.ones(batch).astype(np.float32)
        zeros=np.zeros(batch).astype(np.float32)
        labels=np.random.rand(data.shape[0],8).astype(np.float32)

        for i in range(epoch):
            with tqdm(total=data.shape[0]) as pbar:
                while pbar.n+batch<pbar.total:
                    datum=data[pbar.n:pbar.n+batch];label=labels[pbar.n:pbar.n+batch];

                    with tf.GradientTape() as tape:
                        dis=self.dis(self.gen(label))
                        dis=keras.losses.binary_crossentropy(zeros,dis)
                        dis=tf.reduce_sum(dis) / batch
                        grad=tape.gradient(dis,self.dis.trainable_variables)
                        grad,_ = tf.clip_by_global_norm(grad, 15)
                        optimizer.apply_gradients(zip(grad,self.dis.trainable_variables))
                        del tape

                    with tf.GradientTape() as tape:
                        dis=self.dis(datum)
                        dis=keras.losses.binary_crossentropy(ones,dis)
                        dis=tf.reduce_sum(dis) / batch
                        grad=tape.gradient(dis,self.dis.trainable_variables)
                        grad,_ = tf.clip_by_global_norm(grad, 15)
                        optimizer.apply_gradients(zip(grad,self.dis.trainable_variables))
                        del tape

                    with tf.GradientTape() as tape:
                        dis_gen=self.dis(self.gen(label))
                        dis_gen=keras.losses.binary_crossentropy(ones,dis_gen)
                        dis_gen=tf.reduce_sum(dis_gen) / batch
                        grad=tape.gradient(dis_gen,self.gen.trainable_variables)
                        grad,_ = tf.clip_by_global_norm(grad, 15)
                        optimizer.apply_gradients(zip(grad,self.gen.trainable_variables))
                        del tape

                    pbar.update(batch)

            dis1=self.dis(self.gen(labels[:batch]))
            dis2=self.dis(data[:batch])
            dis1=tf.reduce_sum(keras.losses.binary_crossentropy (zeros,dis1)) / batch
            dis2=tf.reduce_sum(keras.losses.binary_crossentropy (ones,dis2)) / batch
            print("\nke(fake,tru):",dis1.numpy(),dis2.numpy())
            tf2img(self.pred(predbatch),os.path.join(args.outdir,"1"),epoch=i,ext=".png")

            self.dis.save_weights(os.path.join(args.outdir,"disw.hdf5"))
            self.gen.save_weights(os.path.join(args.outdir,"genw.hdf5"))

parser = argparse.ArgumentParser()
parser.add_argument('-t', '--train' ,help="train_data",default="./lfw")
parser.add_argument('-o', '--outdir' ,help="outdir",default="./output")
parser.add_argument('-b', '--batch' ,help="batch",default=32,type=int)
parser.add_argument('-p', '--predbatch' ,help="batch_size_of_prediction",default=8,type=int)
parser.add_argument('-e', '--epoch' ,help="epochs",default=30,type=int)
args = parser.parse_args()

if __name__ == '__main__':
    tf_ini()
    mkdiring(args.outdir)
    img=img2np(ffzk(args.train),64)
    gans=gan()
    gans.train(img,epoch=args.epoch,batch=args.batch,predbatch=args.predbatch)