text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time     : 2018/10/15 21:09
# @Author   : Iydon
# @File     : course4.3.py

import numpy as np
from Poly import Poly

def numerical_quadrature(xs:list, fxs:list, ab:list=[]):
    # Lagrange element
    def Lagrange(i):
        def prod(lst, result=1):
            return prod(lst[:-1], result*lst[-1]) if lst else result
        num = prod([Poly([1,-x]) for x in xs[:i]+xs[i+1:]])
        den = prod([(xs[i]-x) for x in xs[:i]+xs[i+1:]])
        return num / den
    # function main
    if not ab:
        ab = [xs[0], xs[-1]]
    result = 0
    for i in range(len(xs)):
        F = (Lagrange(i)*Poly([1])).integrate().lambdify()
        result += (F(ab[-1])-F(ab[0])) * fxs[i]
    return result

def trapezoidal_rule(xs:list, fxs:list) -> float:
    """
    Trapezoidal Rule:
    /int_{x_0}^{x_1}f(x)dx = (h/2)[f(x_0)+f(x_1)] - E(f)
    E(f) = (h^3/12)f''(/xi)
    """
    return (xs[-1]-xs[0]) / 2 * sum(fxs)

def simpson_rule(xs:list, fxs:list) -> float:
    """
    Simpson's Rule:
    /int_{x_0}^{x_2}f(x)dx = (h/3)[f(x_0)+4f(x_1)+f(x_2)] - E(f)
    E(f) = (h^5/90)f^{(4)}(/xi)
    """
    return (xs[-1]-xs[0]) / 6 * (fxs[0]+4*fxs[1]+fxs[2])

def closed_newton_cotes(xs, fxs):
    """
    len(xs)==len(fxs)
    """
    return numerical_quadrature(xs, fxs, [xs[0],xs[-1]])

def open_newton_cotes(xs, fxs):
    """
    len(xs)==len(fxs)+2
    """
    return numerical_quadrature(xs[1:-1], fxs, [xs[0],xs[-1]])

# numberical quadrature
r = numerical_quadrature([1,2,3], [1, 8, 27])
print(r)

# trapezoidal and simpson
f = lambda x: x*x
xs = [0, 2]
print(trapezoidal_rule(xs, [f(x) for x in xs]))
xs = [0, 1, 2]
print(simpson_rule(xs, [f(x) for x in xs]))

# closed newton-cotes formula
pi = np.pi
f  = np.sin
xs = [0, pi/4]
print(closed_newton_cotes(xs, [f(x) for x in xs]))
xs = [0, pi/8, pi/4]
print(closed_newton_cotes(xs, [f(x) for x in xs]))
xs = [0, pi/12, pi/6, pi/4]
print(closed_newton_cotes(xs, [f(x) for x in xs]))
xs = [0, pi/16, pi/8, 3*pi/16, pi/4]
print(closed_newton_cotes(xs, [f(x) for x in xs]))

# open newton-cotes formula
pi = np.pi
f  = np.sin
xs = [0, pi/8, pi/4]
print(open_newton_cotes(xs, [f(x) for x in xs[1:-1]]))
xs = [0, pi/12, pi/6, pi/4]
print(open_newton_cotes(xs, [f(x) for x in xs[1:-1]]))
xs = [0, pi/16, pi/8, 3*pi/16, pi/4]
print(open_newton_cotes(xs, [f(x) for x in xs[1:-1]]))
xs = [0, pi/20, pi/10, 3*pi/20, pi/5, pi/4]
print(open_newton_cotes(xs, [f(x) for x in xs[1:-1]]))

# exercise
f  = lambda x: x**4
xs = [0.5, 1]
print(trapezoidal_rule(xs, [f(x) for x in xs]))
xs = [0.5, 0.75, 1]
print(simpson_rule(xs, [f(x) for x in xs]))

pi = np.pi
f  = lambda x: x*np.sin(x)
xs = [0, pi/4]
print(trapezoidal_rule(xs, [f(x) for x in xs]))
xs = [0, pi/8, pi/4]
print(simpson_rule(xs, [f(x) for x in xs]))

xs  = [0, 2]
fxs = [2, 2]
print(trapezoidal_rule(xs, fxs))
xs  = [0, 1, 2]
fxs = [2, 1/2, 2]
print(simpson_rule(xs, fxs))

max_stars_repo_name: Iydon/NumericalAnalysisNotes

max_issues_repo_name: iydon/NumericalAnalysisNotes

max_forks_repo_name: iydon/NumericalAnalysisNotes


text:
# encoding utf-8
"""
Top level preprocessing classes.
"""
import logging

import numpy as np
import pandas as pd
from sklearn.decomposition import PCA
from sklearn.preprocessing import LabelEncoder

from automatminer.utils.pkg import (
    AutomatminerError,
    compare_columns,
    check_fitted,
    set_fitted,
)
from automatminer.utils.log import (
    log_progress,
    AMM_LOG_TRANSFORM_STR,
    AMM_LOG_FIT_STR,
)
from automatminer.utils.ml import regression_or_classification, AMM_REG_NAME
from automatminer.base import DFTransformer
from automatminer.preprocessing.feature_selection import (
    TreeFeatureReducer,
    rebate,
    lower_corr_clf,
)

__authors__ = [
    "Alex Dunn <ardunn@lbl.gov>",
    "Alireza Faghaninia <alireza@lbl.gov>",
    "Alex Ganose <aganose@lbl.gov>",
]

logger = logging.getLogger(__name__)

class DataCleaner(DFTransformer):
    """
    Transform a featurized dataframe into an ML-ready dataframe.

    Works by first removing samples not having a target value (if desired), then
    dropping features with high nan rates. Finally, removing or otherwise
    handling nans for individual samples (a relatively uncommon occurrence).

    Args:
        max_na_frac (float): The maximum fraction (0.0 - 1.0) of samples for a
            given feature allowed. Columns containing a higher nan fraction are
            handled according to feature_na_method.
        feature_na_method (str): Defines how to handle features (column) with
            higher na fraction than max_na_frac. "drop" for dropping these
            features. "fill" for filling these features with pandas bfill and
            ffill. "mean" to fill categorical variables and mean for numerical
            variables. Alternatively, specify a number to replace the nans,
            e.g. 0. If all samples are nan, feature will be dropped regardless.
        encode_categories (bool): If True, retains features which are
            categorical (data type is string or object) and then
            one-hot encodes them. If False, drops them.
        encoder (str): choose a method for encoding the categorical
            variables. Current options: 'one-hot' and 'label'.
        drop_na_targets (bool): Drop samples containing target values which are
            na.
        na_method_fit (str, float, int): Set the na_method for samples in fit.
            Select one of the following methods: "fill" (use pandas fillna with
            ffill and bfill, sequentially), "ignore" (totally ignore nans in
            samples), "drop" (drop any remaining samples having a nan feature),
            "mean" (fills categorical variables, takes means of numerical).
            Alternatively, specify a number to replace the nans, e.g. 0.
        na_method_transform (str, float, int): The same as na_method_fit, but
            for transform.

    Attributes:
        max_problem_col_warning_threshold (float): The max number of
            "problematic" columns (as a fraction of total columns) which are nan
            which are allowed before logging a warning.

        The following attrs are set during fitting.

        object_cols (list): The features identified as objects/categories
        number_cols (list): The features identified as numerical
        fitted_df (pd.DataFrame): The fitted dataframe
        fitted_target (str): The target variable in the dataframe.
        dropped_features (list): The features which were dropped.
        dropped_samples (pandas.DataFrame): A dataframe of samples to be dropped
        warnings ([str]): A list of warnings accumulated during fitting.
    """

    def __init__(
        self,
        max_na_frac=0.01,
        feature_na_method="drop",
        encode_categories=True,
        encoder="one-hot",
        drop_na_targets=True,
        na_method_fit="drop",
        na_method_transform="fill",
    ):
        self.max_na_frac = max_na_frac
        self.feature_na_method = feature_na_method
        self.encoder = encoder
        self.encode_categories = encode_categories
        self.drop_na_targets = drop_na_targets
        self.na_method_fit = na_method_fit
        self.na_method_transform = na_method_transform
        self._reset_attrs()
        self.dropped_features = None
        self.object_cols = None
        self.number_cols = None
        self.fitted_df = None
        self.fitted_target = None
        self.dropped_samples = None
        self.max_problem_col_warning_threshold = 0.3
        self.warnings = []
        super(DataCleaner, self).__init__()

    @property
    def retained_features(self):
        """
        The features retained during fitting, which may be used to craft the
        dataframe during transform.

        Returns:
            (list): The list of features retained.
        """
        return self.fitted_df.columns.tolist()

    @log_progress(logger, AMM_LOG_FIT_STR)
    @set_fitted
    def fit(self, df, target):
        """
        Determine a sequence of preprocessing steps to clean a dataframe.

        Args:
            df (pandas.DataFrame): Contains features and the target_key
            target (str): The name of the target in the dataframe

        Returns: self
        """

        logger.info(
            self._log_prefix + "Cleaning with respect to samples with sample "
            "na_method '{}'".format(self.na_method_fit)
        )
        if target not in df.columns:
            raise AutomatminerError(
                "Target {} must be contained in df.".format(target)
            )

        self._reset_attrs()
        df = self.to_numerical(df, target)
        df = self.handle_na(df, target, self.na_method_fit)
        self.fitted_df = df
        self.fitted_target = target
        return self

    @log_progress(logger, AMM_LOG_TRANSFORM_STR)
    @check_fitted
    def transform(self, df, target):
        """
        Apply the sequence of preprocessing steps determined by fit, with the
        option to change the na_method for samples.

        Args:
            df (pandas.DataFrame): Contains features and the target_key
            target (str): The name of the target in the dataframe

        Returns (pandas.DataFrame)
        """
        logger.info(
            self._log_prefix + "Cleaning with respect to samples with sample "
            "na_method '{}'".format(self.na_method_transform)
        )

        if target != self.fitted_target:
            raise AutomatminerError(
                "The transformation target {} is not the same as the fitted "
                "target {}".format(target, self.fitted_target)
            )

        # We assume the two targets are the same from here on out
        df = self.to_numerical(df, target)
        df = self.handle_na(
            df, target, self.na_method_transform, coerce_mismatch=True
        )

        # Ensure the order of columns is identical
        if target in df.columns:
            logger.info(self._log_prefix + "Reordering columns...")
            df = df[self.fitted_df.columns]
        else:
            logger.info(
                self._log_prefix + "Target not found in df columns. Ignoring..."
            )
            reordered_cols = self.fitted_df.drop(columns=[target]).columns
            df = df[reordered_cols]
        return df

    def fit_transform(self, df, target, **fit_kwargs):
        self.fit(df, target, **fit_kwargs)
        return self.fitted_df

    def handle_na(self, df, target, na_method, coerce_mismatch=True):
        """
        First pass for handling cells without values (null or nan). Additional
        preprocessing may be necessary as one column may be filled with
        median while the other with mean or mode, etc.

        Args:
            df (pandas.DataFrame): The dataframe containing features
            target (str): The key defining the ML target.
            coerce_mismatch (bool): If there is a mismatch between the fitted
                dataframe columns and the argument dataframe columns, create
                and drop mismatch columns so the dataframes are matching. If
                False, raises an error. New columns are instantiated as all
                zeros, as most of the time this is a onehot encoding issue.
            na_method (str): How to deal with samples still containing nans
                after troublesome columns are already dropped. Default is
                'drop'. Other options are from pandas.DataFrame.fillna:
                {‘bfill’, ‘pad’, ‘ffill’}, or 'ignore' to ignore nans.
                Alternatively, specify a value to replace the nans, e.g. 0.

        Returns:
            (pandas.DataFrame) The cleaned df
        """
        logger.info(
            self._log_prefix + "Before handling na: {} samples, {} features"
            "".format(*df.shape)
        )

        # Drop targets containing na before further processing
        if self.drop_na_targets and target in df.columns:
            clean_df = df.dropna(axis=0, how="any", subset=[target])
            self.dropped_samples = df[~df.index.isin(clean_df.index)]
            logger.info(
                self._log_prefix
                + "{} samples did not have target values. They were "
                "dropped.".format(len(self.dropped_samples))
            )
            df = clean_df

        # Remove features failing the max_na_frac limit
        feats0 = set(df.columns)
        if not self.is_fit:
            logger.info(
                self._log_prefix + "Handling feature na by max na threshold of {} "
                "with method '{}'.".format(self.max_na_frac, self.feature_na_method)
            )
            threshold = int((1 - self.max_na_frac) * len(df))
            all_problem_cols = df.columns[df.isnull().mean() > self.max_na_frac]
            n_problem_cols = all_problem_cols.shape[0]
            n_total_cols = df.shape[1]
            problem_col_frac = n_problem_cols / n_total_cols
            if problem_col_frac >= self.max_problem_col_warning_threshold:
                warning = (
                    f"Fraction {problem_col_frac} of all columns had nan "
                    f"percentages exceeding the nan threshold of "
                    f"{self.max_na_frac}. It is likely featurization was not "
                    f"effective. Please ensure your featurization input "
                    f"objects (e.g., compositions) are applicable for "
                    f"AutoFeaturizer! Examine your input data and pipeline "
                    f"in detail."
                )
                logger.error(self._log_prefix + warning)
                self.warnings.append(warning)
            if self.feature_na_method == "drop":
                df = df.dropna(axis=1, thresh=threshold)
            else:
                df = df.dropna(axis=1, thresh=1)
                problem_cols = df.columns[df.isnull().mean() > self.max_na_frac]
                dfp = df[problem_cols]
                if self.feature_na_method == "fill":
                    dfp = dfp.fillna(method="ffill")
                    dfp = dfp.fillna(method="bfill")
                elif self.feature_na_method == "mean":
                    # Take the mean of all numeric columns
                    dfpn = dfp[
                        [ncol for ncol in dfp.columns if ncol in self.number_cols]
                    ]
                    dfpn = dfpn.fillna(value=dfpn.mean())
                    dfp[dfpn.columns] = dfpn

                    # Simply fill one hot encoded columns
                    dfp = dfp.fillna(method="ffill")
                    dfp = dfp.fillna(method="bfill")
                else:
                    dfp = dfp.fillna(value=self.feature_na_method)
                df[problem_cols] = dfp

            if len(df.columns) < len(feats0):
                feats = set(df.columns)
                n_feats = len(feats0) - len(feats)
                napercent = self.max_na_frac * 100
                feat_names = feats0 - feats
                logger.info(
                    self._log_prefix
                    + "These {} features were removed as they had more "
                    "than {}% missing values: {}".format(
                        n_feats, napercent, feat_names
                    )
                )
        else:
            mismatch = compare_columns(self.fitted_df, df, ignore=target)
            if mismatch["mismatch"]:
                logger.warning(
                    self._log_prefix + "Mismatched columns found in dataframe "
                    "used for fitting and argument dataframe."
                )
                if coerce_mismatch:
                    logger.warning(
                        self._log_prefix + "Coercing mismatched columns..."
                    )
                    if mismatch["df1_not_in_df2"]:  # in fitted, not in arg
                        logger.warning(
                            self._log_prefix
                            + "Assuming missing columns in argument df are "
                            "one-hot encoding issues. Setting to zero the "
                            "following new columns:\n{}".format(
                                mismatch["df1_not_in_df2"]
                            )
                        )
                        for c in self.fitted_df.columns:
                            if c not in df.columns and c != target:
                                # Interpret as one-hot problems...
                                df[c] = np.zeros((df.shape[0]))
                    if mismatch["df2_not_in_df1"]:  # arg cols not in fitted
                        logger.warning(
                            self._log_prefix
                            + "Following columns are being dropped:\n{}".format(
                                mismatch["df2_not_in_df1"]
                            )
                        )
                        df = df.drop(columns=mismatch["df2_not_in_df1"])
                else:
                    raise AutomatminerError(
                        "Mismatch between columns found in "
                        "arg dataframe and dataframe used "
                        "for fitting!"
                    )

            # handle the case where all samples of transformed df are nan but
            # feature is required by fitted input df, and these is no way to
            # impute by samples or drop...
            nan_cols = [c for c in df.columns if df[c].isna().all()]
            if nan_cols:
                logger.error(
                    self._log_prefix + "Columns {} are all nan "
                    "in transform df but are required by the fit "
                    "df. Using mean values of fitted df to "
                    "impute transformed df. This may result in "
                    "highly erroenous imputed values!"
                    "".format(nan_cols)
                )
                for col in nan_cols:
                    mean_val = self.fitted_df[col].mean()
                    df[col] = [mean_val] * df.shape[0]

        self.dropped_features = [c for c in feats0 if c not in df.columns.values]

        # Handle all rows that still contain any nans
        if na_method == "drop":
            clean_df = df.dropna(axis=0, how="any")
            self.dropped_samples = pd.concat(
                (df[~df.index.isin(clean_df.index)], self.dropped_samples),
                axis=0,
                sort=True,
            )
            df = clean_df
        elif na_method == "ignore":
            pass
        elif na_method == "fill":
            df = df.fillna(method="ffill")
            df = df.fillna(method="bfill")
        elif na_method == "mean":
            # Samples belonging in number columns are averaged to replace na
            dfn = df[[ncol for ncol in df.columns if ncol in self.number_cols]]
            dfn = dfn.fillna(value=dfn.mean())
            df[dfn.columns] = dfn

            # the rest are simply filled
            df = df.fillna(method="ffill")
            df = df.fillna(method="bfill")
        else:
            df = df.fillna(value=na_method)
        logger.info(
            self._log_prefix
            + "After handling na: {} samples, {} features".format(*df.shape)
        )
        return df

    def to_numerical(self, df, target):
        """
        Transforms non-numerical columns to numerical columns which are
        machine learning-friendly.

        Args:
            df (pandas.DataFrame): The dataframe containing features
            target (str): The key defining the ML target.

        Returns:
            (pandas.DataFrame) The numerical df
        """
        logger.info(
            self._log_prefix + "Replacing infinite values with nan for easier "
            "screening."
        )
        df = df.replace([np.inf, -np.inf], np.nan)
        self.number_cols = []
        self.object_cols = []
        for c in df.columns.values:
            try:
                if df[c].dtype == bool:
                    df[c] = df[c].astype(int)
                else:
                    df[c] = pd.to_numeric(df[c])
                if c != target:
                    self.number_cols.append(c)
            except (TypeError, ValueError):
                # The target is most likely strings which are not numeric.
                # Prevent target being encoded
                if c != target:
                    self.object_cols.append(c)

        if target in df.columns:
            target_df = df[[target]]
        else:
            target_df = pd.DataFrame()
        number_df = df[self.number_cols]
        object_df = df[self.object_cols]
        if self.encode_categories and self.object_cols:
            if self.encoder == "one-hot":
                logger.info(
                    self._log_prefix
                    + "One-hot encoding used for columns {}".format(
                        object_df.columns.tolist()
                    )
                )
                object_df = pd.get_dummies(object_df).apply(pd.to_numeric)
            elif self.encoder == "label":
                logger.info(
                    self._log_prefix
                    + "Label encoding used for columns {}".format(
                        object_df.columns.tolist()
                    )
                )
                for c in object_df.columns:
                    object_df[c] = LabelEncoder().fit_transform(object_df[c])
                logger.warning(
                    self._log_prefix
                    + "LabelEncoder used for categorical colums. For access to "
                    "the original labels via inverse_transform, encode "
                    "manually and set retain_categorical to False"
                )
            return pd.concat([target_df, number_df, object_df], axis=1)
        else:
            return pd.concat([target_df, number_df], axis=1)

    def _reset_attrs(self):
        """
        Reset all fit-dependent attrs.

        Returns:
            None
        """
        self.dropped_features = None
        self.object_cols = None
        self.number_cols = None
        self.fitted_df = None
        self.fitted_target = None
        self.dropped_samples = None

class FeatureReducer(DFTransformer):
    """
    Perform feature reduction on a clean dataframe.

    Args:
        reducers ((str)): The set of feature reduction operations to be
            performed on the data. The order of strings determines the order
            in which the reducers will be applied. Valid reducer strings are
            the following:
                'corr': Removes any cross-correlated features having corr.
                    coefficients larger than a threshold value. Retains
                    feature names.

                'tree': Perform iterative feature reduction via a tree-based
                    feature reduction, using ._feature_importances implemented
                    in sklearn. Retains feature names.

                'rebate': Perform ReliefF feature reduction using the skrebate
                    package. Retains feature names.

                'pca': Perform Principal Component Analysis via
                    eigendecomposition. Note the feature labels will be renamed
                    to "PCA Feature X" if pca is present anywhere in the feature
                    reduction scheme!

            Example: Apply tree-based feature reduction, then pca:
                reducers = ('tree', 'pca')
        corr_threshold (float): The correlation threshold between any two
            features needed for one to be removed (calculated with R).
        tree_importance_percentile (float): the selected percentile (between 0.0
            and 1.0)of the features sorted (descending) based on their
            importance.
        n_pca_features (int, float): If int, the number of features to be
            retained by PCA. If float, the fraction of features to be retained
            by PCA once the dataframe is passed to it (i.e., 0.5 means PCA
            retains half of the features it is passed). PCA must be present in
            the reducers. 'auto' automatically determines the number of features
            to retain.
        n_rebate_features (int, float): If int, the number of ReBATE relief
            features to be retained. If float, the fraction of features to be
            retained by ReBATE once it is passed the dataframe (i.e., 0.5 means
            ReBATE retains half of the features it is passed). ReBATE must be
            present in the reducers.
        keep_features (list, None): A list of features that will not be removed.
            This option does nothing if PCA feature removal is present.
        remove_features (list, None): A list of features that will be removed.
            This option does nothing if PCA feature removal is present.

    Attributes:
        The following attrs are set during fitting.

        removed_features (dict): The keys are the feature reduction methods
            applied. The values are the feature labels removed by that feature
            reduction method.
        retained_features (list): The features retained.
        reducer_params (dict): The keys are the feature reduction methods
            applied. The values are the parameters used by each feature reducer.
    """

    def __init__(
        self,
        reducers=("pca",),
        corr_threshold=0.95,
        tree_importance_percentile=0.90,
        n_pca_features="auto",
        n_rebate_features=0.3,
        keep_features=None,
        remove_features=None,
    ):

        for reducer in reducers:
            if reducer not in ["corr", "tree", "rebate", "pca"]:
                raise ValueError(
                    "Reducer {} not found in known reducers!".format(reducer)
                )

        self.reducers = reducers
        self.corr_threshold = corr_threshold
        self.n_pca_features = n_pca_features
        self.tree_importance_percentile = tree_importance_percentile
        self.n_rebate_features = n_rebate_features
        self._keep_features = keep_features or []
        self._remove_features = remove_features or []
        self.removed_features = {}
        self.retained_features = []
        self.reducer_params = {}
        self._pca = None
        self._pca_feats = None
        super(FeatureReducer, self).__init__()

    @log_progress(logger, AMM_LOG_FIT_STR)
    @set_fitted
    def fit(self, df, target):
        missing_remove_features = [
            c for c in self._remove_features if c not in df.columns
        ]
        missing_keep_features = [
            c for c in self._keep_features if c not in df.columns
        ]
        for features, name in [
            (missing_remove_features, "remove"),
            (missing_keep_features, "keep"),
        ]:
            if features:
                logger.warning(
                    self._log_prefix
                    + "Asked to {} some features that do not exist in the "
                    "dataframe. Skipping the following features:\n{}".format(
                        name, features
                    )
                )

        reduced_df = df
        for r in self.reducers:
            X = df.drop(columns=[target])
            y = df[target]
            if r == "corr":
                reduced_df = self.rm_correlated(df, target, self.corr_threshold)
                reduced_df = reduced_df.drop(columns=[target])
            if r == "tree":
                tbfr = TreeFeatureReducer(
                    importance_percentile=self.tree_importance_percentile,
                    mode=regression_or_classification(y),
                )
                reduced_df = tbfr.fit_transform(X, y).copy(deep=True)
                self.reducer_params[r] = {
                    "importance_percentile": tbfr.importance_percentile,
                    "mode": tbfr.mode,
                    "random_state": tbfr.rs,
                }
            elif r == "rebate":
                if isinstance(self.n_rebate_features, float):
                    logger.info(
                        self._log_prefix
                        + "Retaining fraction {} of current {} features.".format(
                            self.n_rebate_features, df.shape[1] - 1
                        )
                    )
                    self.n_rebate_features = int(
                        df.shape[1] * self.n_rebate_features
                    )
                logger.info(
                    self._log_prefix
                    + "ReBATE MultiSURF* running: retaining {} numerical "
                    "features.".format(self.n_rebate_features)
                )
                reduced_df = rebate(df, target, n_features=self.n_rebate_features)
                reduced_df = reduced_df.copy(deep=True)
                logger.info(
                    self._log_prefix
                    + "ReBATE MultiSURF* completed: retained {} numerical "
                    "features.".format(len(reduced_df.columns))
                )
                logger.debug(
                    self._log_prefix + "ReBATE MultiSURF* gave the following "
                    "features: {}".format(reduced_df.columns.tolist())
                )
                self.reducer_params[r] = {"algo": "MultiSURF* Algorithm"}
            elif r == "pca":
                n_samples, n_features = X.shape
                if self.n_pca_features == "auto":
                    if n_samples < n_features:
                        logger.warning(
                            self._log_prefix
                            + "Number of samples ({}) is less than number of "
                            "features ({}). Setting n_pca_features equal to "
                            "n_samples.".format(n_samples, n_features)
                        )
                        self._pca = PCA(n_components=n_samples, svd_solver="full")
                    else:
                        logger.info(
                            self._log_prefix
                            + "PCA automatically determining optimal number of "
                            "features using Minka's MLE."
                        )
                        self._pca = PCA(n_components="mle", svd_solver="auto")
                else:
                    if isinstance(self.n_pca_features, float):
                        logger.info(
                            self._log_prefix
                            + "Retaining fraction {} of current {} features."
                            "".format(self.n_pca_features, df.shape[1])
                        )
                        self.n_pca_features = int(df.shape[1] * self.n_pca_features)
                    if self.n_pca_features > n_samples:
                        logger.warning(
                            self._log_prefix
                            + "Number of PCA features interpreted as {}, which"
                            "  is more than the number of samples ({}). "
                            "n_pca_features coerced to equal n_samples."
                            "".format(self.n_pca_features, n_samples)
                        )
                        self.n_pca_features = n_samples
                    logger.info(
                        self._log_prefix
                        + "PCA running: retaining {} numerical features."
                        "".format(self.n_pca_features)
                    )
                    self._pca = PCA(
                        n_components=self.n_pca_features, svd_solver="auto"
                    )
                self._pca.fit(X.values, y.values)
                matrix = self._pca.transform(X.values)
                pca_feats = ["PCA {}".format(i) for i in range(matrix.shape[1])]
                self._pca_feats = pca_feats
                reduced_df = pd.DataFrame(
                    columns=pca_feats, data=matrix, index=X.index
                )
                logger.info(
                    self._log_prefix + "PCA completed: retained {} numerical "
                    "features.".format(len(reduced_df.columns))
                )

            retained = reduced_df.columns.values.tolist()
            removed = [
                c for c in df.columns.values if c not in retained and c != target
            ]

            self.removed_features[r] = removed
            if target not in reduced_df:
                reduced_df.loc[:, target] = y.tolist()
            df = reduced_df

        all_removed = [c for r, rf in self.removed_features.items() for c in rf]
        all_kept = [c for c in df.columns.tolist() if c != target]
        save_from_removal = [c for c in self._keep_features if c in all_removed]
        for_force_removal = [c for c in self._remove_features if c in all_kept]

        if save_from_removal:
            logger.info(
                self._log_prefix + "Saving features from removal. "
                "Saved features:\n{}".format(save_from_removal)
            )

        if for_force_removal:
            logger.info(
                self._log_prefix + "Forcing removal of features. "
                "Removed features: \n{}".format(for_force_removal)
            )
            self.removed_features["manual"] = for_force_removal

        self.retained_features = [
            c for c in all_kept if c not in self._remove_features or c != target
        ]
        return self

    @log_progress(logger, AMM_LOG_TRANSFORM_STR)
    @check_fitted
    def transform(self, df, target):
        if target not in df.columns:
            logger.warning(
                self._log_prefix + "Target not found in columns to transform."
            )
            X = df
        else:
            X = df.drop(columns=target)
        for r, f in self.removed_features.items():
            if r == "pca":
                matrix = self._pca.transform(X)
                X = pd.DataFrame(columns=self._pca_feats, data=matrix, index=X.index)
            else:
                X = X.drop(columns=[c for c in f if c not in self._keep_features])
        if target in df:
            X.loc[:, target] = df[target].values
        return X

    def rm_correlated(self, df, target, r_max=0.95):
        """
        A feature selection method that remove those that are cross correlated
        by more than threshold.

        Args:
            df (pandas.DataFrame): The dataframe containing features, target_key
            target (str): the name of the target column/feature
            r_max (0<float<=1): if R is greater than this value, the
                feature that has lower correlation with the target is removed.

        Returns (pandas.DataFrame):
            the dataframe with the highly cross-correlated features removed.
        """
        mode = regression_or_classification(df[target])
        corr = abs(df.corr())
        if mode == AMM_REG_NAME:
            corr = corr.sort_values(by=target)
        rm_feats = []
        for feat in corr.columns:
            if feat == target:
                continue
            for idx, corval in zip(corr.index, corr[feat]):
                if np.isnan(corval):
                    break
                if idx == feat or idx in rm_feats:
                    continue
                else:
                    if corval >= r_max:
                        if mode == AMM_REG_NAME:
                            if corr.loc[idx, target] > corr.loc[feat, target]:
                                removed_feat = feat
                            else:
                                removed_feat = idx
                        else:  # mode is classification
                            removed_feat = lower_corr_clf(df, target, feat, idx)
                        if removed_feat not in rm_feats:
                            rm_feats.append(removed_feat)
                            logger.debug(
                                self._log_prefix + '"{}" correlates strongly with '
                                '"{}"'.format(feat, idx)
                            )
                            logger.debug(
                                self._log_prefix
                                + 'removing "{}"...'.format(removed_feat)
                            )
                        if removed_feat == feat:
                            break
        if len(rm_feats) > 0:
            df = df.drop(rm_feats, axis=1)
            logger.info(
                self._log_prefix
                + "{} features removed due to cross correlation more than {}"
                "".format(len(rm_feats), r_max)
            )
            logger.debug(
                self._log_prefix + "Features removed by cross-correlation were: {}"
                "".format(rm_feats)
            )
        return df

ext: py

max_stars_repo_name: sgbaird/automatminer

max_issues_repo_name: sgbaird/automatminer

max_forks_repo_name: sgbaird/automatminer

reason: import numpy


text:
'''
Making labels from the xlsx files from the 2014/12 new data.

Information to extract

sequence information

labels in each image

each column

5: image name
6: coordinate label

'''
import xlrd
import os
import scipy.misc
import matplotlib.pylab as plt
import numpy as np
import cPickle as pkl

def get_info(xlsx_dir):
    '''
    TODO: extract more useful information
    '''
    flag_onlyfirst = True

    xlsx_list = [file_name for file_name in os.listdir(xlsx_dir)
                 if '.xlsx' == file_name[-5:]]

    label_dict = {}

    for xlsx_name in xlsx_list:
        xlsx_path = os.path.join(xlsx_dir, xlsx_name)
        workbook = xlrd.open_workbook(xlsx_path)
        worksheet = workbook.sheet_by_index(0)
        num_rows = worksheet.nrows

        for ind in range(1, num_rows):
            base_status = 0

            if flag_onlyfirst:
                base_status = int(worksheet.cell_value(ind, 7))

            if not base_status:

                img_name = worksheet.cell_value(ind, 5)
                label_str = worksheet.cell_value(ind, 6)
                xy_list = [[int(val) for val in item.split(',')]
                           for item in label_str.split('|')] \
                    if len(label_str) else []

                label_dict[img_name] = xy_list

    return label_dict

def show_image(img_dir, img_name, label_dict):

    img_path = os.path.join(img_dir, img_name)

    img = scipy.misc.imread(img_path)

    plt.figure()
    plt.imshow(img)
    if len(label_dict[img_name]):
        points = np.array(label_dict[img_name]) / 100. * [640., 480.]
        plt.hold('on')
        plt.scatter(points[:, 0], points[:, 1])
    plt.show()

if __name__ == '__main__':

    xlsx_dir = "/mnt/data/datasets/automoth_1412"

    label_dict = get_info(xlsx_dir)

    num_moth_dict = {}
    num_moth_list = []
    img_name_list = sorted(label_dict.keys())

    for img_name in img_name_list:
        num_moth_dict[img_name] = len(label_dict[img_name])
        num_moth_list.append(len(label_dict[img_name]))

    img_name = img_name_list[np.argmax(num_moth_list)]

    # label_path = os.path.join(xlsx_dir, 'labels.pkl')
    # with open(label_path, 'wb') as f:
    #     pkl.dump(label_dict, f)

    xlsx_path = "/mnt/data/datasets/automoth_1412/Some_Farms.xlsx"
    img_dir = "/mnt/data/datasets/automoth_1412/imgs"
    # img_name = '6272_186176.jpg'
    # img_name = '2985_126898.jpg'

    show_image(img_name=img_name, img_dir=img_dir, label_dict=label_dict)

    img_ind = np.where(np.array(img_name_list) == img_name)[0][0]

    show_image(img_name=img_name_list[img_ind - 1], img_dir=img_dir, label_dict=label_dict)
    show_image(img_name=img_name_list[img_ind - 4], img_dir=img_dir, label_dict=label_dict)

    # if only use the first image of each sequence (578 sequences in total)
    # then only have 27 images with moth and 155 moths

max_stars_repo_name: uoguelph-mlrg/cnn-moth-detection

max_issues_repo_name: uoguelph-mlrg/cnn-moth-detection

max_forks_repo_name: uoguelph-mlrg/cnn-moth-detection

reason: import numpy,import scipy


text:
'''
DPMixModel.py
Bayesian parametric mixture model with a unbounded number of components K

Attributes
-------
  K        : # of components
  alpha0   : scalar concentration hyperparameter of Dirichlet process prior

  qalpha0 : K-length vector, params for variational factor q(v)
  qalpha1 : K-length vector, params for variational factor q(v)
            q(v[k]) ~ Beta(qalpha1[k], qalpha0[k])

  truncType : str type of truncation for the Dirichlet Process
              'z' : truncate on the assignments [default]
           or 'v' : truncate stick-breaking distribution
'''
import numpy as np

from bnpy.allocmodel import AllocModel
from bnpy.suffstats import SuffStatBag
from bnpy.util import NumericUtil
from bnpy.util import gammaln, digamma, EPS

class DPMixModel(AllocModel):

  ######################################################### Constructors
  #########################################################
  def __init__(self, inferType, priorDict=None):
    if inferType == 'EM':
      raise ValueError('EM not supported for DPMixModel')
    self.inferType = inferType
    if priorDict is None:
      self.alpha0 = 1.0 # Uniform!
      self.alpha1 = 1.0
      self.truncType = 'z'
    else:
      self.set_prior(priorDict)
    self.K = 0

  def set_prior(self, PriorParamDict):
    self.alpha1 = 1.0
    self.alpha0 = PriorParamDict['alpha0']
    self.truncType = PriorParamDict['truncType']

  def set_helper_params( self ):
    ''' Set dependent attributes given primary global params.
        For DP mixture, this means precomputing digammas.
    '''
    DENOM = digamma(self.qalpha0 + self.qalpha1)
    self.ElogV      = digamma(self.qalpha1) - DENOM
    self.Elog1mV    = digamma(self.qalpha0) - DENOM

    if self.truncType == 'v':
      self.qalpha1[-1] = 1
      self.qalpha0[-1] = EPS # avoid digamma(0), which is way too HUGE
      self.ElogV[-1] = 0  # log(1) => 0
      self.Elog1mV[-1] = np.log(1e-40) # log(0) => -INF, never used

		# Calculate expected mixture weights E[ log w_k ]
    self.Elogw = self.ElogV.copy() #copy so we can do += without modifying ElogV
    self.Elogw[1:] += self.Elog1mV[:-1].cumsum()

  ######################################################### Accessors
  #########################################################
  def get_keys_for_memoized_local_params(self):
    ''' Return list of string names of the LP fields
        that this object needs to memoize across visits to a particular batch
    '''
    return list()

  ######################################################### Local Params
  #########################################################
  def calc_local_params(self, Data, LP, **kwargs):
    ''' Calculate local parameters for each data item and each component.
        This is part of the E-step.

        Args
        -------
        Data : bnpy data object with Data.nObs observations
        LP : local param dict with fields
              E_log_soft_ev : Data.nObs x K array
                  E_log_soft_ev[n,k] = log p(data obs n | comp k)

        Returns
        -------
        LP : local param dict with fields
              resp : Data.nObs x K array whose rows sum to one
              resp[n,k] = posterior responsibility that comp. k has for data n
    '''
    lpr = LP['E_log_soft_ev']
    lpr += self.Elogw
    # Calculate exp in numerically stable manner (first subtract the max)
    #  perform this in-place so no new allocations occur
    NumericUtil.inplaceExpAndNormalizeRows(lpr)
    LP['resp'] = lpr
    assert np.allclose(lpr.sum(axis=1), 1)
    return LP

  ######################################################### Suff Stats
  #########################################################
  def get_global_suff_stats(self, Data, LP,
                             doPrecompEntropy=False,
                             doPrecompMergeEntropy=False, mPairIDs=None):
    ''' Calculate the sufficient statistics for global parameter updates
        Only adds stats relevant for this allocModel.
        Other stats are added by the obsModel.

        Args
        -------
        Data : bnpy data object
        LP : local param dict with fields
              resp : Data.nObs x K array,
                       where resp[n,k] = posterior resp of comp k
        doPrecompEntropy : boolean flag
                      indicates whether to precompute ELBO terms in advance
                      used for memoized learning algorithms (moVB)
        doPrecompMergeEntropy : boolean flag
                      indicates whether to precompute ELBO terms in advance
                      for all possible merges of pairs of components
                      used for optional merge moves

        Returns
        -------
        SS : SuffStats for K components, with field
              N : vector of length-K,
                   effective number of observations assigned to each comp
    '''
    Nvec = np.sum(LP['resp'], axis=0)
    SS = SuffStatBag(K=Nvec.size, D=Data.dim)
    SS.setField('N', Nvec, dims=('K'))
    if doPrecompEntropy:
      ElogqZ_vec = self.E_logqZ(LP)
      SS.setELBOTerm('ElogqZ', ElogqZ_vec, dims=('K'))
    if doPrecompMergeEntropy:
      # Hmerge : KxK matrix of entropies for all possible pair-wise merges
      # for example, if we had only 3 components {0,1,2}
      # Hmerge = [ 0 H(0,1) H(0,2)
      #            0   0    H(1,2)
      #            0   0      0 ]
      #  where H(i,j) is entropy if components i and j merged.
      Hmerge = np.zeros((self.K, self.K))
      for jj in range(self.K):
        compIDs = np.arange(jj+1, self.K)
        Rcombo = LP['resp'][:,jj][:,np.newaxis] + LP['resp'][:,compIDs]
        Hmerge[jj,compIDs] = np.sum(Rcombo*np.log(Rcombo+EPS), axis=0)
      SS.setMergeTerm('ElogqZ', Hmerge, dims=('K','K'))
    return SS

  ######################################################### Global Params
  #########################################################
  def update_global_params_VB( self, SS, **kwargs ):
    ''' Updates global params (stick-breaking Beta params qalpha1, qalpha0)
          for conventional VB learning algorithm.
        New parameters have exactly the number of components specified by SS.
    '''
    self.K = SS.K
    qalpha1 = self.alpha1 + SS.N
    qalpha0 = self.alpha0 * np.ones(self.K)
    qalpha0[:-1] += SS.N[::-1].cumsum()[::-1][1:]
    self.qalpha1 = qalpha1
    self.qalpha0 = qalpha0
    self.set_helper_params()

  def update_global_params_soVB( self, SS, rho, **kwargs ):
    ''' Update global params (stick-breaking Beta params qalpha1, qalpha0).
        for stochastic online VB.
    '''
    assert self.K == SS.K
    qalpha1 = self.alpha1 + SS.N
    qalpha0 = self.alpha0 * np.ones( self.K )
    qalpha0[:-1] += SS.N[::-1].cumsum()[::-1][1:]

    self.qalpha1 = rho * qalpha1 + (1-rho) * self.qalpha1
    self.qalpha0 = rho * qalpha0 + (1-rho) * self.qalpha0
    self.set_helper_params()

  def set_global_params(self, hmodel=None, K=None, qalpha1=None,
                              qalpha0=None, **kwargs):
    ''' Directly set global parameters qalpha0, qalpha1 to provided values
    '''
    if hmodel is not None:
      self.K = hmodel.allocModel.K
      self.qalpha1 = hmodel.allocModel.qalpha1
      self.qalpha0 = hmodel.allocModel.qalpha0
      self.set_helper_params()
      return
    if type(qalpha1) != np.ndarray or qalpha1.size != K or qalpha0.size != K:
      raise ValueError("Bad DP Parameters")
    self.K = K
    self.qalpha1 = qalpha1
    self.qalpha0 = qalpha0
    self.set_helper_params()

  ######################################################### Evidence
  #########################################################
  def calc_evidence(self, Data, SS, LP=None ):
    '''
    '''
    evV = self.E_logpV() - self.E_logqV()
    if SS.hasELBOTerm('ElogqZ'):
      evZq = np.sum(SS.getELBOTerm('ElogqZ'))
    else:
      evZq = np.sum(self.E_logqZ(LP))
    if SS.hasAmpFactor():
      evZ = self.E_logpZ(SS) -  SS.ampF * evZq
    else:
      evZ = self.E_logpZ(SS) - evZq
    return evZ + evV

  def E_logpZ(self, SS):
    return np.inner( SS.N, self.Elogw )

  def E_logqZ(self, LP):
    return np.sum(LP['resp'] * np.log(LP['resp']+EPS), axis=0)

  def E_logpV( self ):
    logNormC = gammaln(self.alpha0 + self.alpha1) \
                    - gammaln(self.alpha0) - gammaln(self.alpha1)
    logBetaPDF = (self.alpha1-1)*self.ElogV + (self.alpha0-1)*self.Elog1mV
    if self.truncType == 'z':
	    return self.K*logNormC + logBetaPDF.sum()
    elif self.truncType == 'v':
      return self.K*logNormC + logBetaPDF[:-1].sum()

  def E_logqV( self ):
    logNormC = gammaln(self.qalpha0 + self.qalpha1) \
                      - gammaln(self.qalpha0) - gammaln(self.qalpha1)
    logBetaPDF = (self.qalpha1-1)*self.ElogV + (self.qalpha0-1)*self.Elog1mV
    if self.truncType == 'z':
      return logNormC.sum() + logBetaPDF.sum()
    elif self.truncType == 'v':
      # skip last entry because entropy of Beta(1,0) = 0
      return logNormC[:-1].sum() + logBetaPDF[:-1].sum()

  ######################################################### IO Utils
  #########################################################   for humans
  def get_info_string( self):
    ''' Returns one-line human-readable terse description of this object
    '''
    msgPattern = 'DP mixture with K=%d. Concentration alpha0= %.2f'
    return msgPattern % (self.K, self.alpha0)

  ######################################################### IO Utils
  #########################################################   for machines
  def to_dict(self):
    return dict(qalpha1=self.qalpha1, qalpha0=self.qalpha0)

  def from_dict(self, myDict):
    self.inferType = myDict['inferType']
    self.K = myDict['K']
    self.qalpha1 = myDict['qalpha1']
    self.qalpha0 = myDict['qalpha0']
    if self.qalpha0.ndim == 0:
      self.qalpha0 = self.qalpha1[np.newaxis]
    if self.qalpha0.ndim == 0:
      self.qalpha0 = self.qalpha0[np.newaxis]
    self.set_helper_params()

  def get_prior_dict(self):
    return dict(alpha1=self.alpha1, alpha0=self.alpha0, K=self.K,
                  truncType=self.truncType)


text:
from PIL import Image, ImageEnhance, ImageOps
import numpy as np
import random

from torchvision import transforms

""" 1. Graduatally increase magnitude
    2. Increase number of operationis in one policy
"""

class PolicySeq(object):
    """magnitude increase when each subpolicy has two operations
    """
    def __init__(self, mag_idx=4, prob=0.5, n_op=2, fillcolor=(128, 128, 128)):
        """

        Args:
        ----------
            mag idx:
            fillcolor (tuple, optional):  Defaults to (128, 128, 128).
        """
        self.op_pool = ["autocontrast", "equalize", "invert"]
        #self.op_pool = ["posterize",  "sharpness", "contrast", "brightness", "color", "solarize"]   # "posterize",  "sharpness", "contrast", "brightness", "color", solarize
        #self.op_pool = ["autocontrast", "equalize", "invert", "posterize",  "sharpness", "contrast", "brightness", "color", "solarize"]

        self.prob = prob
        self.n_op = n_op
        self.magnitude_idx = mag_idx
        self.fillcolor = fillcolor

    def __call__(self, img):
        op_seq = []
        index_seq = []
        # repeat allow
        # for _ in range(self.n_op):
        #     policy_idx = random.randint(0, len(self.op_pool) - 1)
        #     op_seq.append((self.prob, self.op_pool[policy_idx], self.magnitude_idx))

        # repeat not allow
        op_choices = random.sample(self.op_pool, self.n_op)
        for op_name in op_choices:
            op_seq.append((self.prob, op_name, self.magnitude_idx))

        policy = SubPolicySeq(op_seq, self.fillcolor)

        return policy(img)

    def __repr__(self):
        return "Augment Magnitude Increase Policy"

class PolicySeqBlackBG(object):
    """magnitude increase when each subpolicy has two operations
    """
    def __init__(self, mag_idx=4, prob=0.5, n_op=2, fillcolor=(128, 128, 128)):
        """

        Args:
        ----------
            mag idx:
            fillcolor (tuple, optional):  Defaults to (128, 128, 128).
        """
        self.op_pool = ["autocontrast", "equalize", "invert"]
        #self.op_pool = ["posterize",  "sharpness", "contrast", "brightness", "color", "solarize"]   # "posterize",  "sharpness", "contrast", "brightness", "color", solarize
        #self.op_pool = ["autocontrast", "equalize", "invert", "posterize",  "sharpness", "contrast", "brightness", "color", "solarize"]

        self.prob = prob
        self.n_op = n_op
        self.magnitude_idx = mag_idx
        self.fillcolor = fillcolor
        self.toPIL = transforms.ToPILImage()
        self.transform = transforms.Compose([
            transforms.CenterCrop(127),
            transforms.Resize(size=224, interpolation=2),
            transforms.ToTensor()
        ])

    def __call__(self, img):
        img_origin_arr = np.asarray(img)
        img = img.convert('RGB')
        op_seq = []

        # repeat not allow
        op_choices = random.sample(self.op_pool, self.n_op)
        for op_name in op_choices:
            op_seq.append((self.prob, op_name, self.magnitude_idx))

        policy = SubPolicySeq(op_seq, self.fillcolor)
        img_aug = policy(img)
        img_aug_arr = np.asarray(img_aug)

        img_aug_arr = img_aug_arr.copy()
        img_aug_arr[img_origin_arr[:, :, 3] == 0] = [0, 0, 0]
        img_aug = self.toPIL(img_aug_arr)
        img_aug = self.transform(img_aug)

        return img_aug

    def __repr__(self):
        return "Augment Magnitude Increase Policy"

class GeometricPolicySeq(object):
    """magnitude increase when each subpolicy has two operations
    """
    def __init__(self, n_op=2, rotate_degree=30, out_size=127, fillcolor=(128, 128, 128)):
        """

        Args:
        ----------
            mag idx:
            fillcolor (tuple, optional):  Defaults to (128, 128, 128).
        """
        self.op_pool = [transforms.RandomCrop(out_size),
                transforms.RandomHorizontalFlip(),
                transforms.RandomRotation((-1 * rotate_degree, rotate_degree))] #

        self.n_op = n_op
        self.fillcolor = fillcolor
        self.out_size = out_size
        self.center_crop = transforms.CenterCrop(out_size)

    def __call__(self, img):

        # repeat not allow
        op_choices = random.sample(self.op_pool, self.n_op)
        for op in op_choices:
            img = op(img)

        if img.size[0] > self.out_size:
            img = self.center_crop(img)

        return img

    def __repr__(self):
        return "Geometric Increase Policy"

class GeometricPolicyMag(object):
    """magnitude increase when each subpolicy has two operations
    """
    def __init__(self, mag_idx=0, max_angle=60, max_trans_ratio=0.3, level=3, out_size=127, fillcolor=(128, 128, 128)):
        """

        Args:
        ----------
            mag idx:
            fillcolor (tuple, optional):  Defaults to (128, 128, 128).
        """

        self.mag_idx = mag_idx
        degree = (max_angle / level) * mag_idx
        trans_ratio = (max_trans_ratio / level) * mag_idx
        print(f"Random Geometric Augmentation, Leval:{mag_idx}, Rot:{degree}, Trans:{trans_ratio}")
        self.transform = transforms.RandomAffine(degrees=degree, translate=(trans_ratio, trans_ratio))
        self.fillcolor = fillcolor

        self.center_crop = transforms.CenterCrop(out_size)
        self.toTensor = transforms.ToTensor()

    def __call__(self, img):
        img = self.transform(img)
        img = self.center_crop(img)

        return img

    def __repr__(self):
        return "Geometric Increase Policy"

class ColorAugFore(object):
    """

    Args:
        object ([type]): [description]

    Returns:
        [type]: [description]
    """
    def __init__(self, magnitude):
        self.magnitude = magnitude     # 0 10 20 30
        self.toPIL = transforms.ToPILImage()
    def __call__(self, img):
        img_raw = np.array(img)
        img_arr = img_raw[:, :, :3]
        r_aug = 2 * self.magnitude * np.random.rand() - self.magnitude
        g_aug = 2 * self.magnitude * np.random.rand() - self.magnitude
        b_aug = 2 * self.magnitude * np.random.rand() - self.magnitude
        img_arr[img_raw[:, :, 3] != 0, 0] = img_arr[img_raw[:, :, 3] != 0, 0] + r_aug
        img_arr[img_raw[:, :, 3] != 0, 1] = img_arr[img_raw[:, :, 3] != 0, 1] + g_aug
        img_arr[img_raw[:, :, 3] != 0, 2] = img_arr[img_raw[:, :, 3] != 0, 2] + b_aug

        img_arr = np.clip(img_arr, 0, 255)
        img = self.toPIL(img_arr)

        return img

    def __repr__(self):
        return super().__repr__()

class SubPolicySeq(object):
    def __init__(self, op_seq, fillcolor=(128, 128, 128)):
        self.ranges = {
            "shearX": np.linspace(0, 0.3, 10),
            "shearY": np.linspace(0, 0.3, 10),
            "translateX": np.linspace(0, 150 / 331, 10),
            "translateY": np.linspace(0, 150 / 331, 10),
            "rotate": np.linspace(0, 30, 10),
            "color": np.linspace(0.0, 0.9, 10),
            "posterize": np.round(np.linspace(8, 4, 10), 0).astype(np.int),
            "solarize": np.linspace(256, 0, 10),
            "contrast": np.linspace(0.0, 0.9, 10),
            "sharpness": np.linspace(0.0, 0.9, 10),
            "brightness": np.linspace(0.0, 0.9, 10),
            "autocontrast": [0] * 10,
            "equalize": [0] * 10,
            "invert": [0] * 10
        }

        # from https://stackoverflow.com/questions/5252170/specify-image-filling-color-when-rotating-in-python-with-pil-and-setting-expand
        def rotate_with_fill(img, magnitude):
            rot = img.convert("RGBA").rotate(magnitude)
            return Image.composite(rot, Image.new("RGBA", rot.size, (128,) * 4), rot).convert(img.mode)

        self.func = {
            "shearX": lambda img, magnitude: img.transform(
                img.size, Image.AFFINE, (1, magnitude * random.choice([-1, 1]), 0, 0, 1, 0),
                Image.BICUBIC, fillcolor=fillcolor),
            "shearY": lambda img, magnitude: img.transform(
                img.size, Image.AFFINE, (1, 0, 0, magnitude * random.choice([-1, 1]), 1, 0),
                Image.BICUBIC, fillcolor=fillcolor),
            "translateX": lambda img, magnitude: img.transform(
                img.size, Image.AFFINE, (1, 0, magnitude * img.size[0] * random.choice([-1, 1]), 0, 1, 0),
                fillcolor=fillcolor),
            "translateY": lambda img, magnitude: img.transform(
                img.size, Image.AFFINE, (1, 0, 0, 0, 1, magnitude * img.size[1] * random.choice([-1, 1])),
                fillcolor=fillcolor),
            "rotate": lambda img, magnitude: rotate_with_fill(img, magnitude),
            "color": lambda img, magnitude: ImageEnhance.Color(img).enhance(1 + magnitude * random.choice([-1, 1])),
            "posterize": lambda img, magnitude: ImageOps.posterize(img, magnitude),
            "solarize": lambda img, magnitude: ImageOps.solarize(img, magnitude),
            "contrast": lambda img, magnitude: ImageEnhance.Contrast(img).enhance(
                1 + magnitude * random.choice([-1, 1])),
            "sharpness": lambda img, magnitude: ImageEnhance.Sharpness(img).enhance(
                1 + magnitude * random.choice([-1, 1])),
            "brightness": lambda img, magnitude: ImageEnhance.Brightness(img).enhance(
                1 + magnitude * random.choice([-1, 1])),
            "autocontrast": lambda img, magnitude: ImageOps.autocontrast(img),
            "equalize": lambda img, magnitude: ImageOps.equalize(img),
            "invert": lambda img, magnitude: ImageOps.invert(img)
        }
        self.op_seq = op_seq

    def __call__(self, img):

        for op in self.op_seq:
            p, op_name, magnitude_idx = op
            operation = self.func[op_name]
            magnitude = self.ranges[op_name][magnitude_idx]
            if random.random() < p:
                #print(op_name, magnitude, "Reported")
                img = operation(img, magnitude)
        return img

    def __repr__(self):
        rep = ""
        for op in self.op_seq:
            rep += f"({op[0]} {op[1]} {op[2]})   "

        return rep

class SubPolicy(object):
    def __init__(self, p1, operation1, magnitude_idx1, p2, operation2, magnitude_idx2, fillcolor=(128, 128, 128)):
        ranges = {
            "shearX": np.linspace(0, 0.3, 10),
            "shearY": np.linspace(0, 0.3, 10),
            "translateX": np.linspace(0, 150 / 331, 10),
            "translateY": np.linspace(0, 150 / 331, 10),
            "rotate": np.linspace(0, 30, 10),
            "color": np.linspace(0.0, 0.9, 10),
            "posterize": np.round(np.linspace(8, 4, 10), 0).astype(np.int),
            "solarize": np.linspace(256, 0, 10),
            "contrast": np.linspace(0.0, 0.9, 10),
            "sharpness": np.linspace(0.0, 0.9, 10),
            "brightness": np.linspace(0.0, 0.9, 10),
            "autocontrast": [0] * 10,
            "equalize": [0] * 10,
            "invert": [0] * 10
        }

        # from https://stackoverflow.com/questions/5252170/specify-image-filling-color-when-rotating-in-python-with-pil-and-setting-expand
        def rotate_with_fill(img, magnitude):
            rot = img.convert("RGBA").rotate(magnitude)
            return Image.composite(rot, Image.new("RGBA", rot.size, (128,) * 4), rot).convert(img.mode)

        func = {
            "shearX": lambda img, magnitude: img.transform(
                img.size, Image.AFFINE, (1, magnitude * random.choice([-1, 1]), 0, 0, 1, 0),
                Image.BICUBIC, fillcolor=fillcolor),
            "shearY": lambda img, magnitude: img.transform(
                img.size, Image.AFFINE, (1, 0, 0, magnitude * random.choice([-1, 1]), 1, 0),
                Image.BICUBIC, fillcolor=fillcolor),
            "translateX": lambda img, magnitude: img.transform(
                img.size, Image.AFFINE, (1, 0, magnitude * img.size[0] * random.choice([-1, 1]), 0, 1, 0),
                fillcolor=fillcolor),
            "translateY": lambda img, magnitude: img.transform(
                img.size, Image.AFFINE, (1, 0, 0, 0, 1, magnitude * img.size[1] * random.choice([-1, 1])),
                fillcolor=fillcolor),
            "rotate": lambda img, magnitude: rotate_with_fill(img, magnitude),
            "color": lambda img, magnitude: ImageEnhance.Color(img).enhance(1 + magnitude * random.choice([-1, 1])),
            "posterize": lambda img, magnitude: ImageOps.posterize(img, magnitude),
            "solarize": lambda img, magnitude: ImageOps.solarize(img, magnitude),
            "contrast": lambda img, magnitude: ImageEnhance.Contrast(img).enhance(
                1 + magnitude * random.choice([-1, 1])),
            "sharpness": lambda img, magnitude: ImageEnhance.Sharpness(img).enhance(
                1 + magnitude * random.choice([-1, 1])),
            "brightness": lambda img, magnitude: ImageEnhance.Brightness(img).enhance(
                1 + magnitude * random.choice([-1, 1])),
            "autocontrast": lambda img, magnitude: ImageOps.autocontrast(img),
            "equalize": lambda img, magnitude: ImageOps.equalize(img),
            "invert": lambda img, magnitude: ImageOps.invert(img)
        }

        self.p1 = p1
        self.operation1 = func[operation1]
        self.magnitude1 = ranges[operation1][magnitude_idx1]
        self.p2 = p2
        self.operation2 = func[operation2]
        self.magnitude2 = ranges[operation2][magnitude_idx2]

    def __call__(self, img):
        if random.random() < self.p1: img = self.operation1(img, self.magnitude1)
        if random.random() < self.p2: img = self.operation2(img, self.magnitude2)
        return img

if __name__ == "__main__":
    pass

max_stars_repo_name: nsfzyzz/dispersion-score

max_issues_repo_name: nsfzyzz/dispersion-score

max_forks_repo_name: nsfzyzz/dispersion-score


text:
#!/usr/bin/env python
"""
Evaluate the stability (i.e. agreement) between a set of term rankings generated on the same dataset, using Average Term Stability (ATS).

Sample usage:
python eval-term-stability.py models/base/ranks*.pkl
python eval-term-stability.py models/base/
"""
import os, sys
import logging as log
from optparse import OptionParser
import numpy as np
from prettytable import PrettyTable
import unsupervised.nmf, unsupervised.rankings, unsupervised.util

# --------------------------------------------------------------

def main():
	parser = OptionParser(usage="usage: %prog [options] rank_file1|directory1 ...")
	parser.add_option("-t", "--top", action="store", type="int", dest="top", help="number of top terms to use", default=10)
	parser.add_option("-o","--output", action="store", type="string", dest="out_path", help="path for CSV output file", default=None)
	# Parse command line arguments
	(options, args) = parser.parse_args()
	if( len(args) < 1 ):
		parser.error( "Must specify one or more term rankings or directories" )
	log.basicConfig(level=20, format='%(message)s')

	# Get list of all specified term ranking files
	file_paths = []
	for path in args:
		if not os.path.exists( path ):
			log.error("No such file or directory: %s" % path )
			sys.exit(1)
		if os.path.isdir(path):
			log.info("Searching %s for term ranking files" % path )
			for dir_path, dirs, files in os.walk(path):
				for fname in files:
					if fname.startswith("ranks") and fname.endswith(".pkl"):
						file_paths.append( os.path.join( dir_path, fname ) )
		else:
			file_paths.append( path )
	file_paths.sort()
	if len(file_paths) == 0:
		log.error("No term ranking files found to validate")
		sys.exit(1)
	log.info( "Processing %d topic models ..." % len(file_paths) )

	# Load cached ranking sets
	all_term_rankings = []
	for rank_path in file_paths:
		log.debug( "Loading term ranking set from %s ..." % rank_path )
		(term_rankings,labels) = unsupervised.util.load_term_rankings( rank_path )
		log.debug( "Set has %d rankings covering %d terms" % ( len(term_rankings), unsupervised.rankings.term_rankings_size( term_rankings ) ) )
		# do we need to truncate the number of terms in the ranking?
		if options.top > 1:
			term_rankings = unsupervised.rankings.truncate_term_rankings( term_rankings, options.top )
			log.debug( "Truncated to %d -> set now has %d rankings covering %d terms" % ( options.top, len(term_rankings), unsupervised.rankings.term_rankings_size( term_rankings ) ) )
		all_term_rankings.append( term_rankings )

	r = len(all_term_rankings)
	metric = unsupervised.rankings.JaccardBinary()
	matcher = unsupervised.rankings.RankingSetAgreement( metric )

	# Perform pairwise comparisons evaluation for all models
	log.info( "Evaluating stability %d base term rankings with %s and top %d terms ..." % (r, str(metric), options.top) )
	all_scores = []
	for i in range(r):
		for j in range(i+1,r):
			try:
				score = matcher.similarity( all_term_rankings[i], all_term_rankings[j] )
				all_scores.append( score )
			except Exception as e:
				log.warning("Error occurred comparing pair (%d,%d): %s" % ( i, j, str(e) ) )
	log.info("Compared %d pairs of term rankings" % len(all_scores) )

	# Get overall score across all pairs
	all_scores = np.array( all_scores )
	tab = PrettyTable( ["statistic","stability"] )
	tab.align["statistic"] = "l"
	tab.add_row( [ "mean", "%.3f" % all_scores.mean() ] )
	tab.add_row( [ "median", "%.3f" %  np.median(all_scores) ] )
	tab.add_row( [ "sdev", "%.3f" % all_scores.std() ] )
	tab.add_row( [ "min", "%.3f" % all_scores.min() ] )
	tab.add_row( [ "max", "%.3f" % all_scores.max() ] )
	log.info( tab )

	# Write to CSV?
	if not options.out_path is None:
		log.info("Writing summary of results to %s" % options.out_path)
		unsupervised.util.write_table( options.out_path, tab )

# --------------------------------------------------------------

if __name__ == "__main__":
	main()

max_stars_repo_name: derekgreene/topic-ensemble

max_issues_repo_name: derekgreene/topic-ensemble

max_forks_repo_name: derekgreene/topic-ensemble


text:
"""Script to extract all identifiers from a set of PPI files and convert them
to UniProt ACs.

NOTE: the local mapping option cannot discern between reviewed and unreviewed
      entries!

"""

import argparse
import numpy as np
import pandas as pd
import sys

from pathlib import Path

from phppipy.ppi_tools import id_mapper
from phppipy.ppi_tools import ppi_import
from phppipy.ppi_tools import ppi_filter

parser = argparse.ArgumentParser(
    description='Script to extract all identifiers from a set of PPI files and convert them to UniProt ACs',
    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
    '-i',
    '--input',
    dest='input',
    type=str,
    required=True,
    help='Directory with PPI interaction files.')
parser.add_argument(
    '-m',
    '--mapping',
    dest='mapping',
    type=str,
    required=False,
    help='Full mapping file from EBI GOA. Omitting this will default to the online UniProt mapping service.')
parser.add_argument(
    '-o',
    '--output',
    dest='output',
    type=str,
    required=True,
    help='Output directory. File is saved as "all_identifiers.txt"')
args = parser.parse_args()

# Import PPI files
input_dir = Path(args.input)
mitab_files = input_dir.glob('*.mitab')
ppi_df_list = [ppi_import.read_mi_tab(i) for i in mitab_files if i.is_file()]
phisto_files = input_dir.glob('phi*.csv')
mi_file = input_dir / 'mi.obo'
ppi_df_list.extend(
    [ppi_import.read_mitab_phisto(i, mi_file) for i in phisto_files])

# Merge PPI datasets
for i in ppi_df_list:
    i['origin'] = i.name
ppi_df = pd.concat(ppi_df_list, axis=0, join='outer', ignore_index=True, sort=True)

# remap to UniProt AC
id_mapper.check_unique_identifier(ppi_df)
out_mappings_dir = Path(args.output) / 'mapping'
if args.mapping:
    full_mapping_path = Path(args.mapping)
else:
    full_mapping_path = None
try:
    out_mappings_dir.mkdir(parents=True, exist_ok=False)
    # omitting args.mapping defaults it to None, which prompts the map2uniprot function to use the online service
    id_mapper.map2uniprot(ppi_df, out_mappings_dir, reviewed_only=True, full_mapping_file=full_mapping_path)
except FileExistsError:
    print(
        'Warning: supplied output directory already contains a "mapping" directory, aborting operation.'
    )
    sys.exit(1)

# remove multiple mappings
ppi_df = id_mapper.remove_mult(ppi_df)
# TODO: save these somewhere or deal with them in a way

# create unique identifier by combining xrefs
ppi_filter.unique_identifier(ppi_df)

# remove duplicates
ppi_df = ppi_df.drop_duplicates(subset=['xref_partners_sorted'], keep='first')
ppi_df = ppi_df.reset_index(drop=True)

# remove non-UniProt identifiers
ppi_df = ppi_df.loc[(ppi_df.xref_A.str.contains('uniprot'))
                    & (ppi_df.xref_B.str.contains('uniprot'))]
ppi_df = ppi_df.reset_index(drop=True)

# save protein list for filtering gaf/interpro files
all_identifiers = pd.Series(
    pd.unique(ppi_df[['xref_A',
                      'xref_B']].values.ravel('K'))).str.split(':').str.get(1)
out_identifiers = Path(args.output) / 'all_identifiers.txt'
out_identifiers.parent.mkdir(parents=True, exist_ok=True)

with out_identifiers.open('w') as out:
    for i in all_identifiers:
        out.write("{}\n".format(i))

print('Saved list of all UniProtACs to {}'.format(out_identifiers))

max_stars_repo_name: pmoris/host-pathogen-ppi-fim

max_issues_repo_name: pmoris/host-pathogen-ppi-fim

max_forks_repo_name: pmoris/host-pathogen-ppi-fim


text:
from typing import Callable

import numpy as np
from torch.utils.data import TensorDataset

from knodle.trainer.trainer import Trainer

class AutoTrainer:
    """ The factory class for creating training executors
    See See https://medium.com/@geoffreykoh/implementing-the-factory-
    pattern-via-dynamic-registry-and-python-decorators-479fc1537bbe
    """

    registry = {}
    """ Internal registry for available trainers """

    def __init__(self, name, **kwargs):
        self.trainer = self.create_trainer(name, **kwargs)

    def train(
            self,
            model_input_x: TensorDataset = None, rule_matches_z: np.ndarray = None,
            dev_model_input_x: TensorDataset = None, dev_gold_labels_y: TensorDataset = None
    ):
        return self.trainer.train(model_input_x, rule_matches_z, dev_model_input_x, dev_gold_labels_y)

    def test(self, test_features: TensorDataset, test_labels: TensorDataset):
        return self.trainer.test(test_features, test_labels)

    @classmethod
    def create_trainer(cls, name: str, **kwargs) -> Trainer:
        """ Factory command to create the executor """

        exec_class = cls.registry[name]
        executor = exec_class(**kwargs)
        return executor

    @classmethod
    def register(cls, name: str) -> Callable:
        def inner_wrapper(wrapped_class: Trainer) -> Callable:
            cls.registry[name] = wrapped_class
            return wrapped_class

        return inner_wrapper

max_stars_repo_name: michael-aloys/knodle

max_issues_repo_name: michael-aloys/knodle

max_forks_repo_name: michael-aloys/knodle


text:
from __future__ import print_function

from tensorflow.python.keras.models import Model, load_model
from tensorflow.python.keras.layers import Input, LSTM, Dense, Embedding
import numpy as np

from src.utils import one_hot_encode, add_n

class Seq2Seq:
    def __init__(self, num_encoder_tokens, num_decoder_tokens, start_token, end_token,
                 latent_dim=256, projection='one_hot', emb_dim=64,
                 restore_path=None):
        self.num_encoder_tokens = num_encoder_tokens
        self.num_decoder_tokens = num_decoder_tokens

        self.start_token = start_token
        self.end_token = end_token

        if restore_path:
            self.train_model = load_model(restore_path + '/train_model.s2s.h5')
            self.encoder_model = load_model(restore_path + '/encoder_model.s2s.h5')
            self.decoder_model = load_model(restore_path + '/decoder_model.s2s.h5')
            return

        # Define an input sequence and process it.
        encoder_inputs = Input(shape=(None,), name='encoder_input')
        if projection == 'one_hot':
            embedding = Embedding(
                num_encoder_tokens+1,
                num_encoder_tokens,
                weights=[np.concatenate((np.zeros(shape=(1, num_encoder_tokens)), np.identity(num_encoder_tokens)))],
                trainable=False,
                mask_zero=True,
                name='encoder_embedding'
            )
        elif projection == 'word2vec':
            embedding = Embedding(
                num_encoder_tokens + 1,
                emb_dim,
                mask_zero=True,
                name='encoder_embedding'
            )
        else:
             raise Exception("projection method not recognized")
        encoder_emb = embedding(encoder_inputs)
        encoder = LSTM(latent_dim, return_state=True, name='encoder_lstm')
        encoder_outputs, state_h, state_c = encoder(encoder_emb)
        # We discard `encoder_outputs` and only keep the states.
        encoder_states = [state_h, state_c]

        # Set up the decoder, using `encoder_states` as initial state.
        decoder_inputs = Input(shape=(None,), name='decoder_input')
        if projection == 'one_hot':
            embedding = Embedding(
                num_decoder_tokens+1,
                num_decoder_tokens,
                weights=[np.concatenate((np.zeros(shape=(1, num_decoder_tokens)), np.identity(num_decoder_tokens)))],
                trainable=False,
                mask_zero=True,
                name='decoder_embedding'
            )
        elif projection == 'word2vec':
            embedding = Embedding(
                num_decoder_tokens+1,
                emb_dim,
                mask_zero=True,
                name='decoder_embedding'
            )
        else:
            raise Exception("projection method not recognized")
        decoder_emb = embedding(decoder_inputs)
        # We set up our decoder to return full output sequences,
        # and to return internal states as well. We don't use the
        # return states in the training model, but we will use them in inference.
        decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True, name='decoder_lstm')
        decoder_outputs, _, _ = decoder_lstm(decoder_emb,
                                             initial_state=encoder_states)
        decoder_dense = Dense(self.num_decoder_tokens, activation='softmax', name='decoder_softmax')
        decoder_outputs = decoder_dense(decoder_outputs)

        # Define the model that will turn
        # `encoder_input_data` & `decoder_input_data` into `decoder_target_data`
        self.train_model = Model([encoder_inputs, decoder_inputs], decoder_outputs)

        # Define sampling models
        self.encoder_model = Model(encoder_inputs, encoder_states)

        decoder_state_input_h = Input(shape=(latent_dim,), name='decoder_state_input_h')
        decoder_state_input_c = Input(shape=(latent_dim,), name='decoder_state_input_c')
        decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]
        decoder_outputs, state_h, state_c = decoder_lstm(
            decoder_emb, initial_state=decoder_states_inputs)
        decoder_states = [state_h, state_c]
        decoder_outputs = decoder_dense(decoder_outputs)
        self.decoder_model = Model(
            [decoder_inputs] + decoder_states_inputs,
            [decoder_outputs] + decoder_states)

    def pre_process_encoder_input(self, input):
        # return one_hot_encode(input, self.max_encoder_seq_length, self.num_encoder_tokens)
        return add_n(input, self.max_encoder_seq_length, 1)

    def pre_process_decoder_input(self, input):
        # return one_hot_encode(input, self.max_decoder_seq_length, self.num_decoder_tokens)
        return add_n(input, self.max_decoder_seq_length, 1)

    def pre_process_decoder_target(self, input):
        # return one_hot_encode(input, self.max_decoder_seq_length, self.num_decoder_tokens)
        return add_n(input, self.max_decoder_seq_length, 0)

    def train(self, encoder_input_seqs, decoder_input_seqs, decoder_target_seqs, batch_size=64, epochs=100):
        # Run training
        self.max_encoder_seq_length = max([len(seq) for seq in encoder_input_seqs])
        self.max_decoder_seq_length = max([len(seq) for seq in decoder_input_seqs])
        encoder_input_data = self.pre_process_encoder_input(encoder_input_seqs)
        print('encoder_input_data.shape: ', encoder_input_data.shape)
        decoder_input_data = self.pre_process_decoder_input(decoder_input_seqs)
        print('decoder_input_data.shape: ', decoder_input_data.shape)
        decoder_target_data = self.pre_process_decoder_target(decoder_target_seqs)
        print('decoder_target_data.shape: ', decoder_target_data.shape)
        self.train_model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy')
        self.train_model.fit([encoder_input_data, decoder_input_data], decoder_target_data,
                  batch_size=batch_size,
                  epochs=epochs,
                  validation_split=0.2)
        # Save model
        self.train_model.save('train_model.s2s.h5')
        self.encoder_model.save('encoder_model.s2s.h5')
        self.decoder_model.save('decoder_model.s2s.h5')

    def encode(self, input_seq):
        # encoder_input_data = one_hot_encode(input_seq, self.max_encoder_seq_length, self.num_encoder_tokens)
        encoder_input_data = self.pre_process_encoder_input(input_seq)
        return self.encoder_model.predict(encoder_input_data)

    def decode(self, states_value):
        # Generate empty target sequence of length 1.
        # target_seq = np.zeros((1, 1, self.num_decoder_tokens))
        # Populate the first token of target sequence with the start token.
        # target_seq[0, 0, self.start_token] = 1.
        target_seq = np.zeros((1, 1))
        target_seq[0, 0] = self.start_token + 1
        # Sampling loop for a batch of sequences
        # (to simplify, here we assume a batch of size 1).
        stop_condition = False
        decoded_seq = []
        while not stop_condition:
            output_tokens, h, c = self.decoder_model.predict(
                [target_seq] + states_value)

            # Sample a token
            sampled_token_index = np.argmax(output_tokens[0, -1, :])
            decoded_seq += [sampled_token_index]

            # Exit condition: either hit max length
            # or find stop token.
            if (sampled_token_index == self.end_token or
                    len(decoded_seq) > self.max_decoder_seq_length):
                stop_condition = True

            # Update the target sequence (of length 1).
            # target_seq = np.zeros((1, 1, self.num_decoder_tokens))
            # target_seq[0, 0, sampled_token_index] = 1.
            target_seq = np.zeros((1, 1))
            target_seq[0, 0] = sampled_token_index + 1

            # Update states
            states_value = [h, c]

        return decoded_seq

    def predict(self, input_seq):
        # Encode the input as state vectors.
        states_value = self.encode(input_seq)
        prediction = self.decode(states_value)

        return prediction


text:
import ald
import numpy as np
import h5py

U0 = 1.0
tauR = 1.0
alpha = 1.2
particle = ald.Pareto(U0=U0, tauR=tauR, alpha=alpha)
# sedimentation velocity
Us = 0.1
# sedimentation is downward with a negative sign.
flow = ald.ConstantUy(U=-Us)
Lx = 1.0
# length in y is important
Ly = 20 * U0 * tauR
# wall at y=0.
domain = ald.Box(left=-Lx / 2, right=Lx / 2, bottom=0, top=Ly)

ic = ald.InitialConfig(
    x=ald.Uniform(domain.left, domain.right),
    y=ald.Uniform(domain.bottom, domain.top),
    theta=ald.Uniform(0, 2 * np.pi),
)

cfg = ald.Config(particle, domain, N=204_800, dt=1e-4, Nt=4_000_000)

kernel = ald.RTPAboveWallKernel()
compiler = ald.RTPCompiler(kernel, cfg, flow, ic)

compiler.compile()

simulator = ald.RTPSimulator(cfg, compiler)
if isinstance(particle, ald.Pareto):
    tag = "pareto"
elif isinstance(particle, ald.ExponentialRTP):
    tag = "rtp"
else:
    tag = ""

file = "U{:.3f}tauR{:.3f}sediment_{}.h5".format(U0, tauR, tag)
# create an empty file
# with h5py.File(file, "w") as f:
#    pass

# save configuraton.
# callback controller
# x is periodic, unwrap to save the absolute position
# y no need to do so
configsaver = ald.ConfigSaver(
    ald.RangedRunner.from_backward_count(stop=cfg.Nt, freq=10000, count=200),
    file,
    variables=["x", "y"],
    unwrap=[True, False],
)

# range to compute stats on configuration and print time.
runner = ald.RangedRunner(start=0, stop=cfg.Nt, freq=10000)
callbacks = [configsaver, ald.ETA(runner)]

simulator.run(cfg, callbacks=callbacks)

# save particle, domain and simulation attributes.
cfg.save2h5(file)
# save mean variance of x
# x.save2h5(file, "x")
# y.save2h5(file, "y")


text:
import tensorflow as tf
import numpy as np
from tensorflow.keras.utils import to_categorical

from ml_utils.keras import get_states, set_states #for RNN compatibility
from ml_utils.model_builders import dense_stack

def create_traces(model):
  traces = []

  for var in model.trainable_variables:
    trace = tf.Variable(np.zeros(var.shape, dtype="float32"), trainable=False)
    traces.append(trace)

  return traces

#here, y_true should be some action that we took instead of y_pred (ie, from exploration)
def update_traces(model, states, traces, x, y_true, loss, lambda_=0.9):
  """states=the original states at the beginning of the step"""
  #save current states
  saved_states = get_states(model)
  #set to pre-step state
  set_states(model, states)
  with tf.GradientTape() as tape:
    y_pred = model(x)
    loss_value = loss(y_true, y_pred)
    grads = tape.gradient(loss_value, model.trainable_variables)
    for grad, trace in zip(grads, traces):
      trace.assign(trace * lambda_ + grad * (1 - lambda_))# + tf.clip_by_norm(grad, clip_norm = clip_norm))
  #now that we calculated our traces, restore state to the old post-step state
  #  (this probably doesnt need to be done unless the model is stochastic or something...)
  set_states(model, saved_states)

def step_weights(model, traces, lr, reward):
  #step direction
  alpha = lr * reward
  for weight, trace in zip(model.trainable_variables, traces):
    weight.assign_add(-trace * alpha) #gradient descent modulated by reward

def step_weights_opt(model, traces, reward, optimizer):
  traces = [t * reward for t in traces] #modulate by reward
  #then apply normally
  optimizer.apply_gradients(zip(traces, model.trainable_variables))

## UTILS ##
def categorical_crossentropy(y_true, y_pred):
  #yes, this is (probably) bad practice
  _epsilon = 1e-7 #TODO: hardcoded epsilon
  #use keras trick to recover logits from softmax:
  #https://github.com/keras-team/keras/issues/11801
  y_pred = tf.clip_by_value(y_pred, _epsilon, 1 - _epsilon)
  y_pred = tf.log(y_pred)
  return tf.losses.softmax_cross_entropy(y_true, y_pred)

def binary_crossentropy(y_true, y_pred):
  return tf.keras.losses.binary_crossentropy(y_true, y_pred)

def explore_continuous(x, noise_stdev=0.1):
  return x + np.random.normal(0, noise_stdev, x.shape).astype("float32") #y u output float64

def explore_discrete(x, epsilon=0.01):
  x = np.array(x)
  x = x / np.sum(x)
  #choose randomly
  action = np.random.choice(len(x), p=x)
  #add eps.greedy on top
  if np.random.random() < epsilon:
    action = np.random.choice(len(x))

  return to_categorical(action, len(x))

def explore_multibinary(xs, epsilon=0.01):
  action = [np.random.choice([0, 1], p=[1-x, x]) for x in xs]
  action = [(np.random.choice([0, 1]) if np.random.random() < epsilon else x) for x in action]

  return np.array(action)

#TODO: tuple and multidiscrete

max_stars_repo_name: tehZevo/pget

max_issues_repo_name: tehZevo/pget

max_forks_repo_name: tehZevo/pget


text:
import os
import numpy as np
import time
import argparse
import yaml
import torch
import fasttext
from transformers import BertModel
from gensim.models import KeyedVectors
from feature_extraction import TextFeatureExtraction
from feature_extraction import bert_embeddings
from utils import load_text_embeddings
from utils import load_text_dataset, check_balance,\
    convert_to_fasttext_data, save_model,\
    train_basic_segment_classifier

script_dir = os.path.dirname(__file__)
eps = np.finfo(float).eps
seed = 500
if not script_dir:
    with open(r'./config.yaml') as file:
        config = yaml.load(file, Loader=yaml.FullLoader)
else:
    with open(script_dir + '/config.yaml') as file:
        config = yaml.load(file, Loader=yaml.FullLoader)

config = config['text_classifier']

def basic_segment_classifier(data, feature_extractor, pretrained, out_model):
    """
    Reads the config file and trains a classifier. It stores a model_dict
    containing the following information:
        - classifier: the output classifier
        - classifier_type: the type of the output classifier (i.e basic)
        - classifier_classnames: the name of the classes
        - embedding_model: path to the used embeddings model
        - embeddings_limit: possible limit on embeddings vectors

    :param data: csv file with one column transcriptions (text samples)
                   and one column labels
    :param feature_extractor: an initialized TextFeatureExtraction object
    :param pretrained: the path of embeddings model
    :param out_model: name of the output model
    :return None
    """

    np.random.seed(seed)

    print('--> Loading Dataset...')
    transcriptions, labels, classnames = load_text_dataset(
        data, config['hop_samples'])

    is_imbalanced = check_balance(labels)

    # extract features based on pretrained fasttext model

    if pretrained == "bert":
        use_cuda = torch.cuda.is_available()
        device = torch.device("cuda:0" if use_cuda else "cpu")
        bert = BertModel.from_pretrained('bert-base-cased', output_hidden_states=True)
        total_features, _, max_len = bert_embeddings(transcriptions, labels, bert, device=device)
    else:
        total_features, labels = feature_extractor.transform(transcriptions,labels)

    clf = train_basic_segment_classifier(
        total_features, labels, is_imbalanced, config, seed)

    model_dict = {}
    model_dict['classifier_type'] = 'basic'
    model_dict['classifier'] = clf
    model_dict['classifier_classnames'] = classnames
    model_dict['embedding_model'] = pretrained
    if pretrained != "bert":
        model_dict['embeddings_limit'] = feature_extractor.embeddings_limit
    else:
        model_dict['max_len'] = max_len
    out_folder = config['out_folder']
    if out_model is None:
        save_model(model_dict,out_folder, name="basic_classifier")
    else:
        save_model(model_dict,out_folder, out_model=out_model)

    return None

def train_fasttext_segment_classifier(data, embeddings_limit, out_model):
    """
        Reads the config file and uses the pretrained_embedding_vectors
        variable to train a classifier. It stores the
        generated classifier with name fasttext_classifier_{time}.ftz
        and a model_dict containing the following information:
            - fasttext_model: path to the output classifier
            - classifier_type: the type of the output classifier (i.e fasttext)
            - classifier_classnames: the name of the classes

        :param data: csv file with one column transcriptions (text samples)
                    and one column labels with labels written as __label__name
                    (eg. __label__act_low)
        :param embeddings_limit: a possible limit to the embeddings vectors
        :param out_model: name of the output model
        :return None
        """

    np.random.seed(seed)

    class_file_name = out_model + "_classenames.csv"

    pretrained_embedding_vectors = config['pretrained_embedding_vectors']
    print("--> Loading the text embeddings model")
    word_model = KeyedVectors.load_word2vec_format(
        pretrained_embedding_vectors, limit=embeddings_limit)
    word_model.save_word2vec_format('tmp.vec')
    print('--> Loading Dataset...')
    transcriptions, labels, classnames = load_text_dataset(
        data, config['hop_samples'])

    convert_to_fasttext_data(labels, transcriptions)

    print("--> Training classifier using fasttext")
    model = fasttext.train_supervised(
        input='train.txt', epoch=25, lr=1.0,
        wordNgrams=2, verbose=2, minCount=1,
        loss="softmax", dim=300, pretrainedVectors='tmp.vec',
        seed=seed)

    print("--> Performance measures on test set")
    num_of_samples, precision, recall = model.test("test.txt")
    print("Number of samples:", num_of_samples)
    print("Precision:", precision)
    print("Recall", recall)

    #timestamp = time.ctime()
    name = "fasttext_classifier_{}.ftz".format(out_model)
    out_folder = config["out_folder"]
    if not script_dir:
        if not os.path.exists(out_folder):
            os.makedirs(out_folder)
        out_path = os.path.join(out_folder, name)
    else:
        out_folder = os.path.join(script_dir, out_folder)
        if not os.path.exists(out_folder):
            os.makedirs(out_folder)
        out_path = os.path.join(out_folder, name)

    model.save_model(out_path)
    model_dict = {}
    model_dict['classifier_type'] = 'fasttext'
    model_dict['fasttext_model'] = out_path
    model_dict['classifier_classnames'] = classnames

    if out_model is None:
        save_model(model_dict,out_folder, name="FASTTEXT")
    else:
        save_model(model_dict,out_folder, out_model=out_model)

    return None

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("-a", "--annotation", required=True,
                        help="the path of our data samples "
                             "(csv with column transcriptions "
                             "and one column labels)")
    parser.add_argument("-p", "--pretrained", required=True,
                        help="the path of fasttext pretrained model "
                             "(.bin file)")
    parser.add_argument("-o", "--outputmodelname", required=False,
                        default="SVM",
                        help="the name of final model")
    parser.add_argument('-l', '--embeddings_limit', required=False,
                        default=None, type=int,
                        help='Limit of embeddings used. '
                             'Prefered value: 500000')

    args = parser.parse_args()

    if args.pretrained == "bert":
        basic_segment_classifier(args.annotation, None, args.pretrained,
                                 args.outputmodelname)
    else:
        if config['fasttext']:
            train_fasttext_segment_classifier(
                args.annotation, args.embeddings_limit, args.outputmodelname)

        elif config['svm'] or config['xgboost']:
            word_model = load_text_embeddings(args.pretrained,
                                              args.embeddings_limit)
            feature_extractor = TextFeatureExtraction(word_model,
                                                      args.embeddings_limit)
            basic_segment_classifier(args.annotation, feature_extractor, args.pretrained,
                                     args.outputmodelname)
        else:
            print('SVM and fasttext are the only supported classifiers.')

max_stars_repo_name: tyiannak/readys

max_issues_repo_name: tyiannak/readys

max_forks_repo_name: tyiannak/readys