text:
# -*- coding: utf-8 -*-
# -------------------------------------------------------------------------
# 文件目的：演示通过前向逐步回归算法来控制过拟合
# 创建日期：2018/2/3
# -------------------------------------------------------------------------

import numpy as np
from bk.pa.common import open_url
from sklearn import linear_model
from sklearn.metrics import roc_curve, auc
import pylab as pl

"""
前向逐步回归算法
"""

target_url = "http://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv"

def xattrSelect(x, idxSet):
    """

    :param x:
    :param idxSet:
    :return:
    """
    xOut = []
    for row in x:
        xOut.append([row[i] for i in idxSet])
    return xOut

# ----------------------------------------------------------------------------
# ~ Main
if __name__ == '__main__':

    # 存放标签和数据到列表中
    xList = []
    labels = []  # 判别标志
    names = []  # 字段标题

    # 首行存储字段标题
    firstLine = True

    # 读入Winequality数据集，并且将其解析为包含标签与属性的记录
    with open_url(target_url) as data:
        for line in data:
            if firstLine:
                names = line.strip().split(";")
                firstLine = False
            else:
                row = line.strip().split(";")
                labels.append(float(row[-1]))
                row.pop()
                floatRow = [float(num) for num in row]
                xList.append(floatRow)

    # 将数据集分为2个子集：
    indices = range(len(xList))  # 总行数
    # 测试集xListTest包含1/3的数据
    xListTest = [xList[i] for i in indices if i % 3 == 0]
    # 训练集xListTrain包含2/3的数据
    xListTrain = [xList[i] for i in indices if i % 3 != 0]
    labelsTest = [labels[i] for i in indices if i % 3 == 0]
    labelsTrain = [labels[i] for i in indices if i % 3 != 0]

    # 检查下各数据形态
    xTrain = np.array(xListTrain);
    yTrain = np.array(labelsTrain)
    xTest = np.array(xListTest);
    yTest = np.array(labelsTest)

    print("Shape of xTrain array", xTrain.shape)
    print("Shape of yTrain array", yTrain.shape)
    print("Shape of xTest array", xTest.shape)
    print("Shape of yTest array", yTest.shape)

    # 构建属性集
    attributeList = []
    index = range(len(xList[1]))
    indexSet = set(index)
    indexSeq = []
    oosError = []

    for i in index:
        attSet = set(attributeList)

        attTrySet = indexSet - attSet

        attTry = [ii for ii in attTrySet]
        errorList = []
        attTemp = []

        for iTry in attTry:
            attTemp = [] + attributeList
            attTemp.append(iTry)

max_stars_repo_name: oraocp/pystat

max_issues_repo_name: oraocp/pystat

max_forks_repo_name: oraocp/pystat


text:
from __future__ import absolute_import
import sys
sys.path.append("install/lib-faiss")  # noqa
import numpy
import sklearn.preprocessing
import ctypes
import faiss
from ann_benchmarks.algorithms.base import BaseANN

class Faiss(BaseANN):
    def query(self, v, n):
        if self._metric == 'angular':
            v /= numpy.linalg.norm(v)
        D, I = self.index.search(numpy.expand_dims(
            v, axis=0).astype(numpy.float32), n)
        return I[0]

    def batch_query(self, X, n):
        if self._metric == 'angular':
            X /= numpy.linalg.norm(X)
        self.res = self.index.search(X.astype(numpy.float32), n)

    def get_batch_results(self):
        D, L = self.res
        res = []
        for i in range(len(D)):
            r = []
            for l, d in zip(L[i], D[i]):
                if l != -1:
                    r.append(l)
            res.append(r)
        return res

class FaissLSH(Faiss):
    def __init__(self, metric, n_bits):
        self._n_bits = n_bits
        self.index = None
        self._metric = metric
        self.name = 'FaissLSH(n_bits={})'.format(self._n_bits)

    def fit(self, X):
        if X.dtype != numpy.float32:
            X = X.astype(numpy.float32)
        f = X.shape[1]
        self.index = faiss.IndexLSH(f, self._n_bits)
        self.index.train(X)
        self.index.add(X)

class FaissPQ(Faiss):
    def __init__(self, metric, n_M, n_bits):
        self._metric = metric
        self._n_M = n_M
        self._n_bits = n_bits
        self.name = self.__str__()

    def fit(self, X):
        print("start to fit, X.shape[1]=%d, summary=%s" % (X.shape[1], self.__str__()))

        index = faiss.IndexPQ(
            X.shape[1], self._n_M, self._n_bits, faiss.METRIC_L2)
        index.train(X)
        index.add(X)
        self.index = index

    def __str__(self):
        return 'FaissPQ(n_M=%d, n_bits=%d)' % (self._n_M, self._n_bits)

ext: py

max_stars_repo_name: cynricshu/ann-benchmarks

max_issues_repo_name: cynricshu/ann-benchmarks

max_forks_repo_name: cynricshu/ann-benchmarks

reason: import numpy


text:
import pandas as pd
import numpy as np
import datetime
import math

store_dct = {1: 'Marion', 2: 'Olive', 3: 'U-Village', 4: 'Lenora',
             5: 'Thomas-Boren', 6: 'Fremont', 7: 'Bellevue City Center',
             8: 'One Union', 9: 'Pioneer Square', 10: 'INTNL'}

def clean_up_dataframe(df):
    '''
    Function to clean up dataframe from csv

    Input: pandas dataframe

    Output: pandas dataframe
    '''
    df.rename(columns=lambda x: x.replace(' ', '_').lower(), inplace=True)
    df['dob'] = pd.to_datetime(df['dob']).dt.date
    df.rename(columns={'dob': 'date'}, inplace=True)
    df['ordermodename'] = (
        df['ordermodename'].replace({'In Store': 'instore',
                                     '(UnSpecified)': 'instore',
                                     'Online': 'online',
                                     'Online Pick Up': 'online',
                                     'Phone In': 'online',
                                     'Phone Order': 'online',
                                     'Take Out': 'online',
                                     'Future': 'online',
                                     'FlyBuy': 'third_party',
                                     'UberEats': 'third_party',
                                     'Uber Eats': 'third_party',
                                     'Postmates': 'third_party',
                                     'Caviar': 'third_party',
                                     'Doordash': 'third_party',
                                     'Catering': 'catering',
                                     'Wholesale': 'omit_category',
                                     '(DS Adjust)': 'omit_category',
                                     'Commissary': 'omit_category'}))
    df = df.set_index(pd.DatetimeIndex(df['date']))
    return df[['locationid', 'locationname', 'day_of_week', 'ordermodename',
               'net_sales']]

def store_filter(df, store=1):
    '''
    Function to pull sales data for a specific store

    Input Parameters
    ----------------
    df: pandas dataframe
        pandas dataframe converted from csv with Sales, Guests, Checks, Entrees by day

    store: int
        Between 1 and 10, corresponding to a specific store as listed below:

        ['WA-001 3rd and Marion',
        'WA-002 6th and Olive',
        'WA-003 U-Village',
        'WA-004 Lenora-6th',
        'WA-005 Thomas-Boren',
        'WA-006 Fremont',
        'WA-007 - Bellevue City Center',
        'WA 008 - One Union',
        'WA-009 Pioneer Square',
        'WA-10 INTNL']

    Output: pandas dataframe of sales data from just a specific store
    '''
    mask = df['locationid'].apply(lambda x: x == store)
    return df[mask]

def get_instore_sales_data_by_type(df, sales_type=['instore']):
    '''
    Function to find in store sales data for a certain type of sales visit

    Input Parameters
    ----------------
    df: pandas dataframe
        pandas dateframe converted from csv with Sales, Guests, Checks, Entrees by day

    sales_type: list of str
        must be a list of string(s), to show the type of sale from the store
        from the following list of choices:
        instore = ['In Store', '(UnSpecified)']
        online = ['Online', 'Online Pick Up', 'Phone In', 'Phone Order', 'Take Out', 'Future']
        third_party = ['FlyBuy', 'UberEats', 'Postmates', 'Caviar','Uber Eats', 'Doordash']
        catering = ['Catering']
        omit_category = ['Wholesale', '(DS Adjust)', 'Commissary']

    Output: pandas dataframe with type of sales data
    '''
    df = df.loc[df['ordermodename'].isin(sales_type)]
    return df[['date', 'day_of_week','ordermodename', 'net_sales']]

def get_sales_data_by_day(df):
    '''
    Function to find total in store sales data by day.

    Input Parameters
    ----------------
    df: pandas dataframe
        pandas dateframe converted from csv with Sales, Guests, Checks, Entrees by day

    Output: pandas dataframe with type of sales data
    '''
    sales_df = df.resample('D').sum()
    sales_df = sales_df[['net_sales']]
    sales_df = sales_df.drop(sales_df[sales_df['net_sales'] < 200].index)
    combined_df = sales_df.join(df['day_of_week']).drop_duplicates()
    return combined_df

def date_to_nth_day(date):
    '''
    Function to convert a datetime day to a number

    Input: datetime date

    Output: int
    '''
    date = pd.to_datetime(date)
    new_year_day = pd.Timestamp(year=date.year, month=1, day=1)
    return (date - new_year_day).days + 1

def assign_sine_vector(day):
    '''
    Function will create a sine vector based on the day of the year (d) such that
    vector = sin((2 * pi * d) / 365

    Input: date

    Output: sine vector
    '''
    return math.sin((2 * math.pi * date_to_nth_day(day)) / 365)

def assign_cosine_vector(day):
    '''
    Function will create a cosine vector based on the day of the year (d) such that
    vector = cos((2 * pi * d) / 365

    Input: date

    Output: cosine vector
    '''
    return math.cos((2 * math.pi * date_to_nth_day(day)) / 365)

max_stars_repo_name: rsherer/dressing-for-the-weather

max_issues_repo_name: rsherer/dressing-for-the-weather

max_forks_repo_name: rsherer/dressing-for-the-weather


text:
import matplotlib.pyplot as plt
import numpy as np
import os

def load_data(filename):
    pts = []
    f = open(filename, "rb")
    for line in f:
        pts.append(float(line.strip()))
    f.close()
    return pts

dataset = 'hrsc'
weights_path = 'weights_'+dataset+''

###############################################
# Load data
train_pts = load_data(os.path.join(weights_path, 'train_loss.txt'))
# val_pts = load_data(os.path.join(weights_path, 'val_loss.txt'))

def draw_loss():
    x = np.linspace(0, len(train_pts), len(train_pts))
    plt.plot(x,train_pts,'ro-',label='train')
    # plt.plot(x,val_pts,'bo-',label='val')
    # plt.axis([0,len(train_pts), 0.02, 0.08])
    plt.legend(loc='upper right')

    plt.xlabel('Epochs')
    plt.ylabel('Loss')

    plt.show()

def draw_loss_ap():
    ap05_pts = load_data(os.path.join(weights_path, 'ap_list.txt'))

    x = np.linspace(0,len(train_pts),len(train_pts))
    x1 = np.linspace(0, len(train_pts), len(ap05_pts))

    fig, ax1 = plt.subplots()

    color = 'tab:red'
    ax1.set_xlabel('Epochs')
    ax1.set_ylabel('Loss', color=color)
    ax1.plot(x, train_pts, 'ro-',label='train')
    ax1.tick_params(axis='y', labelcolor=color)
    plt.legend(loc = 'lower right')
    ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis
    color = 'tab:blue'
    ax2.set_ylabel('AP', color=color)  # we already handled the x-label with ax1
    ax2.plot(x1, ap05_pts, 'go-',label='AP@05')
    ax2.tick_params(axis='y', labelcolor=color)

    fig.tight_layout()  # otherwise the right y-label is slightly clipped
    plt.legend(loc = 'upper right')
    plt.show()

if __name__ == '__main__':
    # draw_loss()
    draw_loss_ap()


text:
"""
Programmer  :   EOF
File        :   trainingAdaBoost.py
Date        :   2015.12.29
E-mail      :   jasonleaster@163.com

"""

from config   import POSITIVE_SAMPLE
from config   import NEGATIVE_SAMPLE
from config   import TRAINING_IMG_HEIGHT
from config   import TRAINING_IMG_WIDTH
from config   import FEATURE_FILE_TRAINING
from config   import FEATURE_NUM
from config   import ADABOOST_LIMIT
from config   import ADABOOST_CACHE_FILE
from config   import DEBUG_MODEL
from config   import TRAINING_FACE
from config   import TRAINING_NONFACE

from haarFeature import Feature
from image       import ImageSet
from adaboost    import AdaBoost
from adaboost    import getCachedAdaBoost

import os
import numpy

Face    = ImageSet(TRAINING_FACE,    sampleNum = POSITIVE_SAMPLE)
nonFace = ImageSet(TRAINING_NONFACE, sampleNum = NEGATIVE_SAMPLE)

tot_samples = Face.sampleNum + nonFace.sampleNum

haar   = Feature(TRAINING_IMG_WIDTH, TRAINING_IMG_HEIGHT)

if os.path.isfile(FEATURE_FILE_TRAINING + ".npy"):

    _mat = numpy.load(FEATURE_FILE_TRAINING + ".npy")

else:
    if DEBUG_MODEL is True:
        _mat = numpy.zeros((haar.featuresNum, tot_samples))

        for i in xrange(Face.sampleNum):
            featureVec = haar.calFeatureForImg(Face.images[i])
            for j in xrange(haar.featuresNum):
                _mat[j][i                     ]  = featureVec[j]

        for i in xrange(nonFace.sampleNum):
            featureVec = haar.calFeatureForImg(nonFace.images[i])
            for j in xrange(haar.featuresNum):
                _mat[j][i + Face.sampleNum] = featureVec[j]

        numpy.save(FEATURE_FILE_TRAINING, _mat)
    else:
        from mapReduce import map
        from mapReduce import reduce

        map(Face, nonFace)
        _mat = reduce()

mat = _mat

featureNum, sampleNum = _mat.shape

assert sampleNum  == (POSITIVE_SAMPLE + NEGATIVE_SAMPLE)
assert featureNum == FEATURE_NUM

Label_Face    = [+1 for i in xrange(POSITIVE_SAMPLE)]
Label_NonFace = [-1 for i in xrange(NEGATIVE_SAMPLE)]

label = numpy.array(Label_Face + Label_NonFace)

cache_filename = ADABOOST_CACHE_FILE + str(0)

if os.path.isfile(cache_filename):
    model = getCachedAdaBoost(mat     = _mat,
                              label   = label,
                              filename= cache_filename,
                              limit   = ADABOOST_LIMIT)
else:
    model = AdaBoost(mat, label, limit = ADABOOST_LIMIT)
    model.train()
    model.saveModel(cache_filename)

print model

max_stars_repo_name: onkarsiyag/face

max_issues_repo_name: onkarsiyag/face

max_forks_repo_name: onkarsiyag/face


text:
# -*- coding: utf-8 -*-
"""
@file
@brief Implémente la classe @see cl ConstraintKMeans.
"""
import bisect
from collections import Counter
from pandas import DataFrame
import numpy
import scipy.sparse
from scipy.spatial.distance import cdist
from sklearn.metrics.pairwise import euclidean_distances
from sklearn.utils.extmath import row_norms
from ._kmeans_022 import (
    _centers_dense, _centers_sparse, _labels_inertia_skl)

def linearize_matrix(mat, *adds):
    """
    Linearizes a matrix into a new one
    with 3 columns value, row, column.
    The output format is similar to
    :epkg:`csr_matrix` but null values are kept.

    @param      mat     matrix
    @param      adds    additional square matrices
    @return             new matrix

    *adds* defines additional matrices, it adds
    columns on the right side and fill them with
    the corresponding value taken into the additional
    matrices.
    """
    if scipy.sparse.issparse(mat):
        if isinstance(mat, scipy.sparse.csr_matrix):
            max_row = mat.shape[0]
            res = numpy.empty((len(mat.data), 3 + len(adds)), dtype=mat.dtype)
            row = 0
            for i, v in enumerate(mat.data):
                while row < max_row and i >= mat.indptr[row]:
                    row += 1
                res[i, 0] = v
                a, b = row - 1, mat.indices[i]
                res[i, 1] = a
                res[i, 2] = b
                for k, am in enumerate(adds):
                    res[i, k + 3] = am[a, b]
            return res
        raise NotImplementedError(  # pragma: no cover
            "This kind of sparse matrix is not handled: {0}".format(type(mat)))
    else:
        n = mat.shape[0]
        c = mat.shape[1]
        ic = numpy.arange(mat.shape[1])
        res = numpy.empty((n * c, 3 + len(adds)), dtype=mat.dtype)
        for i in range(0, n):
            a = i * c
            b = (i + 1) * c
            res[a:b, 1] = i
            res[a:b, 2] = ic
        res[:, 0] = mat.ravel()
        for k, am in enumerate(adds):
            res[:, 3 + k] = am.ravel()
        return res

def constraint_kmeans(X, labels, sample_weight, centers, inertia,
                      iter, max_iter,  # pylint: disable=W0622
                      strategy='gain', verbose=0, state=None,
                      learning_rate=1., history=False, fLOG=None):
    """
    Completes the constraint :epkg:`k-means`.

    @param      X               features
    @param      labels          initialized labels (unused)
    @param      sample_weight   sample weight
    @param      centers         initialized centers
    @param      inertia         initialized inertia (unused)
    @param      iter            number of iteration already done
    @param      max_iter        maximum of number of iteration
    @param      strategy        strategy used to sort observations before
                                mapping them to clusters
    @param      verbose         verbose
    @param      state           random state
    @param      learning_rate   used by strategy `'weights'`
    @param      history         return list of centers accross iterations
    @param      fLOG            logging function (needs to be specified otherwise
                                verbose has no effects)
    @return                     tuple (best_labels, best_centers, best_inertia,
                                iter, all_centers)
    """
    if labels.dtype != numpy.int32:
        raise TypeError(  # pragma: no cover
            "Labels must be an array of int not '{0}'".format(labels.dtype))

    if strategy == 'weights':
        return _constraint_kmeans_weights(
            X, labels, sample_weight, centers, inertia, iter,
            max_iter, verbose=verbose, state=state,
            learning_rate=learning_rate, history=history, fLOG=fLOG)
    else:
        if isinstance(X, DataFrame):
            X = X.values
        x_squared_norms = row_norms(X, squared=True)
        counters = numpy.empty((centers.shape[0],), dtype=numpy.int32)
        limit = X.shape[0] // centers.shape[0]
        leftover = X.shape[0] - limit * centers.shape[0]
        leftclose = numpy.empty((centers.shape[0],), dtype=numpy.int32)
        n_clusters = centers.shape[0]
        distances_close = numpy.empty((X.shape[0],), dtype=X.dtype)
        best_inertia = None
        best_iter = None
        all_centers = []

        # association
        _constraint_association(leftover, counters, labels, leftclose, distances_close,
                                centers, X, x_squared_norms, limit, strategy, state=state)

        if sample_weight is None:
            sw = numpy.ones((X.shape[0],))
        else:
            sw = sample_weight

        if scipy.sparse.issparse(X):
            _centers_fct = _centers_sparse
        else:
            _centers_fct = _centers_dense

        while iter < max_iter:
            # compute new clusters
            centers = _centers_fct(
                X, sw, labels, n_clusters, distances_close)

            if history:
                all_centers.append(centers)

            # association
            _constraint_association(
                leftover, counters, labels, leftclose, distances_close,
                centers, X, x_squared_norms, limit, strategy, state=state)

            # inertia
            _, inertia = _labels_inertia_skl(
                X=X, sample_weight=sw, x_squared_norms=x_squared_norms,
                centers=centers, distances=distances_close)

            iter += 1
            if verbose and fLOG:  # pragma: no cover
                fLOG("CKMeans %d/%d inertia=%f" % (iter, max_iter, inertia))

            # best option so far?
            if best_inertia is None or inertia < best_inertia:
                best_inertia = inertia
                best_centers = centers.copy()
                best_labels = labels.copy()
                best_iter = iter

            # early stop
            if (best_inertia is not None and inertia >= best_inertia and
                    iter > best_iter + 5):
                break

        return (best_labels, best_centers, best_inertia, None,
                iter, all_centers)

def constraint_predictions(X, centers, strategy, state=None):
    """
    Computes the predictions but tries
    to associates the same numbers of points
    in each cluster.

    @param      X           features
    @param      centers     centers of each clusters
    @param      strategy    strategy used to sort point before
                            mapping them to a cluster
    @param      state       random state
    @return                 labels, distances, distances_close
    """
    if isinstance(X, DataFrame):
        X = X.values
    x_squared_norms = row_norms(X, squared=True)
    counters = numpy.empty((centers.shape[0],), dtype=numpy.int32)
    limit = X.shape[0] // centers.shape[0]
    leftover = X.shape[0] - limit * centers.shape[0]
    leftclose = numpy.empty((centers.shape[0],), dtype=numpy.int32)
    distances_close = numpy.empty((X.shape[0],), dtype=X.dtype)
    labels = numpy.empty((X.shape[0],), dtype=numpy.int32)

    distances = _constraint_association(
        leftover, counters, labels, leftclose,
        distances_close, centers, X, x_squared_norms,
        limit, strategy, state=state)

    return labels, distances, distances_close

def _constraint_association(leftover, counters, labels, leftclose, distances_close,
                            centers, X, x_squared_norms, limit, strategy, state=None):
    """
    Completes the constraint :epkg:`k-means`.

    @param      X               features
    @param      labels          initialized labels (unused)
    @param      centers         initialized centers
    @param      x_squared_norms norm of *X*
    @param      limit           number of point to associate per cluster
    @param      leftover        number of points to associate at the end
    @param      counters        allocated array
    @param      leftclose       allocated array
    @param      labels          allocated array
    @param      distances_close allocated array
    @param      strategy        strategy used to sort point before
                                mapping them to a cluster
    @param      state           random state
    """
    if strategy in ('distance', 'distance_p'):
        return _constraint_association_distance(
            leftover, counters, labels, leftclose, distances_close,
            centers, X, x_squared_norms, limit, strategy, state=state)
    if strategy in ('gain', 'gain_p'):
        return _constraint_association_gain(
            leftover, counters, labels, leftclose, distances_close,
            centers, X, x_squared_norms, limit, strategy, state=state)
    raise ValueError("Unknwon strategy '{0}'.".format(
        strategy))  # pragma: no cover

def _compute_strategy_coefficient(distances, strategy, labels):
    """
    Creates a matrix
    """
    if strategy in ('gain', 'gain_p'):
        ar = numpy.arange(distances.shape[0])
        dist = distances[ar, labels]
        return distances - dist[:, numpy.newaxis]
    raise ValueError(  # pragma: no cover
        "Unknwon strategy '{0}'.".format(strategy))

def _randomize_index(index, weights):
    """
    Randomizes index depending on the value.
    Swap indexes.
    Modifies *index*.
    """
    maxi = weights.max()
    mini = weights.min()
    diff = max(maxi - mini, 1e-5)
    rand = numpy.random.rand(weights.shape[0])
    for i in range(1, index.shape[0]):
        ind1 = index[i - 1]
        ind2 = index[i]
        w1 = weights[ind1]
        w2 = weights[ind2]
        ratio = abs(w2 - w1) / diff * 0.5
        if rand[i] >= ratio + 0.5:
            index[i - 1], index[i] = ind2, ind1
            weights[i - 1], weights[i] = w2, w1

def _switch_clusters(labels, distances):
    """
    Tries to switch clusters.
    Modifies *labels* inplace.

    @param      labels      labels
    @param      distances   distances
    """
    perm = numpy.random.permutation(numpy.arange(0, labels.shape[0]))
    niter = 0
    modif = 1
    while modif > 0 and niter < 10:
        modif = 0
        niter += 1
        for i_ in range(labels.shape[0]):
            for j_ in range(i_ + 1, labels.shape[0]):
                i = perm[i_]
                j = perm[j_]
                c1 = labels[i]
                c2 = labels[j]
                if c1 == c2:
                    continue
                d11 = distances[i, c1]
                d12 = distances[i, c2]
                d21 = distances[j, c1]
                d22 = distances[j, c2]
                if d11**2 + d22**2 > d21**2 + d12**2:
                    labels[i], labels[j] = c2, c1
                    modif += 1

def _constraint_association_distance(leftover, counters, labels, leftclose, distances_close,
                                     centers, X, x_squared_norms, limit, strategy, state=None):
    """
    Completes the constraint *k-means*,
    the function sorts points by distance to the closest
    cluster and associates them into that order.
    It deals first with the further point and maps it to
    the closest center.

    @param      X               features
    @param      labels          initialized labels (unused)
    @param      centers         initialized centers
    @param      x_squared_norms norm of *X*
    @param      limit           number of point to associate per cluster
    @param      leftover        number of points to associate at the end
    @param      counters        allocated array
    @param      leftclose       allocated array
    @param      labels          allocated array
    @param      distances_close allocated array
    @param      strategy        strategy used to sort point before
                                mapping them to a cluster
    @param      state           random state (unused)
    """

    # initialisation
    counters[:] = 0
    leftclose[:] = -1
    labels[:] = -1

    # distances
    distances = euclidean_distances(
        centers, X, Y_norm_squared=x_squared_norms, squared=True)
    distances = distances.T
    distances0 = distances.copy()
    maxi = distances.ravel().max() * 2
    centers_index = numpy.argsort(distances, axis=1)

    while labels.min() == -1:
        mini = numpy.min(distances, axis=1)
        sorted_index = numpy.argsort(mini)
        _randomize_index(sorted_index, mini)

        nover = leftover
        for ind in sorted_index:
            if labels[ind] >= 0:
                continue
            for c in centers_index[ind, :]:
                if counters[c] < limit:
                    # The cluster still accepts new points.
                    counters[c] += 1
                    labels[ind] = c
                    distances[ind, c] = maxi
                    break
                if nover > 0 and leftclose[c] == -1:
                    # The cluster may accept one point if the number
                    # of clusters does not divide the number of points in X.
                    counters[c] += 1
                    labels[ind] = c
                    nover -= 1
                    leftclose[c] = 0
                    distances[ind, c] = maxi
                    break

    _switch_clusters(labels, distances0)
    distances_close[:] = distances[numpy.arange(X.shape[0]), labels]
    return distances0

def _constraint_association_gain(leftover, counters, labels, leftclose, distances_close,
                                 centers, X, x_squared_norms, limit, strategy, state=None):
    """
    Completes the constraint *k-means*.
    Follows the method described in `Same-size k-Means Variation
    <https://elki-project.github.io/tutorial/same-size_k_means>`_.

    @param      X               features
    @param      labels          initialized labels (unused)
    @param      centers         initialized centers
    @param      x_squared_norms norm of *X*
    @param      limit           number of points to associate per cluster
    @param      leftover        number of points to associate at the end
    @param      counters        allocated array
    @param      leftclose       allocated array
    @param      labels          allocated array
    @param      distances_close allocated array
    @param      strategy        strategy used to sort point before
                                mapping them to a cluster
    @param      state           random state

    See `Same-size k-Means Variation <https://elki-project.github.io/tutorial/same-size_k_means>`_.
    """
    # distances
    distances = euclidean_distances(
        centers, X, Y_norm_squared=x_squared_norms, squared=True)
    distances = distances.T

    if strategy == 'gain_p':
        labels[:] = numpy.argmin(distances, axis=1)
    else:
        # We assume labels comes from a previous iteration.
        pass

    strategy_coef = _compute_strategy_coefficient(distances, strategy, labels)
    distance_linear = linearize_matrix(distances, strategy_coef)
    sorted_distances = distance_linear[distance_linear[:, 3].argsort()]
    distances_close[:] = 0

    # counters
    ave = limit
    counters[:] = 0
    for i in labels:
        counters[i] += 1
    leftclose[:] = counters[:] - ave
    leftclose[leftclose < 0] = 0
    nover = X.shape[0] - ave * counters.shape[0]
    sumi = nover - leftclose.sum()
    if sumi != 0:
        if state is None:
            state = numpy.random.RandomState()  # pylint: disable=E1101

        def loopf(h, sumi):
            if sumi < 0 and leftclose[h] > 0:  # pylint: disable=R1716
                sumi -= leftclose[h]
                leftclose[h] = 0
            elif sumi > 0 and leftclose[h] == 0:
                leftclose[h] = 1
                sumi += 1
            return sumi

        it = 0
        while sumi != 0:
            h = state.randint(0, counters.shape[0])
            sumi = loopf(h, sumi)
            it += 1
            if it > counters.shape[0] * 2:
                break
        for h in range(counters.shape[0]):
            if sumi == 0:
                break
            sumi = loopf(h, sumi)

    transfer = {}

    for i in range(0, sorted_distances.shape[0]):
        gain = sorted_distances[i, 3]
        ind = int(sorted_distances[i, 1])
        dest = int(sorted_distances[i, 2])
        cur = labels[ind]
        if distances_close[ind]:
            continue
        if cur == dest:
            continue
        if ((counters[dest] < ave + leftclose[dest]) and
                (counters[cur] > ave + leftclose[cur])):
            labels[ind] = dest
            counters[cur] -= 1
            counters[dest] += 1
            distances_close[ind] = 1  # moved
        else:
            cp = transfer.get((dest, cur), [])
            while len(cp) > 0:
                g, destind = cp[0]
                if distances_close[destind]:
                    del cp[0]
                else:
                    break
            if len(cp) > 0:
                g, destind = cp[0]
                if g + gain < 0:
                    del cp[0]
                    labels[ind] = dest
                    labels[destind] = cur
                    add = False
                    distances_close[ind] = 1  # moved
                    distances_close[destind] = 1  # moved
                else:
                    add = True
            else:
                add = True
            if add:
                # We add the point to the list of points willing to transfer.
                if (cur, dest) not in transfer:
                    transfer[cur, dest] = []
                gain = sorted_distances[i, 3]
                bisect.insort(transfer[cur, dest], (gain, ind))

    neg = (counters < ave).sum()
    if neg > 0:
        raise RuntimeError(  # pragma: no cover
            "The algorithm failed, counters={0}".format(counters))

    _switch_clusters(labels, distances)
    distances_close[:] = distances[numpy.arange(X.shape[0]), labels]

    return distances

def _constraint_kmeans_weights(X, labels, sample_weight, centers, inertia, it,
                               max_iter, verbose=0, state=None, learning_rate=1.,
                               history=False, fLOG=None):
    """
    Runs KMeans iterator but weights cluster among them.

    @param      X               features
    @param      labels          initialized labels (unused)
    @param      sample_weight   sample weight
    @param      centers         initialized centers
    @param      inertia         initialized inertia (unused)
    @param      it              number of iteration already done
    @param      max_iter        maximum of number of iteration
    @param      verbose         verbose
    @param      state           random state
    @param      learning_rate   learning rate
    @param      history         keeps all centers accross iterations
    @param      fLOG            logging function (needs to be specified otherwise
                                verbose has no effects)
    @return                     tuple (best_labels, best_centers, best_inertia, weights, it)
    """
    if isinstance(X, DataFrame):
        X = X.values
    n_clusters = centers.shape[0]
    best_inertia = None
    best_iter = None
    weights = numpy.ones(centers.shape[0])
    if sample_weight is None:
        sw = numpy.ones((X.shape[0],))
    else:
        sw = sample_weight

    if scipy.sparse.issparse(X):
        _centers_fct = _centers_sparse
    else:
        _centers_fct = _centers_dense

    total_inertia = _inertia(X, sw)
    all_centers = []

    while it < max_iter:

        # compute new clusters
        centers = _centers_fct(
            X, sw, labels, n_clusters, None)
        if history:
            all_centers.append(centers)

        # association
        labels = _constraint_association_weights(X, centers, sw, weights)
        if len(set(labels)) != centers.shape[0]:
            if verbose and fLOG:  # pragma: no cover
                if isinstance(verbose, int) and verbose >= 10:
                    fLOG("CKMeans new weights: w=%r" % weights)
                else:
                    fLOG("CKMeans new weights")
            weights[:] = 1
            labels = _constraint_association_weights(X, centers, sw, weights)

        # inertia
        inertia, diff = _labels_inertia_weights(
            X, centers, sw, weights, labels, total_inertia)
        if numpy.isnan(inertia):
            raise RuntimeError(  # pragma: no cover
                "nanNobs={} Nclus={}\ninertia={}\nweights={}\ndiff={}\nlabels={}".format(
                    X.shape[0], centers.shape[0], inertia, weights, diff,
                    set(labels)))

        # best option so far?
        if best_inertia is None or inertia < best_inertia:
            best_inertia = inertia
            best_centers = centers.copy()
            best_labels = labels.copy()
            best_weights = weights.copy()
            best_iter = it

        # moves weights
        weights, _ = _adjust_weights(X, sw, weights, labels,
                                     learning_rate / (it + 10))

        it += 1
        if verbose and fLOG:
            if isinstance(verbose, int) and verbose >= 10:
                fLOG("CKMeans %d/%d inertia=%f (%f T=%f) dw=%r w=%r" % (
                    it, max_iter, inertia, best_inertia, total_inertia,
                    diff, weights))
            elif isinstance(verbose, int) and verbose >= 5:
                hist = Counter(labels)
                fLOG("CKMeans %d/%d inertia=%f (%f) hist=%r" % (
                    it, max_iter, inertia, best_inertia, hist))
            else:
                fLOG("CKMeans %d/%d inertia=%f (%f T=%f)" % (  # pragma: no cover
                    it, max_iter, inertia, best_inertia, total_inertia))

        # early stop
        if (best_inertia is not None and inertia >= best_inertia and
                it > best_iter + 5 and numpy.abs(diff).sum() <= weights.shape[0] / 2):
            break

    return (best_labels, best_centers, best_inertia, best_weights,
            it, all_centers)

def _constraint_association_weights(X, centers, sw, weights):
    """
    Associates points to clusters.

    @param      X           features
    @param      centers     centers
    @param      sw          sample weights
    @param      weights     cluster weights
    @return                 labels
    """
    dist = cdist(X, centers) * weights.reshape((1, -1))
    index = numpy.argmin(dist, axis=1)
    return index

def _inertia(X, sw):
    """
    Computes total weighted inertia.

    @param      X               features
    @param      sw              sample weights
    @return                     inertia
    """
    bary = numpy.mean(X, axis=0)
    diff = X - bary
    norm = numpy.linalg.norm(diff, axis=1)
    if sw is not None:
        norm *= sw
    return sw.sum()

def _labels_inertia_weights(X, centers, sw, weights, labels, total_inertia):
    """
    Computes weighted inertia. It also adds a fraction
    of the whole inertia depending on how balanced the
    clusters are.

    @param      X               features
    @param      centers         centers
    @param      sw              sample weights
    @param      weights         cluster weights
    @param      labels          labels
    @param      total_inertia   total inertia
    @return                     inertia
    """
    www, exp, _ = _compute_balance(X, sw, labels, centers.shape[0])
    www -= exp
    wwwa = numpy.square(www)
    ratio = wwwa.sum() ** 0.5 / X.shape[0]
    dist = cdist(X, centers) * weights.reshape((1, -1)) * sw.reshape((-1, 1))
    return dist.sum() + ratio * total_inertia, www

def _compute_balance(X, sw, labels, nbc=None):
    """
    Computes weights difference.

    @param      X           features
    @param      sw          sample weights
    @param      labels      known labels
    @param      nbc         number of clusters
    @return                 (weights per cluster, expected weight,
                            total weight)
    """
    if nbc is None:
        nbc = labels.max() + 1
    N = numpy.float64(nbc)
    www = numpy.zeros((nbc, ), dtype=numpy.float64)
    if sw is None:
        for la in labels:
            www[la] += 1
    else:
        for w, la in zip(sw, labels):
            www[la] += w
    exp = www.sum() / N
    return www, exp, N

def _adjust_weights(X, sw, weights, labels, lr):
    """
    Changes *weights* mapped to every cluster.
    *weights < 1* are used for big clusters,
    *weights > 1* are used for small clusters.

    @param      X           features
    @param      centers     centers
    @param      sw          sample weights
    @param      weights     cluster weights
    @param      lr          learning rate
    @param      labels      known labels
    @return                 labels
    """
    www, exp, N = _compute_balance(X, sw, labels, weights.shape[0])

    for i in range(0, weights.shape[0]):
        nw = (www[i] - exp) / exp
        delta = nw * lr
        weights[i] += delta
        N += delta

    return weights / N * weights.shape[0], www

max_stars_repo_name: sdpython/mlinsights

max_issues_repo_name: sdpython/mlinsights

max_forks_repo_name: sdpython/mlinsights

reason: import numpy,import scipy,from scipy


text:
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import models
import utils
import sys
import os
import logging
import argparse

logging.basicConfig(level=logging.INFO)

# Parameter setting
# example of tuning by argparser: python train.py --IN_SIZE=40 --USE_CMVN
parser = argparse.ArgumentParser()
parser.add_argument('--IN_SIZE',     type=int, default=13)             # input mfcc dim
parser.add_argument('--HIDDEN_SIZE', type=int, default=128)            # unit num of layer
parser.add_argument('--NUM_STACK',   type=int, default=5)              # number of layers
parser.add_argument('--USE_CMVN',    action='store_true')              # whether applying CMVN, default is False
parser.add_argument('--BATCH_SIZE',  type=int, default=256)
parser.add_argument('--PARAM_FILE',  type=str)                         # parameter of trained model
parser.add_argument('--EVAL_LIST',  type=str)                          # evaluation file list
parser.add_argument('--MFCC_ROOT',  type=str)                          # root path of mfcc feature files
args = parser.parse_args()

IN_SIZE       = args.IN_SIZE
NUM_CLASS     = 10
HIDDEN_SIZE   = args.HIDDEN_SIZE
NUM_STACK     = args.NUM_STACK
USE_CMVN      = args.USE_CMVN
BATCH_SIZE    = args.BATCH_SIZE
PARAM_FILE    = args.PARAM_FILE
EVAL_LIST     = args.EVAL_LIST
MFCC_ROOT     = args.MFCC_ROOT

# Build up model and batch generator
device      = 'cuda' if torch.cuda.is_available() else 'cpu'   # check available gpu
model       = models.Classifier(IN_SIZE, NUM_CLASS, HIDDEN_SIZE, NUM_STACK, 0.0).to(device) # build model (same structure as trained model)
model.load_state_dict(torch.load(PARAM_FILE)) # load parameters from trained model
batch_test  = utils.Batch_generator(MFCC_ROOT, EVAL_LIST, BATCH_SIZE) # data batch generator for evaluation data

# Print out setting
logging.info('Batch_size: {}'.format(BATCH_SIZE))
logging.info('Hidden size: {}'.format(HIDDEN_SIZE))
logging.info('Num stack: {}'.format(NUM_STACK))
logging.info('Use cmvn: {}'.format(USE_CMVN))

# Training part
with torch.no_grad(): # disable gradient calculation, reduce memory consumption
    model.eval()
    total_num   = 0 # total num of test data
    correct_num = 0 # corrected prediction num
    while True:
        inputs, labels, lengths, epoch = next(batch_test) # generate a data batch
        if USE_CMVN:
            inputs  = utils.Apply_cmvn(torch.from_numpy(inputs).to(device)) # use cmvn
        else:
            inputs  = torch.from_numpy(inputs).to(device)
        inputs  = nn.utils.rnn.pack_padded_sequence(inputs, lengths, batch_first=True) # pack the padded sequence (remove the redundancy padding)
        labels  = torch.from_numpy(labels).to(device)
        outputs = model(inputs)
        total_num   += len(outputs)
        correct_num += torch.bincount(torch.abs(torch.argmax(outputs, dim=1) - labels))[0] # compute the number of corrected prediction
        if epoch == 2: break
    score = correct_num.float() / total_num
    logging.info('accuracy: {:.04f}'.format(score))
    with open('score.txt', 'w') as f:
        f.write('recognition rate = {:.1%}\n'.format(score))

logging.info('done')

max_stars_repo_name: RRFHOUDEN/tut-asr-voicecommand

max_issues_repo_name: RRFHOUDEN/tut-asr-voicecommand

max_forks_repo_name: RRFHOUDEN/tut-asr-voicecommand


text:
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import mxnet as mx
import numpy as np
from gluonnlp.metric import MaskedAccuracy, LengthNormalizedLoss
from mxnet.test_utils import assert_almost_equal

def test_acc():
    pred = mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])
    label = mx.nd.array([0, 1, 1])
    mask = mx.nd.array([1, 1, 0])
    metric = MaskedAccuracy()
    metric.update([label], [pred], [mask])
    _, acc = metric.get()
    matched = (np.argmax(pred.asnumpy(), axis=1) == label.asnumpy()) * mask.asnumpy()
    valid_count = mask.asnumpy().sum()
    expected_acc = 1.0 * matched.sum() / valid_count
    assert acc == expected_acc

    metric = MaskedAccuracy()
    metric.update([label], [pred])
    _, acc = metric.get()
    matched = (np.argmax(pred.asnumpy(), axis=1) == label.asnumpy())
    valid_count = len(label)
    expected_acc = 1.0 * matched.sum() / valid_count
    assert acc == expected_acc

def test_normalized_loss(rtol=1e-5, atol=1e-5):
    tgt_valid_length = mx.nd.array([1, 3, 2, 7])
    loss = mx.nd.array([1.1, 2.5, 3.8, 5.3])
    metric = LengthNormalizedLoss()
    metric.update([0, tgt_valid_length], loss)
    _, metric_loss = metric.get()
    expected_loss = loss.asnumpy().sum() / tgt_valid_length.asnumpy().sum()
    assert_almost_equal(metric_loss, expected_loss, rtol=rtol, atol=atol)

    tgt_valid_length = mx.nd.array([8, 4, 2, 7])
    loss = mx.nd.array([8.7, 2.3, 1.8, 9.3])
    metric = LengthNormalizedLoss()
    metric.update([0, tgt_valid_length], loss)
    _, metric_loss = metric.get()
    expected_loss = loss.asnumpy().sum() / tgt_valid_length.asnumpy().sum()
    assert_almost_equal(metric_loss, expected_loss, rtol=rtol, atol=atol)

max_stars_repo_name: StrayBird-ATSH/gluon-nlp

max_issues_repo_name: StrayBird-ATSH/gluon-nlp

max_forks_repo_name: StrayBird-ATSH/gluon-nlp


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Sep 10 19:38:35 2019

@author: gaurav

    inFile  : ../data/train/df_fe_without_preprocessing_train.csv
    outFile : ../data/train/nlp_features_train.csv

"""

import warnings
warnings.filterwarnings("ignore")
import numpy as np
import pandas as pd

import os
import re

from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from bs4 import BeautifulSoup

from fuzzywuzzy import fuzz
from sklearn.manifold import TSNE
from wordcloud import WordCloud, STOPWORDS
from os import path

#https://stackoverflow.com/questions/12468179/unicodedecodeerror-utf8-codec-cant-decode-byte-0x9c
if os.path.isfile('../data/train/df_fe_without_preprocessing_train.csv'):
    df = pd.read_csv("../data/train/df_fe_without_preprocessing_train.csv",encoding='latin-1')
    df = df.fillna('')
    df.head()
else:
    print("get df_fe_without_preprocessing_train.csv from drive or run the previous notebook")

    # To get the results in 4 decemal points
SAFE_DIV = 0.0001

STOP_WORDS = stopwords.words("english")

def preprocess(x):
    x = str(x).lower()
    x = x.replace(",000,000", "m").replace(",000", "k").replace("′", "'").replace("’", "'")\
                           .replace("won't", "will not").replace("cannot", "can not").replace("can't", "can not")\
                           .replace("n't", " not").replace("what's", "what is").replace("it's", "it is")\
                           .replace("'ve", " have").replace("i'm", "i am").replace("'re", " are")\
                           .replace("he's", "he is").replace("she's", "she is").replace("'s", " own")\
                           .replace("%", " percent ").replace("₹", " rupee ").replace("$", " dollar ")\
                           .replace("€", " euro ").replace("'ll", " will")
    x = re.sub(r"([0-9]+)000000", r"\1m", x)
    x = re.sub(r"([0-9]+)000", r"\1k", x)

    porter = PorterStemmer()
    pattern = re.compile('\W')

    if type(x) == type(''):
        x = re.sub(pattern, ' ', x)

    if type(x) == type(''):
        x = porter.stem(x)
        example1 = BeautifulSoup(x)
        x = example1.get_text()

    return x

def get_token_features(q1, q2):
    token_features = [0.0]*10

    # Converting the Sentence into Tokens:
    q1_tokens = q1.split()
    q2_tokens = q2.split()

    if len(q1_tokens) == 0 or len(q2_tokens) == 0:
        return token_features
    # Get the non-stopwords in Questions
    q1_words = set([word for word in q1_tokens if word not in STOP_WORDS])
    q2_words = set([word for word in q2_tokens if word not in STOP_WORDS])

    #Get the stopwords in Questions
    q1_stops = set([word for word in q1_tokens if word in STOP_WORDS])
    q2_stops = set([word for word in q2_tokens if word in STOP_WORDS])

    # Get the common non-stopwords from Question pair
    common_word_count = len(q1_words.intersection(q2_words))

    # Get the common stopwords from Question pair
    common_stop_count = len(q1_stops.intersection(q2_stops))

    # Get the common Tokens from Question pair
    common_token_count = len(set(q1_tokens).intersection(set(q2_tokens)))

    token_features[0] = common_word_count / (min(len(q1_words), len(q2_words)) + SAFE_DIV)
    token_features[1] = common_word_count / (max(len(q1_words), len(q2_words)) + SAFE_DIV)
    token_features[2] = common_stop_count / (min(len(q1_stops), len(q2_stops)) + SAFE_DIV)
    token_features[3] = common_stop_count / (max(len(q1_stops), len(q2_stops)) + SAFE_DIV)
    token_features[4] = common_token_count / (min(len(q1_tokens), len(q2_tokens)) + SAFE_DIV)
    token_features[5] = common_token_count / (max(len(q1_tokens), len(q2_tokens)) + SAFE_DIV)

    # Last word of both question is same or not
    token_features[6] = int(q1_tokens[-1] == q2_tokens[-1])

    # First word of both question is same or not
    token_features[7] = int(q1_tokens[0] == q2_tokens[0])

    token_features[8] = abs(len(q1_tokens) - len(q2_tokens))

    #Average Token Length of both Questions
    token_features[9] = (len(q1_tokens) + len(q2_tokens))/2
    return token_features

# get the Longest Common sub string

def get_longest_substr_ratio(a, b):
    strs = list(distance.lcsubstrings(a, b))
    if len(strs) == 0:
        return 0
    else:
        return len(strs[0]) / (min(len(a), len(b)) + 1)

def extract_features(df):
    # preprocessing each question
    df["question1"] = df["question1"].fillna("").apply(preprocess)
    df["question2"] = df["question2"].fillna("").apply(preprocess)

    print("token features...")

    # Merging Features with dataset

    token_features = df.apply(lambda x: get_token_features(x["question1"], x["question2"]), axis=1)

    df["cwc_min"]       = list(map(lambda x: x[0], token_features))
    df["cwc_max"]       = list(map(lambda x: x[1], token_features))
    df["csc_min"]       = list(map(lambda x: x[2], token_features))
    df["csc_max"]       = list(map(lambda x: x[3], token_features))
    df["ctc_min"]       = list(map(lambda x: x[4], token_features))
    df["ctc_max"]       = list(map(lambda x: x[5], token_features))
    df["last_word_eq"]  = list(map(lambda x: x[6], token_features))
    df["first_word_eq"] = list(map(lambda x: x[7], token_features))
    df["abs_len_diff"]  = list(map(lambda x: x[8], token_features))
    df["mean_len"]      = list(map(lambda x: x[9], token_features))

    #Computing Fuzzy Features and Merging with Dataset

    # do read this blog: http://chairnerd.seatgeek.com/fuzzywuzzy-fuzzy-string-matching-in-python/
    # https://stackoverflow.com/questions/31806695/when-to-use-which-fuzz-function-to-compare-2-strings
    # https://github.com/seatgeek/fuzzywuzzy
    print("fuzzy features..")

    df["token_set_ratio"]       = df.apply(lambda x: fuzz.token_set_ratio(x["question1"], x["question2"]), axis=1)
    # The token sort approach involves tokenizing the string in question, sorting the tokens alphabetically, and
    # then joining them back into a string We then compare the transformed strings with a simple ratio().
    df["token_sort_ratio"]      = df.apply(lambda x: fuzz.token_sort_ratio(x["question1"], x["question2"]), axis=1)
    df["fuzz_ratio"]            = df.apply(lambda x: fuzz.QRatio(x["question1"], x["question2"]), axis=1)
    df["fuzz_partial_ratio"]    = df.apply(lambda x: fuzz.partial_ratio(x["question1"], x["question2"]), axis=1)
    df["longest_substr_ratio"]  = df.apply(lambda x: get_longest_substr_ratio(x["question1"], x["question2"]), axis=1)
    return df

if os.path.isfile('../data/train/nlp_features_train.csv'):
    df = pd.read_csv("../data/train/nlp_features_train.csv",encoding='latin-1')
    df.fillna('')
else:
    print("Extracting features for train:")
    df = pd.read_csv("../data/train/train.csv")
    df = extract_features(df)
    df.to_csv("../data/train/nlp_features_train.csv", index=False)
df.head(2)


text:
import pytest
import numpy as np
import pandas as pd
import networkx as nx

from neuprint import Client, default_client, set_default_client
from neuprint import (fetch_skeleton, heal_skeleton, reorient_skeleton, skeleton_df_to_nx, skeleton_df_to_swc, skeleton_swc_to_df)

from neuprint.tests import NEUPRINT_SERVER, DATASET

@pytest.fixture(scope='module')
def client():
    c = Client(NEUPRINT_SERVER, DATASET)
    set_default_client(c)
    assert default_client() is c
    return c

@pytest.fixture
def linear_skeleton():
    """
    A test fixture to produce a fake 'skeleton'
    with no branches, just 10 nodes in a line.
    """
    rows = np.arange(1,11)
    coords = np.zeros((10,3), dtype=int)
    coords[:,0] = rows**2
    radii = rows.astype(np.float32)
    links = [-1, *range(1,10)]

    df = pd.DataFrame({'rowId': rows,
                       'x': coords[:,0],
                       'y': coords[:,1],
                       'z': coords[:,2],
                       'radius': radii,
                       'link': links})
    return df

def test_skeleton_df_to_nx(linear_skeleton):
    g = skeleton_df_to_nx(linear_skeleton, directed=False)
    assert not isinstance(g, nx.DiGraph)
    expected_edges = linear_skeleton[['rowId', 'link']].values[1:]
    expected_edges.sort(axis=1)
    assert (np.array(g.edges) == expected_edges).all()

    g = skeleton_df_to_nx(linear_skeleton, directed=True)
    assert isinstance(g, nx.DiGraph)
    assert (np.array(g.edges) == linear_skeleton[['rowId', 'link']].values[1:]).all()

    g = skeleton_df_to_nx(linear_skeleton, with_attributes=True)
    assert (np.array(g.edges) == linear_skeleton[['rowId', 'link']].values[1:]).all()
    for row in linear_skeleton.itertuples():
        attrs = g.nodes[row.rowId]
        assert tuple(attrs[k] for k in [*'xyz', 'radius']) == (row.x, row.y, row.z, row.radius)

def test_skeleton_df_to_swc(linear_skeleton):
    swc = skeleton_df_to_swc(linear_skeleton)
    roundtrip_df = skeleton_swc_to_df(swc)
    assert (roundtrip_df == linear_skeleton).all().all()

def test_reorient_skeleton(linear_skeleton):
    s = linear_skeleton.copy()
    reorient_skeleton(s, 10)
    assert (s['link'] == [*range(2,11), -1]).all()

    s = linear_skeleton.copy()
    reorient_skeleton(s, xyz=(100,0,0))
    assert (s['link'] == [*range(2,11), -1]).all()

    s = linear_skeleton.copy()
    reorient_skeleton(s, use_max_radius=True)
    assert (s['link'] == [*range(2,11), -1]).all()

def test_reorient_broken_skeleton(linear_skeleton):
    broken_skeleton = linear_skeleton.copy()
    broken_skeleton.loc[2, 'link'] = -1
    broken_skeleton.loc[7, 'link'] = -1

    s = broken_skeleton.copy()
    reorient_skeleton(s, 10)
    assert (s['link'].iloc[7:10] == [9,10,-1]).all()

    # reorienting shouldn't change the number of roots,
    # though they may change locations.
    assert len(s.query('link == -1')) == 3

def test_heal_skeleton(linear_skeleton):
    broken_skeleton = linear_skeleton.copy()
    broken_skeleton.loc[2, 'link'] = -1
    broken_skeleton.loc[7, 'link'] = -1

    healed_skeleton = heal_skeleton(broken_skeleton)
    assert (healed_skeleton == linear_skeleton).all().all()

def test_heal_skeleton_with_threshold(linear_skeleton):
    broken_skeleton = linear_skeleton.copy()
    broken_skeleton.loc[2, 'link'] = -1
    broken_skeleton.loc[7, 'link'] = -1

    healed_skeleton = heal_skeleton(broken_skeleton, 10.0)

    # With a threshold of 10, the first break could be healed,
    # but not the second.
    expected_skeleton = linear_skeleton.copy()
    expected_skeleton.loc[7, 'link'] = -1
    assert (healed_skeleton == expected_skeleton).all().all()

def test_fetch_skeleton(client):
    orig_df = fetch_skeleton(5813027016, False)
    healed_df = fetch_skeleton(5813027016, True)

    assert len(orig_df) == len(healed_df)
    assert (healed_df['link'] == -1).sum() == 1
    assert healed_df['link'].iloc[0] == -1

@pytest.mark.skip("Need to write a test for skeleton_segments()")
def test_skeleton_segments(linear_skeleton):
    pass

if __name__ == "__main__":
    args = ['-s', '--tb=native', '--pyargs', 'neuprint.tests.test_skeleton']
    #args += ['-k', 'heal_skeleton']
    pytest.main(args)

max_stars_repo_name: umayaml/neuprint-python

max_issues_repo_name: umayaml/neuprint-python

max_forks_repo_name: umayaml/neuprint-python

reason: import numpy,import networkx