text:
"""
Utilities and helper functions for the Optimal Mouse Grouping program.
"""

import os
from pathlib import Path
from typing import Dict, List

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

########################################################################################################################

def load_and_verify_mouse_id_and_tumor_size_data_frame(
        excel_file_path: str, orig_id_column_name: str, orig_tumor_size_column_name: str,
        id_column_name: str, tumor_size_column_name: str
) -> pd.DataFrame:
    """
    Load input data from an Excel file. Input data is a list of mice with a Mouse ID and a Tumor Size [mm^3] per mouse.
    :param excel_file_path: Path to the input Excel file (should end with .xlsx).
    :param orig_id_column_name: Name of the ID column in the input Excel file.
    :param orig_tumor_size_column_name:  Name of the Tumor Size column in the input Excel file.
    :param id_column_name: Name of the ID column that will be used throughout the analysis.
    :param tumor_size_column_name: Name of the Tumor Size column that will be used throughout the analysis.
    :return: A data frame of the input Excel file contents, specifically the mouse IDs and the tumor sizes.
    """
    # Verify Excel file existence
    if not os.path.isfile(excel_file_path):
        excel_full_file_path = Path(excel_file_path).resolve()
        raise FileNotFoundError(f"Excel file not found: \"{excel_file_path}\". Full path: \"{excel_full_file_path}\"")

    # Load data frame via pandas
    df: pd.DataFrame = pd.read_excel(excel_file_path)

    # Verify column names
    if orig_id_column_name not in df.columns.to_list():
        raise ValueError(f"Column \"{orig_id_column_name}\" not found in input Excel file. Please fix.")
    if orig_tumor_size_column_name not in df.columns.to_list():
        raise ValueError(f"Column \"{orig_tumor_size_column_name}\" not found in input Excel file. Please fix.")
    if orig_id_column_name == orig_tumor_size_column_name:
        raise ValueError(f"ID and Tumor Size column names cannot be the same: \"{orig_id_column_name}\". Please fix.")

    # Check that all IDs are unique
    if len(df[orig_tumor_size_column_name].unique()) != len(df[orig_tumor_size_column_name]):
        raise ValueError("Not all Mouse IDs in the input Excel file are unique! Please fix.")

    # Check that there are no missing values
    if df[orig_id_column_name].isna().sum() != 0:
        raise ValueError(f"Column \"{orig_id_column_name}\" contains missing values. Please fix.")
    if df[orig_tumor_size_column_name].isna().sum() != 0:
        raise ValueError(f"Column \"{orig_tumor_size_column_name}\" contains missing values. Please fix.")

    # Check that all tumor sizes are non-negative
    if not (df[orig_tumor_size_column_name] >= 0).all():
        raise ValueError("All tumor sizes must be non-negative. Please fix.")

    # Rename columns
    df = df.rename(columns={
        orig_id_column_name: id_column_name,
        orig_tumor_size_column_name: tumor_size_column_name
    })
    return df

########################################################################################################################

def compute_group_sizes(num_mice: int, min_group_size: int) -> np.ndarray:
    """
    Given the number of mice, compute all the group sizes so that there is a minimum of `min_group_size` in each group.
    One or more groups might contain one or more mice more than the minimum number.

    Example 1: 44 mice and a minimum group size of 5 result in four groups of 6 and four groups of 5.
    Example 2: 14 mice and a minimum group size of 5 result in two groups of 7.

    :param num_mice: The total number of mice to divide into groups.
    :param min_group_size: The minimum size of the groups that the mice should be divided into.
    :return: Array of group sizes, one group size (integer) per group.
    """
    num_groups, num_remainders = np.divmod(num_mice, min_group_size)
    group_sizes: np.ndarray = np.repeat(min_group_size, repeats=num_groups)
    # Divide initial remainders onto the groups
    num_remainders_per_group = int(np.floor(num_remainders / num_groups))
    # Add these to the groups
    group_sizes += num_remainders_per_group
    # Compute final remainders
    num_final_remainders = num_remainders - num_groups * num_remainders_per_group
    assert num_final_remainders == num_mice - group_sizes.sum()
    assert num_final_remainders < num_groups
    # Add the final remainders to some of the groups
    group_sizes[:num_final_remainders] += 1
    # Verify and return the computed group sizes
    assert group_sizes.sum() == num_mice
    assert np.all(group_sizes >= min_group_size)
    return group_sizes

def compute_group_means(df: pd.DataFrame, value_colname: str, group_colname: str) -> pd.Series:
    """
    Compute the mean of one column, grouped by another column.
    Used here to compute the mean tumor size across generated mouse groups.
    :param df: Input data frame of IDs and Tumor Sizes (read from the input Excel file).
    :param value_colname: Name of the column that contains the values to group.
    :param group_colname: Name of the column that contains the grouping variable.
    :return: The computed group means, e.g. the tumor size means per mouse group.
    """
    group_means: pd.Series = df.groupby(group_colname)[value_colname].mean()
    return group_means

########################################################################################################################

def print_group_sizes(group_sizes: np.ndarray) -> None:
    """
    Print the group sizes with one group per line plus a total at the end.
    :param group_sizes: Array of group sizes, one group size (integer) per group.
    """
    num_in_total: int = int(np.sum(group_sizes))
    print("Group sizes:")
    for group_id, num_in_group in enumerate(group_sizes, start=1):
        print(f"- Group {group_id}: {num_in_group} mice")
    print(f"- ({num_in_total} mice in total)\n")

def construct_mouse_groups_data_frame(
        df: pd.DataFrame, id_column_name: str, tumor_size_column_name: str, group_column_name: str
) -> pd.DataFrame:
    """
    Construct a data frame that summarized the mouse groups, including the mean tumor size for each group etc.
    :param df: Input data frame of IDs and Tumor Sizes (read from the input Excel file).
    :param id_column_name: Name of the ID column.
    :param tumor_size_column_name: Name of the Tumor Size column.
    :param group_column_name: Name of the Group column.
    :return: A data frame summarizing the mouse groups.
    """
    group_means: pd.Series = compute_group_means(df, tumor_size_column_name, group_column_name)
    overall_tumor_size_mean: np.float64 = df[tumor_size_column_name].mean()
    groups_diffs: np.ndarray = group_means.values - overall_tumor_size_mean
    mouse_ids_in_group: pd.Series = df.groupby(group_column_name)[id_column_name].unique()
    # https://stackoverflow.com/a/45307119
    num_mice_in_group: List[int] = [len(lst) for lst in mouse_ids_in_group]
    mouse_ids_in_group_str: List[str] = [", ".join(map(str, lst)) for lst in mouse_ids_in_group]
    group: List[int] = list(df.groupby(group_column_name).indices.keys())
    df_dict = {
        "group": group,
        "num_mice_in_group": num_mice_in_group,
        "mouse_ids_in_group": mouse_ids_in_group_str,
        "tumor_size_mean": group_means,
        "overall_mean_diff": groups_diffs,
    }
    df_groups = pd.DataFrame(df_dict)
    return df_groups

########################################################################################################################

def save_mouse_grouping_as_xlsx(df_sorted: pd.DataFrame, df_groups: pd.DataFrame, xlsx_file_path: str) -> None:
    """
    Create an Excel document of the mouse grouping results.
    It contains two sheets:
    - Sheet 1: "mouse_grouping" - A list of mice with information about each (Group, Mouse ID, Tumor Size).
    - Sheet 2: "group_statistics" - A list of mouse groups with information, see construct_mouse_groups_data_frame().
    :param df_sorted: A data frame containing the list of mice with information about each.
    :param df_groups: A data frame containing the list of mouse groups with information.
    :param xlsx_file_path: File path to write the XLSX file to.
    """
    data_frames_by_sheet_name = {
        "mouse_grouping": df_sorted,
        "group_statistics": df_groups,
    }
    column_formats_by_sheet_name = get_column_formats_by_sheet_name()

    # Create Excel file and format cells
    with pd.ExcelWriter(xlsx_file_path, engine="xlsxwriter") as writer:  # type: ignore  # pylint: disable=E0110
        for sheet_name, df in data_frames_by_sheet_name.items():
            # Write data frame as sheet
            df.to_excel(writer, sheet_name=sheet_name, index=False)
            # Define header formatting
            _modify_header_row_format(df, writer, sheet_name)
            # Define column formats
            _modify_column_number_formats(df, writer, sheet_name, column_formats_by_sheet_name[sheet_name])
            # Autoadjust column widths
            _adjust_column_width(df, writer, sheet_name)

    print(f"Mouse grouping results saved to \"{xlsx_file_path}\".")

########################################################################################################################

def get_column_formats_by_sheet_name() -> Dict[str, Dict[str, Dict[str, str]]]:
    """
    Get the Excel cell formats for each column in each of the sheets in the output XLSX file.
    :return: Nested dictionary of Excel cell formats.
    """
    # Cell formats
    format_str: Dict[str, str] = {}
    format_int: Dict[str, str] = {"num_format": "#0"}
    format_float2: Dict[str, str] = {"num_format": "#,##0.00"}
    format_float4: Dict[str, str] = {"num_format": "#,####0.0000"}

    # Column formats by sheet name
    column_formats_by_sheet_name = {
        "mouse_grouping": {
            "group": format_int,
            "mouse_id": format_int,
            "tumor_size": format_float2,
        },
        "group_statistics": {
            "group": format_int,
            "num_mice_in_group": format_int,
            "mouse_ids_in_group": format_str,
            "tumor_size_mean": format_float2,
            "overall_mean_diff": format_float4,
        },
    }
    return column_formats_by_sheet_name

########################################################################################################################

def _modify_header_row_format(df: pd.DataFrame, writer: pd.ExcelWriter, sheet_name: str) -> None:
    """
    Format the header row (light blue background color, 1 px border).
    :param df: Data frame that the sheet was created from.
    :param writer: The active ExcelWriter object that was just used to create the sheet from the data frame.
    :param sheet_name: Name of the sheet that will be modified.
    """
    # Colored header row
    header_fmt = writer.book.add_format({"bg_color": "#DDEEFF", "border": 1})
    header_fmt.set_align("center")
    header_fmt.set_align("vcenter")
    header_fmt.set_bold()
    for col_name in df:
        col_idx = df.columns.get_loc(col_name)
        writer.sheets[sheet_name].write_string(0, col_idx, col_name, cell_format=header_fmt)

def _adjust_column_width(df: pd.DataFrame, writer: pd.ExcelWriter, sheet_name: str, extra_width: int = 2) -> None:
    """
    Automatically adjust the width of the columns in an Excel sheet.
    Inspired by https://stackoverflow.com/a/61617835
    :param df: Data frame that the sheet was created from.
    :param writer: The active ExcelWriter object that was just used to create the sheet from the data frame.
    :param sheet_name: Name of the sheet that will be modified.
    """
    for col_name in df:
        if pd.api.types.is_float_dtype(df[col_name]):
            longest_value = df[col_name].map('${:,.2f}'.format).astype(str).map(len).max()
        else:
            longest_value = df[col_name].astype(str).map(len).max()
        column_length = max(longest_value, len(col_name)) + extra_width
        col_idx = df.columns.get_loc(col_name)
        writer.sheets[sheet_name].set_column(col_idx, col_idx, width=column_length)

def _modify_column_number_formats(df: pd.DataFrame, writer: pd.ExcelWriter, sheet_name: str,
                                  column_formats_by_colum_name: Dict[str, Dict[str, str]]) -> None:
    """
    Modify the number formats of selected columns in an Excel sheet.
    :param df: Data frame that the sheet was created from.
    :param writer: The active ExcelWriter object that was just used to create the sheet from the data frame.
    :param sheet_name: Name of the sheet that will be modified.
    :param column_formats_by_colum_name: Dictionary of column formats (dicts like {"num_format": "#0"}) by column names.
    """
    # Formats need to be added to the workbook before use
    workbook = writer.book
    for col_name, col_format in column_formats_by_colum_name.items():
        # Add formats to workbook before use
        fmt = workbook.add_format(col_format)
        # Set text alignment, see https://xlsxwriter.readthedocs.io/format.html#set_align
        fmt.set_align("center")
        fmt.set_align("vcenter")
        # Set column format
        col_idx = df.columns.get_loc(col_name)
        writer.sheets[sheet_name].set_column(col_idx, col_idx, cell_format=fmt)

########################################################################################################################

def move_column_inplace(df: pd.DataFrame, col_name: str, pos: int) -> None:
    """
    Move a data frame column in-place, i.e. modifying the given data frame.
    :param df: A data frame.
    :param col_name: The name of the column to move.
    :param pos: The desired position of the column, e.g. 0 to move it to the front.
    """
    # https://stackoverflow.com/a/58686641
    column: pd.Series = df.pop(col_name)  # type: ignore
    df.insert(pos, column.name, column)

########################################################################################################################

def plot_mouse_groups(
        df: pd.DataFrame, tumor_size_column_name: str, group_column_name: str, plot_file_path: str
) -> None:
    """
    Plot the resulting mouse groups as a violin plot with a swarm plot on top. The plot will be saved as a PNG-file.
    :param df: A data frame of mice with information about each (Group, Mouse ID, Tumor Size).
    :param tumor_size_column_name: Name of the Tumor Size column.
    :param group_column_name: Name of the Group column.
    :param plot_file_path: File path to write the PNG to.
    """
    # Use default seaborn theme for plots
    sns.set_theme()

    # Define height and aspect ratio of plots
    h, a = 7, 1.6

    title = "Optimal Mouse Grouping\n(optimized for equal tumor size average across groups)"

    group_means = compute_group_means(df, tumor_size_column_name, group_column_name)
    group_means_df = pd.DataFrame(group_means)
    group_means_df.reset_index(level=0, inplace=True)

    g = sns.catplot(x=group_column_name, y=tumor_size_column_name, kind="violin", inner=None, bw=0.3, data=df,
                    height=h, aspect=a)
    sns.swarmplot(x=group_column_name, y=tumor_size_column_name, color="k", size=7, data=df, ax=g.ax)
    g.ax.set_title(title, weight="bold")
    g.ax.set_xlabel(group_column_name.capitalize(), weight="bold")
    g.ax.set_ylabel("Tumor Size [mm\u00b3]", weight="bold")

    # Add group means as markers
    # https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.plot.html
    g.ax.plot(group_means_df[tumor_size_column_name], "o", ms=12, mew=2, mec="w", mfc="r")

    # Set tight layout with a little extra padding
    # https://stackoverflow.com/a/14307273/1447415
    g.tight_layout(pad=0.4, w_pad=0.5, h_pad=1.0)

    # Save as PNG
    if not plot_file_path.lower().endswith(".png"):
        raise ValueError("Plot file path must end with \".png\".")

    g.savefig(plot_file_path)
    print(f"Mouse grouping plot saved to \"{plot_file_path}\".")

    # Close figure
    plt.close(g.fig)

########################################################################################################################

max_stars_repo_name: johandicap/optimal-mouse-grouping

max_issues_repo_name: johandicap/optimal-mouse-grouping

max_forks_repo_name: johandicap/optimal-mouse-grouping


text:
# Copyright 2017. Allen Institute. All rights reserved
#
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
# following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following
# disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote
# products derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
import h5py
import numpy as np
import pandas as pd
import os

def read_spk_txt(f_name):

    '''

    Parameters
    ----------
    f_name: string
        Full path to a file containing cell IDs and spike times.

    Returns
    -------
    A dataframe containing two columns: spike times and cell IDs.

    Usage:
    x = read_spk_txt('output/spk.dat')

    '''

    df = pd.read_csv(f_name, header=None, sep=' ')
    df.columns = ['t', 'gid']

    return df

def read_spk_h5(f_name):

    '''

    Parameters
    ----------
    f_name: string
        Full path to a file containing cell IDs and spike times.

    Returns
    -------
    A dataframe containing two columns: spike times and cell IDs.

    Usage:
    x = read_spk_h5('output/spk.h5')

    '''

    f = h5py.File(f_name, 'r' , libver='latest')
    spikes = {}

    t = np.array([])
    gids = np.array([])
    for i, gid in enumerate(f.keys()):  # save spikes of all gids
        if (i % 1000 == 0):
            print(i)
        spike_times = f[gid][...]
        t = np.append(t, spike_times)
        gids = np.append(gids, np.ones(spike_times.size)*int(gid))

    f.close()

    df = pd.DataFrame(columns=['t', 'gid'])
    df['t'] = t
    df['gid'] = gids

    return df

def spikes_to_mean_f_rate(cells_f, spk_f, t_window, **kwargs):

    '''

    Parameters
    ----------
    cells_f: string
        Full path to a file containing information about all cells (in particular, all cell IDs,
        and not just those that fired spikes in a simulation).
    spk_f: string
        Full path to a file containing cell IDs and spike times.
    t_window: a tuple of two floats
        Start and stop time for the window within which the firing rate is computed.
    **kwargs
    spk_f_type: string with accepted values 'txt' or 'h5'
        Type of the file from which spike times should be extracted.

    Assumptions
    -----------
    It is assumed here that TIME IS in ms and the RATES ARE RETURNED in Hz.

    Returns
    -------
    A dataframe containing a column of cell IDs and a column of corresponding
    average firing rates.

    Usage:
    x = spikes_to_mean_f_rate('../network_model/cells.csv', 'output/spk.dat', (500.0, 3000.0))

    '''

    # Make sure the time window's start and stop times are reasonable.
    t_start = t_window[0]
    t_stop = t_window[1]
    delta_t = t_stop - t_start
    if (delta_t <= 0.0):
        print('spikes_to_mean_f_rate: stop time %f is <= start time %f; exiting.' % (t_stop, t_start))
        quit()

    # Read information about all cells.
    cells_df = pd.read_csv(cells_f, sep=' ')
    gids = cells_df['id'].values

    # By default, the spk file type is "None", in which case it should be chosen
    # based on the extension of the supplied spk file name.
    spk_f_type = kwargs.get('spk_f_type', None)
    if (spk_f_type == None):
        spk_f_ext = spk_f.split('.')[-1]
        if (spk_f_ext in ['txt', 'dat']):
            spk_f_type = 'txt' # Assume this is an ASCII file.
        elif (spk_f_ext in ['h5']):
            spk_f_type = 'h5' # Assume this is an HDF5 file.
        else:
            print('spikes_to_mean_f_rate: unrecognized file extension.  Use the flag spk_f_type=\'txt\' or \'h5\' to override this message.  Exiting.')
            quit()

    # In case the spk_f_type was provided directly, check that the value is among those the code recognizes.
    if (spk_f_type not in ['txt', 'h5']):
        print('spikes_to_mean_f_rate: unrecognized value of spk_f_type.  The recognized values are \'txt\' or \'h5\'.  Exiting.')
        quit()

    # Read spikes.
    # If the spike file has zero size, create a dataframe with all rates equal to zero.
    # Otherwise, use spike times from the file to fill the dataframe.
    if (os.stat(spk_f).st_size == 0):
        f_rate_df = pd.DataFrame(columns=['gid', 'f_rate'])
        f_rate_df['gid'] = gids
        f_rate_df['f_rate'] = np.zeros(gids.size)
    else:
        # Use the appropriate function to read the spikes.
        if (spk_f_type == 'txt'):
            df = read_spk_txt(spk_f)
        elif(spk_f_type == 'h5'):
            df = read_spk_h5(spk_f)

        # Keep only those entries that have spike times within the time window.
        df = df[(df['t'] >= t_start) & (df['t'] <= t_stop)]

        # Compute rates.
        f_rate_df = df.groupby('gid').count() * 1000.0 / delta_t # Time is in ms and rate is in Hz.
        f_rate_df.columns = ['f_rate']
        # The 'gid' label is now used as index (after the groupby operation).
        # Convert it to a column; then change the index name to none, as in default.
        f_rate_df['gid'] = f_rate_df.index
        f_rate_df.index.names = ['']

        # Find cell IDs from the spk file that are not in the cell file.
        # Remove them from the dataframe with rates.
        gids_not_in_cells_f = f_rate_df['gid'].values[~np.in1d(f_rate_df['gid'].values, gids)]
        f_rate_df = f_rate_df[~f_rate_df['gid'].isin(gids_not_in_cells_f)]

        # Find cell IDs from the cell file that do not have counterparts in the spk file
        # (for example, because those cells did not fire).
        # Add these cell IDs to the dataframe; fill rates with zeros.
        gids_not_in_spk = gids[~np.in1d(gids, f_rate_df['gid'].values)]
        f_rate_df = f_rate_df.append(pd.DataFrame(np.array([gids_not_in_spk, np.zeros(gids_not_in_spk.size)]).T, columns=['gid', 'f_rate']))

        # Sort the rows according to the cell IDs.
        f_rate_df = f_rate_df.sort('gid', ascending=True)

    return f_rate_df

# Tests.

#x = spikes_to_mean_f_rate('/data/mat/yazan/corticalCol/ice/sims/column/build/net_structure/cells.csv', '/data/mat/yazan/corticalCol/ice/sims/column/full_preliminary_runs/output008/spikes.txt', (500.0, 2500.0))
#print x

#x = spikes_to_mean_f_rate('/data/mat/yazan/corticalCol/ice/sims/column/build/net_structure/cells.csv', '/data/mat/yazan/corticalCol/ice/sims/column/full_preliminary_runs/output008/spikes.h5', (500.0, 2500.0))
#print x

#x = spikes_to_mean_f_rate('/data/mat/yazan/corticalCol/ice/sims/column/build/net_structure/cells.csv', '/data/mat/yazan/corticalCol/ice/sims/column/full_preliminary_runs/output008/spikes.txt', (500.0, 2500.0), spk_f_type='txt')
#print x

ext: py

max_stars_repo_name: tjbanks/bmtk

max_issues_repo_name: tjbanks/bmtk

max_forks_repo_name: tjbanks/bmtk

reason: import numpy


text:
# model file for you to edit
import pandas as pd
import pandas as pd1
import numpy as np
from sklearn import tree
from sklearn.model_selection import cross_validate, cross_val_predict
from sklearn.model_selection import train_test_split
from pathlib import Path
global X_train, y_train, X, y, X_test, y_test
import logging
import click

logging.basicConfig(filename="logfilename.log",
                    level=logging.DEBUG,
                    format="%(asctime)s - %(levelname)s - %(message)s",
                    datefmt="%d/%m/%Y %I:%M:%S %p",
                    filemode='w',
                    force = True)

def read_logfile(log_filepath: str):
    with open(log_filepath) as f:
        for line in f:
            print(line)

# Created a data processing function

def data_processing(filepath: str, SplitRatio: float, limit: float):
    """This is the function to select the relevent data and splitting data into test and train"""
    try:
        logging.info("INFO: File path exists")
        df = pd.read_csv(filepath, sep=',')
        # Shuffle the datarows randomly, to be sure that the ordering of rows is somewhat random:
        df = df.sample(frac=1)
        # Use the method "drop nan" on the DataFrame itself to force it to remove rows with null/nan-values:
        df = df.dropna(axis=0, how='any')
        print('Price limit set for classification: ${}'.format(limit))
        # Select the columns/features from the Pandas dataframe that we want to use in the model:
        X = np.array(df[['Age', 'KM']])  # we only take the first two features.
        y = 1*np.array(df['Price']>limit)
        X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=SplitRatio)
        return X_train, X_test, y_train, y_test
    except FileNotFoundError:
        print("WARNING: File path doesn't exist")
    except ValueError:
        print("WARNING: Value not interger")

# Create a linear regression model that we can train:
# This function is to train the model, with training data and CV as input to the fuction and it returns the traing results
def model_training(X_train, y_train, CV):
    """This function is to train the model, with training data and CV as input to the fuction and it returns the traing results"""
    clf = tree.DecisionTreeClassifier(max_depth=3)
    #clf.fit(X_train, y_train)
    # Train the model using CV and multiple scoring on the data we have prepared:
    cv_results = cross_validate(clf, # Provide our model to the CV-function
                            X_train, # Provide all the features (in real life only the training-data)
                            y_train, # Provide all the "correct answers" (in real life only the training-data)
                            scoring=('f1', 'precision', 'recall', 'accuracy'),
                            cv=CV # Cross-validate using 5-fold (K-Fold method) cross-validation splits
                           )
    return cv_results

# Print some information about the linear model and its parameters:

#this function is to print the results of the training model
def print_results(cv_results):
    """This function is to print the results of the training model"""
    #print(clf)
    F1_pos   = cv_results['test_f1']
    P_pos   = cv_results['test_precision']
    R_pos   = cv_results['test_recall']
    A   = cv_results['test_accuracy']
    logging.info('\n-------------- Scores ---------------')
    logging.info('Average F1:\t {:.2f} (+/- {:.2f})'.format(F1_pos.mean(), F1_pos.std()))
    logging.info('Average Precision (y positive):\t {:.2f} (+/- {:.2f})'.format(P_pos.mean(), P_pos.std()))
    logging.info('Average Recall (y positive):\t {:.2f} (+/- {:.2f})'.format(R_pos.mean(), R_pos.std()))
    logging.info('Average Accuracy:\t {:.2f} (+/- {:.2f})'.format(A.mean(), A.std()))

@click.command()
@click.option('--limit', default=None)

def model(limit):
    df1 = pd1.read_csv('Config.csv', sep=',')
    #printing configuration file
    logging.info(df1)
    #assigning variables
    file_path =df1["DATA_PATH"][0]
    SplitRatio=df1["SplitRatio"][0]
    if limit is None:
        limit=df1["CostLimit"][0]
    limit=int(limit)
    CV=df1["CV"][0]
    if Path(file_path).exists():
        logging.info('INFO: File path exists')
        X_train, X_test, y_train, y_test = data_processing(file_path, SplitRatio, limit)
        cv_results = model_training(X_train, y_train, CV)
        print_results(cv_results)
    else:
        logging.info('WARNING: File path does not exist')
    read_logfile("logfilename.log")

if __name__ == '__main__':
    model()


text:
# 此文件和test.py文件类似，不过可以看到更详细中间输出结果，可以用来调试

from tools.image_input import read_img_file
import tensorflow as tf
from tools.cnn_model import cnn_model_fn
from tools.cnn_model import cnn_symbol_classifier
from tools import normalize_matrix_value,get_spatial_relationship,\
    verify_spatial_relationship,get_candidates,sort_characters
import process
from config import FILELIST
import numpy as np
from matplotlib import pyplot as plt
from tools.img_preprocess import read_img_and_convert_to_binary,binary_img_segment
import cv2
import my_parser
import tools
from calculator import *

#test
original_img, binary_img = read_img_and_convert_to_binary('./testImgs/easy +/3.jpg')

symbols = binary_img_segment(binary_img,original_img)
symbols = process.detect_uncontinous_symbols(symbols,binary_img)
length = len(symbols)
column = length/3+1
index = 1
# for symbol in symbols:
#     # print(symbol)
#     plt.subplot(column,3,index)
#     plt.imshow(symbol['src_img'], cmap='gray')
#     plt.title(index), plt.xticks([]), plt.yticks([])
#     index += 1
# temp_img = original_img[:, :, ::-1]
# # cv2.imshow('img',temp_img)
# # cv2.waitKey(0)
# # cv2.destroyAllWindows()
# plt.subplot(column,3,index)
# plt.imshow(temp_img, cmap = 'gray', interpolation = 'bicubic')
# plt.title(index),plt.xticks([]), plt.yticks([])
# plt.show()

symbols_to_be_predicted = normalize_matrix_value([x['src_img'] for x in symbols])

predict_input_fn = tf.estimator.inputs.numpy_input_fn(
    x={"x": np.array(symbols_to_be_predicted)},
    shuffle=False)

predictions = cnn_symbol_classifier.predict(input_fn=predict_input_fn)
print(predictions)
characters = []
for i,p in enumerate(predictions):
    # print(p['classes'],FILELIST[p['classes']])
    candidates = get_candidates(p['probabilities'])
    characters.append({'location':symbols[i]['location'],'candidates':candidates})
#print([x['location'] for x in characters])
# print(characters)

sorted_characters = sort_characters(characters)
print('排序前的字符列表')
print(characters)
print('排序后的字符序列')
print([[x['location'],x['candidates']] for x in sorted_characters])
tokens = process.group_into_tokens(sorted_characters)
print('识别出的token')
print(tokens)
# 先将每一个token初始化成一个树节点，得到一个节点列表node_list
node_list = my_parser.characters_to_nodes(sorted_characters)

parser_tree = my_parser.decompose(node_list)
print(parser_tree)
latex_str = post_order(parser_tree)
print(latex_str)
print(parser_tree['value'])
# parser_tree = my_parser.my_parser(sorted_characters)
# for i in range(10):
#     print()
# print('识别的表达式：')
# latex_str = tools.print_parser_tree(parser_tree,"")
# print()
# value = calculate(parser_tree)
# print('计算结果：',value)
#
# print('转化成的latex语句:')
#
expression_str = r'$result:'+latex_str+'='+latex(parser_tree['value'])+'$'
print(expression_str)
import os
if not os.path.exists('./result'):
    os.mkdir('./result')
plt.text(0.1,0.9,expression_str,fontsize=20)

plt.xticks([]),plt.yticks([])
plt.savefig('./result/1.jpg')
plt.close()

max_stars_repo_name: SOFTowaha/mathascode

max_issues_repo_name: SOFTowaha/mathascode

max_forks_repo_name: SOFTowaha/mathascode


text:
import pickle
from nltk.data import load
import pandas as pd
import numpy as np
import re
import string
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from sklearn.feature_extraction.text import TfidfVectorizer

from .gettweets import get_tweets_search

# from sklearn.model_selection import train_test_split
# from nltk.stem import PorterStemmer
# from nltk.stem import WordNetLemmatizer
# ML Libraries

# from sklearn.metrics import accuracy_score
# from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC

# Global Parameters
# nltk.download('stopwords')
# nltk.download('punkt')
# nltk.download('wordnet')
stop_words = set(stopwords.words('english'))

def preprocess_tweet_text(tweet):
    tweet.lower()
    # Remove urls
    tweet = re.sub(r"http\S+|www\S+|https\S+", '', tweet, flags=re.MULTILINE)
    # Remove user @ references and '#' from tweet
    tweet = re.sub(r'\@\w+|\#','', tweet)
    # Remove punctuations
    tweet = tweet.translate(str.maketrans('', '', string.punctuation))
    # Remove stopwords
    tweet_tokens = word_tokenize(tweet)
    filtered_words = [w for w in tweet_tokens if not w in stop_words]

    # ps = PorterStemmer()
    # stemmed_words = [ps.stem(w) for w in filtered_words]
    # lemmatizer = WordNetLemmatizer()
    # lemma_words = [lemmatizer.lemmatize(w, pos='a') for w in stemmed_words]

    return " ".join(filtered_words)

def load_model(filename):
    filename = 'app/model/'+filename
    infile = open(filename, 'rb')
    file = pickle.load(infile)
    infile.close()
    return file

def int_to_string(sentiment):
    if sentiment == 0:
        return "Negative"
    elif sentiment == 2:
        return "Neutral"
    else:
        return "Positive"

def get_sentiment(tweets = get_tweets_search()):
    # tweets = get_tweets_search()
    # tweets = [preprocess_tweet_text(i.text) for i in tweets]

    # vector = pickle.load(open('app/model/vector.pkl', 'rb'))
    vector = load_model('vector')
    # C:\Users\EMMA PRAISE\Documents\Account Audit\tweeter\app\model\vector
    tweet_transform = vector.transform(np.array(tweets).ravel())

    model = load_model('model')
    prediction = model.predict(tweet_transform)
    sentiment = [int_to_string(i) for i in prediction]
    return sentiment

def to_Dataframe(tweets):
    tweets = [preprocess_tweet_text(i.text) for i in tweets]
    sentiment = get_sentiment(tweets)
    data = {'Tweets':tweets, 'Sentiment': sentiment}
    dataframe = pd.DataFrame(data= data)
    dataframe.to_csv('app/export/hello.csv')
    return dataframe

# get_sentiment()

max_stars_repo_name: emmapraise/tweekners

max_issues_repo_name: emmapraise/tweekners

max_forks_repo_name: emmapraise/tweekners


text:
"""
Example: Forming of 1st order soliton pulse (sech) from gaussian input
by interaction of group velocity dispersion and self-phase modulation.

Interface based on MATLAB code published in
'Supercontinuum Generation in Optical Fibers'
by J. M. Dudley and J. R. Taylor, available at http://scgbook.info/.
"""

import numpy as np
import gnlse
import matplotlib.pyplot as plt

if __name__ == '__main__':
    setup = gnlse.gnlse.GNLSESetup()

    # Numerical parameters
    ###########################################################################
    # number of grid time points
    setup.resolution = 2**13
    # time window [ps]
    setup.time_window = 12.5
    # number of distance points to save
    setup.z_saves = 200
    # relative tolerance for ode solver
    setup.rtol = 1e-6
    # absoulte tolerance for ode solver
    setup.atol = 1e-6

    # Physical parameters
    ###########################################################################
    # Central wavelength [nm]
    setup.wavelength = 835
    # Nonlinear coefficient [1/W/m]
    setup.nonlinearity = 0.11
    # Dispersion: derivatives of propagation constant at central wavelength
    # n derivatives of betas are in [ps^n/m]
    betas = np.array([-11.830e-3])
    # Input pulse: pulse duration [ps]
    tFWHM = 0.050
    t0 = tFWHM / 2 / np.sqrt(np.log(2))  # for dispersive length calculation

    # 3rd order soliton conditions
    ###########################################################################
    # Dispersive length
    LD = t0 ** 2 / np.abs(betas[0])
    # Non-linear length for 1st order soliton
    LNL = LD / (1 ** 2)
    # Input pulse: peak power [W]
    power = 1 / (LNL * setup.nonlinearity)
    # Fiber length [m]
    setup.fiber_length = 10 * LD
    # Type of impulse:  gaussian
    setup.impulse_model = gnlse.GaussianEnvelope(power, tFWHM)
    # Loss coefficient [dB/m]
    loss = 0
    # Type of dyspersion operator: build from Taylor expansion
    setup.dispersion_model = gnlse.DispersionFiberFromTaylor(loss, betas)

    # Type of Ramman scattering function: None (default)
    # Selftepening: not accounted
    setup.self_steepening = False

    # Simulation
    ###########################################################################
    solver = gnlse.gnlse.GNLSE(setup)
    solution = solver.run()

    # Visualization
    ###########################################################################
    plt.figure(figsize=(15, 7))

    plt.subplot(1, 2, 1)
    gnlse.plot_wavelength_vs_distance(solution, WL_range=[780, 900])
    plt.subplot(1, 2, 2)
    gnlse.plot_delay_vs_distance(solution, time_range=[-.5, .5])

    plt.show()

max_stars_repo_name: JiEumKim/github

max_issues_repo_name: JiEumKim/github

max_forks_repo_name: JiEumKim/github


text:
import numpy as np
import mido as m
import importing_midi as midi
import json

with open('midi_json_mini.json', 'r') as midiJ:
    json_data = json.load(midiJ)

shape = tuple(json_data['shape'])
length = json_data['lengths']
midi_memmap = "POP909-Dataset-master/POP909/memmap_mini.dat"
fpath = np.memmap(midi_memmap, dtype='float32', mode='w+', shape=shape)

for num in range(1, len(json_data['y'])):
    midi_file = m.MidiFile(f"POP909-Dataset-master/POP909/{num:03d}/{num:03d}.mid", clip=True)
    result_array = midi.mid2arry(midi_file)
    if length == 15000:
        fpath[num, :15000, :] = result_array[:15000]
    else:
        for i in length:
            fpath[num, :i, :] = result_array[:15000]
    print(f'{num:03d} {result_array.shape}')


text:
# qbic

import attr
import random
import re

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader
from transformers.optimization import AdamW, WarmupLinearSchedule

from solvers.torch_helpers import (
    ModelTrainer,
    Field,
    BatchCollector,
    AccuracyCounter,
    init_bert,
    load_bert,
    load_bert_tokenizer,
)
from solvers.solver_helpers import fix_spaces, singleton, AbstractSolver

@attr.s(frozen=True)
class Example(object):
    tokens = attr.ib()
    token_ids = attr.ib()
    segment_ids = attr.ib()
    mask = attr.ib()
    position = attr.ib()
    label_id = attr.ib(default=None)

    def __len__(self):
        return len(self.token_ids)

@attr.s(frozen=True)
class TrainConfig(object):
    learning_rate = attr.ib(default=1e-5)
    train_batch_size = attr.ib(default=32)
    test_batch_size = attr.ib(default=32)
    epoch_count = attr.ib(default=10)
    warm_up = attr.ib(default=0.1)

def _get_optimizer(model, train_size, config):
    num_total_steps = int(train_size / config.train_batch_size * config.epoch_count)
    num_warmup_steps = int(num_total_steps * config.warm_up)

    optimizer = AdamW(model.parameters(), lr=config.learning_rate, correct_bias=False)
    scheduler = WarmupLinearSchedule(
        optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps
    )

    return optimizer, scheduler

def _get_batch_collector(device, is_train=True):
    if is_train:
        vector_fields = [Field("label_id", torch.float32)]
    else:
        vector_fields = []

    vector_fields.append(Field("position", torch.long))

    return BatchCollector(
        matrix_fields=[
            Field("token_ids", np.int64),
            Field("segment_ids", np.int64),
            Field("mask", np.int64),
        ],
        vector_fields=vector_fields,
        device=device,
    )

class RubertClassifier(nn.Module):
    def __init__(self, bert, output_name, bert_output_dim=768):
        super(RubertClassifier, self).__init__()

        self._bert = bert
        self._dropout = nn.Dropout(0.3)
        self._predictor = nn.Linear(bert_output_dim, 1)
        self._output_name = output_name

    def forward(self, batch):
        outputs, pooled_outputs = self._bert(
            input_ids=batch["token_ids"],
            attention_mask=batch["mask"],
            token_type_ids=batch["segment_ids"],
        )

        row_positions = torch.arange(0, outputs.shape[0])
        outputs = outputs[row_positions, batch["position"]]

        status_logits = self._predictor(self._dropout(outputs)).squeeze(-1)

        loss = 0.0
        if self._output_name + "_id" in batch:
            loss = F.binary_cross_entropy_with_logits(
                status_logits, batch[self._output_name + "_id"]
            )

        return {self._output_name + "_logits": status_logits, "loss": loss}

class ClassifierTrainer(ModelTrainer):
    def on_epoch_begin(self, *args, **kwargs):
        super(ClassifierTrainer, self).on_epoch_begin(*args, **kwargs)

        self._accuracies = [AccuracyCounter()]

    def on_epoch_end(self):
        info = super(ClassifierTrainer, self).on_epoch_end()

        return "{}, {}".format(
            info, ", ".join(str(score) for score in self._accuracies)
        )

    def forward_pass(self, batch):
        outputs = self._model(batch)

        predictions = (outputs["label_logits"] > 0.0).float()
        assert predictions.shape == batch["label_id"].shape

        self._accuracies[0].update(predictions, batch["label_id"])

        info = ", ".join(str(score) for score in self._accuracies)
        return outputs["loss"], info

class Solver(AbstractSolver):
    def __init__(self, bert_path="data/models/bert/rubert/qbic"):
        self.bert_path = bert_path
        self._tokenizer = load_bert_tokenizer(self.bert_path)
        self._device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
        self._batch_collector = _get_batch_collector(self._device, is_train=False)
        self._model = RubertClassifier(init_bert(self.bert_path), output_name="label").to(
            self._device
        )

    @staticmethod
    def _get_sentences(text):
        for sent in text.split("."):
            if re.search("\(\d+\)", sent):
                yield fix_spaces(sent.strip()) + "."

    def _convert_task(self, task):
        text = task["text"].replace("?", ".").replace("\xa0", "")
        sentence = " ".join(list(self._get_sentences(text)))

        correct = None
        if "solution" in task:
            if "correct" in task["solution"]:
                correct = sorted([int(idx) - 1 for idx in task["solution"]["correct"]])
            else:
                correct = sorted(
                    [int(idx) - 1 for idx in task["solution"]["correct_variants"][0]]
                )

        tokens, positions = ["[CLS]"], []
        prev_match_end = 0
        for match in re.finditer("\(\d+\)", sentence):
            tokens.extend(
                self._tokenizer.tokenize(
                    sentence[prev_match_end : match.start()].strip()
                )
            )
            positions.append(len(tokens))
            tokens.append("[MASK]")
            prev_match_end = match.end()

        tokens.extend(self._tokenizer.tokenize(sentence[prev_match_end:].strip()))
        tokens.append("[SEP]")
        assert all(tokens[position] == "[MASK]" for position in positions)

        token_ids = self._tokenizer.convert_tokens_to_ids(tokens)
        assert token_ids[0] == 101 and token_ids[-1] == 102
        assert len(token_ids) == len(tokens)

        for position_id, position in enumerate(positions):
            label_id = None
            if correct is not None:
                label_id = int(position_id in correct)

            yield Example(
                tokens=tokens,
                token_ids=token_ids,
                segment_ids=[0] * len(token_ids),
                mask=[1] * len(token_ids),
                position=position,
                label_id=label_id,
            )

    def _prepare_examples(self, tasks):
        examples = []
        for task in tasks:
            for example in self._convert_task(task):
                examples.append(example)
        return examples

    def fit(self, tasks, test_tasks=None):
        print("START FIT")
        train_examples = self._prepare_examples(tasks)
        test_examples = None
        if test_tasks is not None:
            test_examples = self._prepare_examples(test_tasks)

        config = TrainConfig()
        self._model = RubertClassifier(load_bert(self.bert_path), output_name="label").to(
            self._device
        )

        batch_collector = _get_batch_collector(self._device, is_train=True)

        train_loader = DataLoader(
            train_examples,
            batch_size=config.train_batch_size,
            shuffle=True,
            collate_fn=batch_collector,
            pin_memory=False,
        )
        test_loader, test_batches_per_epoch = None, 0
        if test_examples is not None:
            test_loader = DataLoader(
                test_examples,
                batch_size=config.test_batch_size,
                shuffle=False,
                collate_fn=batch_collector,
                pin_memory=False,
            )
            test_batches_per_epoch = int(len(test_examples) / config.test_batch_size)

        optimizer, scheduler = _get_optimizer(self._model, len(train_examples), config)

        trainer = ClassifierTrainer(self._model, optimizer, scheduler, use_tqdm=True)
        trainer.fit(
            train_iter=train_loader,
            train_batches_per_epoch=int(len(train_examples) / config.train_batch_size),
            val_iter=test_loader,
            val_batches_per_epoch=test_batches_per_epoch,
            epochs_count=config.epoch_count,
        )

    def save(self, path="data/models/solvers/solver17/punctuator.pkl"):
        torch.save(self._model.state_dict(), path)

    @singleton
    def load(self, path="data/models/solvers/solver17/punctuator.pkl"):
        weights = torch.load(path, map_location=self._device)
        self._model.load_state_dict(weights)
        self._model.eval()
        self.is_loaded = True

    def predict_from_model(self, task):

        examples = list(self._convert_task(task))
        model_inputs = self._batch_collector(examples)

        with torch.no_grad():
            model_prediction = self._model(model_inputs)["label_logits"].cpu().numpy()

        choices = task["question"]["choices"]
        prediction = [
            str(choices[ind]["id"])
            for ind, logit in enumerate(model_prediction)
            if logit > 0.0
        ]

        return prediction


text:
import neural_network_lyapunov.examples.quadrotor2d.quadrotor_2d as\
    quadrotor_2d

import unittest
import numpy as np
import torch
import scipy.integrate

class TestQuadrotor2D(unittest.TestCase):
    def test_dynamics_equilibrium(self):
        plant = quadrotor_2d.Quadrotor2D(torch.float64)
        u = plant.u_equilibrium
        xdot = plant.dynamics(np.zeros((6, )), u)
        np.testing.assert_allclose(xdot, np.zeros((6, )))

    def test_dynamics(self):
        plant = quadrotor_2d.Quadrotor2D(torch.float64)

        def check_dynamics(x, u):
            assert (isinstance(x, torch.Tensor))
            xdot = plant.dynamics(x, u)
            xdot_np = plant.dynamics(x.detach().numpy(), u.detach().numpy())
            np.testing.assert_allclose(xdot_np, xdot.detach().numpy())

        check_dynamics(torch.tensor([1, 2, 3, 4, 5, 6], dtype=torch.float64),
                       torch.tensor([7, 8], dtype=torch.float64))
        check_dynamics(
            torch.tensor([1, -2, 3, -4, 5, -6], dtype=torch.float64),
            torch.tensor([7, -8], dtype=torch.float64))

    def test_linearized_dynamics(self):
        plant = quadrotor_2d.Quadrotor2D(torch.float64)

        def check_linearized_dynamics(x, u):
            assert (isinstance(x, torch.Tensor))
            A, B = plant.linearized_dynamics(x, u)
            xdot = plant.dynamics(x, u)
            for i in range(6):
                if x.grad is not None:
                    x.grad.zero_()
                if u.grad is not None:
                    u.grad.zero_()
                xdot[i].backward(retain_graph=True)
                Ai_expected = x.grad.detach().numpy() if x.grad is not None\
                    else np.zeros((6,))
                np.testing.assert_allclose(A[i, :].detach().numpy(),
                                           Ai_expected)
                Bi_expected = u.grad.detach().numpy() if u.grad is not None\
                    else np.zeros((2,))
                np.testing.assert_allclose(B[i, :].detach().numpy(),
                                           Bi_expected)
            # Make sure numpy and torch input give same result.
            A_np, B_np = plant.linearized_dynamics(x.detach().numpy(),
                                                   u.detach().numpy())
            np.testing.assert_allclose(A_np, A.detach().numpy())
            np.testing.assert_allclose(B_np, B.detach().numpy())

        check_linearized_dynamics(
            torch.tensor([1, 2, 3, 4, 5, 6],
                         dtype=torch.float64,
                         requires_grad=True),
            torch.tensor([7, 8], dtype=torch.float64, requires_grad=True))
        check_linearized_dynamics(
            torch.tensor([-1, -2, 3, 4, 5, 6],
                         dtype=torch.float64,
                         requires_grad=True),
            torch.tensor([7, -8], dtype=torch.float64, requires_grad=True))

    def test_lqr_control(self):
        plant = quadrotor_2d.Quadrotor2D(torch.float64)
        x_star = np.zeros((6, ))
        u_star = plant.u_equilibrium.detach().numpy()
        Q = np.diag([10, 10, 10, 1, 1, plant.length / 2. / np.pi])
        R = np.array([[0.1, 0.05], [0.05, 0.1]])
        K, S = plant.lqr_control(Q, R, x_star, u_star)
        result = scipy.integrate.solve_ivp(
            lambda t, x: plant.dynamics(x, K @ (x - x_star) + u_star), (0, 10),
            np.array([0.1, 0.1, -0.1, 0.2, 0.2, -0.3]))
        np.testing.assert_allclose(result.y[:, -1], x_star, atol=1E-6)

if __name__ == "__main__":
    unittest.main()

max_issues_repo_name: StanfordASL/neural-network-lyapunov

max_forks_repo_name: StanfordASL/neural-network-lyapunov

reason: import numpy,import scipy


text:
# Iteratively:
# Create the transition for one pair of songs
#   - Time wrapping (progressivelly better)
#   - Key changing (explore strategies)
#   - Align all the sequence according to modified beats (try to do it with downbeats)
#   - Volume fades to mix both

import numpy as np
import rubberband as rb
from utils import convert

store_path_transition_times = "../output/mix_name_info.txt"

def create_transitions(queue):

    mix = queue[0]

    print("Creating_transitions...")

    for i in range(1, len(queue)):

        print(f"\tMixing tracks {i} and {i+1}...")
        mix, transition_time = mix_pair(mix, queue[i])

        with open(store_path_transition_times, "a") as myFile:
            myFile.write(f"Transition {i} at time {convert(transition_time)}.\n")

    return mix

def mix_pair(previous_mix, next_song):
    """
    output
        mix = {
        name ([string])
        audio_array (np.array)
        sampling_rate ([double])
        ...
        real_bpm ([Int])
        estimated_bpm ([Int])
        estimated_key ([String])
        cue_points (np.array)
        }
    """

    # selecting the actual cue-points from all the posibilities
    previous_mix_cue_point = select_cue_points(previous_mix)

    print("\t\tAligning songs...")
    next_song_aligned = align(next_song)

    print("\t\tMixing beats...")
    previous_mix_stretched,next_song_stretched,previous_ending,next_beginning = time_wrap(previous_mix, next_song_aligned, previous_mix_cue_point)

    #print("\t\tTransposing keys...")
    # previous_mix, next_song = key_change(previous_mix_stretched, next_song_stretched)

    print("\t\tFading transition...")
    previous_mix_faded, next_song_faded = fade(previous_mix_stretched, next_song_stretched)

    print("\t\tCombining tracks...")
    mix = combine_songs(previous_mix_faded, next_song_faded, previous_ending)

    return mix, previous_ending/previous_mix["frame_rate"]

def select_cue_points(previous_mix):

    cue_point = np.zeros_like(previous_mix["cue_points"])
    possible_idx = np.where(previous_mix["cue_points"] == 1)[0]

    flag = False
    i = 1
    while flag == False:
        # select first cue point that are at least 20s from end.
        if (len(previous_mix["audio_array"]) - possible_idx[-i]) / previous_mix["frame_rate"] >= 20:
            cue_point[possible_idx[-i]] = 1
            flag = True
        i += 1

    return cue_point

def align(next_song):

    first_beat = np.where(next_song["beat_times"] == 1)[0][0]

    new_next = next_song.copy()

    new_next["audio_array"] = next_song["audio_array"][first_beat:]
    new_next["beat_times"] = next_song["beat_times"][first_beat:]
    new_next["cue_points"] = next_song["cue_points"][first_beat:]

    return new_next

def time_wrap(previous_mix, next_song, previous_mix_cue_point):

    transition_length_seconds = 20
    avg_bpm = (previous_mix["estimated_bpm"] + next_song["estimated_bpm"]) / 2

    ending_stretching_ratio = previous_mix["estimated_bpm"] / avg_bpm
    beginning_stretching_ratio = next_song["estimated_bpm"] / avg_bpm

    cue_point_idx = np.where(previous_mix_cue_point == 1)[0][0]

    transition_length_prev_frames_stretched = transition_length_seconds * previous_mix["frame_rate"]
    transition_length_prev_frames = int(transition_length_prev_frames_stretched / ending_stretching_ratio)

    transition_length_next_frames_stretched = transition_length_seconds * next_song["frame_rate"]
    transition_length_next_frames = int(transition_length_next_frames_stretched / beginning_stretching_ratio)

    ending_audio = previous_mix["audio_array"][cue_point_idx : cue_point_idx + transition_length_prev_frames]
    ending_beats = previous_mix["beat_times"][cue_point_idx : cue_point_idx + transition_length_prev_frames]
    beginning_audio = next_song["audio_array"][:transition_length_next_frames]
    beginning_beats = next_song["beat_times"][:transition_length_next_frames]

    ending_audio_stretched = rb.stretch(np.array(ending_audio, dtype=previous_mix["dtype"]),rate=previous_mix["frame_rate"],ratio=ending_stretching_ratio,crispness=6,formants=False,precise=True)
    beginning_audio_stretched = rb.stretch(np.array(beginning_audio, dtype=next_song["dtype"]),rate=next_song["frame_rate"],ratio=beginning_stretching_ratio,crispness=6,formants=False,precise=True)

    ending_beats_stretched = stretch_beats(ending_beats, ending_stretching_ratio, ending_audio_stretched.size)
    beginning_beats_stretched = stretch_beats(beginning_beats, beginning_stretching_ratio, beginning_audio_stretched.size)

    previous_mix["estimated_bpm"] = next_song["estimated_bpm"]

    new_previous = previous_mix.copy()

    new_previous["audio_array"] = np.concatenate((new_previous["audio_array"][:cue_point_idx], ending_audio_stretched))
    new_previous["beat_times"] = np.concatenate((new_previous["beat_times"][:cue_point_idx], ending_beats_stretched))
    new_previous["cue_points"] = np.concatenate((new_previous["cue_points"][:cue_point_idx],np.zeros(ending_audio_stretched.size, dtype=previous_mix["cue_points"].dtype)))

    new_next = next_song.copy()
    new_next["audio_array"] = np.concatenate((beginning_audio_stretched, new_next["audio_array"][transition_length_next_frames:]))
    new_next["beat_times"] = np.concatenate((beginning_beats_stretched, new_next["beat_times"][transition_length_next_frames:]))
    new_next["cue_points"] = np.concatenate((np.zeros(beginning_audio_stretched.size, dtype=next_song["cue_points"].dtype),next_song["cue_points"][transition_length_next_frames:]))

    return (new_previous,new_next,new_previous["audio_array"][:cue_point_idx].size,beginning_audio_stretched.size)

def stretch_beats(beat_times, stretching_ratio, desired_length):

    new_beats = []

    zero_sequence_length = 0
    for i in beat_times:
        if i == 0:
            zero_sequence_length += 1
        elif i == 1:
            new_beats += [0] * int(zero_sequence_length * stretching_ratio)
            new_beats += [1]
            zero_sequence_length = 0

    diff = desired_length - len(new_beats)
    if diff > 0:
        new_beats += [0] * diff

    return np.array(new_beats, dtype=int)

def key_change(previous_mix, next_song, previous_mix_cue_point, next_song_cue_point):

    # rubberband

    # Choose to change the key of next_song completely or only the transition part

    return previous_mix, next_song

def fade(previous_mix, next_song):

    fade_seconds = 20
    fade_frames = fade_seconds * previous_mix["frame_rate"]

    for i in range(fade_frames):

        #exponential fade
        #previous_mix["audio_array"][-i] = previous_mix["audio_array"][-i] * (1.1 - np.exp(2.398 * (1 - i / fade_frames)) * 0.1)
        #next_song["audio_array"][i] = next_song["audio_array"][i] * (0.1 * np.exp(2.398 * i / fade_frames) - 0.1)

        #linear fade
        previous_mix["audio_array"][-i] = previous_mix["audio_array"][-i] * i/fade_frames
        next_song["audio_array"][i] = next_song["audio_array"][i] * i/fade_frames

    return previous_mix, next_song

def combine_songs(previous_mix, next_song, previous_ending):

    mix = previous_mix.copy()

    next_audio_padded = np.pad(next_song["audio_array"], (previous_ending, 0), constant_values=0)
    next_beat_padded = np.pad(next_song["beat_times"], (previous_ending, 0), constant_values=0)
    next_cue_padded = np.pad(next_song["cue_points"], (previous_ending, 0), constant_values=0)

    mix["audio_array"] = next_audio_padded
    mix["beat_times"] = next_beat_padded
    mix["cue_points"] = next_cue_padded

    mix["audio_array"][: previous_mix["audio_array"].size] += previous_mix["audio_array"]
    mix["beat_times"][: previous_mix["beat_times"].size] += previous_mix["beat_times"]
    mix["cue_points"][: previous_mix["cue_points"].size] += previous_mix["cue_points"]

    return mix


text:
import os
import sys
from sqlalchemy import Column, Float, Integer, String,Boolean
from sqlalchemy.dialects.mysql import BLOB as Blob
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine,and_
from sqlalchemy.orm import sessionmaker
from sqlalchemy.sql import select
import sqlalchemy

import numpy as np;

Base = declarative_base()

class Pascal3D(Base):
    __tablename__ = 'pascal3d'
    # id = Column(Integer, primary_key=True,autoincrement=True)
    idx = Column(Integer,nullable=False,primary_key=True);
    img_path = Column(String,nullable=False,primary_key=True);
    layer = Column(String,nullable=False,primary_key=True);
    nn_experiment_timestamp = Column(String,nullable=False,primary_key=True);
    class_id = Column(String,nullable=False);
    class_idx= Column(Integer,nullable=False);
    caffe_model=Column(String,nullable=False);

    azimuth= Column(Float);
    neighbor_index=Column(Blob);
    neighbor_distance=Column(Blob);
    trainedClass=Column(Boolean);
    azimuth_differences=Column(Blob);

class Pascal3D_Manipulator(object):

    def __init__(self,path_to_db):
        self.db_path=path_to_db;
        self.engine=create_engine(self.db_path);
        if not os.path.exists(path_to_db):
            Base.metadata.create_all(self.engine);
        # print self.engine
        Base.metadata.bind=self.engine;
        self.session=None;

    def openSession(self):
        DBSession=sessionmaker(bind=self.engine);
        self.session=DBSession();

    def closeSession(self):
        self.session.commit();
        self.session.close();
        self.session=None;

    def _getBlobsFromArrays(self,azimuth_differences,neighbor_distance,neighbor_index):
        if azimuth_differences is not None:
            azimuth_differences=np.array(azimuth_differences,dtype='float64').tostring()
        if neighbor_distance is not None:
            neighbor_distance=np.array(neighbor_distance,dtype='float64').tostring()
        if neighbor_index is not None:
            neighbor_index=np.array(neighbor_index,dtype='int64').tostring()
        return azimuth_differences,neighbor_distance,neighbor_index

    def insert(self,idx,img_path,layer,nn_experiment_timestamp,class_id,class_idx,caffe_model, azimuth=None,neighbor_index=None,neighbor_distance=None,trainedClass=None,azimuth_differences=None,commitFlag=True):

        if self.session is None:
            raise Exception('Open a Session First');

        azimuth_differences,neighbor_distance,neighbor_index=self._getBlobsFromArrays(azimuth_differences,neighbor_distance,neighbor_index);

        new_val = Pascal3D(idx=idx,img_path=img_path,nn_experiment_timestamp=nn_experiment_timestamp,layer=layer,class_id=class_id,class_idx=class_idx,caffe_model=caffe_model, azimuth=azimuth, neighbor_index=neighbor_index, neighbor_distance=neighbor_distance,trainedClass=trainedClass,azimuth_differences=azimuth_differences)

        self.session.add(new_val)
        if commitFlag:
            self.session.commit();

        return True;

    def filter(self,criterion):
        if self.session is None:
            raise Exception('Open a Session First');
        vals=self.session.query(Pascal3D).filter(*criterion).all();
        vals=[self._getArraysFromBlobs(val) for val in vals];
        return vals

    def _getArraysFromBlobs(self,val):
        if val.neighbor_index is not None:
            val.neighbor_index = np.fromstring(val.neighbor_index,dtype='int64')

        if val.neighbor_distance is not None:
            val.neighbor_distance = np.fromstring(val.neighbor_distance,dtype='float64')

        if val.azimuth_differences is not None:
            val.azimuth_differences = np.fromstring(val.azimuth_differences,dtype='float64')

        return val;

    def update(self,criterion,updateVals):

        if self.session is None:
            raise Exception('Open a Session First');

        keys_blobs=[Pascal3D.azimuth_differences,Pascal3D.neighbor_distance,Pascal3D.neighbor_index]

        x=self._getBlobsFromArrays(updateVals.get(keys_blobs[0],None),updateVals.get(keys_blobs[1],None),updateVals.get(keys_blobs[2],None));

        for idx,key_blob in enumerate(keys_blobs):
            if key_blob in updateVals:
                updateVals[key_blob]=x[idx];

        self.session.query(Pascal3D).filter(*criterion).update(updateVals);
        return True

    def _getArrayFromBlob(self,column,blob):
        if column.key=='neighbor_index':
            arr = np.fromstring(blob,dtype='int64')
        elif column.key=='azimuth_differences' or  column.key=='neighbor_distance':
            arr = np.fromstring(blob,dtype='float64')
        else:
            arr=None
        return arr

    def select(self,toSelect,criterion=None,distinct=False):
        if self.session is None:
            raise Exception('Open a Session First');

        if criterion is None:
            vals=self.session.execute(select(toSelect,distinct=distinct))
        else:
            if len(criterion)==1:
                vals=self.session.execute(select(toSelect,distinct=distinct).where(*criterion))
            else:

                vals=self.session.execute(select(toSelect,distinct=distinct).where(and_(*criterion)))

        val_list=[];
        for val in vals:
            val=list(val);
            for idx,field in enumerate(val):
                if str(toSelect[idx].type)=='VARCHAR':
                    val[idx]=str(field);
                if str(toSelect[idx].type)=='BLOB':
                    val[idx]=self._getArrayFromBlob(toSelect[idx],field)
            val=tuple(val);
            val_list.append(val);

        return val_list

def main():

    # Session.rollback();
    mani=Pascal3D_Manipulator('sqlite:///temp/pascal3d.db');
    mani.openSession();
    # for idx in range(10):
    #     try:
    #         mani.insert(idx=idx,img_path='img_path',layer='layer',nn_experiment_timestamp='nn_experiment_timestamp',class_id='class_id',class_idx=idx,caffe_model='caffe_model', azimuth=0.0,neighbor_index=range(10),neighbor_distance=np.random.rand(10),trainedClass=True,azimuth_differences=np.random.rand(10))
    #     except Exception:
    #         mani.session.rollback();
    #         print 'Hello';

    vals=mani.select((Pascal3D.img_path,Pascal3D.class_id,Pascal3D.azimuth_differences,Pascal3D.neighbor_distance,Pascal3D.neighbor_index),criterion=(Pascal3D.img_path=='img_path',),distinct=False)

    for val in vals:
        print val
    #     for field in val:
    #         print field, type(field)
    #     print '___'
    #     # for val_key in val.__dict__.keys():
        #     print val_key,val.__dict__[val_key]
    #
    mani.closeSession();

if __name__=='__main__':
    main();

max_stars_repo_name: maheenRashid/caffe

max_issues_repo_name: maheenRashid/caffe

max_forks_repo_name: maheenRashid/caffe


text:
"""Agilent B2902A Precision Source/Measure Unit"""

import testgear.base_classes as base
import numpy as np

class B2902A(base.source):
    def init(self):
        self.set_timeout(10)
        self.idstr = self.query("*IDN?").strip()

    def set_output(self, voltage=None, current=None, enabled=True, frequency=0, resistance=None, fourWire=False, channel=1):
        pass

    def get_output(self):
        """return an object which reflects the output conditions"""
        obj = base.output_status()
        return obj

max_stars_repo_name: PhilippCo/testgear

max_issues_repo_name: PhilippCo/testgear

max_forks_repo_name: PhilippCo/testgear


text:
#author Vipul Vaibhaw

import torch
import torch.nn as nn
# Linear Layer in pytorch is equivalent fully connected layer or dense
import torch.nn.functional as F
# 1st layer to be a nn.Conv2d(layer), F.conv2d(custom layer)
import torch.optim as optim

#x = torch.tensor(([5.1,4.3,2.5],[5.1,4.3,2.5],[5.1,4.3,2.5]),
#    dtype=torch.float) # 3X3 tensor

# size 4X3

#x.size() # size of my tensor

# converting from numpy to pytorch tensor

#import numpy as np

#a = np.array([1,2,3]) # numpy array
#b = torch.from_numpy(a)

#print(b)

#y = torch.rand(3,3) # initialized a random tensor of size 3X3
# take values from normal distribution of mean 0 and std dev 1.

#add_x_and_y = torch.add(x,y)

#print(add_x_and_y)

#x1 = torch.tensor(([5.1,4.3],[5.1,4.3],[5.1,4.3]),
#    dtype=torch.float) # 2X3 tensor

#z1 = x1.view(-1,2)

#print(z1)
#print(z1.size())

import torchvision as tv
import torchvision.transforms as transforms

dataset_transform = transforms.Compose([tv.transforms.ToTensor(),
            tv.transforms.Normalize((0.4914, 0.4822, 0.4465), (0.247, 0.243, 0.261))])
# Assignment - read about batch norm and
# how it is different from normalization,
# how to do it in pytorch!

trainset = tv.datasets.CIFAR10(root="./data",
            train=True, transform=dataset_transform, download=True)

dataloader = torch.utils.data.DataLoader(trainset,
            batch_size=4,num_workers=4)

class ConvModel(nn.Module):
    def __init__(self):
        super(ConvModel,self).__init__()

        self.conv1 = nn.Conv2d(3,6,5)
        self.pool = nn.MaxPool2d(2,2)
        self.conv2 = nn.Conv2d(6,16,5)
        self.fc1 = nn.Linear(16*5*5,200)
        self.fc2 = nn.Linear(200,100)
        self.fc3 = nn.Linear(100,10)

    def forward(self,x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1,16*5*5)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

net = ConvModel()
print(net)

loss_func = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(),lr=0.001,
            momentum=0.9,weight_decay= 1e-6,nesterov=True)

# training
for epoch in range(2):
    running_loss = 0.0
    for i,data in enumerate(dataloader,0):

        inputs,labels = data
        optimizer.zero_grad()

        #forward prop
        ouputs = net(inputs)
        loss = loss_func(ouputs,labels)

        #backward
        loss.backward() #autograd in pytorch
        optimizer.step()

        running_loss += loss.item()

        if i % 2000 == 1999:
            print("[epoch: %d, minibatch: %5d] loss: %.3f"%(epoch+1,i+1,running_loss/2000))
            running_loss = 0.0

print("Yay!! Training Finished!")

max_stars_repo_name: chetanseth/pytorch

max_issues_repo_name: chetanseth/pytorch

max_forks_repo_name: chetanseth/pytorch

reason: import numpy,from numpy


text:
import numpy as np
import time

from neu_ro_arm.robot.robot_arm import RobotArm
from neu_ro_arm.camera.camera import Camera
from neu_ro_arm.camera.gui import GUI, ShowCubes
from neu_ro_arm.camera import camera_utils

# I just picked a convenient spot to place arm
START_ARM_JPOS = [-0.15498, -1.0932, 1.386, 1.3362, 0.01256]

def decide_cube_id(camera):
    # setup visualizer so you can see camera image with identified cubes
    # when the user enters a number on the keypad, that will be returned
    modifier = ShowCubes(camera.unpack_configs(False), include_id=True)
    exit_keys = [ord(str(i)[0]) for i in range(12)]
    exit_key = camera.gui.show(modifier_fns=[modifier],
                            exit_keys=exit_keys,
                            window_name='Press cube number to grab it',
                           )

    # escape key was pressed so stop program
    if exit_key == 27:
        exit()

    cube_id = exit_keys.index(exit_key)
    return cube_id

def pickup_cube(camera, robot, cube_id):
    # identify all cubes in current image
    img = camera.get_image()
    cubes = camera_utils.find_cubes(img, camera._mtx, camera._dist_coeffs, camera._cam2world)

    # filter the list of cubes for the one with the right cube_id
    cube = next((c for c in cubes if c.id_==cube_id), None)
    if cube is None:
        print(f'Could not find the cube={cube_id}')
        return

    # get cube location
    cube_pos = cube.pos

    # perform correction
    cube_pos[0] += POSITION_CORRECTION[0] # x
    cube_pos[1] += POSITION_CORRECTION[1] # y
    cube_pos[2] += POSITION_CORRECTION[2] # z

    # go grab it with robot
    robot.open_gripper()
    robot.move_hand_to(cube_pos)
    time.sleep(0.2)
    robot.close_gripper()

    # return to start position
    robot.move_arm_jpos(START_ARM_JPOS)

    # drop block
    robot.open_gripper()

if __name__ == "__main__":
    # you will have to adjust this for your own robot
    POSITION_CORRECTION = [0, -0.018, 0.025]

    camera = Camera()
    robot = RobotArm()
    robot.move_arm_jpos(START_ARM_JPOS)

    cube_id = decide_cube_id(camera)

    pickup_cube(camera, robot, cube_id)

max_stars_repo_name: dmklee/nuro-arm

max_issues_repo_name: dmklee/learning-robotics

max_forks_repo_name: dmklee/nuro-arm


text:
# THE FOLLOWING CODE CAN BE USED IN YOUR SAGEMAKER NOTEBOOK TO TEST AN UPLOADED IMAGE TO YOUR S3 BUCKET AGAINST YOUR MODEL

import os
import urllib.request
import boto3
from IPython.display import Image
import cv2
import json
import numpy as np

# input the S3 bucket you are using for this project and the file path for a folder and file that contains your uploaded test image
test_image_bucket = 'deeplens-sagemaker-socksortingeast'
test_image_name = 'testimages/image0.jpeg'

tmp_file_name = 'tmp-test-image-jpg'
resized_file_name = 'resized-test-image.jpg'
s3 = boto3.client('s3')
with open(tmp_file_name, 'wb') as f:
    s3.download_fileobj(test_image_bucket, test_image_name, f)

# width
W = 500
oriimg = cv2.imread(tmp_file_name)
height, width, depth = oriimg.shape
# scale the image
imgScale = W/width
newX,newY = oriimg.shape[1].imgScale, oriimg.shape[0]*imgScale
newimg = cv2.resize(oriimg, (int(newX),int(newY)))
cv2.imwrite(resized_file_name, newimg)

with open(resized_file_name, 'rb') as f:
    payload = f.read()
    payload = bytearray(payload)

result = json.loads(ic_classifier.predict(payload, initial_args={'ContentType': 'application/x-image'}))
# find the index of the class that matches the test image with the highest probability
index = np.argmax(result)
# input your own output categories
object_categories = ['BlueStripes', 'DarkGray', 'IronMan']
print("Result: label - " + object_categories[index] + ", probability - " + str(result[index]))
print()
print(result)
print(ic._current_job_name)
Image(resized_file_name)


text:
import math
import warnings

import numpy as np
import scipy.signal as sig

def _parse_window(nperseg, noverlap, window='boxcar'):
    """ Get window function """
    if window == 'rect' or window =='rectangular':
        window = 'boxcar'
    window_fn = vars(sig)[window]
    if window != 'boxcar':
        return window_fn(nperseg) * (nperseg - noverlap) / nperseg * 2
    else:
        return window_fn(nperseg) * (nperseg - noverlap) / nperseg

def resample(data, fs, new_fs, window='hamming', n=60):
    """ Resample using FIR filter. Method borrowed from gwpy.timeseries.TimeSeries
    Parameters
    ----------
    data: `numpy.ndarray`
        timeseries data for resampling
    fs: `float`
        sampling rate in Hz
    new_fs: `float`
        new sampling rate in Hz
    window: `str`, `numpy.ndarray`
    n: `int`
        number of taps
    Returns
    -------
    `numpy.ndarray`
        resampled timeseries data
    """
    factor = (fs / new_fs)
    if math.isclose(factor, 1., rel_tol=1e-09, abs_tol=0.):
        warnings.warn(
            "resample() rate matches current sample_rate ({}), returning "
            "input data unmodified; please double-check your "
            "parameters".format(fs),
            UserWarning,
        )
        return data

    # if integer down-sampling, use decimate
    if factor.is_integer():
        filt = sig.firwin(n + 1, 1./factor, window=window)
        new = sig.filtfilt(filt, [1.], data)
        return new[::int(factor)]
    # otherwise use Fourier filtering
    else:
        nsamp = int(len(data) * new_fs / fs)
        new = sig.resample(data, nsamp, window=window)
        return new

def bandpass(data, fs, fl, fh, order=None, axis=-1):
    """ Apply Butterworth bandpass filter using scipy.signal.sosfiltfilt method
    Parameters
    ----------
    data: array
    fs: sampling frequency
    fl, fh: low and high frequency for bandpass
    axis: axis to apply the filter on

    Returns:
    --------
    data_filt: filtered array
    """
    if order is None:
        order = 8

    # Make filter
    nyq = fs/2.
    low, high = fl/nyq, fh/nyq  # normalize frequency
    z, p, k = sig.butter(order, [low, high], btype='bandpass', output='zpk')
    sos = sig.zpk2sos(z, p, k)

    # Apply filter and return output
    data_filt = sig.sosfiltfilt(sos, data, axis=axis)
    return data_filt

def overlap_add(data, noverlap, window, verbose=True):
    """ Concatenate timeseries using the overlap-add method
    Parameters
    -----------
    data: `numpy.ndarray` of shape (N, nperseg)
        array of timeseries segments to be concatenate
    noverlap: `int`
        number of overlapping samples between each segment in `data`
    window: `str`, `numpy.ndarray`

    Returns
    --------
    `numpy.ndarray`
        concatenated timeseries
    """
    # Get dimension
    N, nperseg = data.shape
    stride = nperseg - noverlap

    # Get window function
    if isinstance(window, str):
        window = _parse_window(nperseg, noverlap, window)

    # Concatenate timeseries
    nsamp = int((N - 1) * stride + nperseg)
    new = np.zeros(nsamp)
    for i in range(N):
        new[i * stride: i * stride + nperseg] += data[i] * window
    return new

def filter_add(data, fs, flow, fhigh, order=8, axis=-1):
    """ Sub bandpass filter
    Parameters
    ----------
    data: `numpy.ndarray`
    fs: `float`
        sampling frequency in Hz
    flow: `float`
        list of low frequencies
    fhigh: `float`
        list of high frequencies
    Returns
    -------
    `numpy.ndarray`
        Data with sub bandpass filter
    """
    if isinstance(flow, (int, float)):
        flow = (flow, )
    if isinstance(fhigh, (int, float)):
        fhigh = (fhigh, )

    new = np.zeros_like(data)
    for fl, fh in zip(flow, fhigh):
        new += bandpass(data, fs, fl, fh, order=order, axis=axis)
    return new

max_stars_repo_name: KAGRA-TW-ML/deepclean-prod

max_issues_repo_name: KAGRA-TW-ML/deepclean-prod

max_forks_repo_name: KAGRA-TW-ML/deepclean-prod