text:
# train.py
# Training operations.

import json
from argparse import Namespace
from typing import Dict, List, Tuple

import numpy as np
import optuna
import torch
import torch.nn as nn
from numpyencoder import NumpyEncoder
from sklearn.metrics import (
    precision_recall_curve,
    precision_recall_fscore_support,
)

from tagifai import data, models, utils
from tagifai.config import logger

class Trainer(object):
    """Object used to facilitate training."""

    def __init__(
        self,
        model,
        device=torch.device("cpu"),
        loss_fn=None,
        optimizer=None,
        scheduler=None,
        trial=None,
    ):

        # Set params
        self.model = model
        self.device = device
        self.loss_fn = loss_fn
        self.optimizer = optimizer
        self.scheduler = scheduler
        self.trial = trial

    def train_step(self, dataloader):
        """Train step.
        Args:
            dataloader (torch.utils.data.DataLoader): Torch dataloader to load batches from.
        """
        # Set model to train mode
        self.model.train()
        loss = 0.0

        # Iterate over train batches
        for i, batch in enumerate(dataloader):

            # Step
            batch = [item.to(self.device) for item in batch]  # Set device
            inputs, targets = batch[:-1], batch[-1]
            self.optimizer.zero_grad()  # Reset gradients
            z = self.model(inputs)  # Forward pass
            J = self.loss_fn(z, targets)  # Define loss
            J.backward()  # Backward pass
            self.optimizer.step()  # Update weights

            # Cumulative Metrics
            loss += (J.detach().item() - loss) / (i + 1)

        return loss

    def eval_step(self, dataloader):
        """Evaluation (val / test) step.
        Args:
            dataloader (torch.utils.data.DataLoader): Torch dataloader to load batches from.
        """
        # Set model to eval mode
        self.model.eval()
        loss = 0.0
        y_trues, y_probs = [], []

        # Iterate over val batches
        with torch.no_grad():
            for i, batch in enumerate(dataloader):

                # Step
                batch = [item.to(self.device) for item in batch]  # Set device
                inputs, y_true = batch[:-1], batch[-1]
                z = self.model(inputs)  # Forward pass
                J = self.loss_fn(z, y_true).item()

                # Cumulative Metrics
                loss += (J - loss) / (i + 1)

                # Store outputs
                y_prob = torch.sigmoid(z).cpu().numpy()
                y_probs.extend(y_prob)
                y_trues.extend(y_true.cpu().numpy())

        return loss, np.vstack(y_trues), np.vstack(y_probs)

    def predict_step(self, dataloader: torch.utils.data.DataLoader):
        """Prediction (inference) step.
        Note:
            Loss is not calculated for this loop.
        Args:
            dataloader (torch.utils.data.DataLoader): Torch dataloader to load batches from.
        """
        # Set model to eval mode
        self.model.eval()
        y_trues, y_probs = [], []

        # Iterate over val batches
        with torch.no_grad():
            for i, batch in enumerate(dataloader):

                # Forward pass w/ inputs
                batch = [item.to(self.device) for item in batch]  # Set device
                inputs, y_true = batch[:-1], batch[-1]
                z = self.model(inputs)

                # Store outputs
                y_prob = torch.sigmoid(z).cpu().numpy()
                y_probs.extend(y_prob)
                y_trues.extend(y_true.cpu().numpy())

        return np.vstack(y_trues), np.vstack(y_probs)

    def train(
        self,
        num_epochs: int,
        patience: int,
        train_dataloader: torch.utils.data.DataLoader,
        val_dataloader: torch.utils.data.DataLoader,
    ) -> Tuple:
        """Training loop.
        Args:
            num_epochs (int): Maximum number of epochs to train for (can stop earlier based on performance).
            patience (int): Number of acceptable epochs for continuous degrading performance.
            train_dataloader (torch.utils.data.DataLoader): Dataloader object with training data split.
            val_dataloader (torch.utils.data.DataLoader): Dataloader object with validation data split.
        Raises:
            optuna.TrialPruned: Early stopping of the optimization trial if poor performance.
        Returns:
            The best validation loss and the trained model from that point.
        """

        best_val_loss = np.inf
        best_model = None
        _patience = patience
        for epoch in range(num_epochs):
            # Steps
            train_loss = self.train_step(dataloader=train_dataloader)
            val_loss, _, _ = self.eval_step(dataloader=val_dataloader)
            self.scheduler.step(val_loss)

            # Pruning based on the intermediate value
            if self.trial:
                self.trial.report(val_loss, epoch)
                if self.trial.should_prune():
                    logger.info("Unpromising trial pruned!")
                    raise optuna.TrialPruned()

            # Early stopping
            if val_loss < best_val_loss:
                best_val_loss = val_loss
                best_model = self.model
                _patience = patience  # reset _patience
            else:
                _patience -= 1
            if not _patience:  # 0
                logger.info("Stopping early!")
                break

            # Logging
            logger.info(
                f"Epoch: {epoch+1} | "
                f"train_loss: {train_loss:.5f}, "
                f"val_loss: {val_loss:.5f}, "
                f"lr: {self.optimizer.param_groups[0]['lr']:.2E}, "
                f"_patience: {_patience}"
            )
        return best_val_loss, best_model

def find_best_threshold(y_true: np.ndarray, y_prob: np.ndarray) -> float:
    """Determine the best threshold for maximum f1 score.
    Usage:
    ```python
    # Find best threshold
    _, y_true, y_prob = trainer.eval_step(dataloader=train_dataloader)
    args.threshold = find_best_threshold(y_true=y_true, y_prob=y_prob)
    ```
    Args:
        y_true (np.ndarray): True labels.
        y_prob (np.ndarray): Probability distribution for predicted labels.
    Returns:
        Best threshold for maximum f1 score.
    """
    precisions, recalls, thresholds = precision_recall_curve(
        y_true.ravel(), y_prob.ravel()
    )
    f1s = (2 * precisions * recalls) / (precisions + recalls)
    return thresholds[np.argmax(f1s)]

def get_performance(
    y_true: np.ndarray, y_pred: np.ndarray, classes: List
) -> Dict:
    """Per-class performance metrics.
    Args:
        y_true (np.ndarray): True class labels.
        y_pred (np.ndarray): Predicted class labels.
        classes (List): List of all unique classes.
    Returns:
        Dictionary of overall and per-class performance metrics.
    """
    # Get metrics
    performance = {"overall": {}, "class": {}}
    metrics = precision_recall_fscore_support(y_true, y_pred)

    # Overall performance
    performance["overall"]["precision"] = np.mean(metrics[0])
    performance["overall"]["recall"] = np.mean(metrics[1])
    performance["overall"]["f1"] = np.mean(metrics[2])
    performance["overall"]["num_samples"] = np.float64(np.sum(metrics[3]))

    # Per-class performance
    for i in range(len(classes)):
        performance["class"][classes[i]] = {
            "precision": metrics[0][i],
            "recall": metrics[1][i],
            "f1": metrics[2][i],
            "num_samples": np.float64(metrics[3][i]),
        }

    return performance

def train(
    args: Namespace,
    train_dataloader: torch.utils.data.DataLoader,
    val_dataloader: torch.utils.data.DataLoader,
    model: nn.Module,
    device: torch.device,
    class_weights: Dict,
    trial: optuna.trial._trial.Trial = None,
) -> Tuple:
    """Train a model.
    Args:
        args (Namespace): Parameters for data processing and training.
        train_dataloader (torch.utils.data.DataLoader): train data loader.
        val_dataloader (torch.utils.data.DataLoader): val data loader.
        model (nn.Module): Initialize model to train.
        device (torch.device): Device to run model on.
        class_weights (Dict): Dictionary of class weights.
        trial (optuna.trial._trial.Trail, optional): Optuna optimization trial. Defaults to None.
    Returns:
        The best trained model, loss and performance metrics.
    """
    # Define loss
    class_weights_tensor = torch.Tensor(np.array(list(class_weights.values())))
    loss_fn = nn.BCEWithLogitsLoss(weight=class_weights_tensor)

    # Define optimizer & scheduler
    optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
        optimizer, mode="min", factor=0.05, patience=5
    )

    # Trainer module
    trainer = Trainer(
        model=model,
        device=device,
        loss_fn=loss_fn,
        optimizer=optimizer,
        scheduler=scheduler,
        trial=trial,
    )

    # Train
    best_val_loss, best_model = trainer.train(
        args.num_epochs, args.patience, train_dataloader, val_dataloader
    )

    # Find best threshold
    _, y_true, y_prob = trainer.eval_step(dataloader=train_dataloader)
    args.threshold = find_best_threshold(y_true=y_true, y_prob=y_prob)

    return args, best_model, best_val_loss

def evaluate(
    dataloader: torch.utils.data.DataLoader,
    model: nn.Module,
    device: torch.device,
    threshold: float,
    classes: List,
) -> Dict:
    """Evaluate performance on data.
    Args:
        dataloader (torch.utils.data.DataLoader): Dataloader with the data your want to evaluate.
        model (nn.Module): Trained model.
        threshold (float): Precision recall threshold determined during training.
        classes (List): List of unique classes.
        device (torch.device): Device to run model on. Defaults to CPU.
    Returns:
        Performance metrics.
    """
    # Determine predictions using threshold
    trainer = Trainer(model=model, device=device)
    y_true, y_prob = trainer.predict_step(dataloader=dataloader)
    y_pred = np.array([np.where(prob >= threshold, 1, 0) for prob in y_prob])

    # Evaluate
    performance = get_performance(
        y_true=y_true, y_pred=y_pred, classes=classes
    )

    return performance

def run(args: Namespace, trial: optuna.trial._trial.Trial = None) -> Dict:
    """Operations for training.
    1. Set seed
    2. Set device
    3. Load data
    4. Clean data
    5. Preprocess data
    6. Encode labels
    7. Split data
    8. Tokenize inputs
    9. Create dataloaders
    10. Initialize model
    11. Train model
    12. Evaluate model
    Args:
        args (Namespace): Input arguments for operations.
        trial (optuna.trial._trial.Trail, optional): Optuna optimization trial. Defaults to None.
    Returns:
        Artifacts to save and load for later.
    """
    # 1. Set seed
    utils.set_seed(seed=args.seed)

    # 2. Set device
    device = utils.set_device(cuda=args.cuda)

    # 3. Load data
    df, projects_dict, tags_dict = data.load(
        shuffle=args.shuffle, num_samples=args.num_samples
    )

    # 4. Clean data
    df, tags_dict, tags_above_frequency = data.clean(
        df=df, tags_dict=tags_dict, min_tag_freq=args.min_tag_freq
    )

    # 5. Preprocess data
    df.text = df.text.apply(data.preprocess, lower=args.lower, stem=args.stem)

    # 6. Encode labels
    y, class_weights, label_encoder = data.encode_labels(labels=df.tags)

    # 7. Split data
    utils.set_seed(seed=args.seed)  # needed for skmultilearn
    X_train, X_val, X_test, y_train, y_val, y_test = data.split(
        X=df.text.to_numpy(), y=y, train_size=args.train_size
    )

    # 8. Tokenize inputs
    X_train, tokenizer = data.tokenize_text(
        X=X_train, char_level=args.char_level
    )
    X_val, _ = data.tokenize_text(
        X=X_val, char_level=args.char_level, tokenizer=tokenizer
    )
    X_test, _ = data.tokenize_text(
        X=X_test, char_level=args.char_level, tokenizer=tokenizer
    )

    # 9. Create dataloaders
    train_dataloader = data.get_dataloader(
        data=[X_train, y_train],
        max_filter_size=args.max_filter_size,
        batch_size=args.batch_size,
    )
    val_dataloader = data.get_dataloader(
        data=[X_val, y_val],
        max_filter_size=args.max_filter_size,
        batch_size=args.batch_size,
    )
    test_dataloader = data.get_dataloader(
        data=[X_test, y_test],
        max_filter_size=args.max_filter_size,
        batch_size=args.batch_size,
    )

    # 10. Initialize model
    model = models.initialize_model(
        args=args,
        vocab_size=len(tokenizer),
        num_classes=len(label_encoder),
        device=device,
    )

    # 11. Train model
    logger.info(
        f"Arguments: {json.dumps(args.__dict__, indent=2, cls=NumpyEncoder)}"
    )
    args, model, loss = train(
        args=args,
        train_dataloader=train_dataloader,
        val_dataloader=val_dataloader,
        model=model,
        device=device,
        class_weights=class_weights,
        trial=trial,
    )

    # 12. Evaluate model
    device = torch.device("cpu")
    performance = evaluate(
        dataloader=test_dataloader,
        model=model.to(device),
        device=device,
        threshold=args.threshold,
        classes=label_encoder.classes,
    )

    return {
        "args": args,
        "label_encoder": label_encoder,
        "tokenizer": tokenizer,
        "model": model,
        "loss": loss,
        "performance": performance,
    }

def objective(args: Namespace, trial: optuna.trial._trial.Trial) -> float:
    """Objective function for optimization trials.
    Args:
        args (Namespace): Input arguments for each trial (see `config/args.json`) for argument names.
        trial (optuna.trial._trial.Trial): Optuna optimization trial.
    Returns:
        F1 score from evaluating the trained model on the test data split.
    """
    # Paramters (to tune)
    args.embedding_dim = trial.suggest_int("embedding_dim", 128, 512)
    args.num_filters = trial.suggest_int("num_filters", 128, 512)
    args.hidden_dim = trial.suggest_int("hidden_dim", 128, 512)
    args.dropout_p = trial.suggest_uniform("dropout_p", 0.3, 0.8)
    args.lr = trial.suggest_loguniform("lr", 5e-5, 5e-4)

    # Train (can move some of these outside for efficiency)
    logger.info(f"\nTrial {trial.number}:")
    logger.info(json.dumps(trial.params, indent=2))
    artifacts = run(args=args, trial=trial)

    # Set additional attributes
    args = artifacts["args"]
    performance = artifacts["performance"]
    logger.info(json.dumps(performance["overall"], indent=2))
    trial.set_user_attr("threshold", args.threshold)
    trial.set_user_attr("precision", performance["overall"]["precision"])
    trial.set_user_attr("recall", performance["overall"]["recall"])
    trial.set_user_attr("f1", performance["overall"]["f1"])

    return performance["overall"]["f1"]

max_stars_repo_name: saketkattu/AI-tags

max_issues_repo_name: saketkattu/AI-tags

max_forks_repo_name: saketkattu/AI-tags

reason: import numpy,from numpy


text:
from flask import Flask, render_template, redirect, jsonify
import numpy as np
import pandas as pd
import json

import os
import psycopg2
DATABASE_URL = os.environ['DATABASE_URL']

import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func, inspect

from config import password
# engine = create_engine(f"postgresql://postgres:{password}@localhost:5432/Poverty_and_wellness")
engine = psycopg2.connect(DATABASE_URL, sslmode='require')
session = Session(bind=engine)

# GET DATA FROM SQL and creating endpoints:
# Approach 1: Use sqlalchemy reflection method to convert tables to python classes and query them
Base = automap_base()
Base.prepare(engine, reflect=True)

# Save reference to the table
povState = Base.classes.poverty
eduState = Base.classes.education

# start the flask server
app=Flask(__name__)

@app.route("/states")
def states():
    # link  to the DB
    session = Session(engine)
   # Query all states
    results = session.query(povState.state).all()
    session.close()
    # Convert list of tuples into normal list
    all_states = list(np.ravel(results))

    return jsonify(all_states)

#/
@app.route("/poverty")
def poverty():

    session = Session(engine)

    results = session.query(povState.pov_rate).all()
    session.close()

    all_pov = list(np.ravel(results))

    return jsonify(all_pov)

@app.route("/education")
def education():

    session = Session(engine)

    results = session.query(eduState.college_per).all()
    session.close()

    # Convert list of tuples into normal list
    all_edu = list(np.ravel(results))

    return jsonify(all_edu)

# A different approach to getting the data from sql: Connect to the database and use pandas to read tables
# into dataframes and then create objects from there
@app.route("/")
def index():

    rank_data_df = pd.read_sql_query("Select * from state_health_rankings",  conn)
    rank_data_df.drop("rank", inplace=True, axis=1)
    slim_df=rank_data_df.drop([rank_data_df.index[-1], rank_data_df.index[-2]])
    slim_df.rename(columns = {"score":"Variance from Nat'l Average", 'state':'State'}, inplace = True)
    rank_table=slim_df.to_html(index=False)

    return render_template("index.html", table=rank_table)

@app.route("/map_data",methods=["GET","POST"])
def map_data():

    map_data_df = pd.read_sql("Select * from map_coordinates",conn)
    map_data_df.rename(columns = {'long':'lon'}, inplace = True)
    map_data_dict=map_data_df.to_dict(orient='records')
    map_json = jsonify(map_data_dict)

    return(map_json)

@app.route("/crime_data")
def crime_data():
    crime_data_df = pd.read_sql("Select * from violent_crime",conn)
    crime_data_df.rename(columns = {'abbr':'State_Abbreviation','state': 'State', 'pop': 'Population',
    'vio_crime': 'Violent_Crime','vio_rate':'Violent_Crime_Rate','vio_rank':'Violent_Crime_Rank'}, inplace = True)
    crime_data_dict=crime_data_df.to_dict(orient='records')
    crime_json = jsonify(crime_data_dict)

    return crime_json

# Approach 3: apply sqlalchemy engine.execute method to manipulate sql tables directly without reflection:
@app.route("/food_deserts",methods=["GET","POST"])
def food_deserts():
    # engine = create_engine(f"postgresql://postgres:{password}@localhost:5432/Poverty_and_wellness")
    engine = psycopg2.connect(DATABASE_URL, sslmode='require')
    session = Session(bind=engine)
    food_data = engine.execute('SELECT f.state, f.abbr, f.desert_rate, p.pov_rate from food_deserts as f INNER JOIN poverty as p ON f.state = p.state').fetchall()

    food_dict_list=[]
    for state in food_data:

        food_dict = {}
        food_dict["state"] = state[0]
        food_dict["abbr"] = state[1]
        food_dict["desert_rate"]=state[2]
        food_dict["desert_rank"]=(sorted(food_data, key = lambda x: x[2],reverse=True)).index(state)+1
        food_dict["pov_rate"]=state[3]
        food_dict["pov_rank"]=(sorted(food_data, key = lambda x: x[3],reverse=True)).index(state)+1

        food_dict_list.append(food_dict)

    food_json=json.dumps(food_dict_list, indent=4, sort_keys=True)
    return(food_json)

@app.route("/combined_data")
def combined_data():

    # engine = create_engine(f"postgresql://postgres:{password}@localhost:5432/Poverty_and_wellness")
    engine = psycopg2.connect(DATABASE_URL, sslmode='require')
    session=Session(engine)
    combined_data = engine.execute('SELECT e.state, e.college_per, f.desert_rate, s.score, p.pov_rate, v.vio_rate from education as e INNER JOIN food_deserts as f ON e.state = f.state INNER JOIN state_health_rankings as s ON f.state = s.state INNER JOIN poverty as p ON s.state = p.state INNER JOIN violent_crime as v ON p.state = v.state ORDER BY e.state').fetchall()

    combined_dict_list=[]
    for state  in combined_data:

        combined_dict = {}
        combined_dict["State"] = state[0]
        combined_dict["College Grad Rate"] = state[1]
        combined_dict["College Grad Rank"]=(sorted(combined_data, key = lambda x: x[1],reverse=True)).index(state)+1
        combined_dict["Food Desert Rate"] = state[2]
        combined_dict["Food Desert Rank"]=(sorted(combined_data, key = lambda x: x[2],reverse=True)).index(state)+1
        combined_dict["Health Score"] = state[3]
        combined_dict["Health Rank"]=(sorted(combined_data, key = lambda x: x[3],reverse=True)).index(state)+1
        combined_dict["Poverty Rate"] = state[4]
        combined_dict["Poverty Rank"]=(sorted(combined_data, key = lambda x: x[4],reverse=True)).index(state)+1
        combined_dict["Violent Crime Rate"]=state[5]
        combined_dict["Violent Crime Rank"]=(sorted(combined_data, key = lambda x: x[5],reverse=True)).index(state)+1

        combined_dict_list.append(combined_dict)

    combined_json=json.dumps(combined_dict_list, indent=4, sort_keys=True)
    return(combined_json)

if __name__ == "__main__":
    app.run(debug=True)

ext: py

reason: import numpy


text:
#!/usr/bin/env python2
#
# This file is part of the dune-hdd project:
#   https://github.com/pymor/dune-hdd
# Copyright Holders: Felix Schindler
# License: BSD 2-Clause License (http://opensource.org/licenses/BSD-2-Clause)

from __future__ import division, print_function

from functools import partial
import numpy as np

from pymor.algorithms.greedy import greedy
from pymor.core.logger import getLogger
from pymor.playground.algorithms import gram_schmidt_block_basis_extension
from pymor.vectorarrays.block import BlockVectorArray

from simdb.run import add_values

from dune.pymor.la.container import make_listvectorarray

from OS2015_SISC__6_2__estimators import ReducedEstimator, reduce_with_estimator

def offline_phase(cfg, data):
    logger = getLogger('.OS2015_SISC__6_2.offline_phase')
    logger.setLevel('INFO')

    discretization = data['discretization']
    example        = data['example']
    local_products = data['local_products']
    norm           = data['norm']
    mu_bar_dune    = data['mu_bar_dune']
    mu_hat_dune    = data['mu_hat_dune']
    wrapper        = data['wrapper']

    if cfg['training_sampling_strategy'] == 'random':
        training_samples = list(discretization.parameter_space.sample_randomly(cfg['num_training_samples']))
    elif cfg['training_sampling_strategy'] == 'uniform':
        training_samples = list(discretization.parameter_space.sample_uniformly(cfg['num_training_samples']))
    add_values(training_samples=training_samples)
    if cfg['compute_some_solution_norms'] and len(training_samples) > 0:
        logger.info('Computing solution norms:')
        if norm is not None:
            solution_norms = [norm(discretization.solve(mu)) for mu in training_samples]
        else:
            solution_norms = [discretization.solve(mu).l2_norm()[0] for mu in training_samples]
        logger.info('  range:              [{}, {}]'.format(np.amin(solution_norms), np.amax(solution_norms)))
        logger.info('  mean:                {}'.format(np.mean(solution_norms)))
        add_values(solution_norms=solution_norms)

    extension_algorithm=partial(gram_schmidt_block_basis_extension, product=local_products)
    initial_basis = discretization.functionals['rhs'].source.empty()._blocks
    if cfg['initialize_basis_with'] >= 0:
        logger.info('Initializing local bases of up to order {} ...'.format(cfg['initialize_basis_with']))
        one = make_listvectorarray(wrapper[example.project('1')])
        one = BlockVectorArray([discretization.localize_vector(one, ss)
                                for ss in np.arange(discretization.num_subdomains)])
        initial_basis, _ = extension_algorithm(initial_basis, one)
    if cfg['initialize_basis_with'] >= 1:
        xx = make_listvectorarray(wrapper[example.project('x[0]')])
        xx = BlockVectorArray([discretization.localize_vector(xx, ss)
                               for ss in np.arange(discretization.num_subdomains)])
        initial_basis, _ = extension_algorithm(initial_basis, xx)
        yy = make_listvectorarray(wrapper[example.project('x[1]')])
        yy = BlockVectorArray([discretization.localize_vector(yy, ss)
                               for ss in np.arange(discretization.num_subdomains)])
        initial_basis, _ = extension_algorithm(initial_basis, yy)
        xy = make_listvectorarray(wrapper[example.project('x[0]*x[1]')])
        xy = BlockVectorArray([discretization.localize_vector(xy, ss)
                               for ss in np.arange(discretization.num_subdomains)])
        initial_basis, _ = extension_algorithm(initial_basis, xy)
    if cfg['initialize_basis_with'] >= 2:
        logger.warn('Ignoring initialize_basis_with higher than 1: {}'.format(cfg['initialize_basis_with']))
    logger.info('')

    reduced_estimator = ReducedEstimator(
            discretization, example, wrapper, mu_hat_dune, mu_bar_dune, norm, cfg['estimator_compute'], cfg['estimator_return'])
    greedy_data = greedy(discretization,
                         partial(reduce_with_estimator,
                                 reduced_estimator=reduced_estimator),
                         training_samples,
                         initial_basis=initial_basis,
                         use_estimator=cfg['greedy_use_estimator'],
                         error_norm=norm,
                         extension_algorithm=extension_algorithm,
                         max_extensions=cfg['greedy_max_extensions'],
                         target_error=cfg['greedy_target_error'])
    add_values(time=greedy_data['time'],
               max_err_mus=greedy_data['max_err_mus'],
               extensions=greedy_data['extensions'],
               max_errs=greedy_data['max_errs'],
               offline_estimator_data=reduced_estimator.data)
    rd, rc = greedy_data['reduced_discretization'], greedy_data['reconstructor']
    basis, basis_mus = greedy_data['basis'], greedy_data['max_err_mus']

    print('')
    logger.info('Offline phase finished.')
    logger.info('Basis sizes range from {} to {}.'.format(np.min([len(bb) for bb in basis]),
                                                          np.max([len(bb) for bb in basis])))
    return {'basis': basis,
            'basis_mus': basis_mus,
            'rc': rc,
            'rd': rd}

max_stars_repo_name: pymor/dune-hdd

max_issues_repo_name: dune-community/dune-hdd

max_forks_repo_name: pymor/dune-hdd


text:
import os
import numpy as np
import tensorflow as tf
import tensorflow_datasets as tfds
from utils import datasets_util as du

class DatasetsTest(tf.test.TestCase):
  """Tests some functions in the datasets_util."""
  def setUp(self):
    self.usps_path = "/home/dbash_google_com/datasets/usps.h5"
    self.data_dir = "/home/dbash_google_com/datasets/"

  def _check_split(self, split, train_ds, test_ds):
    """Checks if the train and test data are of the of correct proportion."""
    num_train = tf.data.experimental.cardinality(train_ds).numpy()
    num_test = tf.data.experimental.cardinality(test_ds).numpy()
    num_total = num_train + num_test
    self.assertEqual(num_train, np.ceil(split * num_total))
    self.assertEqual(num_test, np.floor((1 - split) * num_total))

  def test_usps(self):
    """Tests if get_usps_tf_dataset doesn't crash and returns
    two tf.data.Dataset objects."""
    train_ds, test_ds = du.get_usps_tf_dataset(self.usps_path)
    self.assertIsInstance(train_ds, tf.data.Dataset,
                          "tf.data.Dataset object not returned")
    self.assertIsInstance(test_ds, tf.data.Dataset,
                          "tf.data.Dataset object not returned")

  def test_split_fn(self):
    """Checks if get_split_dataset returns splits of correct proportion."""
    split = 0.6
    datasets, _ = du.get_split_dataset("quickdraw_bitmap", self.data_dir, split)
    train_ds, test_ds = datasets
    self._check_split(split, train_ds, test_ds)

  def test_office(self):
    """Checks if the get_office_datasets function returns a
    tf.data.Dataset object."""
    office_path = os.path.join(self.data_dir, "office")
    split = 0.3
    ds_dict = du.get_office_datasets(office_path, split=split)
    key_list = list(ds_dict.keys())
    self.assertAllEqual(key_list, ["amazon", "dslr", "webcam"])

    for domain in ds_dict:
      train_ds, test_ds, classes = ds_dict[domain]
      self.assertIsInstance(train_ds, tf.data.Dataset,
                            "tf.data.Dataset object not returned")
      self.assertIsInstance(test_ds, tf.data.Dataset,
                            "tf.data.Dataset object not returned")
      self.assertEqual(len(classes), 31)
      self._check_split(1. - split, train_ds, test_ds)

  def test_domain_net(self):
    """Checks if the get_domain_net_datasets function returns a
    tf.data.Dataset object."""
    domain_net_path = os.path.join(self.data_dir, "domain_net")
    split = 0.3
    ds_dict = du.get_domain_net_datasets(domain_net_path, split=split)
    key_list = list(ds_dict.keys())
    self.assertAllEqual(key_list, ["clipart", "infograph", "painting",
                                   "real", "sketch"])

    for domain in ds_dict:
      train_ds, test_ds, classes = ds_dict[domain]
      self.assertIsInstance(train_ds, tf.data.Dataset,
                            "tf.data.Dataset object not returned")
      self.assertIsInstance(test_ds, tf.data.Dataset,
                            "tf.data.Dataset object not returned")
      self.assertEqual(len(classes), 345)
      self._check_split(1. - split, train_ds, test_ds)

  def test_preprocess(self):
    """Tests if the preprocess_dataset function returns correct objects."""
    image_size = [64, 64]
    datasets, _ = tfds.load("cifar10", data_dir=self.data_dir,
                            with_info=True,
                            as_supervised=True)
    train_ds = datasets["train"]
    train_x, train_y = du.preprocess_dataset(
        train_ds, image_size, 10, augment=False, repeat=False)

    for image, label in tf.data.Dataset.zip(tuple([train_x, train_y])).take(1):
      self.assertEqual(image.shape, (64, 64, 3))
      self.assertEqual(label.shape, (10,))

max_stars_repo_name: googleinterns/loop-project

max_issues_repo_name: googleinterns/loop-project

max_forks_repo_name: googleinterns/loop-project


text:
import librosa
import matplotlib.pyplot as plt
import numpy as np
import sys, csv, pdb

# h5_path = sys.argv[1]
# data=h5py.File(h5_path,'r')

# filename = sys.argv[2]

sr = 44100
audio_path = sys.argv[1]
audio, y = librosa.load(audio_path, sr=sr)

x_values=[]
y_values=[]

contour_path = sys.argv[2]
with open(contour_path, 'r') as contourCsv:
	reader = csv.reader(contourCsv)
	for row in reader:
		x_values.append(float(row[0]))
		y_values.append(int(float(row[1])))
		print(float(row[0]), int(float(row[1])))

total_entries =len(audio)
total_time = total_entries/sr
time_axis = np.linspace(0, total_time, total_entries)

fig, ax1 = plt.subplots()

color = 'tab:blue'
ax1.set_xlabel('time (s)')
ax1.set_ylabel('sample', color=color)
ax1.plot(time_axis, audio, color=color)
ax1.tick_params(axis='y', labelcolor=color)

ax2 = ax1.twinx()

color = 'tab:orange'
ax2.set_ylabel('pitch', color=color)
ax2.plot(x_values, y_values, color=color)
ax2.tick_params(axis='y', labelcolor=color)

fig.tight_layout()
plt.show()


text:
"""
Module which groups all the auxiliary functions for transforming each point of
a time series regarding their vicinity to a descriptor or descriptors.
The resultant of the application of these functions to a time series is a
2d array in which the first dim represents the descriptors and the other
the different elements of the system.
"""

import numpy as np

############################# auxiliary functions #############################
###############################################################################
def aggregating_ups_downs(Yt, n, desc='normal'):
    ## TODO: 2 ways. Measure or number
    logi = len(Yt.shape) == 1
    Yt = np.atleast_2d(Yt).T
    nneur = Yt.shape[1]
    neur = np.arange(nneur)
    Ytt = np.zeros(Yt.shape)
    n1 = (n-1)*2+1
    for neu in neur:
        idx = np.where(Yt[:, neu] != 0)[0]
        for i in range(idx.shape[0]-n1+1):
            y = Yt[idx[i]:idx[i]+n1, neu]
            if desc == 'normal':
                Ytt[idx[i]] = np.sum(y)
            elif desc == 'shape':
                Ytt[idx[i]] = np.sum(y)/float(len(y))
    if logi:
        Ytt = Ytt.squeeze()
    return Ytt

def collapse3waveform(waveform_diff, event_t=None):
    """Funcions to collapse a waveform in 3 stages, pre-spike, spike and
    post-spike in order to be able to identify the spikes easily.
    The input it is supposed that is ups_downs discretized.
    """

    collapsed = np.zeros(3)
    if event_t is None:
        event_t = np.argmax(waveform_diff)

    collapsed[0] = np.sum(waveform_diff[:event_t])
    collapsed[1] = waveform_diff[event_t]
    collapsed[2] = np.sum(waveform_diff[event_t+1:])
    return collapsed

def collapse3waveform_matrix(waveform, event_t, axisn=0):
    """Funcions to collapse a waveform in 3 stages, pre-spike, spike and
    post-spike in order to be able to identify the spikes easily.

    Parameters
    ---------
    waveform: array_like, shape(Nf, M)
        representation of the waveform or part of the time series we want to
        transform and extracting representative features of the agrregated
        state of pre-event, event and post-event situation.
    event_t: int
        the index of the time in which the interested event occurs.
    axisn: int, optional
        the axis in which is represented the time.

    Returns
    -------
    collapsed: array_like, shape (3, M)
        collapsed representation of the waveform passed.
    """

    # How many waveforms we have
    axisn2 = (axisn - 1) % 2
    m = waveform.shape[axisn2]
    # Initialization
    collapsed = np.zeros((3, m))

    # Computation of the features
    collapsed[0, :] = np.sum(waveform[:event_t, :], axisn).reshape(-1)
    collapsed[1, :] = waveform[event_t, :].reshape(-1)
    collapsed[2, :] = np.sum(waveform[event_t+1:, :], axisn).reshape(-1)
    collapsed = collapsed.T
    return collapsed


text:
import os
import h5py
import argparse
import json
import numpy as np

from scipy.misc import imread, imresize
from tqdm import tqdm
from collections import Counter
from random import seed, choice, sample

def create_input_files(
    dataset, karpathy_json_path, image_folder, captions_per_image, min_word_freq, output_folder,
    max_len=100
):
    """Creates input files for training, validation, and test data.

    Args:
        dataset: name of dataset, one of 'coco', 'flickr8k', 'flickr30k'
        karpathy_json_path: path of Karpathy JSON file with splits and captions
        image_folder: folder with downloaded images
        captions_per_image: number of captions to sample per image
        min_word_freq: words occuring less frequently than this threshold are binned as <unk>s
        output_folder: folder to save files
        max_len: don't sample captions longer than this length
    """

    assert dataset in {'coco', 'flickr8k', 'flickr30k'}

    # Read Karpathy JSON
    with open(karpathy_json_path, 'r') as j:
        data = json.load(j)

    # Read image paths and captions for each image
    train_image_paths = []
    train_image_captions = []
    val_image_paths = []
    val_image_captions = []
    test_image_paths = []
    test_image_captions = []
    word_freq = Counter()

    for img in data['images']:
        captions = []
        for c in img['sentences']:
            # Update word frequency
            word_freq.update(c['tokens'])
            if len(c['tokens']) <= max_len:
                captions.append(c['tokens'])

        if len(captions) == 0:
            continue

        path = os.path.join(image_folder, img['filepath'], img['filename']) if dataset == 'coco' else os.path.join(
            image_folder, img['filename'])

        if img['split'] in {'train', 'restval'}:
            train_image_paths.append(path)
            train_image_captions.append(captions)
        elif img['split'] in {'val'}:
            val_image_paths.append(path)
            val_image_captions.append(captions)
        elif img['split'] in {'test'}:
            test_image_paths.append(path)
            test_image_captions.append(captions)

    # Sanity check
    assert len(train_image_paths) == len(train_image_captions)
    assert len(val_image_paths) == len(val_image_captions)
    assert len(test_image_paths) == len(test_image_captions)

    # Create word map
    words = [w for w in word_freq.keys() if word_freq[w] > min_word_freq]
    word_map = {k: v + 1 for v, k in enumerate(words)}
    word_map['<unk>'] = len(word_map) + 1
    word_map['<start>'] = len(word_map) + 1
    word_map['<end>'] = len(word_map) + 1
    word_map['<pad>'] = 0

    # Create a base/root name for all output files
    base_filename = dataset + '_' + str(captions_per_image) + '_cap_per_img_' + str(min_word_freq) + '_min_word_freq'

    # Save word map to a JSON
    with open(os.path.join(output_folder, 'WORDMAP_' + base_filename + '.json'), 'w') as j:
        json.dump(word_map, j)

    # Sample captions for each image, save images to HDF5 file, and captions and their lengths to JSON files
    seed(123)
    for impaths, imcaps, split in [(train_image_paths, train_image_captions, 'TRAIN'),
                                   (val_image_paths, val_image_captions, 'VAL'),
                                   (test_image_paths, test_image_captions, 'TEST')]:

        with h5py.File(os.path.join(output_folder, split + '_IMAGES_' + base_filename + '.hdf5'), 'a') as h:
            # Make a note of the number of captions we are sampling per image
            h.attrs['captions_per_image'] = captions_per_image

            # Create dataset inside HDF5 file to store images
            images = h.create_dataset('images', (len(impaths), 3, 256, 256), dtype='uint8')

            print("\nReading %s images and captions, storing to file...\n" % split)

            enc_captions = []
            caplens = []

            for i, path in enumerate(tqdm(impaths)):

                # Sample captions
                if len(imcaps[i]) < captions_per_image:
                    captions = imcaps[i] + [choice(imcaps[i]) for _ in range(captions_per_image - len(imcaps[i]))]
                else:
                    captions = sample(imcaps[i], k=captions_per_image)

                # Sanity check
                assert len(captions) == captions_per_image

                # Read images
                img = imread(impaths[i])
                if len(img.shape) == 2:
                    img = img[:, :, np.newaxis]
                    img = np.concatenate([img, img, img], axis=2)
                img = imresize(img, (256, 256))
                img = img.transpose(2, 0, 1)
                assert img.shape == (3, 256, 256)
                assert np.max(img) <= 255

                # Save image to HDF5 file
                images[i] = img

                for j, c in enumerate(captions):
                    # Encode captions
                    enc_c = [word_map['<start>']] + [word_map.get(word, word_map['<unk>']) for word in c] + [
                        word_map['<end>']] + [word_map['<pad>']] * (max_len - len(c))

                    # Find caption lengths
                    c_len = len(c) + 2

                    enc_captions.append(enc_c)
                    caplens.append(c_len)

            # Sanity check
            assert images.shape[0] * captions_per_image == len(enc_captions) == len(caplens)

            # Save encoded captions and their lengths to JSON files
            with open(os.path.join(output_folder, split + '_CAPTIONS_' + base_filename + '.json'), 'w') as j:
                json.dump(enc_captions, j)

            with open(os.path.join(output_folder, split + '_CAPLENS_' + base_filename + '.json'), 'w') as j:
                json.dump(caplens, j)

if __name__ == '__main__':
    BASE_DIR = os.path.dirname(os.path.abspath(__file__))

    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--dataset', default='flickr8k',
        choices=['coco', 'flickr8k', 'flickr30k'],
        help='dataset type'
    )
    parser.add_argument(
        '--json', default=os.path.join(BASE_DIR, 'data', 'dataset_flickr8k.json'),
        help='path to karpathy json file'
    )
    parser.add_argument(
        '--image_folder', default=os.path.join(BASE_DIR, 'data', 'images'),
        help='path to folder containing images'
    )
    parser.add_argument(
        '--captions_per_image', type=int, default=5,
        help='number of captions to sample per image'
    )
    parser.add_argument(
        '--min_word_freq', type=int, default=5,
        help='words occuring less frequently than this threshold are binned as <unk>s'
    )
    parser.add_argument(
        '--output_folder', default=os.path.join(BASE_DIR, 'data'),
        help='folder to save files'
    )
    parser.add_argument(
        '--max_len', type=int, default=50,
        help='don\'t sample captions longer than this length'
    )
    args = parser.parse_args()

    # Create input files (along with word map)
    create_input_files(
        dataset=args.dataset,
        karpathy_json_path=args.json,
        image_folder=args.image_folder,
        captions_per_image=args.captions_per_image,
        min_word_freq=args.min_word_freq,
        output_folder=args.output_folder,
        max_len=args.max_len,
    )

max_stars_repo_path: 12 - Image Captioning/create_input_files.py

reason: import numpy,from scipy


text:
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

from phone_data_utils import UPB_XLIM_EASTING, UPB_YLIM_NORTHING

pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

CFG_FILE = "cfg.yaml"
CFG_EXTRA_FILE = "cfg_extra.yaml"

SPEED_FILE = "speed.csv"
STEER_FILE = "steer.csv"
PHONE_FILE = "phone.log.pkl"

def main():
    exp = "/media/nemodrive0/Samsung_T5/nemodrive/18_nov/session_0/1542537659_log"
    phone = pd.read_pickle("{}/phone.log.pkl".format(exp))
    steer = pd.read_csv("{}/steer.csv".format(exp))
    speed = pd.read_csv("{}/speed.csv".format(exp))

    offset_tp = 1543059543.52

# ==================================================================================================
# Analyze bdd
import json

with open("/media/nemodrive0/Samsung_T5/BDD_SAMPLE/tfrecords_from_train/02b6d2a0-54fc4ce8.json") as f:
    data = json.load(f)

fig = plt.figure()
df = pd.DataFrame(data["locations"])
df.course.plot()

# ==================================================================================================
# Plot gps

    gps_unique = phone.groupby(['loc_tp']).head(1)

    fig = plt.figure()
    plt.plot(gps_unique.easting - gps_unique.easting.min(), gps_unique.northing -
                gps_unique.northing.min(), zorder=-1)
    plt.scatter(gps_unique.easting - gps_unique.easting.min(), gps_unique.northing -
                gps_unique.northing.min(), s=1.5, c="r", zorder=1)
    plt.show()
    plt.axes().set_aspect('equal')

# ==================================================================================================
#  Plot steering

    fig = plt.figure()
    plt.scatter(steer.tp, steer.steer, s=3.5)
    plt.show()

# ==================================================================================================
#  Plot Magnetometer

    fig = plt.figure()
    plt.plot(phone.tp - offset_tp, phone.trueHeading)

    fig = plt.figure()
    plt.plot(steer.tp - offset_tp, steer.steer)

# ==================================================================================================
#  Plot Accelerometer

    acc = pd.DataFrame.from_dict(list(phone["acceleration"].values))
    acc["tp"] = phone["tp"]
    phone_start_tp = acc.tp.min()
    acc["tp_rel"] = acc.tp - phone_start_tp
    # acc["tp_rel"] = acc.tp - camer_start_time

    tp_start = 11*60+ 45. + 1542296336.71 - 30.
    acc_sel = acc[(acc.tp >= tp_start) & (acc.tp < tp_start + 30.)]

    fig, ax = plt.subplots(1, 3)
    ax[0].plot(acc_sel.tp_rel, acc_sel.x)
    ax[1].plot(acc_sel.tp_rel, acc_sel.y)
    ax[2].plot(acc_sel.tp_rel, acc_sel.z)

    fig, ax = plt.subplots()
    ax.plot(acc.tp_rel, acc.z)
    ax.set_title("acc_z")

    fig, ax = plt.subplots()
    ax.plot(acc.tp_rel, acc.y)
    ax.set_title("acc_y")

    fig, ax = plt.subplots()
    ax.plot(acc.tp_rel, acc.x)
    ax.set_title("acc_x")

    speed["tp_rel"] = speed.tp - phone_start_tp
    fig, ax = plt.subplots()
    ax.plot(speed.tp_rel, speed.mps)
    ax.set_title("speed")
# ==================================================================================================
#  Plot Accelerometer

    userAcceleration = pd.DataFrame.from_dict(list(phone["userAcceleration"].values))
    userAcceleration["tp"] = phone["tp"]
    phone_start_tp = acc.tp.min()
    userAcceleration["tp_rel"] = userAcceleration.tp - phone_start_tp

    tp_start = 11*60+ 45. + 1542296336.71 - 30.
    acc_sel = acc[(acc.tp >= tp_start) & (acc.tp < tp_start + 30.)]

    fig, ax = plt.subplots()
    ax.plot(userAcceleration.tp_rel, userAcceleration.z)

# ==================================================================================================
#  Plot Accelerometer

    gravity = pd.DataFrame.from_dict(list(phone["gravity"].values))
    gravity["tp"] = phone["tp"]
    phone_start_tp = acc.tp.min()
    gravity["tp_rel"] = gravity.tp - phone_start_tp

    fig, ax = plt.subplots()
    ax.plot(gravity.tp_rel, gravity.z)
    ax.set_title("gravity")

# ==================================================================================================
#  Plot attiude
    attiude = pd.DataFrame.from_dict(list(phone["attitudeEularAngles"].values))
    attiude["tp"] = phone["tp"]
    phone_start_tp = phone.tp.min()
    attiude["tp_rel"] = attiude.tp - phone_start_tp

    camera_2_start = 1542296336.71
    tp_start = 450 + camera_2_start

    attiude_sel = attiude[(attiude.tp >= tp_start) & (attiude.tp < tp_start + 300.)]

    fig, ax = plt.subplots(1, 3)
    ax[0].plot(attiude_sel.tp_rel, attiude_sel.x)
    ax[1].plot(attiude_sel.tp_rel, attiude_sel.y)
    ax[2].plot(attiude_sel.tp_rel, attiude_sel.z)

    fig, ax = plt.subplots()
    ax.plot(attiude.tp_rel, np.cumsum(attiude.y.values))

# ==================================================================================================
#  Interesting data segments
base_exp_path = "/media/nemodrive0/Samsung_T5/nemodrive"

DATA = [
    {
        "exp_path": f"{base_exp_path}/24_nov/session0/1543059528_log",
        "gps_unique_idx": [
            [57, 92],
            [89, 103],
            # [342, 371],  # MAYBE NOT THAT STRAIGHT
            [604, 625],
            [782, 820],
            [1217, 1229],
            [1335, 1356]
        ]
    },
    {
        "exp_path": f"{base_exp_path}/18_nov/session_1/1542549716_log",
        "gps_unique_idx": [
            [149, 158],
            [202, 215],
            [207, 246],
            [242, 265],
            [258, 287],
            [304, 332],
            [322, 341],
            [663, 700],
            [680, 700],
            [692, 729],
            [723, 747],
            [743, 780],
            [926, 959],
            [1214, 1284],
            [1279, 1362],  # Large loop closure
            [1660, 1717],  # Parallel road driving with full turn
            [1777, 1795],
            [2144, 2194],  # Bad driving on straight road
            [2216, 2243],  # Circle around round "island"
            [2269, 2319],  # Max steering full circle turn
        ]
    }
]

phone_splits = []
steer_splits = []
speed_splits = []

for data_dict in DATA:
    experiment_path = data_dict["exp_path"]
    gps_unique_idx = data_dict["gps_unique_idx"]

    phone = pd.read_pickle("{}/phone.log.pkl".format(experiment_path))
    steer = pd.read_csv("{}/steer.csv".format(experiment_path))
    speed = pd.read_csv("{}/speed.csv".format(experiment_path))

    gps_unique = phone.groupby(['loc_tp']).head(1)

    for i, j in gps_unique_idx:
        tp_start, tp_end = gps_unique.iloc[i]["tp"], gps_unique.iloc[j]["tp"]

        phone_split = phone[(phone.tp >= tp_start) & (phone.tp < tp_end)]
        steer_split = steer[(steer.tp >= tp_start) & (steer.tp < tp_end)]
        speed_split = speed[(speed.tp >= tp_start) & (speed.tp < tp_end)]

        phone_splits.append(phone_split)
        steer_splits.append(steer_split)
        speed_splits.append(speed_split)

LOOPS = [
    {
        "exp_path": f"{base_exp_path}/18_nov/session_0/1542537659_log",
        "reference_time":"camera_2_timestamp",
        "tp": [
            # ["00:00.0", "10:46.12"], # 2 loops
            ["02:43.18", "06:47.10"],
            ["06:36.02", "10:31.19"],
            ["11:17.26", "15:00.9"],
            ["14:58.1", "18:29.26"],
        ]
    },
    {
        "exp_path": f"{base_exp_path}/18_nov/session_1/1542549716_log",
        "reference_time":"camera_2_timestamp",
        "tp": [
            ["05:40.29", "09:26.15"],
            ["19:21.26", "21:12.21"],
            ["22:06.27", "23:09.23"],
        ]
    },
    {
        "exp_path": f"{base_exp_path}/24_nov/session0/1543059528_log",
        "reference_time":"camera_2_timestamp",
        "tp": [
            ["01:27.5", "07:04.16"],
            ["06:38.6", "12:13.29"],
            ["23:49.6", "27:45.2"],
        ]
    }
]

STRAIGHT_LINE = [
    {
        "exp_path": f"{base_exp_path}/15_nov/1542296320_log",
        "reference_time":"camera_2_timestamp",
        "tp": [
            ["22:21.27", "22:50.08"],
            ["23:37.24", "23:43.17"],
        ]
    },
    {
        "exp_path": f"{base_exp_path}/18_nov/session_0/1542537659_log",
        "reference_time":"camera_2_timestamp",
        "tp": [
            ["06:09.24", "06:38.23"],
            ["08:33.14", "08:45.19"],
            ["10:07.20", "10:30.26"],
            ["11:20.30", "11:48.25"],
            ["15:00.11", "15:24.20"],
            ["20:59.11", "20:14.03"],
            ["21:38.13", "21:57.29"],
            ["22:16.11", "22:32.20"],
            ["23:07.21", "23:19.21"],
            ["23:46.08", "24:01.14"],
            ["24:22.23", "24:29.09"],
            ["24:44.06", "24:59.12"],
            ["26:26.18", "26:40.25"],
            ["30:52.01", "31:10.23"],
        ]
    },
    {
        "exp_path": f"{base_exp_path}/18_nov/session_1/1542549716_log",
        "reference_time":"camera_2_timestamp",
        "tp": [
            ["03:09.09", "03:17.29"],
            ["04:28.06", "04:48.20"],
            ["05:41.29", "06:10.13"],
            ["07:28.25", "07:36.11"],
            ["12:03.07", "12:20.06"],
            ["12:38.23", "12:48.20"],
            ["13:02.27", "13:07.12"],
            ["13:20.25", "13:38.13"],
            ["14:33.05", "14:38.08"],
            ["16:05.14", "16:16.20"],
            ["16:33.05", "16:43.12"],
            ["17:52.06", "18:08.11"],
            ["18:53.14", "19:03.07"],
            ["21:00.25", "22:03.26"],
            ["23:14.13", "23:30.3"],
            ["23:57.5", "24:06.27"],
            ["24:21.24", "24:37.29"],
            ["27:23.21", "27:49.29"],
            ["30:50.16", "31:02.18"],
        ]
    },
    {
        "exp_path": f"{base_exp_path}/24_nov/session0/1543059528_log",
        "reference_time":"camera_2_timestamp",
        "tp": [
            ["02:10.13", "02:45.25"],
            ["11:44.4", "12:01.01"],
            ["15:20.13", "15:31.19"],
            ["18:56.28", "19:15.01"],
            ["19:42.13", "19:56.22"],
            ["20:32.16", "20:43.01"],
            ["22:05.25", "22:16.04"],
            ["22:05.25", "22:16.04"],
        ]
    },
    {
        "exp_path": f"{base_exp_path}/25_nov/session_0/1543134132_log",
        "reference_time":"camera_2_timestamp",
        "tp": [
            ["03:10.13", "03:17.10"],
            ["03:29.25", "03:41.07"],
            ["05:03.25", "05:18.01"],
            ["06:11.13", "06:37.07"],
            ["10:14.22", "10:43.25"],
            ["12:10.07", "12:26.04"],
            ["12:50.28", "13:08.04"],
            ["14:38.25", "14:46.25"],
            ["17:08.10", "17:18.01"],
            ["17:30.22", "17:45.25"],
            ["18:50.19", "19:01.25"],
            ["19:10.4", "19:20.22"],
            ["21:58.4", "22:10.01"],
            ["22:30.19", "22:40.13"],
            ["23:37.19", "23:52.28"],
            ["24:05.28", "24:20.28"],
        ]
    },
]

def parse_video_time_format(s, fps=30.):
    """ Format MM:SS.f """
    m, sf = s.split(":")
    m = float(m)
    s, f = [float(x) for x in sf.split(".")]

    time_interval = m * 60. + s + 1. /fps * f
    return time_interval

# # SHOULD use margins for save cut FOR STRAIGHT LINE
# margin_right = 1.0
# margin_left = 1.

# # SHOULD use margins for save cut
margin_right = 0.0
margin_left = 0.0

phone_splits = []
steer_splits = []
speed_splits = []

for data_dict in LOOPS:  # STRAIG
    experiment_path = data_dict["exp_path"]

    phone = pd.read_pickle("{}/phone.log.pkl".format(experiment_path))
    steer = pd.read_csv("{}/steer.csv".format(experiment_path))
    speed = pd.read_csv("{}/speed.csv".format(experiment_path))
    reference_time = data_dict['reference_time']
    time_intervals = data_dict["tp"]

    with open(f"{experiment_path}/{reference_time}") as f:
        camera_start_tp = float(f.read())

    gps_unique = phone.groupby(['loc_tp']).head(1)

    for i, j in time_intervals:
        tp_start = parse_video_time_format(i) + camera_start_tp
        tp_end = parse_video_time_format(j) + camera_start_tp

        phone_split = phone[(phone.tp >= tp_start + margin_left) & (phone.tp < tp_end - margin_right)]
        steer_split = steer[(steer.tp >= tp_start + margin_left) & (steer.tp < tp_end - margin_right)]
        speed_split = speed[(speed.tp >= tp_start + margin_left) & (speed.tp < tp_end - margin_right)]

        phone_splits.append(phone_split)
        steer_splits.append(steer_split)
        speed_splits.append(speed_split)

# ==============================================================================================
# Determine offset_steering ON STRAIGHT LINE

for idx, (phone_s, steer_s, speed_s) in enumerate(zip(phone_splits, steer_splits, speed_splits)):
    phone, steer, speed = phone_s.copy(), steer_s.copy(), speed_s.copy()
    gps_unique_points = phone.groupby(['loc_tp']).head(1)

    fig, ax = plt.subplots(1, 1)
    ax.set_aspect('equal')
    ax.plot(gps_unique_points.easting, gps_unique_points.northing, c="r", zorder=1)
    # ax.plot(steer.tp, steer.can_steer, c="r", zorder=1)
    # fig.suptitle(f"{idx}_{len(gps_unique_points)}")
    # fig.savefig(f"data/Min_off_steering_steer_ratio_{idx}.png")
    plt.show()
    plt.waitforbuttonpress()
    plt.close("all")

for idx, steer in enumerate(steer_splits):
    steer["dataset_idx"] = idx

all_steer: pd.DataFrame = pd.concat(steer_splits)

all_steer.boxplot(column=["can_steer"], by="dataset_idx")

outliers = []
all_steer_wo_o = all_steer[all_steer.dataset_idx.apply(lambda x: x not in outliers)]

all_steer_wo_o.can_steer.plot.box()
all_steer_wo_o.can_steer.median()  # -15.4
all_steer_wo_o.can_steer.mean()  # -15.149503459604759
all_steer_wo_o.can_steer.describe()
# count    80356.000000
# mean       -15.149503
# std          3.221953
# min        -36.200000
# 25%        -16.800000
# 50%        -15.400000
# 75%        -13.600000
# max          1.200000
# Name: can_steer, dtype: float64

# ==============================================================================================
# Determine offset_steering by optimization

from car_utils import get_rotation_and_steering_offset
from multiprocessing import Pool as ThreadPool

all_res = []
no = 0

no += 1

def get_rotation_and_steering_offset_mt(args):
    speed, steer, gps_unique_points, task = args
    print(f"Start {task}...")
    result = get_rotation_and_steering_offset(speed, steer, gps_unique_points, simple=False, idx=task)
    print(f"Done {task}!")
    return result

threads = 11
pool = ThreadPool(threads)
args = []
for idx, (phone_s, steer_s, speed_s) in enumerate(zip(phone_splits, steer_splits, speed_splits)):
    phone, steer, speed = phone_s.copy(), steer_s.copy(), speed_s.copy()
    gps_unique_points = phone.groupby(['loc_tp']).head(1)

    args.append((speed, steer, gps_unique_points, idx))

results = pool.map(get_rotation_and_steering_offset_mt, args)
pool.close()
pool.join()

# ==============================================================================================
# Iterative version

results = []
# i = 3
# phone_s, steer_s, speed_s = phone_splits[3], steer_splits[3], speed_splits[3]
for phone_s, steer_s, speed_s in zip(phone_splits, steer_splits, speed_splits):
    phone, steer, speed = phone_s.copy(), steer_s.copy(), speed_s.copy()
    gps_unique_points = phone.groupby(['loc_tp']).head(1)
    result = get_rotation_and_steering_offset(speed, steer, gps_unique_points)

    results.append(result)
    i += 1
    print(f"Done {i}/{len(phone_splits)}")
# ==============================================================================================

# save data to disk
fld = "/media/nemodrive3/Samsung_T5/nemodrive/optimized_steering_offset/optimize_with_ratio/optim_offset_ratio_18"
steer_optimization = dict({"DATA": DATA, "results": results, "args": args})
np.save(f"{fld}/steer_optimization_{no}.npy", steer_optimization)

# # Load from disk
# steer_optimization = np.load(f"{fld}/steer_optimization.npy").item()
# DATA = steer_optimization["DATA"]
# results = steer_optimization["results"]
# args = steer_optimization["args"]
#
# Run plot
steer_offset_res = []

nrows = 5
ncols = 4
fig, axes = plt.subplots(nrows=nrows, ncols=ncols)

for idx, (new_points, gps_unique, result_opti) in enumerate(results):
    ax = axes[idx//ncols, idx%ncols]

    ax.scatter(new_points.coord_x, new_points.coord_y, s=1., c="r")

    ax.scatter(gps_unique.target_x, gps_unique.target_y, s=1., c="b")
    ax.set_aspect('equal')

    # best_orientation, best_offest_x, best_offest_y, best_steering_offset, best_wheel_steer_ratio = result_opti["x"]
    best_orientation, best_offest_x, best_offest_y, best_steering_offset = result_opti["x"]
    # best_steering_offset = OFFSET_STEERING
    ax.set_title(f"i:{idx} - s:{best_steering_offset}")

    steer_offset_res.append(best_steering_offset)

fig.set_size_inches(18.55, 9.86)
fig.savefig(f"{fld}/calc_wheel_ratio_{no}.png")

# res_info = pd.DataFrame([[*r[2]["x"], r[2]["success"], r[2]["message"], r[2]["nit"]] for r in results],
#                                 columns=["oritentaion", "offset_x", "offset_y", "wheel_steer_ratio",
#                                          "success", "meessage", "nit"])
res_info = pd.DataFrame([[*r[2]["x"], r[2]["success"], r[2]["message"], r[2]["nit"]] for r in results],
                                columns=["oritentaion", "offset_x", "offset_y", "steering_offset",
                                         "success", "meessage", "nit"])
res_info.to_csv(f"{fld}/optimization_info.csv",
                float_format = '%.14f')

all_res.append(res_info)

# Other stupid

fig = plt.figure()
res_info.steering_offset.plot(kind="bar")
fig.suptitle("Calculated steerings by optimization", fontsize=16)

# MERge
res_info = pd.concat(all_res)
res_info = res_info.reset_index()
res_info = res_info.rename_axis({"index":"idx"}, axis=1)

# Wheel ration analysis
ignore_wheel = [0, 1, 5, 6, 8, 13, 16, 17, 18, 19]
wheel_steer_ratio_s = res_info[[x not in ignore_wheel for x in res_info["idx"]]].wheel_steer_ratio

fig = plt.figure()
wheel_steer_ratio_s.plot()

wheel_steer_ratio_s.describe()
# count    40.000000
# mean     18.053225
# std       0.812695
# min      16.955807
# 25%      17.350352
# 50%      17.947226
# 75%      18.581726
# max      19.416291

# Try different steering offsets
from car_utils import get_rotation, get_car_can_path

out_path = ""
maybe_offsets = np.linspace(14.555555555555555555 -0.7, 14.555555555555555555 + 0.7, 1000)
nrows = 5
ncols = 4

out_folder = "/media/nemodrive3/Samsung_T5/nemodrive/optimized_steering_offset/small_variation_18_0"
full_results = []

for offset_sol in maybe_offsets:
    fig, axes = plt.subplots(nrows=nrows, ncols=ncols)

    save_data = []
    for idx, (_a, _r) in enumerate(zip(args, results)):
        speed, steer, gps_unique, task = _a
        best_orientation, best_offest_x, best_offest_y, best_steering_offset = _r[2]["x"]

        can_coord = get_car_can_path(speed.copy(), steer.copy(), steering_offset=offset_sol)

        new_points, gps_unique, result = get_rotation(can_coord.copy(), gps_unique.copy(),
                                                      guess_orientation=best_orientation,
                                                      guess_offest_x=best_offest_x, guess_offest_y=best_offest_y)

        ax = axes[idx // ncols, idx % ncols]

        ax.scatter(new_points.coord_x, new_points.coord_y, s=1., c="r")

        ax.scatter(gps_unique.target_x, gps_unique.target_y, s=1., c="b")
        ax.set_aspect('equal')

        ax.set_title(f"i:{idx}")

        save_info = dict({"new_points": new_points, "gps_unique": new_points, "result": new_points})
        save_data.append(save_info)

        full_results.append([offset_sol, idx, result["nit"], result["success"], result["message"],
                             *result["x"], result["loss"]])

    fig.suptitle(f"Steering offset {offset_sol}")
    fig.set_size_inches(18.55, 9.86)
    fig.savefig(f"{out_folder}/offset_{offset_sol}.png")
    plt.close('all')

    np.save(f"{out_folder}/offset_{offset_sol}.npy", save_data)

full_results_df = pd.DataFrame(full_results, columns=["offset_sol", "idx", "nit", "success", "message", "orientation",
                                                      "offset_x", "offset_y", "loss"])
full_results_df.to_pickle(f"{out_folder}/full_results_df.pkl")

df = full_results_df[full_results_df.idx < 18]
df.groupby("offset_sol")["loss"].sum().plot()
losses = df.groupby("offset_sol")["loss"].sum()
losses.sort_values(ascending=True).head(10)

# maybe_offsets = np.linspace(15.71921921921922 - 0.3, 15.71921921921922 + 0.3, 1000)

# Minimum result 15.72072072072072
# 15.800000       0.000000 # Clearly
# 16.000000       0.000000
# 15.600000       0.000000
# 15.844444    2298.341425 # Anomaly
# 15.533333    2427.975330 # Anomaly
# 15.720721    2483.125663
# 15.720120    2483.125678
# 15.721321    2483.125689
# 15.719520    2483.125709
# 15.722523    2483.125715

# ==============================================================================================
# Minimize loop closer

WHEEL_STEER_RATIO = 18.0
OFFSET_STEERING = 15.45000001

losses = []
var_offset_steering = [15.149503459604759, 15.400000000010011]
var_steer_ratio = np.linspace(17.8, 18.3, 100)

# MULTI PROCESS
from multiprocessing import Pool as ThreadPool
from car_utils import get_car_can_path
import time

def get_losses(args):
    speed, steer, dataset_idx = args
    losses = []
    print(f"Start {dataset_idx}...")
    for offset_steering in var_offset_steering:
        stp = time.time()

        for i, steer_ratio in enumerate(var_steer_ratio):
            can_coord = get_car_can_path(speed.copy(), steer.copy(),
                                         steering_offset=offset_steering, wheel_steer_ratio=steer_ratio)

            loss = np.linalg.norm(can_coord.iloc[-1][["coord_x", "coord_y"]].values)
            losses.append([loss, steer_ratio, offset_steering, dataset_idx])
        print(f"Done {dataset_idx}: {i} ({time.time()-stp})!")
    return losses

threads = 9
pool = ThreadPool(threads)
args = []
for idx, (phone_s, steer_s, speed_s) in enumerate(zip(phone_splits, steer_splits, speed_splits)):
    phone, steer, speed = phone_s.copy(), steer_s.copy(), speed_s.copy()
    args.append((speed, steer, idx))

results = pool.map(get_losses, args)
pool.close()
pool.join()

r = []
for x in results:
    r += x

from mpl_toolkits.mplot3d import Axes3D

losses = pd.DataFrame(r, columns=["loss", "steer_ratio", "offset_steering", "dataset_idx"])

# Plot all datasets
fig = plt.figure()
for idx in range(len(phone_splits)):
    df = losses[losses.dataset_idx == idx]
    ax = fig.add_subplot(3, 4, idx+1, projection='3d')
    ax.scatter(df.steer_ratio, df.offset_steering, df.loss, s=1.)
fig.suptitle("Grid search steer offset and ratio for loops")

# Plot minimum loss coord
all_data_loss = losses.groupby(["steer_ratio", "offset_steering"])["loss"].apply(lambda x: x.pow(4, axis=0).sum())
all_data_loss = all_data_loss.reset_index()
fig = plt.figure()
ax = fig.add_subplot(1,1,1, projection='3d')
ax.scatter(all_data_loss.steer_ratio, all_data_loss.offset_steering, all_data_loss.loss, s=1.)

min_off_s, min_s_r = all_data_loss.loc[all_data_loss.loss.idxmin()][["offset_steering", "steer_ratio"]].values

# SEEMS like median is the best !
# All steering smaller than 18. seems weird !

selection = losses[losses.offset_steering == var_offset_steering[1]]
min_loss = selection.groupby("dataset_idx").loss.idxmin()
selection = selection.loc[min_loss]

min_steer_ratio = selection[selection.steer_ratio > 18.0].steer_ratio.mean()

min_off_s  = var_offset_steering[1]
min_s_r = min_steer_ratio
"""
    BEST RESULT:
        OFFSET: 15.40000000001001
         RATIO: 18.215151515151515

"""
# Plot minimum loss coord
for idx, (phone_s, steer_s, speed_s) in enumerate(zip(phone_splits, steer_splits, speed_splits)):
    phone, steer, speed = phone_s.copy(), steer_s.copy(), speed_s.copy()
    gps_unique_points = phone.groupby(['loc_tp']).head(1)

    # df = losses[losses.dataset_idx == idx]
    # min_idx = df.loss.idxmin()
    # min_conf = df.loc[min_idx]
    # #
    # min_off_s = min_conf.offset_steering
    # min_s_r = min_conf.steer_ratio
    # print(f"IDX: {idx} min_ {min_off_s} _ {min_s_r}")
    # can_coord = get_car_can_path(speed.copy(), steer.copy(),
    #                              steering_offset=min_off_s, wheel_steer_ratio=min_s_r)
    can_coord = get_car_can_path(speed.copy(), steer.copy(),
                                 steering_offset=min_off_s, wheel_steer_ratio=min_s_r)

    fig, ax = plt.subplots(1, 2)
    ax[0].scatter(can_coord.coord_x, can_coord.coord_y, s=1.5, c="r", zorder=1)
    ax[0].set_aspect('equal')
    ax[1].scatter(gps_unique_points.easting, gps_unique_points.northing, s=1.5, c="r", zorder=1)
    ax[1].scatter(gps_unique_points.iloc[0].easting, gps_unique_points.iloc[0].northing, s=1.5, c="b", zorder=1)
    ax[1].scatter(gps_unique_points.iloc[-1].easting, gps_unique_points.iloc[-1].northing, s=1.5, c="black", zorder=1)
    ax[1].set_aspect('equal')
    fig.suptitle(f"Offset_steering: {min_off_s}. Steer ratio: {min_s_r}")
    fig.savefig(f"data/Min_off_steering_steer_ratio_{idx}.png")
    plt.show()
    plt.waitforbuttonpress()
    plt.close("all")

# -- Iterative
dataset_idx = 0

for phone_s, steer_s, speed_s in zip(phone_splits, steer_splits, speed_splits):
    phone, steer, speed = phone_s.copy(), steer_s.copy(), speed_s.copy()
    gps_unique_points = phone.groupby(['loc_tp']).head(1)

    for steer_ratio in var_steer_ratio:
        for offset_steering in var_offset_steering:

            can_coord = get_car_can_path(speed.copy(), steer.copy(),
                                         steering_offset=offset_steering, wheel_steer_ratio=steer_ratio)

            loss = np.linalg.norm(can_coord.iloc[-1][["coord_x", "coord_y"]].values)
            losses.append([loss, steer_ratio, offset_steering, dataset_idx])
            # fig = plt.figure()
            # plt.scatter(can_coord.coord_x, can_coord.coord_y, s=1.5, c="r", zorder=1)
            # plt.axes().set_aspect('equal')
            # plt.show()

    dataset_idx += 1

tp = 0.

gps = gps_unique[gps_unique.tp < tp]
fig = plt.figure()
plt.plot(gps.easting - gps.easting.min(), gps.northing -
         gps.northing.min(), zorder=-1)
plt.scatter(gps.easting - gps.easting.min(), gps.northing -
            gps.northing.min(), s=1.5, c="r", zorder=1)
plt.show()
plt.axes().set_aspect('equal')

# ==============================================================================================
# Plot GPS

results = []
for phone_s, steer_s, speed_s in zip(phone_splits, steer_splits, speed_splits):

    gps_unique = phone_s.groupby(['loc_tp']).head(1)

    fig = plt.figure()
    plt.plot(gps_unique.easting, gps_unique.northing, zorder=-1)
    plt.scatter(gps_unique.easting, gps_unique.northing, s=1.5, c="r", zorder=1)
    plt.xlim(UPB_XLIM_EASTING)
    plt.ylim(UPB_YLIM_NORTHING)
    plt.axes().set_aspect('equal')
    plt.show()
    plt.pause(0.0001)
    k = input()

# ==================================================================================================
# Plot Corrected paths
import glob
from car_utils import get_corrected_path
import os
from data_utils import load_experiment_data
from street_view import ImageWgsHandler

corrected_paths = glob.glob("/media/nemodrive0/Samsung_T5/nemodrive/**/*_log/corrected_path.csv", recursive=True)

corrected_path_dfs = []
gpss = []
for idx, corrected_path in enumerate(corrected_paths):
    experiment_path = os.path.dirname(corrected_path)
    edata = load_experiment_data(experiment_path)

    gps_unique = edata.phone.groupby(['loc_tp']).head(1)
    gpss.append(gps_unique)

    # print("Found path correction file")
    # path_corrections = pd.read_csv(corrected_path)
    # corrected_path_df = get_corrected_path(path_corrections, edata.steer, edata.speed)
    # corrected_path_df["dataset_idx"] = idx
    # corrected_path_dfs.append(corrected_path_df)

corrected_path_dfs = pd.concat(corrected_path_dfs)
gpss = pd.concat(gpss)

#  Plot only points
fig = plt.figure()
plt.scatter(corrected_path_dfs.easting, corrected_path_dfs.northing, s=1.5, c="r", zorder=1)
plt.axes().set_aspect('equal')

#  Plot on image
map_viewer = ImageWgsHandler("util_data/high_res_full_UPB_hybrid.jpg")

fig, ax = map_viewer.plot_wgs_coord(corrected_path_dfs.easting.values, corrected_path_dfs.northing.values, convert_method=1)

# Plot gps
fig, ax = map_viewer.plot_wgs_coord(gpss.easting.values, gpss.northing.values, convert_method=1)

# ==================================================================================================
# Plot Corrected paths
import glob
import numpy as np
import json
import matplotlib.pyplot as plt

good_data_info_f = glob.glob("/media/nemodrive0/Samsung_T5/nemodrive/**/good/export_info.npy", recursive=True)
bad_data_info_f = glob.glob("/media/nemodrive0/Samsung_T5/nemodrive/**/bad/export_info.npy", recursive=True)

low_light_paths = ["15_nov/1542296320_log", "18_nov/session_2/1542563017_log", ]

def load_data(export_infos):

    all_df = []
    for export_info in export_infos:
        e = np.load(export_info).item()
        de = pd.DataFrame([e["cut_intervals"], e["rideIDs"]], index=["cut_interval", "rideID"]).T
        de["g_out_fld"] = e["g_out_fld"]
        de["experiment_path"] = e["experiment_path"]
        all_df.append(de)
    all_df = pd.concat(all_df).reset_index()
    all_df["fileID"] = all_df.rideID.apply(lambda x: x[:16])
    all_df["start_pts"] = all_df.cut_interval.apply(lambda x: x[0])
    all_df["end_pts"] = all_df.cut_interval.apply(lambda x: x[1])
    return all_df

def load_jsons(export_info_df: pd.DataFrame):

    all_data = []
    all_locations = []
    for idx, row in export_info_df.iterrows():
        ride_path = os.path.join(row["g_out_fld"], row["fileID"]+".json")
        with open(ride_path) as f:
            data = json.load(f)

        locations = pd.DataFrame(data["locations"])
        locations["rideID"] = row["rideID"]

        all_data.append(data)
        all_locations.append(locations)

    all_locations = pd.concat(all_locations).reset_index()
    return all_data, all_locations

good_data_info = load_data(good_data_info_f)
bad_data_info = load_data(bad_data_info_f)

df = low_light_sample = good_data_info[good_data_info.experiment_path.apply(lambda x: any([y in x for y in low_light_paths]))]
df = good_data_info
df = bad_data_info

data, locations = load_jsons(df)

print("Duration {} h".format((df.end_pts - df.start_pts).sum()/ 3600.))

# Calculate distance
locations = locations.sort_values("timestamp")
coord = np.array([locations["easting"].values, locations["northing"].values]).transpose()

# select data just from indiv rides rideID
select_ind_rides = (locations.rideID.duplicated().values)[1:]

dist_coord = np.linalg.norm(coord[1:] - coord[:-1], axis=1) # Distances
dist_coord = dist_coord[select_ind_rides]

print("distance in Km: {}".format(dist_coord.sum() / 1000.))

fig = plt.figure()
plt.plot(dist_coord)

fig = plt.figure()
plt.boxplot(dist_coord, labels=["distanec in m"])
plt.title("Distance gap in CAN log data")

# Histogram relative course diff
diff_course_t = 1/3. * 1000. # ms
idx = 0
all_course_diff = []
f_locations = locations[locations.speed > 0]  # Only with speed greater than 0
for idx, row in f_locations.groupby("rideID"):
    row = row.sort_values("timestamp")
    min_tp, max_tp  = row.timestamp.min(), row.timestamp.max()
    groups = row.groupby(pd.cut(row.timestamp, np.arange(min_tp, max_tp, diff_course_t)))
    course_t_next = groups.course.apply(lambda x: x.tail(1)).values
    course_t = groups.course.apply(lambda x: x.head(1)).values

    course_dif = course_t_next - course_t
    course_dif = (course_dif + 180.) % 360. - 180.
    tp_dif = groups.apply(lambda x: x.tail(1)).timestamp.values - groups.apply(lambda x: x.head(1)).timestamp.values

    course_dif = course_dif[tp_dif > diff_course_t/2.]  # Filter too much variance
    all_course_diff.append(course_dif)

all_course_diff = np.concatenate(all_course_diff)

fig = plt.figure()
plt.boxplot(all_course_diff)

fig = plt.figure()
plt.hist(all_course_diff[np.abs(all_course_diff) < all_course_diff.std() * 6], bins=200)

# ==================================================================================================
# --- Box with  cut out lines

f, (ax, ax2) = plt.subplots(2, 1, sharex=True)
ax.set_title(f"Course difference histogram for bins of {diff_course_t:.4} ms")
# plot the same data on both axes
ax.hist(all_course_diff[np.abs(all_course_diff) < all_course_diff.std() * 6], bins=200)
ax2.hist(all_course_diff[np.abs(all_course_diff) < all_course_diff.std() * 6], bins=200)

# ax.set_xlabel('course diff')
# ax.set_ylabel('minutes')
ax2.set_xlabel('course diff')
ax2.set_ylabel('minutes')

# zoom-in / limit the view to different portions of the data
ax.set_ylim(1000., 11000.)  # outliers only
ax2.set_ylim(0., 600.)  # most of the data

labels = [item.get_text() for item in ax.get_yticklabels()]
new_labels = [f"{int(float(x) * diff_course_t/(1000. * 60))}" for x in labels]
ax.set_yticklabels(new_labels)
labels = [item.get_text() for item in ax2.get_yticklabels()]
new_labels = [f"{int(float(x) * diff_course_t/(1000. * 60))}" for x in labels]
ax2.set_yticklabels(new_labels)

# hide the spines between ax and ax2
ax.spines['bottom'].set_visible(False)
ax2.spines['top'].set_visible(False)
ax.xaxis.tick_top()
ax.tick_params(labeltop='off')  # don't put tick labels at the top
ax2.xaxis.tick_bottom()

d = .015  # how big to make the diagonal lines in axes coordinates
# arguments to pass to plot, just so we don't keep repeating them
kwargs = dict(transform=ax.transAxes, color='k', clip_on=False)
ax.plot((-d, +d), (-d, +d), **kwargs)        # top-left diagonal
ax.plot((1 - d, 1 + d), (-d, +d), **kwargs)  # top-right diagonal

kwargs.update(transform=ax2.transAxes)  # switch to the bottom axes
ax2.plot((-d, +d), (1 - d, 1 + d), **kwargs)  # bottom-left diagonal
ax2.plot((1 - d, 1 + d), (1 - d, 1 + d), **kwargs)  # bottom-right diagonal
# ==================================================================================================

t = []
for i in groups.index.values:
    t.append(i.right - i.left)

fig = plt.figure()
plt.scatter(locations.easting, locations.northing,  s=1.5)
plt.show()
plt.axes().set_aspect('equal')


text:
#!/usr/bin/env python3

import sys
import math
import pprint
import os
import re
import doctest
import itertools
import types
import logging
from collections import deque
from collections import defaultdict
#import networkx as nx
from copy import deepcopy
try:
   import matplotlib.pyplot as plt
except ImportError:
   plt = None

# create absolute mydir
mydir = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.join(mydir, '../lib'))

from advent import *
from intcode import *

def isvalid(buffer, v):
    # print("Checking ", v, buffer)
    for x, y in itertools.permutations(buffer, 2):
        #print(x, y)
        if x == y:
            continue
        if x + y == v:
            # print("Found ", x, y)
            return True
    return False

if __name__ == '__main__':
    if len(sys.argv) == 2 and sys.argv[1] == "TEST":
        import doctest
        doctest.testmod()
        sys.exit(0)

    logging.basicConfig(level=logging.INFO)
    path = "day9.txt"
    size = 25
    if len(sys.argv) > 1:
        path = sys.argv[1]
    if len(sys.argv) > 2:
        size = int(sys.argv[2])

    with open(path) as f:
        data = deque(int(x) for x in f.readlines())

    print(data)
    buffer = deque()
    while len(buffer) < size:
        buffer.append(data.popleft())

    while len(data):
        v = data.popleft()
        if not isvalid(buffer, v):
            print("Not valid!", v)
        buffer.append(v)
        buffer.popleft()

reason: import networkx


text:
import numpy as np

class Change(object):
    def __init__(self, metric_count=100):
        super().__init__()
        self._persistence = {}
        self._metric_count = metric_count

    def delta(self, name, value):
        name += '.delta'
        prev_value = self._persistence.get(name, value)
        self._persistence[name] = value
        return value - prev_value

    def every(self, name, value, interval):
        if interval <= 0:
            return False
        name += '.every'
        next_value = self._persistence.setdefault(name, value) + interval
        if value < next_value:
            return False
        self._persistence[name] = value
        return True

    def moving_metrics(self, name, value, std=True, over=None):
        name += '.moving'
        history = self._persistence.setdefault(name, [])
        over = over or self._metric_count
        while len(history) >= over:
            history.pop(0)
        history.append(value)
        mean = np.mean(history)
        if not std:
            return mean
        return mean, np.std(history)

    def reset(self, name):
        for key in list(self._persistence):
            if not key.startswith(name + '.'):
                continue
            del self._persistence[key]

max_stars_repo_name: deep-fry/mayo

max_issues_repo_name: kypomon/mayo

max_forks_repo_name: kypomon/mayo


text:
"""
Copyright 2020 The OneFlow Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import os
import unittest
import numpy as np

import oneflow as flow
import oneflow.unittest

def _test_graph_lazy_inplace(test_case, x, y):
    class LazyInplaceAdd(flow.nn.Graph):
        def __init__(self):
            super().__init__()

        def build(self, x, y):
            x += y
            return x

    z = LazyInplaceAdd()(x, y)
    test_case.assertTrue(np.allclose(z.numpy(), (x + y).numpy(), 1e-05, 1e-05))

@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
@flow.unittest.skip_unless_1n1d()
class TestLocalInplace(oneflow.unittest.TestCase):
    def test_graph_inplace_gpu(test_case):
        x = flow.randn(10, 10, device=flow.device("cuda"))
        y = flow.ones(10, device=flow.device("cuda"))
        _test_graph_lazy_inplace(test_case, x, y)

    def test_graph_inplace_cpu(test_case):
        x = flow.randn(10, 10, device=flow.device("cpu"))
        y = flow.ones(10, device=flow.device("cpu"))
        _test_graph_lazy_inplace(test_case, x, y)

@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
@flow.unittest.skip_unless_1n2d()
class TestConsistentInplace(oneflow.unittest.TestCase):
    def test_graph_inplace_gpu(test_case):
        x = flow.randn(
            10,
            10,
            placement=flow.placement("cuda", ranks=[0, 1]),
            sbp=flow.sbp.split(1),
        )
        y = flow.ones(
            10, placement=flow.placement("cuda", ranks=[0, 1]), sbp=flow.sbp.broadcast
        )
        _test_graph_lazy_inplace(test_case, x, y)

    def test_graph_inplace_cpu(test_case):
        x = flow.randn(
            10, 10, placement=flow.placement("cpu", ranks=[0, 1]), sbp=flow.sbp.split(1)
        )
        y = flow.ones(
            10, placement=flow.placement("cpu", ranks=[0, 1]), sbp=flow.sbp.broadcast
        )
        _test_graph_lazy_inplace(test_case, x, y)

if __name__ == "__main__":
    unittest.main()


text:
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2021 Jose Gonzalez ~ All rights reserved. MIT license.
import sys

import numpy as np
import pygame
from OpenGL.GL import *
from OpenGL.GLU import *
from pygame.constants import *

from pyvmol import MoleculeLoad

class MolViewer:

    def __init__(self, file, style: str = 'Default'):
        self.style = style
        self.size_radius = 5
        self.mol = MoleculeLoad(file)
        self.coordinates, self.radios, self.colors = self.mol.load()
        self.connectivity = self.mol.connect()
        self.x, self.y, self.z = self.centroid()
        self.width = None
        self.height = None
        self.init()
        self.sphere = gluNewQuadric()
        self.cylinder = gluNewQuadric()
        self.display()

    def coords(self):
        return self.coordinates

    @staticmethod
    def init():
        pygame.init()
        viewport = (600, 600)
        pygame.display.set_caption('PyVMolecule')
        pygame.display.set_mode(viewport, OPENGL | DOUBLEBUF)

        # Light
        glLightfv(GL_LIGHT0, GL_POSITION, (-40, 200, 100, 0.0))
        glLightfv(GL_LIGHT0, GL_AMBIENT, (0.4, 0.4, 0.4, 1.0))
        glLightfv(GL_LIGHT0, GL_DIFFUSE, (0.5, 0.5, 0.5, 1.0))
        glEnable(GL_LIGHT0)
        glEnable(GL_LIGHTING)
        glEnable(GL_COLOR_MATERIAL)
        glEnable(GL_DEPTH_TEST)
        glShadeModel(GL_SMOOTH)
        # glShadeModel(GL_FLAT)
        glMatrixMode(GL_PROJECTION)
        glLoadIdentity()
        width, height = viewport
        gluPerspective(45, width / float(height), 1, 100.0)

    def centroid(self):
        array = np.array(self.coordinates)
        length = array.shape[0]
        sum_x = np.sum(array[:, 0])
        sum_y = np.sum(array[:, 1])
        sum_z = np.sum(array[:, 2])
        return sum_x / length, sum_y / length, sum_z / length

    def display(self):
        rx, ry = (self.x, self.y)
        tx, ty = (0, 0)
        # ADJUST ZOOM ACCORDING TO FIGURE SIZE !!!!!!!
        # zoom_position = (self.z * (100 / (self.x + self.y + self.z))) + 25
        zoom_position = 45
        rotate = move = False
        glMatrixMode(GL_MODELVIEW)
        run = True
        while run:
            for e in pygame.event.get():
                if e.type == QUIT:
                    run = False
                elif e.type == KEYDOWN and e.key == K_ESCAPE:
                    run = False
                elif e.type == MOUSEBUTTONDOWN:
                    if e.button == 4:
                        zoom_position = max(1, zoom_position - 1)
                    elif e.button == 5:
                        zoom_position += 1
                    elif e.button == 1:
                        rotate = True
                    elif e.button == 3:
                        move = True
                elif e.type == MOUSEBUTTONUP:
                    if e.button == 1:
                        rotate = False
                    elif e.button == 3:
                        move = False
                elif e.type == MOUSEMOTION:
                    i, j = e.rel
                    if rotate:
                        rx += i
                        ry += j
                    if move:
                        tx += i
                        ty -= j
            glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
            glLoadIdentity()
            glTranslate(tx / 20., ty / 20., - zoom_position)
            glRotate(ry, 1, 0, 0)
            glRotate(rx, 0, 1, 0)
            if self.style == 'Default':
                self.draw_spheres()
                self.draw_lines()
            elif self.style == 'wire':
                self.draw_lines()
            elif self.style == 'cpk':
                self.draw_spheres()
                self.size_radius = 1.5
            else:
                self.draw_spheres()
                self.draw_lines()
            pygame.display.flip()

    def get_coords(self, num: int):
        return self.coordinates[num - 1]

    def get_colors(self, num: int):
        return self.colors[num - 1]

    def draw_lines(self):
        for connect in self.connectivity:
            coord1 = self.get_coords(connect[0])
            coord2 = self.get_coords(connect[1])
            color1 = self.get_colors(connect[0])
            color2 = self.get_colors(connect[1])
            glBegin(GL_LINES)
            glColor3f(color1[0], color1[1], color1[2])
            glVertex3f(coord1[0] - self.x, coord1[1] -
                       self.y, coord1[2] - self.z)
            glColor3f(color2[0], color2[1], color2[2])
            glVertex3f(coord2[0] - self.x, coord2[1] -
                       self.y, coord2[2] - self.z)
            glEnd()
            glLineWidth(1.5)

    def draw_spheres(self):
        i = 0
        for coord in self.coordinates:
            glPushMatrix()
            glTranslate(coord[0] - self.x, coord[1] -
                        self.y, coord[2] - self.z)
            glColor3f(self.colors[i][0], self.colors[i][1], self.colors[i][2])
            gluSphere(self.sphere, self.radios[i] / self.size_radius, 20, 20)
            glPopMatrix()
            i += 1

if __name__ == '__main__':
    try:
        MolViewer(sys.argv[1], sys.argv[2])
    except IndexError:
        try:
            MolViewer(sys.argv[1])
        except IndexError:
            print('FileNotFoundError')
    except OSError:
        print('FileNotFoundError')


text:
#!/usr/bin/env python

# Imports
import numpy as np
from strum import strum

# Sequences representing some of the variability of the
# FOXA1 binding site.
training_sequences = [
    "CTGTGCAAACA",
    "CTAAGCAAACA",
    "CTGTGCAAACA",
    "CTGTGCAAACA",
    "CAGAGCAAACA",
    "CTAAGCAAACA",
    "CTGTGCAAACA",
    "CAATGTAAACA",
    "CTGAGTAAATA",
]

# Initialize a new StruM object, using the basic features
# from the DiProDB table.
motif = strum.StruM(mode='basic')

# Use the training sequences to define the StruM,
# ensuring that that the variation of all position-specific
# features is at least 10e-5 (lim)
motif.train(training_sequences, lim=10**-5)

# Print out what the PWM would look like for these sequences
motif.print_PWM(True)

# Define an example sequence to analyze.
test_sequence = "ACGTACTGCAGAGCAAACAACTGATCGGATC"
# Reverse complement it, as the best match may not be on
# the forward strand.
reversed_test = strum.rev_comp(test_sequence)

# Get a score of the similarity for each kmer in the test
# sequence to the StruM.
forward_scores = motif.score_seq(test_sequence)
reverse_scores = motif.score_seq(reversed_test)

# Find the best match.
## Determine the strand the best match lies on
if np.max(reverse_scores) > np.max(forward_scores):
    score, seq = reverse_scores, reversed_test
    strand = "-"
else:
    score, seq = forward_scores, test_sequence
    strand = "+"
## Find position in the sequence
best_pos = np.argmax(score)
best_seq = seq[best_pos : best_pos + motif.k]
best_score = score[best_pos]

# Print the results to screen
print "\n", strand, best_pos, best_seq, best_score

max_stars_repo_name: pdeford/StructuralMotifs

max_issues_repo_name: pdeford/StructuralMotifs

max_forks_repo_name: pdeford/StructuralMotifs


text:
import numpy as np
from scipy.optimize import curve_fit
from ahoy import directions, rudders, noise_measurers
import test

class TestRotationRudders2D(test.TestBase):
    rudders_cls = rudders.RotationRudders2D
    dim = 2

    def setUp(self):
        super(TestRotationRudders2D, self).setUp()
        self.dt = 0.005
        self.noise_0 = 2.0
        self.t_rot_expect = 1.0 / self.noise_0
        self.noise_measurer = noise_measurers.NoiseMeasurer(self.noise_0)

    def test_rudder_autocorrelation(self):
        n = 10000
        t_max = 0.5

        ds = directions.directions_factory(n, self.dim, aligned_flag=True)
        rudders = self.rudders_cls(self.noise_measurer)

        u_0 = ds.u
        ts = np.arange(0.0, t_max, self.dt)
        mean_dots = []
        for t in ts:
            rudders.rotate(ds, self.dt, self.rng)
            mean_dot = np.mean(np.sum(u_0 * ds.u, axis=-1))
            mean_dots.append(mean_dot)

        def exp(t, t_rot):
            return np.exp(-t / t_rot)

        popt, pcov = curve_fit(exp, ts, mean_dots)
        t_rot_actual = popt[0]
        self.assertAlmostEqual(t_rot_actual, self.t_rot_expect, 1)

    def test_random_seeding(self):
        n = 1000
        num_iterations = 100
        rng_seed = 1

        def get_ds(npy_seed):
            rng = np.random.RandomState(rng_seed)
            np.random.seed(npy_seed)
            ds = directions.directions_factory(n, self.dim, aligned_flag=True)
            ruds = self.rudders_cls(self.noise_measurer)
            for _ in range(num_iterations):
                ruds.rotate(ds, self.dt, rng)
            return ds

        ds_1 = get_ds(2)
        ds_2 = get_ds(3)

        self.assertTrue(np.allclose(ds_1.u, ds_2.u))

class TestTumbleRudders1D(TestRotationRudders2D):
    rudders_cls = rudders.TumbleRudders
    dim = 1

    def test_tumble_rate(self):
        n = 2000
        dt = 1.0
        p = self.rng.uniform(0.0, 1.0, size=n)
        noise_measurer = noise_measurers.NoiseMeasurer(p)

        n_expected = (p * dt).sum()
        # In 1D is a half chance to tumble back to the same direction.
        if self.dim == 1:
            n_expected /= 2.0
        ds = directions.directions_factory(n, self.dim, aligned_flag=False,
                                           rng=self.rng)
        ruds = rudders.TumbleRudders(noise_measurer)
        ruds.rotate(ds, dt, self.rng)
        u_change = np.not_equal(ds.u_0, ds.u)
        n_actual = np.any(u_change, axis=-1).sum()
        err = np.abs(n_expected - n_actual) / n_expected
        self.assertTrue(err < 0.05)

class TestTumbleRudders2D(TestTumbleRudders1D):
    dim = 2

max_stars_repo_name: eddiejessup/ships

max_issues_repo_name: eddiejessup/ships

max_forks_repo_name: eddiejessup/ships


text:
import sys

sys.path.append('../../..')

from definitions import DATA_DIR

import torch
import soundfile as sf
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import math
from sklearn.model_selection import StratifiedKFold

def read_file_data(filename):

    raw_file_data = sf.read(filename)[0]
    # Convert data to be 2-dimensional. If only one channel, copy the channel in the other dimension.

    if len(raw_file_data.shape)==1:
        file_data=np.zeros((raw_file_data.shape[0],2))
        file_data[:,0]=raw_file_data
        file_data[:,1]=raw_file_data
    else:
        file_data=raw_file_data
    # Subample so that the series are of the order of 2000 points
    # subsampling=math.ceil(raw_file_data.shape[0]/2000)
    # file_data=file_data[::subsampling,:]
    return(file_data)

def main(normed_length=2000):
    labels_df=pd.read_csv(DATA_DIR + '/raw/urban_sound/UrbanSound8K/metadata/UrbanSound8K.csv')
    labels_df['filename'] = (labels_df['fold'].apply(lambda x: DATA_DIR+'/raw/urban_sound/UrbanSound8K/audio/fold'
        +str(x) +'/')+labels_df['slice_file_name'].apply(str))

    X = []
    y = []

    for i in range (labels_df.shape[0]):
    # for i in range(10):
        filename=labels_df['filename'].iloc[i]
        Xi=torch.tensor(read_file_data(filename))

        # If normalising length then do thisj
        if normed_length is not False:
            if Xi.shape[0]>normed_length:
                Xi=Xi[:normed_length,:]
            else:
                Xi=torch.cat([Xi, Xi[-1].repeat(normed_length - Xi.shape[0], 1)], dim=0)

        # Into list
        X.append(Xi)
        y.append(labels_df['classID'].iloc[i])
    if normed_length is not False:
        X = torch.stack(X, dim=0)
    y = torch.tensor(y)
    return X, y

if __name__ == '__main__':
    from definitions import *
    from sklearn.model_selection import train_test_split
    X, y = main(normed_length=False)

    idxs = train_test_split(list(range(X.shape[0])))

    save_pickle(X, DATA_DIR + '/interim/other/urban_sound/data.pkl')
    save_pickle(y, DATA_DIR + '/interim/other/urban_sound/labels.pkl')


text:
import argparse
import itertools
import json
import os
from collections import defaultdict

import numpy as np
import tqdm
import torch

from ensemble_boxes_wbf import weighted_boxes_fusion

from lvis import LVIS

"""
bbox
{
    'image_id': 9,
    'category_id': 61,
    'bbox': [303.3284606933594, 1.341711401939392, 330.8099060058594, 239.7283172607422],
    'score': 0.2566065192222595
}
"""

def main(args):
    scales = args.scales.split(' ')
    model_weights = list(map(float, args.weigths.split(' ')))

    assert len(scales) == len(model_weights)
    model_weights = np.array(model_weights)
    model_weights /= model_weights.sum()
    model_weights = model_weights.tolist()

    bbox_dict = defaultdict(list)
    for root, dirs, files in os.walk(args.res_dir):
        scale = root.split('/')[-1]
        if scale not in scales:
            continue
        for name in files:
            prefix = f'{args.prefix}{args.gpu_id}_'
            if name.startswith(prefix):
                file, ext = os.path.splitext(name)
                sub = file.split('_')[-1]
                bbox_dict[sub].append((scale, os.path.join(root, name)))

    lvis_gt = LVIS(args.ann)

    for sub_name, bbox_fn_list in bbox_dict.items():
        SAVE_PATH = os.path.join(args.res_dir, f"wbf_{args.prefix}{args.gpu_id}_{sub_name}.json")
        if os.path.exists(SAVE_PATH):
            continue

        idx = 0
        bbox_list = []
        for scale, fn in bbox_fn_list:
            idx += 1
            print(f"[{idx}/{len(bbox_fn_list)}] loading {fn}")
            with open(fn, 'r') as f:
                sub_res = json.load(f)
            for ann in sub_res:
                ann['scale'] = scale
            bbox_list.append(sub_res)
        bbox = list(itertools.chain.from_iterable(bbox_list))

        img_ids = set()
        img_bbox_map = defaultdict(lambda: defaultdict(list))
        img_scores_map = defaultdict(lambda: defaultdict(list))
        img_labels_map = defaultdict(lambda: defaultdict(list))
        print("img map")
        for ann in tqdm.tqdm(bbox):
            image_id = ann["image_id"]
            image_info = lvis_gt.load_imgs([image_id])[0]
            im_w, im_h = image_info['width'], image_info['height']
            scale = ann["scale"]
            img_ids.add(image_id)
            x1, y1, w, h = ann['bbox']
            x1 /= im_w
            y1 /= im_h
            w /= im_w
            h /= im_h
            img_bbox_map[image_id][scale].append([x1, y1, x1 + w, y1 + h])  # xyxy, [0, 1]
            img_scores_map[image_id][scale].append(ann['score'])
            img_labels_map[image_id][scale].append(ann['category_id'])

        wbf_res = []
        print("wbf")
        for image_id in tqdm.tqdm(list(img_ids)):
            image_info = lvis_gt.load_imgs([image_id])[0]
            im_w, im_h = image_info['width'], image_info['height']

            bboxes_list = img_bbox_map[image_id]
            scores_list = img_scores_map[image_id]
            labels_list = img_labels_map[image_id]

            boxes, scores, labels = weighted_boxes_fusion(
                [bboxes_list[s] for s in scales],
                [scores_list[s] for s in scales],
                [labels_list[s] for s in scales],
                weights=model_weights,
                iou_thr=args.overlap_thresh,
                conf_type=args.conf_type,
            )

            boxes[:, [0, 2]] *= im_w
            boxes[:, [1, 3]] *= im_h
            boxes[:, 2:] -= boxes[:, :2]  # to xywh
            boxes = boxes.tolist()
            scores = scores.tolist()
            labels = labels.tolist()

            for b, s, l in zip(boxes, scores, labels):
                res = {
                    'image_id': image_id,
                    'category_id': l,
                    'bbox': b,
                    'score': s,
                }
                wbf_res.append(res)

        # save
        print(f"save to {SAVE_PATH}")
        with open(SAVE_PATH, 'w') as f:
            json.dump(wbf_res, f)

        del bbox_list
        del bbox
        del wbf_res

if __name__ == "__main__":
    # Parse arguments
    parser = argparse.ArgumentParser(description="wbf_bbox_ms_res")
    parser.add_argument("--local_rank", type=int, default=0)
    parser.add_argument("--gpu_id", type=int, default=0)

    parser.add_argument('--ann', type=str, default='/home/user/Database/MSCOCO2017/annotations/lvis/lvis_v1_val.json')
    parser.add_argument('--res_dir', type=str, default='/home/nieyang/Pet-dev/ckpts/cnn/LVIS/swin/centernet2-mask_SWIN-L-FPN-GCE-64ROI-MASKNORM_fed_rfs_0.5x_ms-pretrained@64ROI/res')
    parser.add_argument('--scales', type=str, default='600 700 800 900 1000 1100 1200')
    parser.add_argument('--weigths', type=str, default='45.6 47.3 48.2 48.5 48.9 48.4 48.6')
    parser.add_argument('--conf_type', choices=('avg', 'max', 'box_and_model_avg', 'absent_model_aware_avg'), type=str, default='avg')
    parser.add_argument('--overlap_thresh', type=float, default=0.65)
    parser.add_argument('--prefix', type=str, default='bbox_')

    args = parser.parse_args()

    torch.cuda.set_device(args.local_rank)
    try:
        torch.distributed.init_process_group(backend="nccl", init_method="env://")
    except:
        pass

    main(args)

max_stars_repo_name: flytocc/lvis-api

max_issues_repo_name: flytocc/lvis-api

max_forks_repo_name: flytocc/lvis-api