text:
"""
@brief PSF characterization and system gain from distribution of
Gaussian fit parameters to Fe55 data.

@author J. Chiang <jchiang@slac.stanford.edu>
"""
from __future__ import print_function
from __future__ import absolute_import
import os
import numpy as np
import astropy.io.fits as fits
import lsst.eotest.image_utils as imutils
from .fe55_psf import PsfGaussFit, psf_sigma_statistics
from .MaskedCCD import MaskedCCD
from .EOTestResults import EOTestResults
from .Fe55GainFitter import Fe55GainFitter

import lsst.pex.config as pexConfig
import lsst.pipe.base as pipeBase

class Fe55Config(pexConfig.Config):
    """Configuration for Fe55 analysis task"""
    chiprob_min = pexConfig.Field("Minimum chi-square probability for cluster fit",
                                  float, default=0.1)
    nsig = pexConfig.Field(
        "Charge cluster footprint threshold in number of standard deviations of noise in bias section", float, default=4)
    temp_set_point = pexConfig.Field("Required temperature (C) set point",
                                     float, default=-95.)
    temp_set_point_tol = pexConfig.Field("Required temperature set point tolerance (degrees C)",
                                         float, default=1.)

    output_dir = pexConfig.Field("Output directory", str, default='.')
    output_file = pexConfig.Field("Output filename", str, default=None)
    eotest_results_file = pexConfig.Field("EO test results filename",
                                          str, default=None)
    fit_xy = pexConfig.Field("Fit sigmas in x- and y-directions separately",
                             bool, default=False)
    verbose = pexConfig.Field("Turn verbosity on", bool, default=True)

class Fe55Task(pipeBase.Task):
    """Task to estimate PSF size and system gain from the distribution of
    Gaussian fit parameters to Fe55 data."""
    ConfigClass = Fe55Config
    _DefaultName = "Fe55Task"

    def fit_gains(self, fitter, gains, gain_errors, sigma_modes, amps=None,
                  hist_nsig=10, dn_range=None):
        "Fit the DN distributions to obtain the system gain per amp."
        my_gains, my_gain_errors, my_sigma_modes = \
            gains, gain_errors, sigma_modes
        if amps is None:
            amps = imutils.allAmps()
        for amp in amps:
            data = fitter.results(min_prob=self.config.chiprob_min, amp=amp)
            dn = data['dn']
            if dn_range is not None:
                dn = dn[np.where((dn >= dn_range[0]) & (dn <= dn_range[1]))]
            if len(dn) > 2:
                try:
                    foo = Fe55GainFitter(dn)
                    kalpha_peak, kalpha_sigma = foo.fit(hist_nsig=hist_nsig)
                    my_gains[amp] = foo.gain
                    my_gain_errors[amp] = foo.gain_error
                except RuntimeError as eobj:
                    print(eobj)
                    continue
                try:
                    sigma = sorted(np.concatenate((data['sigmax'],
                                                   data['sigmay']))*10)
                    mode, median, mean = psf_sigma_statistics(sigma, bins=50,
                                                              range=(2, 6))
                    my_sigma_modes[amp] = float(mode)
                except RuntimeError as eobj:
                    print(eobj)
                    continue
        return my_gains, my_gain_errors, my_sigma_modes

    @pipeBase.timeMethod
    def run(self, sensor_id, infiles, mask_files, bias_frame=None,
            fe55_catalog=None, minClustersPerAmp=None, chiprob_min=0.1,
            accuracy_req=0, hist_nsig=10, linearity_correction=None,
            dn_range=None):
        imutils.check_temperatures(infiles, self.config.temp_set_point_tol,
                                   setpoint=self.config.temp_set_point,
                                   warn_only=True)
        if self.config.verbose and fe55_catalog is None:
            self.log.info("Input files:")
            for item in infiles:
                self.log.info("  %s" % item)
        #
        # Detect and fit 2D Gaussian to Fe55 charge clusters,
        # accumulating the results by amplifier.
        #
        fitter = PsfGaussFit(nsig=self.config.nsig, fit_xy=self.config.fit_xy)
        gains, gain_errors, sigma_modes = {}, {}, {}
        if fe55_catalog is None:
            for infile in infiles:
                if self.config.verbose:
                    self.log.info("processing %s" % infile)
                ccd = MaskedCCD(infile, mask_files=mask_files,
                                bias_frame=bias_frame,
                                linearity_correction=linearity_correction)
                try:
                    seqnum = ccd.md.get('SEQNUM')
                except KeyError:
                    seqnum = 0
                for amp in ccd:
                    if self.config.verbose:
                        self.log.info("  amp %i" % amp)
                    if amp in gain_errors:
                        gain_accuracy = np.abs(gain_errors[amp]/gains[amp])
                        if self.config.verbose:
                            message = "  Relative gain accuracy, dgain/gain " \
                                + "= %.2e" % gain_accuracy
                            self.log.info(message)
                        if gain_accuracy < accuracy_req:
                            # Requested accuracy already obtained, so
                            # skip cluster fitting.
                            continue
                    fitter.process_image(ccd, amp, logger=self.log,
                                         seqnum=seqnum)
                    gains, gain_errors, sigma_modes = \
                        self.fit_gains(fitter, gains, gain_errors, sigma_modes,
                                       amps=[amp], hist_nsig=hist_nsig,
                                       dn_range=dn_range)
            if self.config.output_file is None:
                psf_results = os.path.join(self.config.output_dir,
                                           '%s_psf_results_nsig%i.fits'
                                           % (sensor_id, self.config.nsig))
            else:
                psf_results = self.config.output_file
            if self.config.verbose:
                self.log.info("Writing psf results file to %s" % psf_results)
            fitter.write_results(outfile=psf_results)
            namps = len(ccd)
        else:
            fitter.read_fe55_catalog(fe55_catalog)
            namps = fits.open(fe55_catalog)[0].header['NAMPS']

        results_file = self.config.eotest_results_file
        if results_file is None:
            results_file = os.path.join(self.config.output_dir,
                                        '%s_eotest_results.fits' % sensor_id)

        if self.config.verbose:
            self.log.info("Writing gain and psf sigma results to %s"
                          % results_file)

        results = EOTestResults(results_file, namps=namps)
        for amp in gains:
            results.add_seg_result(amp, 'GAIN', gains[amp])
            results.add_seg_result(amp, 'GAIN_ERROR', gain_errors[amp])
            results.add_seg_result(amp, 'PSF_SIGMA', sigma_modes[amp])
        results.write(clobber=True)

        return gains

max_stars_repo_name: tguillemLSST/eotest

max_issues_repo_name: tguillemLSST/eotest

max_forks_repo_name: tguillemLSST/eotest

reason: import numpy,import astropy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Sep  9 23:22:02 2017

@author: jmmauricio
"""

from pydgrid import grid
from pydgrid.pf import pf_eval,time_serie
from pydgrid.electric import bess_vsc, bess_vsc_eval
from pydgrid.simu import simu, f_eval, ini_eval, run_eval
import matplotlib.pyplot as plt
import numpy as np
import time

def test_line_sequence_z():
    ''' Test that positive sequence phasor calculations match pydgrid result'''
    data = {
        "lines":[
                {"bus_j": "Bus_1",  "bus_k": "Bus_2",  "code": "RX", "m": 1000}
                ],
        "buses":[
        		{"bus": "Bus_1",  "pos_x": 10, "pos_y":  0, "units": "m", "U_kV":0.4},
        		{"bus": "Bus_2",  "pos_x": 200, "pos_y": 0, "units": "m", "U_kV":0.4}
        		],
        "grid_formers":[
        		{"bus": "Bus_1","bus_nodes": [1, 2, 3],
        			"kV": [11.547, 11.547, 11.547],
        			"deg": [0, 120, 240.0]
        		}
        		],
        "loads":[
        		{"bus": "Bus_2" , "kVA": 2000.0, "pf": 0.9,"type":"3P"}
                ],
        "line_codes":
        		{"RX":
        		{"R1":0.8, "X1":0.6,"R0":2.4, "X0":1.8
        		}
        		}
        }

    # pydgrid calculation
    sys1 = grid()
    sys1.read(data)  # Load data
    sys1.pf_solver = 1
    sys1.pf()  # solve power flow
    sys1.get_v()      # post process voltages
    sys1.get_i()      # post process currents
    I_12_pydgrid = sys1.lines[0]['i_j_a_m']

    # positive sequence "manual" calculation
    V_1_m = sys1.buses[0]['v_an']
    V_2_m = sys1.buses[1]['v_an']
    theta_1 = np.deg2rad(sys1.buses[0]['deg_an'])
    theta_2 = np.deg2rad(sys1.buses[1]['deg_an'])
    V_1 = V_1_m*np.exp(1j*theta_1)
    V_2 = V_2_m*np.exp(1j*theta_2)
    R_1 = data["line_codes"]["RX"]["R1"]
    X_1 = data["line_codes"]["RX"]["X1"]
    Z_1 = R_1 + 1j*X_1
    I_12_manual = np.abs((V_1-V_2)/Z_1)

    # comparison
    error = I_12_manual - I_12_pydgrid

    assert abs(error)<0.001
#    phases = ['v_an','v_bn','v_cn']
#    for bus in sys1.buses:
#        for phase in phases:
#            print(bus[phase])

def test_pr_mt_h():
    ''' Test that positive sequence phasor calculations match pydgrid result'''
    data = {
        "lines":[
                {"bus_j": "Bus_1",  "bus_k": "Bus_2",  "code": "RX", "m": 200}
                ],
        "buses":[
        		{"bus": "Bus_1",  "pos_x": 10, "pos_y":  0, "units": "m", "U_kV":0.4},
        		{"bus": "Bus_2",  "pos_x": 200, "pos_y": 0, "units": "m", "U_kV":0.4}
        		],
        "grid_formers":[
        		{"bus": "Bus_1","bus_nodes": [1, 2, 3],
        			"kV": [11.547, 11.547, 11.547],
        			"deg": [0, 120, 240.0]
        		}
        		],
        "grid_feeders":[{"bus": "Bus_2","bus_nodes": [1, 2, 3],
                "kW": [0, 0, 0], "kvar": [0,0,0],
                "kA": [-0.245,-0.245,-0.245], "phi_deg":[-36.8698976, -36.8698976, -36.8698976]}],
        "line_codes":
        		{"RX":
        		{"R1":0.277, "X1":0.11,"R0":0.0, "X0":0.01
        		}
        		}
        }

    # pydgrid calculation
    sys1 = grid()
    sys1.read(data)  # Load data
    sys1.pf_solver = 1
    sys1.pf()  # solve power flow
    sys1.get_v()      # post process voltages
    sys1.get_i()      # post process currents
#    I_12_pydgrid = sys1.lines[0]['i_a_m']
#
#    # positive sequence "manual" calculation
    V_1_m = sys1.buses[0]['v_an']
    V_2_m = sys1.buses[1]['v_an']

    DV = V_1_m - V_2_m
    DU_pydgrid = np.sqrt(3)*DV
    DU_catalog = 24.41

    # comparison
    error = DU_pydgrid - DU_catalog

    assert abs(error)<0.001
#    phases = ['v_an','v_bn','v_cn']
#    for bus in sys1.buses:
#        for phase in phases:
#            print(bus[phase])

if __name__ == "__main__":

#    test_line_sequence_z()
    test_pr_mt_h()

ext: py

max_stars_repo_name: pydgrid/pydgrid

max_issues_repo_name: pydgrid/pydgrid

max_forks_repo_name: pydgrid/pydgrid

reason: import numpy


text:
"""
In this file I will look to do the following

1. Loop thought various threasholds that separate top lowest 90% of values
    a. Optimise a RF for the threashold
    b. Run the RF over the test data
    c. record the ROC_AUC for the RF on the test data
2. Plot the ROC_AUC vs the threashold"""

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import RandomizedSearchCV
from sklearn.metrics import roc_auc_score
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.tree import DecisionTreeClassifier
from evaluate_model import evaluate_model

def find_best_params(dtfm, labels_col, classifier=RandomForestClassifier,
        test_size=0.3, random_state=np.random, logger=False):
    """Returns the best params in terms of roc_auc for an optimised Random Forest
    Trained and tested over a passed in dataset

    USGAE: find_best_params(dtfm, labels_col, test_size=0.3, random_state=np.random, logger=logger)

    INPUTS:
        dtfm: dataframe to train and test model on
        labels_col: column title containing the classification flag

    **Optional** (default)
        test_size: (0.3) proportion of data set to use for testing
        random_state: (np.random) seed used by the random number generator
        logger: (False) wheather to log outputs or not

    OUTPUT:
        roc_auc: (scalar) Area under the receiver operating
            charachteristic curve
    """

    dtfm_labels = dtfm.pop(labels_col)
    # separate the labels from the data
    labels = np.array(dtfm_labels)
    print(dtfm.head())

    # print value counts so we can see how split affects data
    if logger:
        print("Output value count:\n {}".format(dtfm_labels.value_counts()))

    # split data into train and test sets split% test
    train, test, train_labels, test_labels = train_test_split(dtfm, labels, stratify = labels, test_size = test_size, random_state=random_state)

    #imputation of missing values
    train = train.fillna(train.mean())
    test = test.fillna(test.mean())

    c_name = classifier.__name__
    if c_name == 'RandomForestClassifier':
        # Hyperparameter grid
        param_grid = {
            'n_estimators': np.linspace(10, 200, 10).astype(int),
            'max_depth': [None] + list(np.linspace(3, 20, 10).astype(int)),
            'max_features': ['auto', 'sqrt', None] + list(np.arange(0.5, 1, 0.1)),
            'max_leaf_nodes': [None] + list(np.linspace(10, 50, 10).astype(int)),
            'min_samples_split': [2],
            'bootstrap': [True],
            }
    elif c_name == 'DecisionTreeClassifier':
        # Hyperparameter grid
        param_grid = {
            'max_features': ['auto', 'sqrt', None] + list(np.arange(0.5, 1, 0.1)),
            'max_leaf_nodes': [None] + list(np.linspace(10, 50, 10).astype(int)),
            'min_samples_split': [2],
        }
    else:
        raise ValueError('That is not a supported Classifier')

    # Estimator for use in random search
    estimator = classifier(random_state = random_state)
    # Create the random search model
    rs = RandomizedSearchCV(estimator, param_grid, n_jobs = -1,
                         scoring = 'roc_auc', cv = 3,
                         n_iter = 10, verbose = 1,
                         random_state=random_state)
    # Fit
    rs.fit(train, train_labels)

    print("Best params:\n{}".format(rs.best_params_))
    # print result
    if logger:
        print("Best params:\n{}".format(rs.best_params_))

    return rs.best_params_

def roc_auc(dtfm, labels_col, c_params, classifier=RandomForestClassifier,
        test_size=0.3, random_state=np.random, logger=False, optimise=True):
    """Returns the roc_auc for an optimised Random Forest
    Trained and tested over a passed in dataset

    USGAE: roc_auc(dtfm, labels_col, test_size=0.3, random_state=np.random, logger=logger)

    INPUTS:
        dtfm: dataframe to train and test model on
        labels_col: column title containing the classification flag

    **Optional** (default)
        test_size: (0.3) proportion of data set to use for testing
        random_state: (np.random) seed used by the random number generator
        logger: (False) wheather to log outputs or not

    OUTPUT:
        roc_auc: (scalar) Area under the receiver operating
            charachteristic curve
    """

    dtfm_labels = dtfm.pop(labels_col)
    # separate the labels from the data
    labels = np.array(dtfm_labels)

    # print value counts so we can see how split affects data
    if logger:
        print("Output value count:\n {}".format(dtfm_labels.value_counts()))

    # split data into train and test sets split% test
    train, test, train_labels, test_labels = train_test_split(dtfm, labels, stratify = labels, test_size = test_size, random_state=random_state)

    #imputation of missing values
    train = train.fillna(train.mean())
    test = test.fillna(test.mean())

    # Features for feature importances
    features = list(train.columns)
    c_name = classifier.__name__
    if c_name == 'RandomForestClassifier':
        # tune model
        model = classifier(n_estimators=c_params['n_estimators'], max_depth=c_params['max_depth'],
                max_features=c_params['max_features'], max_leaf_nodes=c_params['max_leaf_nodes'],
                min_samples_split=c_params['min_samples_split'], bootstrap=c_params['bootstrap'],
                random_state = random_state)
    elif c_name == 'DecisionTreeClassifier':
        # tune model
        model = classifier(max_features=c_params['max_features'], max_leaf_nodes=c_params['max_leaf_nodes'],
                min_samples_split=c_params['min_samples_split'], random_state=random_state)
    else:
        raise ValueError('That is not a supported Classifier')

    # Fit
    model.fit(train, train_labels)

    train_predictions = model.predict(train)
    train_probs = model.predict_proba(train)[:, 1]

    predictions = model.predict(test)
    probs = model.predict_proba(test)[:, 1]

    [baseline, results, train_results ] = evaluate_model(predictions, probs,
            train_predictions, train_probs,
            test_labels, train_labels, logger=logger)

    # calculate variables of most importance in model
    fi_model = pd.DataFrame({'feature': features,
                'importance': model.feature_importances_})

    if logger:
        print("Features importance in RF:\n{}".format(fi_model.sort_values('importance', 0, ascending=True)))

    return [
            baseline, results,
            train_results, fi_model,
            ]

def test_threasholds(threasholds, dtfm, classifier=RandomForestClassifier,
        dep_key='BLAST_D8', random_state=50, logger=False, optimise=True):
    if logger== True:
        print("Threasholds at which we'll calculte ROC_AUC: \n {}".format(threasholds))

    # ROC_AUC array
    roc_auc_arr = []
    roc_auc_train_arr = []
    # Precision
    precision_arr = []
    precision_train_arr = []
    # Recall
    recall_arr = []
    recall_train_arr = []
    #Accuracy
    accuracy_arr = []
    accuracy_train_arr = []
    # feature importance
    fi_dtfm = pd.DataFrame()
    dtfm_index = 0
    # set base for c_params
    c_params = {'n_estimators': 94, 'min_samples_split': 2, 'max_leaf_nodes': 36, 'max_features': 0.5, 'max_depth': 18, 'bootstrap': True}
    #create loop
    for threashold in threasholds:
        if logger == True:
            print("Optimising a RF for threashold: {}".format(threashold))
        dtfm_temp = dtfm.copy()# make copy of dtfm
        # update blast labels to sit above or below threashold
        dtfm_temp[dep_key] = dtfm_temp[dep_key].where(dtfm[dep_key] >= threashold, other=0)
        dtfm_temp[dep_key] = dtfm_temp[dep_key].where(dtfm[dep_key] < threashold, other=1)

        # bool trigger for multiple optimisations
        if optimise:
            dtfm_optimise = dtfm_temp.copy()
            c_params = find_best_params(dtfm_optimise, dep_key, classifier=classifier, random_state=random_state, logger=logger)

        # maybe run optimise once then use params from optimise for subsequent threasholds
        [
            baseline, results,
            train_results, fi_model,
        ] = roc_auc(dtfm_temp, dep_key, c_params, classifier=classifier,
                random_state=random_state, logger=logger, optimise=optimise)

        # create first index
        if dtfm_index == 0:
            fi_dtfm.insert(0, 'feature', fi_model.feature)

        fi_dtfm.insert(dtfm_index + 1, round(threashold, 2),fi_model.importance)
        dtfm_index += 1

        # ROC_AUC
        roc_auc_arr.append(results['roc'])
        roc_auc_train_arr.append(train_results['roc'])

        # Precision
        precision_arr.append(results['precision'])
        precision_train_arr.append(train_results['precision'])

        # Recall
        recall_arr.append(results['recall'])
        recall_train_arr.append(train_results['recall'])

        # Precision
        accuracy_arr.append(results['accuracy'])
        accuracy_train_arr.append(train_results['accuracy'])

    # redefine index
    fi_dtfm = fi_dtfm.set_index('feature')
    fi_dtfm['mean'] = fi_dtfm.mean(axis=1)
    fi_dtfm = fi_dtfm.sort_values('mean', 0, ascending=True)
    fi_dtfm = fi_dtfm.transpose()

    return [
            fi_dtfm,
            roc_auc_arr,
            roc_auc_train_arr,
            precision_arr,
            precision_train_arr,
            recall_arr,
            recall_train_arr,
            accuracy_arr,
            accuracy_train_arr,
           ]


text:
import logging
import numpy as np

class GaussFull:
    """
    Class to model a speaker by a gaussian with full covariance
    """

    def __init__(self, name, dim):
        self.logger = logging.getLogger(__name__)
        self.name = name
        self.count = 0
        self.dim = dim
        self.stat1 = np.zeros(dim)
        self.stat2 = np.zeros((dim, dim))
        self.cov_log_det = 0
        self.mu = None
        self.mu_dot = np.NAN
        self.cov = None
        self.partial_bic = np.NaN

    def add(self, features):
        """
        Accumulate statistics for *features*
        :param features: numpy.ndarray

        """
        self.count += features.shape[0] # add the number of features
        self.stat1 += features.sum(axis=0)
        self.stat2 += np.dot(features.T, features)

    def _cov_log_det(self):
        """
        Compute the log det of the covariance matrix
        :return:  float
        """
        s, d = np.linalg.slogdet(self.cov)
        return d
        # cov_chol, lower = scipy.linalg.cho_factor(self.cov)
        # return 2.0 * numpy.sum(numpy.log(numpy.diagonal(cov_chol)))

    def compute(self):
        """
        Compute the mean and covariance according the statistique, the log det
         of the covariance and the partial BIC :math:`PBIC`.

        :math:`PBIC_{x}  = \\frac{n_x}{2} \\log|\\Sigma_x|`

        """
        self.mu = self.stat1 / self.count
        tmp = self.mu[:, np.newaxis]
        self.mu_dot = np.dot(tmp, tmp.T)
        self.cov = self.stat2 / self.count - self.mu_dot
        self.cov_log_det = self._cov_log_det()
        self.partial_bic = self.cov_log_det * 0.5 * self.count

    @classmethod
    def merge(cls, m1, m2):
        """
        Merge two models *m1* and *m2*. Compute the new mean (*mu*),
        covariance (*cov*) and PBIC *partial_bic*

        :param m1: a GaussFull object
        :param m2: a GaussFull object
        :return: a GaussFull object
        """
        m = GaussFull(m1.name, m1.dim)
        m.count = m1.count + m2.count
        m.stat1 = m1.stat1 + m2.stat1
        m.stat2 = m1.stat2 + m2.stat2
        m.mu = m.stat1 / m.count
        tmp = m.mu[:, np.newaxis]
        m.cov = m.stat2 / m.count - np.dot(tmp, tmp.T)
        m.cov_log_det = m._cov_log_det()
        m.partial_bic = m.cov_log_det * 0.5 * m.count
        return m

    @classmethod
    def merge_partial_bic(cls, m1, m2):
        """
        Merge statistic accumulators of two a GaussFull objects, compute
        the new log det.

        :param m1: a GaussFull object
        :param m2: a GaussFull object
        :return: the log det
        """
        try:
            count = m1.count + m2.count
            mu = ((m1.stat1 + m2.stat1) / count)[:, np.newaxis]
            cov = (m1.stat2 + m2.stat2) / count - np.dot(mu, mu.T)
            s, d = np.linalg.slogdet(cov)
            # cov_chol, lower = scipy.linalg.cho_factor(cov, overwrite_a=True, check_finite=False)
            # d = 2.0 * numpy.sum(numpy.log(numpy.diagonal(cov_chol)))
            d *= 0.5 * count
            return d
        except:
            logging.warning('Det problem set to NaN ', m1.name, m2.name)
            return np.nan

    @classmethod
    def cst_bic(cls, dim, alpha):
        """
        Compute the BIC constant:

            :math:`cst  = \\frac{1}{2} \\alpha \\left(d + \\frac{d(d+1)}{2}\\right)`

        where :math:`d`is the feature dimension (*dim*)
        and :math:`alpha` a threshold (*alpha*)

        :param dim: the feature dimension
        :param alpha: the threshold
        :return: the constant
        """
        return 0.5 * alpha * (dim + (0.5 * ((dim + 1) * dim)))


text:
# ros package
import rospy
import ros_numpy
import sensor_msgs.point_cloud2 as pc2
from visualization_msgs.msg import MarkerArray, Marker
from sensor_msgs.msg import PointCloud2, PointField
from geometry_msgs.msg import Point

from std_msgs.msg import Header

import numpy as np
import os
import sys
import torch
import time
import glob

from pathlib import Path
from pcdet.datasets import DatasetTemplate
from pyquaternion import Quaternion
from pcdet.models import build_network, load_data_to_gpu
from pcdet.config import cfg, cfg_from_yaml_file
from pcdet.utils import common_utils

class DemoDataset(DatasetTemplate):
    def __init__(self, dataset_cfg, class_names, training=True, root_path=None, logger=None, ext='.bin'):
        """
        Args:
            root_path:
            dataset_cfg:
            class_names:
            training:
            logger:
        """
        super().__init__(
            dataset_cfg=dataset_cfg, class_names=class_names, training=training, root_path=root_path, logger=logger
        )
        self.root_path = root_path
        self.ext = ext
        data_file_list = glob.glob(str(root_path/ f'*{self.ext}')) if self.root_path.is_dir() else [self.root_path]

        data_file_list.sort()
        self.sample_file_list = data_file_list

    def __len__(self):
        return len(self.sample_file_list)

    def __getitem__(self, index):
        if self.ext == '.bin':
            points = np.fromfil(self.sample_file_list[index], dtype=np.float32).reshape(-1,4)
        elif self.ext == '.npy':
            points = np.load(self.sample_file_list[index])
        else:
            raise NotImplementedError

        input_dict = {
            'points': points,
            'frame_id': index,
        }

        data_dict = self.prepare_data(data_dict=input_dict)
        return data_dict

def yaw2quaternion(yaw: float) -> Quaternion:
    return Quaternion(axis=[0,0,1], radians=yaw)

def get_annotations_indices(types, thresh, label_preds, scores):
    indexs = []
    annotation_indices = []
    for i in range(label_preds.shape[0]):
        if label_preds[i] == types:
            indexs.append(i)
    for index in indexs:
        if scores[index] >= thresh:
            annotation_indices.append(index)
    return annotation_indices

def remove_low_score_nu(image_anno, thresh):
    img_filtered_annotations = {}
    label_preds_ = image_anno["pred_labels"].detach().cpu().numpy()
    scores_ = image_anno["pred_scores"].detach().cpu().numpy()

    cat_indices = get_annotations_indices(1, 0.50, label_preds_, scores_)
    truck_indices = get_annotations_indices(2, 0.35, label_preds_, scores_)
    costruction_vehicle_indices = get_annotations_indices(3, 0.20, label_preds_, scores_)
    bus_indices = get_annotations_indices(4, 0.5, label_preds_, scores_)
    trailer_indices = get_annotations_indices(5, 0.2, label_preds_, scores_)
    barrier_indices = get_annotations_indices(6, 0.5, label_preds_, scores_)
    motorcycle_indices = get_annotations_indices(7, 0.5, label_preds_, scores_)
    bicycle_indices = get_annotations_indices(8, 0.15, label_preds_, scores_)
    pedestrain_indices = get_annotations_indices(9, 0.15, label_preds_, scores_)
    traffic_cone_indices = get_annotations_indices(10, 0.15, label_preds_, scores_)

    for key in image_anno.keys():
        if key == 'metadata':
            continue
        img_filtered_annotations[key]=(
            image_anno[key][cat_indices+
                            pedestrain_indices+
                            bicycle_indices+
                            bus_indices+
                            traffic_cone_indices+
                            costruction_vehicle_indices+
                            trailer_indices+
                            barrier_indices+
                            motorcycle_indices+
                            truck_indices])

    return img_filtered_annotations

class Precessor_ROS:
    def __init__(self, config_path, model_path):
        self.points = None
        self.config_path = config_path
        self.model_path = model_path
        self.device = None
        self.net = None

    def initialize(self):
        self.read_config()

    def read_config(self):
        config_path = self.config_path
        cfg_from_yaml_file(self.config_path,cfg)
        self.logger = common_utils.create_logger()
        self.demo_datasets  = DemoDataset(
            dataset_cfg=cfg.DATA_CONFIG, class_names=cfg.CLASS_NAMES, training=False,
            root_path=Path('/home/xjtuiair/KITTI/KITTI/2011_09_26/2011_09_26_drive_0001_sync/velodyne_points/data/0000000097.bin'),
            ext='.bin'
        )

        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        self.net = build_network(model_cfg=cfg.MODEL, num_class=len(cfg.CLASS_NAMES), dataset=self.demo_datasets)
        self.net.load_params_from_file(filename=self.model_path, logger=self.logger, to_cpu=True)
        self.net = self.net.to(self.device).eval()

    def run(self, points):
        t1 = time.time()
        print(f"input points shape: {points.shape}")
        num_features = 4 #kitti model
        #num_features = 5
        self.points = points.reshape([-1,num_features])

        input_dict = {
            'points': self.points,
            'frame_id': 0,
        }

        data_dict = self.demo_datasets.prepare_data(data_dict=input_dict)
        data_dict = self.demo_datasets.collate_batch([data_dict])
        load_data_to_gpu(data_dict)

        torch.cuda.synchronize()
        pred_dicts, _ = self.net.forward(data_dict)
        torch.cuda.synchronize()

        t2 = time.time()
        print(f"net inference cost time: {t2 - t1}")

        pred = remove_low_score_nu(pred_dicts[0], 0.45)
        boxes_lidar = pred['pred_boxes'].detach().cpu().numpy()
        scores = pred['pred_scores'].detach().cpu().numpy()
        types = pred['pred_labels'].detach().cpu().numpy()
        #print(f" pred boxes: { boxes_lidar }")
        #print(f" pred scores: {scores}")
        print(f" pred labels: {types}")

        return scores, boxes_lidar, types

def get_xyz_points(cloud_array, remove_nans=True, dtype=np.float):
    if remove_nans:
        mask = np.isfinite(cloud_array['x']) & np.isfinite(cloud_array['y']) & np.isfinite(cloud_array['z'])
        cloud_array = cloud_array[mask]

    points = np.zeros(cloud_array.shape + (4,), dtype=dtype) # kitti model
    #points = np.zeros(cloud_array.shape + (5,), dtype=dtype)
    #points = np.zeros(cloud_array.shape, dtype=dtype)
    #print(cloud_array['intensity'])
    points[..., 0] = cloud_array['x']
    points[..., 1] = cloud_array['y']
    points[..., 2] = cloud_array['z'] + 2.0
    points[..., 3] = cloud_array['intensity'] / 255.0
    return points

def xyz_array_to_pointcloud2(points_sum, stamp=None, frame_id=None):
    msg = PointCloud2()

    if stamp:
        msg.header.stamp = stamp
    if frame_id:
        msg.header.frame_id = frame_id

    msg.height = 1
    msg.width = points_sum.shape[0]
    msg.fields = [
        PointField('x', 0, PointField.FLOAT32, 1),
        PointField('y', 4, PointField.FLOAT32, 1),
        PointField('z', 8, PointField.FLOAT32, 1),
        PointField('intensity', 12, PointField.FLOAT32, 1)
    ]
    msg.is_bigendian = False
    msg.point_step = 16
    msg.row_step = points_sum.shape[0]
    msg.is_dense = int(np.isfinite(points_sum).all())
    msg.data = np.array(points_sum, np.float32).tobytes()
    return msg

def check_numpy_to_torch(x):
    if isinstance(x, np.ndarray):
        return torch.from_numpy(x).float(), True
    return x, False

def rotate_points_along_z(points, angle):
    """
    Args:
        points: (B, N, 3 + C)
        angle: (B), angle along z-axis, angle increases x ==> y
    Returns:

    """
    points, is_numpy = check_numpy_to_torch(points)
    angle, _ = check_numpy_to_torch(angle)

    cosa = torch.cos(angle)
    sina = torch.sin(angle)
    zeros = angle.new_zeros(points.shape[0])
    ones = angle.new_ones(points.shape[0])
    rot_matrix = torch.stack((
        cosa,  sina, zeros,
        -sina, cosa, zeros,
        zeros, zeros, ones
    ), dim=1).view(-1, 3, 3).float()
    points_rot = torch.matmul(points[:, :, 0:3], rot_matrix)
    points_rot = torch.cat((points_rot, points[:, :, 3:]), dim=-1)
    return points_rot.numpy() if is_numpy else points_rot

def boxes_to_corners_3d(boxes3d):
    """
        7 -------- 4
       /|         /|
      6 -------- 5 .
      | |        | |
      . 3 -------- 0
      |/         |/
      2 -------- 1
    Args:
        boxes3d:  (N, 7) [x, y, z, dx, dy, dz, heading], (x, y, z) is the box center

    Returns:
    """
    boxes3d, is_numpy = check_numpy_to_torch(boxes3d)
    print(is_numpy)
    template = boxes3d.new_tensor((
        [1, 1, -1], [1, -1, -1], [-1, -1, -1], [-1, 1, -1],
        [1, 1, 1], [1, -1, 1], [-1, -1, 1], [-1, 1, 1],
    )) / 2

    corners3d = boxes3d[:, None, 3:6].repeat(1, 8, 1) * template[None, :, :]
    corners3d = rotate_points_along_z(corners3d.view(-1, 8, 3), boxes3d[:, 6]).view(-1, 8, 3)
    corners3d += boxes3d[:, None, 0:3]

    return corners3d.numpy() if is_numpy else corners3d

nums = 0
av_time = 0
def lidar_callback(msg):
    t1 = time.time()
    msg_cloud = ros_numpy.point_cloud2.pointcloud2_to_array(msg)
    frame_id = msg.header.frame_id
    np_p = get_xyz_points(msg_cloud, True)
    print(np_p.size)
    print("  ")
    scores, dt_box_lidar, types = proc_1.run(np_p)
    bbox_corners3d = boxes_to_corners_3d(dt_box_lidar)

    print(bbox_corners3d.shape)

    empty_markers = MarkerArray()
    clear_marker = Marker()
    clear_marker.header.stamp = rospy.Time.now()
    clear_marker.header.frame_id = "velodyne"
    clear_marker.ns = "objects"
    clear_marker.id = 0
    clear_marker.action = clear_marker.DELETEALL
    clear_marker.lifetime = rospy.Duration()
    empty_markers.markers.append(clear_marker)
    pub_bbox_array.publish(empty_markers)

    bbox_arry = MarkerArray()
    if scores.size != 0:
        for i in range(scores.size):
            point_list = []
            bbox = Marker()
            bbox.type = Marker.LINE_LIST
            bbox.ns = "objects"
            bbox.id = i
            box = bbox_corners3d[i]
            q = yaw2quaternion(float(dt_box_lidar[i][6]))
            for j in range(24):
                p = Point()
                point_list.append(p)

            point_list[0].x = float(box[0, 0])
            point_list[0].y = float(box[0, 1])
            point_list[0].z = float(box[0, 2])
            point_list[1].x = float(box[1, 0])
            point_list[1].y = float(box[1, 1])
            point_list[1].z = float(box[1, 2])

            point_list[2].x = float(box[1, 0])
            point_list[2].y = float(box[1, 1])
            point_list[2].z = float(box[1, 2])
            point_list[3].x = float(box[2, 0])
            point_list[3].y = float(box[2, 1])
            point_list[3].z = float(box[2, 2])

            point_list[4].x = float(box[2, 0])
            point_list[4].y = float(box[2, 1])
            point_list[4].z = float(box[2, 2])
            point_list[5].x = float(box[3, 0])
            point_list[5].y = float(box[3, 1])
            point_list[5].z = float(box[3, 2])

            point_list[6].x = float(box[3, 0])
            point_list[6].y = float(box[3, 1])
            point_list[6].z = float(box[3, 2])
            point_list[7].x = float(box[0, 0])
            point_list[7].y = float(box[0, 1])
            point_list[7].z = float(box[0, 2])

            point_list[8].x = float(box[4, 0])
            point_list[8].y = float(box[4, 1])
            point_list[8].z = float(box[4, 2])
            point_list[9].x = float(box[5, 0])
            point_list[9].y = float(box[5, 1])
            point_list[9].z = float(box[5, 2])

            point_list[10].x = float(box[5, 0])
            point_list[10].y = float(box[5, 1])
            point_list[10].z = float(box[5, 2])
            point_list[11].x = float(box[6, 0])
            point_list[11].y = float(box[6, 1])
            point_list[11].z = float(box[6, 2])

            point_list[12].x = float(box[6, 0])
            point_list[12].y = float(box[6, 1])
            point_list[12].z = float(box[6, 2])
            point_list[13].x = float(box[7, 0])
            point_list[13].y = float(box[7, 1])
            point_list[13].z = float(box[7, 2])

            point_list[14].x = float(box[7, 0])
            point_list[14].y = float(box[7, 1])
            point_list[14].z = float(box[7, 2])
            point_list[15].x = float(box[4, 0])
            point_list[15].y = float(box[4, 1])
            point_list[15].z = float(box[4, 2])

            point_list[16].x = float(box[0, 0])
            point_list[16].y = float(box[0, 1])
            point_list[16].z = float(box[0, 2])
            point_list[17].x = float(box[4, 0])
            point_list[17].y = float(box[4, 1])
            point_list[17].z = float(box[4, 2])

            point_list[18].x = float(box[1, 0])
            point_list[18].y = float(box[1, 1])
            point_list[18].z = float(box[1, 2])
            point_list[19].x = float(box[5, 0])
            point_list[19].y = float(box[5, 1])
            point_list[19].z = float(box[5, 2])

            point_list[20].x = float(box[2, 0])
            point_list[20].y = float(box[2, 1])
            point_list[20].z = float(box[2, 2])
            point_list[21].x = float(box[6, 0])
            point_list[21].y = float(box[6, 1])
            point_list[21].z = float(box[6, 2])

            point_list[22].x = float(box[3, 0])
            point_list[22].y = float(box[3, 1])
            point_list[22].z = float(box[3, 2])
            point_list[23].x = float(box[7, 0])
            point_list[23].y = float(box[7, 1])
            point_list[23].z = float(box[7, 2])

            for j in range(24):
                bbox.points.append(point_list[j])
                #print(f"{j} point {point_list[j]}")
            bbox.scale.x = 0.1
            bbox.color.a = 1.0
            bbox.color.r = 1.0
            bbox.color.g = 0.0
            bbox.color.b = 0.0
            bbox.header.stamp = rospy.Time.now()
            bbox.header.frame_id = "velodyne"
            bbox_arry.markers.append(bbox)

    print("total callback time: ", time.time() - t1)
    global nums, av_time
    nums += 1
    av_time += time.time() - t1
    print(f"net inference AV HZ: {1/(av_time/nums)}")

    pub_bbox_array.publish(bbox_arry)
    cloud_array = xyz_array_to_pointcloud2(np_p[:, :4], rospy.Time.now(), "velodyne")
    print(frame_id)
    pub_point2_.publish(cloud_array)

if __name__ == "__main__":

    global proc

    ## config and model path
    #config_path = '/home/xjtuiair/code/OpenLidarPerceptron/tools/cfgs/kitti_models/second.yaml'
    #modle_path = '/home/xjtuiair/KITTI_PCDet/checkpoint_epoch_80.pth'

    #config_path = '/home/xjtuiair/code/OpenLidarPerceptron/tools/cfgs/kitti_models/pointpillar.yaml'
    #modle_path = '/home/xjtuiair/KITTI_PCDet/pointpillar_7728.pth'
    #config_path = '/home/xjtuiair/code/OpenLidarPerceptron/tools/cfgs/kitti_models/pointpillar.yaml'
    #modle_path = '/home/xjtuiair/KITTI_PCDet/pointpillars_1.pth'
    #config_path = '/home/xjtuiair/code/OpenLidarPerceptron/models/pointpillar_rob_11_26/default/pointpillar_rob.yaml'
    #modle_path = '/home/xjtuiair/code/OpenLidarPerceptron/models/pointpillar_rob_11_26/default/pointpillars.pth'

    #config_path = '/home/xjtuiair/code/OpenLidarPerceptron/tools/cfgs/kitti_models/pv_rcnn.yaml'
    #modle_path = '/home/xjtuiair/KITTI_PCDet/pv_rcnn_8369.pth'

    #config_path = '/home/xjtuiair/code/OpenLidarPerceptron/tools/cfgs/nuscenes_models/cbgs_pp_multihead.yaml'
    #modle_path = '/home/xjtuiair/KITTI_PCDet/pp_multihead_nds5823.pth'

    #config_path = '/home/xjtuiair/code/robosense_cbgs_pp_multihead_1/afdet_BR/robosense_cbgs_pp_multihead.yaml'
    #modle_path = '/home/xjtuiair/code/robosense_cbgs_pp_multihead_1/afdet_BR/ckpt/checkpoint_epoch_40.pth'

    config_path = '/home/xjtuiair/datasets/robosense_cbgs_pp_multihead_all/afdet_BR/robosense_cbgs_pp_multihead_all.yaml'
    modle_path =  '/home/xjtuiair/datasets/robosense_cbgs_pp_multihead_all/afdet_BR/checkpoint_epoch_40.pth'

    proc_1 = Precessor_ROS(config_path, modle_path)

    proc_1.initialize()

    rospy.init_node('net_lidar_ros_node')
    sub_lidar_topic = [
        "/velodyne_points",
        "/rslidar_points",
        "/kitti/velo/pointcloud"
    ]
    sub_ = rospy.Subscriber(sub_lidar_topic[1], PointCloud2, lidar_callback, queue_size=1, buff_size=2**24)

    pub_bbox_array = rospy.Publisher('lidar_net_results', MarkerArray, queue_size=1)
    pub_point2_ = rospy.Publisher('lidar_points', PointCloud2, queue_size=1)

    print("lidar net ros start!")
    rate = rospy.Rate(10)
    rospy.spin()
    rate.sleep()

max_stars_repo_name: Xiangzhaohong/LidarNet

max_issues_repo_name: Xiangzhaohong/LidarNet

max_forks_repo_name: Xiangzhaohong/LidarNet


text:
## Unsupervised summarization model -- static summary

from gensim.summarization import summarize
from gensim.models import Word2Vec
import nltk
nltk.download('punkt')
from nltk.corpus import stopwords
nltk.download('stopwords')
import skipthoughts
from skipthoughts import load_model, Encoder
import os
import theano
import theano.tensor as tensor
import pickle as pkl
import numpy as np
import copy
from collections import OrderedDict, defaultdict
from scipy.linalg import norm
from nltk.tokenize import word_tokenize
from sklearn.cluster import KMeans
from sklearn.metrics import pairwise_distances_argmin_min
import pandas as pd

## load dataset
df1 = pd.read_json('df1.json')

def tokenize_text(text):
    """
    Splits the text into individual sentences
    """
    sentences = nltk.sent_tokenize(text)
    return sentences

def skipthought_encode(sentences):
    """
    Obtains sentence embeddings for each sentence in the text
    """
    # print('Loading pre-trained models...')
    model = skipthoughts.load_model()
    encoder = skipthoughts.Encoder(model)
    # print('Encoding sentences...')
    encoded = encoder.encode(sentences)
    return encoded

def summarize(text):
    """
    Performs summarization of text
    """
    summary = []
    # print('Splitting into sentences...')
    token_text = tokenize_text(text)
    # print('Starting to encode...')
    enc_text = skipthought_encode(token_text)
    # print('Encoding Finished')
    n_clusters = int(np.ceil(len(enc_text) * 0.07)) # n_clusters is related to the length of summary
    kmeans = KMeans(n_clusters=n_clusters, random_state=0)
    kmeans = kmeans.fit(enc_text)
    avg = []
    closest = []
    for j in range(n_clusters):
        idx = np.where(kmeans.labels_ == j)[0]
        avg.append(np.mean(idx))
    closest, _ = pairwise_distances_argmin_min(kmeans.cluster_centers_, \
                                               enc_text)
    ordering = sorted(range(n_clusters), key=lambda k: avg[k])
    summary = ' '.join([token_text[closest[idx]] for idx in ordering])
    # print('Clustering Finished')
    return summary

# def extract_summary(id):
#     """
#     Input business_id, get corresponding summary
#     """
#     text = str(business_tip[business_tip['business_id']==id]['tip_text'].values)
#     tip_summary = summarize(text)
#     return tip_summary

def get_summary(df):
    """
    Input dataframe, add new feature 'summary'
    """
    for i in range(len(df)):
        text = df.at[i,'tip_text']
        if len(text) > 0:
            post_text = summarize(text)
        else:
            post_text = ''
        df.at[i,'summary'] = post_text
    return df

data1 = get_summary(df1)
raw1 = data1[['business_id','summary']]
raw1.to_json('data1.json')
print('Finish saving')

max_stars_repo_name: cheraincao/search-engine-Yelp

max_issues_repo_name: cheraincao/search-engine-Yelp

max_forks_repo_name: cheraincao/search-engine-Yelp

reason: import numpy,from scipy,import theano


text:
import streamlit as st
import pandas as pd
import numpy as np
import requests
from plotly.offline import iplot
import plotly.graph_objs as go
import plotly.express as px
from pandas.io.json import json_normalize

# from streamlit.ScriptRunner import StopException, RerunException

fig = go.Figure()
st.write("""
# Coronapp :sunglasses:

""")

st.write('Ponte al dia minuto a minuto como va la propagación del Covid 19 en el mundo y en tu barrio!')

url = 'https://api.covid19api.com/countries'
r = requests.get(url)

df0 = json_normalize(r.json())

top_row = pd.DataFrame({'Country': ['Select a Country'], 'Slug': ['Empty'], 'ISO2': ['E']})
# Concat with old DataFrame and reset the Index.
df0 = pd.concat([top_row, df0]).reset_index(drop=True)

st.sidebar.header('Create/Filter your search')
graph_type = st.sidebar.selectbox('Cases type', ('confirmed', 'deaths', 'recovered'))
# st.sidebar.subheader('Search by country ')
country = st.sidebar.selectbox('Country', df0.Country)
country1 = st.sidebar.selectbox('Compare with another Country', df0.Country)

if st.sidebar.button('Refresh Data'):
    raise RerunException(st.ScriptRequestQueue.RerunData(None))

if country != 'Select a Country':
    slug = df0.Slug[df0['Country'] == country].to_string(index=False)[1:]
    url = 'https://api.covid19api.com/total/dayone/country/' + slug + '/status/' + graph_type
    r = requests.get(url)
    st.write("""# Total """ + graph_type + """ cases in """ + country + """ are: """ + str(r.json()[-1].get("Cases")))
    df = json_normalize(r.json())
    layout = go.Layout(
        title=country + '\'s ' + graph_type + ' cases Data',
        xaxis=dict(title='Date'),
        yaxis=dict(title='Number of cases'), )
    fig.update_layout(dict1=layout, overwrite=True)
    fig.add_trace(go.Scatter(x=df.Date, y=df.Cases, mode='lines', name=country))

    if country1 != 'Select a Country':
        slug1 = df0.Slug[df0['Country'] == country1].to_string(index=False)[1:]
        url = 'https://api.covid19api.com/total/dayone/country/' + slug1 + '/status/' + graph_type
        r = requests.get(url)
        st.write(
            """# Total """ + graph_type + """ cases in """ + country1 + """ are: """ + str(r.json()[-1].get("Cases")))
        df = json_normalize(r.json())
        layout = go.Layout(
            title=country + ' vs ' + country1 + ' ' + graph_type + ' cases Data',
            xaxis=dict(title='Date'),
            yaxis=dict(title='Number of cases'), )
        fig.update_layout(dict1=layout, overwrite=True)
        fig.add_trace(go.Scatter(x=df.Date, y=df.Cases, mode='lines', name=country1))

    st.plotly_chart(fig, use_container_width=True)

else:
    url = 'https://api.covid19api.com/world/total'
    r = requests.get(url)
    total = r.json()["TotalConfirmed"]
    deaths = r.json()["TotalDeaths"]
    recovered = r.json()["TotalRecovered"]
    st.write("""# En el mundo:""")
    st.write(" **Total cases:** " + str(total) + ", **Total deaths:** " + str(deaths) + ", **Total recovered:** " + str(
        recovered))
    x = ["TotalCases", "TotalDeaths", "TotalRecovered"]
    y = [total, deaths, recovered]

    layout = go.Layout(
        title='World Data',
        xaxis=dict(title='Category'),
        yaxis=dict(title='Number of cases'), )

    fig.update_layout(dict1=layout, overwrite=True)
    fig.add_trace(go.Bar(name='World Data', x=x, y=y))
    st.plotly_chart(fig, use_container_width=True)

st.sidebar.subheader(
    """Created by [OScar Martinez](https://www.linkedin.com/in/oscar-martinez-6bb41918) from ASTRÁIN-AI """)

st.write("""

[Source](https://documenter.getpostman.com/view/10808728/SzS8rjbc?version=latest#81447902-b68a-4e79-9df9-1b371905e9fa) is used to get the data in this app.
""")

hide_footer_style = """
<style>
.reportview-container .main footer {visibility: hidden;}
"""
st.markdown(hide_footer_style, unsafe_allow_html=True)

hide_menu_style = """
        <style>
        #MainMenu {visibility: hidden;}
        </style>
        """
st.markdown(hide_menu_style, unsafe_allow_html=True)

max_stars_repo_name: martinezbel/teocasti

max_issues_repo_name: martinezbel/teocasti

max_forks_repo_name: martinezbel/teocasti


text:
import numpy as np
from skimage.color import rgb2gray

def get_seq(lst):
    def f_(lst):
        lst2 = lst[:]
        i = 0
        while i < len(lst2) - 1:
            if lst2[i + 1] - lst2[i] != 1:
                del lst2[i]
            else:
                i += 1
        return lst2

    l = f_(lst)
    if l == lst:
        return lst
    return get_seq(l)

def cropping_box(img, window_size=200, round_factor=0.4, epsilon=1e4):

    assert img.ndim == 2, "Image must be grayscle"
    _, width = img.shape

    img = rgb2gray(img)
    bin_img = np.round(img + round_factor)

    boxes = []
    w = 0
    b = 0
    while w < width + window_size // 2:
        sub_img = bin_img[:, w: w + window_size]

        th = sub_img.sum() / epsilon

        if th >= 1:
            boxes.append(b)

        b += 1
        w += window_size

    filtered_boxes = get_seq(boxes)
    l = filtered_boxes[0] * 200
    u = filtered_boxes[-1] * 200

    return l, u

def crop_img_and_mask(img, mask, window_size=200, round_factor=0.4, epsilon=1e4):

    l, u = cropping_box(img, window_size, round_factor, epsilon)

    cropped_img = img[:, l: u]
    cropped_mask = mask[:, l: u]

    return cropped_img, cropped_mask


text:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from kalman_filter_system_params import KalmanFilter

## IMPORT 1 MINUTE GRANULAR BITCOIN DATA
df = pd.read_csv("../bitcoin/bitcoin.csv")
df.timestamp_readable = [ts[:-3] for ts in df.timestamp_readable]	# Remove seconds from time format

interval_duration = (df["timestamp"].iloc[-1] - df["timestamp"].iloc[0]) / 60 # minutes
number_of_steps = df.shape[0]

## SET DISCRETE STEP LENGTH
#
# Experiment with different granularities for different data sets
#
#granularity = 1	# No filtering
#granularity = 6	# Finest granularity possible with noticable effect. For analysis in timeranges < 1h
#granularity = 30	# For analysing trends in timeranges ~ 5h
#granularity = 60	# For analysing trends in timeranges ~ 10h
#granularity = 120	# For analysing trends in timeranges > 20h (Bitcoin's variance is pretty narrow when zooming out more than ~24 h)

for granularity in [(120, 0)]: #[(6, 18), (30, 16), (60, 12), (120, 0)]:
	discrete_step = (interval_duration/number_of_steps) * (1/granularity[0])

	## INITIAL VALUES
	state_array = np.zeros(number_of_steps)
	state_array_der = np.zeros(number_of_steps)
	x_0 = np.array([[df["value"].iloc[0]], [0]], dtype=float)
	P_0 = np.eye(2)
	x_k = x_0
	P_k = P_0

	## GUESS STANDARD DEVIATION AND INITIATE FILTER
	standard_deviation = 1
	filter = KalmanFilter(P_k, x_k, standard_deviation, discrete_step)

	## FILTER DATA
	for index, row in df.iterrows():
		filter.update(row['value'])
		state_array[index] = filter.x_k[0]
		state_array_der[index] = filter.x_k[1]

	df["filtered"] = state_array

	## CALCULATE DERIVATIVES
	""" df["derivative_1"] = df["filtered"].diff() / df["timestamp"].diff()
	df["derivative_2"] = (df["filtered"] - 2*df["filtered"].shift(1) + df["filtered"].shift(2)) / df["timestamp"].diff() """

	## PLOT

	minutes = df["timestamp"] - df["timestamp"].iloc[0]
	minute_array = minutes.tolist()

	# PLOT SETTINGS
	start = int(number_of_steps * (granularity[1]/20))
	end = int(number_of_steps * (20/20))
	set_ticks = int(len(minute_array[start:end])/5)

	# PLOT NICE FORMAT
	fig, axes = plt.subplots(1, 1, figsize=(14,8))
	axes.plot(minute_array[start:end], df["value"].tolist()[start:end], 'r')
	axes.plot(minute_array[start:end], state_array[start:end], 'b')
	axes.set_title(f'One minute resolution SEK/Bitcoin')
	axes.legend(['Observed', 'Filtered'], loc='lower left')
	axes.grid(True)
	axes.set_xticks(minute_array[start:end:set_ticks])
	axes.set_xticklabels(df["timestamp_readable"].tolist()[start:end:set_ticks])

	plt.savefig(f'bitcoin_{granularity[0]}.png', bbox_inches='tight')


text:
from abc import ABC, abstractmethod
import numpy as np
import skimage.measure
import torch
import math
import cv2
import torchvision.models as models

from PIL import Image
import time
import os
import copy

class FeatureExtractionMethod(ABC):

    @abstractmethod
    def run(self, images, desired_vector_size):
        pass

class VGG16Method_train(FeatureExtractionMethod):

    def __init__(self):
        self.vgg16 = models.vgg16(pretrained = True)
        self.vgg16.cuda()
        self.curr_loss = -1

    def preprocess_data(self, images):
        pass

    def __str__(self):
        return "Extract the features using vgg16 network with training"

    def save(self, save_directory):
        save_file = os.path.join(save_directory, f'vgg16_loss_{self.curr_loss}')
        torch.save(self.vgg16.state_dict(), save_file)
        return

    def train(self, dataloaders):
        criterion = torch.nn.MSELoss()
        optimizer = torch.optim.Adam(self.vgg16.features.parameters(), lr=0.0001)

        device = 'cuda' if torch.cuda.is_available() else 'cpu'
        num_epochs = 10

        self.train_model(self.vgg16, dataloaders, criterion, optimizer, device=device, num_epochs=num_epochs)

    def train_model(self, model, dataloaders, criterion, optimizer, device='cpu', num_epochs=25, is_inception=False):
        ## TODO: update self.curr_loss -> this is used for
        since = time.time()
        val_acc_history = []

        best_model_wts = copy.deepcopy(model.state_dict())
        best_loss = 1000000

        for epoch in range(num_epochs):
            print('Epoch {}/{}'.format(epoch, num_epochs - 1))
            print('-' * 10)

            # Each epoch has a training and validation phase
            for phase in ['train', 'val']:
                if phase == 'train':
                    model.train()  # Set model to training mode
                else:
                    model.eval()  # Set model to evaluate mode

                running_loss = 0.0
                running_corrects = 0

                # Iterate over data.
                for inputs, labels in dataloaders[phase]:
                    inputs = inputs.to(device)
                    labels = labels.to(device)

                    # zero the parameter gradients
                    optimizer.zero_grad()

                    # forward
                    # track history if only in train
                    with torch.set_grad_enabled(phase == 'train'):
                        # Get model outputs and calculate loss
                        # Special case for inception because in training it has an auxiliary output. In train
                        #   mode we calculate the loss by summing the final output and the auxiliary output
                        #   but in testing we only consider the final output.

                        ##outputs = model(inputs)
                        ## TODO: we don't use the classifier
                        outputs = model.features(inputs)
                        outputs = model.avgpool(outputs)

                        #print(f"output shape: {outputs.size()}, labels shape: {labels.size()}")
                        loss = criterion(outputs, labels)

                        _, preds = torch.max(outputs, 1)

                        # backward + optimize only if in training phase
                        if phase == 'train':
                            loss.backward()
                            optimizer.step()

                    # statistics
                    running_loss += loss.item() * inputs.size(0)

                epoch_loss = running_loss / len(dataloaders[phase].dataset)
                self.curr_loss = epoch_loss

                print('{} Loss: {:.4f} '.format(phase, epoch_loss))

                # deep copy the model
                if phase == 'val' and self.curr_loss < best_loss:
                    best_loss = self.curr_loss
                    best_model_wts = copy.deepcopy(model.state_dict())
                if phase == 'val':
                    val_acc_history.append(self.curr_loss)

            print()

        time_elapsed = time.time() - since
        print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
        print('Best val Loss: {:4f}'.format(best_loss))

        # load best model weights
        model.load_state_dict(best_model_wts)
        return model, val_acc_history

    def run(self, images, desired_vector_size):

        #import os
        #os.environ["CUDA_VISIBLE_DEVICES"] = "1"  ## we want to run everything on gpu 1

        vgg16 = models.vgg16(pretrained = True)
        vgg16.cuda()
        images_torch = torch.tensor(images, device = 'cpu').float()
        images_torch = images_torch.permute(0,3,1,2)
        data_loader = torch.utils.data.DataLoader(images_torch, batch_size = 32, shuffle = False)
        outputs = []
        features = []
        for i, data_pack in enumerate(data_loader):
            image_batch = data_pack
            image_batch = image_batch.cuda()
            ## TODO: add in adjustable average pool
            output = vgg16.features(image_batch)
            output = vgg16.avgpool(output)
            output_sum = torch.sum(output, axis = 1)
            output_sum = output_sum.cpu()
            output_np = output_sum.detach().numpy() #n, 512, 9, 9

            outputs.append( output_np )
            features.append( output.cpu().detach().numpy() )

        images_reshaped = np.concatenate(outputs, axis = 0)
        self.image_features = np.concatenate(features, axis = 0)

        images_reshaped = images_reshaped.reshape(images_reshaped.shape[0], images_reshaped.shape[1] * images_reshaped.shape[2])

        if images_reshaped.shape[1] > desired_vector_size:
            return images_reshaped[:,:desired_vector_size]

        elif images_reshaped.shape[1] < desired_vector_size:
            tmp = np.ndarray(shape = (images_reshaped.shape[0], desired_vector_size - images_reshaped.shape[1]))
            images_reshaped = np.concatenate([images_reshaped, tmp], axis = 1)
            return images_reshaped

        else: ## images_reshaped.shape[1] == desired_vector_size
            return images_reshaped

class VGG16Method(FeatureExtractionMethod):

    def __str__(self):
        return "Extract the features using vgg16 network"

    def run(self, images, desired_vector_size):
        import torchvision.models as models
        #import os
        #os.environ["CUDA_VISIBLE_DEVICES"] = "1"  ## we want to run everything on gpu 1

        vgg16 = models.vgg16(pretrained = True)
        vgg16.cuda()
        images_torch = torch.tensor(images, device = 'cpu').float()
        images_torch = images_torch.permute(0,3,1,2)
        data_loader = torch.utils.data.DataLoader(images_torch, batch_size = 32, shuffle = False)
        outputs = []
        for i, data_pack in enumerate(data_loader):
            image_batch = data_pack
            image_batch = image_batch.cuda()
            output = vgg16.features(image_batch)
            output = output.cpu()
            output_np = output.detach().numpy() #n, 512, 9, 9

            outputs.append( np.sum(output_np, axis = 1) )

        images_reshaped = np.concatenate(outputs, axis = 0)
        images_reshaped = images_reshaped.reshape(images_reshaped.shape[0], images_reshaped.shape[1] * images_reshaped.shape[2])

        if images_reshaped.shape[1] > desired_vector_size:
            return images_reshaped[:,:desired_vector_size]

        elif images_reshaped.shape[1] < desired_vector_size:
            tmp = np.ndarray(shape = (images_reshaped.shape[0], desired_vector_size - images_reshaped.shape[1]))
            images_reshaped = np.concatenate([images_reshaped, tmp], axis = 1)
            return images_reshaped

        else: ## images_reshaped.shape[1] == desired_vector_size
            return images_reshaped

class BackgroundSubtractionMethod(FeatureExtractionMethod):

    def __str__(self):
        return "Extract features after using background subtraction"

    def run(self, images, desired_vector_size, debug = False):
        import cv2
        # fgbg only takes grayscale images, we need to convert
        ## check if image is already converted to grayscale -> channels = 1

        segmented_images = np.ndarray(shape=(images.shape[0], images.shape[1], images.shape[2]), dtype=np.uint8)
        print(f"segmented_images shape will be {segmented_images.shape}")
        history = 40
        dist2Threshold = 300
        detectShadows = False
        fgbg = cv2.createBackgroundSubtractorKNN(history=history, dist2Threshold=dist2Threshold,
                                                 detectShadows=detectShadows)

        # first round is to tune the values of the background subtractor
        for ii in range(10):
            image_gray = np.mean(images[ii], axis = 2).astype(np.uint8)
            fgbg.apply(image_gray)

        for ii in range(len(images)):
            image_gray = np.mean(images[ii], axis = 2).astype(np.uint8)
            segmented_images[ii] = fgbg.apply(image_gray)

        ## after we generate the segmented_images, we need to downsample this somehow...
        ## first step, let's just downsample
        import math
        wanted_width = int(math.sqrt(desired_vector_size))
        wanted_height = desired_vector_size // wanted_width
        width_skip_rate = images.shape[1] // wanted_width
        height_skip_rate = images.shape[2] // wanted_height
        images_downscaled = segmented_images[:, ::width_skip_rate, ::height_skip_rate]
        print(f"segmented images shape {segmented_images.shape}")
        print(f"images_downsampled shape {images_downscaled.shape}")
        ## we need to cut off the last element
        if images_downscaled.shape[1] > wanted_height:
            images_downscaled = images_downscaled[:,:wanted_height,:]
        if images_downscaled.shape[2] > wanted_width:
            images_downscaled = images_downscaled[:,:,:wanted_width]
        assert(images_downscaled.shape == (images.shape[0], wanted_height, wanted_width))

        images_reshaped = images_downscaled.reshape(len(images), wanted_width * wanted_height)

        if debug:
            return images_reshaped, segmented_images

        return images_reshaped

class DownSampleLanczosMethod(FeatureExtractionMethod):

    def __init__(self):
        self.name = 'LANCZOS'

    def __str__(self):

        return "Downsample the image by choosing max value in pixel blocks"

    def run(self, images, desired_vector_size):
        assert (desired_vector_size % 10 == 0)
        wanted_width = 10
        wanted_height = desired_vector_size // wanted_width

        size = (wanted_height, wanted_width)
        new_images_arr = np.ndarray(shape = (len(images), wanted_width, wanted_height, images.shape[3]))
        for i in range(len(images)):
            im = images[i]
            pil_im = Image.fromarray(im, 'RGB')
            pil_im = pil_im.resize(size, Image.LANCZOS)
            new_images_arr[i] = np.array(pil_im, dtype = np.uint8)

        ## now we have to reshape this
        new_images_arr = np.mean(new_images_arr, axis = 3)
        images_reshaped = new_images_arr.reshape(len(images), wanted_height * wanted_width)

        return images_reshaped

class DownSampleMaxMethod(FeatureExtractionMethod):

    def __str__(self):
        return "Downsample the image by choosing max value in pixel blocks"

    def run(self, images, desired_vector_size):
        assert(desired_vector_size % 10 == 0)
        wanted_width = 10
        wanted_height = desired_vector_size // wanted_width
        height_box_size = images.shape[1] // wanted_height
        width_box_size = images.shape[2] // wanted_width

        images_reshaped = skimage.measure.block_reduce(images, (1, height_box_size, width_box_size, 3), np.max)
        images_reshaped = np.squeeze(images_reshaped, axis=3)
        images_reshaped = images_reshaped.reshape(len(images), wanted_height * wanted_width)

        return images_reshaped

class DownSampleMeanMethod2(FeatureExtractionMethod):

    def __str__(self):
        return "Downsample the image by avg pixel blocks using block_reduce from skimage"

    def run(self, images, desired_vector_size):
        assert(desired_vector_size % 10 == 0)
        wanted_width = int(math.sqrt(desired_vector_size))
        wanted_height = desired_vector_size // wanted_width
        height_box_size = images.shape[1] // wanted_height
        width_box_size = images.shape[2] // wanted_width

        images_reshaped = skimage.measure.block_reduce(images, (1, height_box_size, width_box_size, 3), np.mean)
        images_reshaped = np.squeeze(images_reshaped, axis = 3)

        ## we need to cut things off if they overflow
        if images_reshaped.shape[1] > wanted_height:
            images_reshaped = images_reshaped[:, :wanted_height, :]
        if images_reshaped.shape[2] > wanted_width:
            images_reshaped = images_reshaped[:, :, :wanted_width]

        images_reshaped = images_reshaped.reshape(len(images), wanted_height * wanted_width)

        return images_reshaped

class DownSampleMeanMethod(FeatureExtractionMethod):
    def __init__(self):
        self.name = 'MEAN'

    def __str__(self):
        return "Downsample the image by avg pixel blocks"

    def run(self, images, desired_vector_size):
        assert(desired_vector_size % 10 == 0)
        import math
        wanted_width = int(math.sqrt(desired_vector_size)) ## will modify this to
        wanted_height = desired_vector_size // wanted_width
        height_box_size = images.shape[1] // wanted_height
        width_box_size = images.shape[2] // wanted_width
        images_downscaled = np.ndarray(shape = (images.shape[0], wanted_height, wanted_width))

        print(images_downscaled.shape)
        for i in range(0, wanted_height):
            for j in range(0, wanted_width):
                start_i = i * height_box_size
                end_i = (i + 1) * height_box_size
                start_j = j * width_box_size
                end_j = (j + 1) * width_box_size
                images_downscaled[:,i,j] = np.mean(images[:, start_i:end_i, start_j:end_j, :], axis = (1,2,3))

        images_reshaped = images_downscaled.reshape(len(images), wanted_width * wanted_height)
        return images_reshaped

class DownSampleSkippingMethod(FeatureExtractionMethod):
    def __init__(self):
        self.name = 'SKIP'

    def __str__(self):
        return "Downsample the image by skipping pixels"

    def run(self, images, desired_vector_size):
        assert(desired_vector_size % 10 == 0)
        wanted_width = 10
        wanted_height = desired_vector_size // wanted_width
        width_skip_rate = images.shape[1] // wanted_width
        height_skip_rate = images.shape[2] // wanted_height
        images_downscaled = images[:, ::width_skip_rate, ::height_skip_rate]
        images_downscaled = np.mean(images_downscaled, axis=3)
        print(images_downscaled.shape)
        images_reshaped = images_downscaled.reshape(len(images), wanted_width * wanted_height)

        return images_reshaped

class DownSampleSkippingCVMethod(FeatureExtractionMethod):

    def __str__(self):
        return "Downsample the image by skipping pixels"

    def run(self, images, desired_vector_size):
        assert(desired_vector_size % 10 == 0)
        wanted_width = 10
        wanted_height = desired_vector_size // wanted_width
        ## we will use the cv resize method to do things
        resized_images = []
        for i in range(len(images)):
            resized_images.append( cv2.resize(images[i], (wanted_width, wanted_height)) )
        images_downscaled = np.stack(resized_images, axis = 0)
        ## make sure stacking works as expected
        assert(images_downscaled.shape == (len(images), 10, 10, 3))
        images_downscaled = np.mean(images_downscaled, axis=3)
        print(images_downscaled.shape)
        images_reshaped = images_downscaled.reshape(len(images), wanted_width * wanted_height)

        return images_reshaped


text:
import numpy as np

import openmdao.api as om
from openaerostruct.structures.utils import norm

class Weight(om.ExplicitComponent):
    """Compute total weight and center-of-gravity location of the spar elements.

    parameters
    ----------
    A[ny-1] : numpy array
        Areas for each FEM element.
    nodes[ny, 3] : numpy array
        Flattened array with coordinates for each FEM node.

    Returns
    -------
    structural_mass : float
        Weight of the wing structure.
    elmenet_weight[ny-1] : float
        Weight of each element.

    """

    def initialize(self):
        self.options.declare("surface", types=dict)

    def setup(self):
        self.surface = surface = self.options["surface"]

        self.ny = ny = surface["mesh"].shape[1]

        self.add_input("A", val=np.ones((self.ny - 1)), units="m**2")
        self.add_input("nodes", val=np.zeros((self.ny, 3)), units="m")

        self.add_output("structural_mass", val=0.0, units="kg")
        self.add_output("element_mass", val=np.zeros((self.ny - 1)), units="kg")

        self.declare_partials("structural_mass", ["A", "nodes"])

        row_col = np.arange(self.ny - 1, dtype=int)
        self.declare_partials("element_mass", "A", rows=row_col, cols=row_col)

        dimensions = 3
        rows = np.empty((dimensions * 2 * (ny - 1)))
        cols = np.empty((dimensions * 2 * (ny - 1)))
        for i in range(ny - 1):
            rows[i * dimensions * 2 : (i + 1) * dimensions * 2] = i
            cols[i * dimensions * 2 : (i + 1) * dimensions * 2] = np.linspace(
                i * dimensions, i * dimensions + (dimensions * 2 - 1), dimensions * 2
            )
        self.declare_partials("element_mass", "nodes", rows=rows, cols=cols)

        self.set_check_partial_options("*", method="cs", step=1e-40)

    def compute(self, inputs, outputs):
        A = inputs["A"]
        nodes = inputs["nodes"]
        mrho = self.surface["mrho"]
        wwr = self.surface["wing_weight_ratio"]

        # Calculate the volume and weight of the structure
        element_volumes = norm(nodes[1:, :] - nodes[:-1, :], axis=1) * A

        # nodes[1:, :] - nodes[:-1, :] this is the delta array of the different between the points
        element_mass = element_volumes * mrho * wwr
        weight = np.sum(element_mass)

        # If the tube is symmetric, double the computed weight
        if self.surface["symmetry"]:
            weight *= 2.0

        outputs["structural_mass"] = weight
        outputs["element_mass"] = element_mass

    def compute_partials(self, inputs, partials):

        A = inputs["A"]
        nodes = inputs["nodes"]
        mrho = self.surface["mrho"]
        wwr = self.surface["wing_weight_ratio"]
        ny = self.ny

        # Calculate the volume and weight of the structure
        const0 = nodes[1:, :] - nodes[:-1, :]
        const1 = np.linalg.norm(const0, axis=1)
        const2 = mrho * wwr

        # First we will solve for dweight_dA
        # Calculate the volume and weight of the total structure
        norms = const1.reshape(1, -1)

        # Multiply by the material density and force of gravity
        dweight_dA = norms * const2

        # Account for symmetry
        if self.surface["symmetry"]:
            dweight_dA *= 2.0

        # Save the result to the jacobian dictionary
        partials["structural_mass", "A"] = dweight_dA

        # Next, we will compute the derivative of weight wrt nodes.
        # Here we're using results from AD to compute the derivative
        # Initialize the reverse seeds.
        nodesb = np.zeros(nodes.shape)
        tempb = (const0) * (A / norms).reshape(-1, 1)
        nodesb[1:, :] += tempb
        nodesb[:-1, :] -= tempb

        # Apply the multipliers for material properties and symmetry
        nodesb *= mrho * wwr

        if self.surface["symmetry"]:
            nodesb *= 2.0

        # Store the flattened array in the jacobian dictionary
        partials["structural_mass", "nodes"] = nodesb.reshape(1, -1)

        # Element_weight Partials
        partials["element_mass", "A"] = const1 * const2

        precalc = np.sum(np.power(const0, 2), axis=1)
        d__dprecalc = 0.5 * precalc ** (-0.5)

        dimensions = 3
        for i in range(ny - 1):
            first_part = const0[i, :] * d__dprecalc[i] * 2 * (-1) * A[i] * const2
            second_part = const0[i, :] * d__dprecalc[i] * 2 * A[i] * const2
            partials["element_mass", "nodes"][i * dimensions * 2 : (i + 1) * dimensions * 2] = np.append(
                first_part, second_part
            )

max_stars_repo_name: lamkina/OpenAeroStruct

max_issues_repo_name: lamkina/OpenAeroStruct

max_forks_repo_name: lamkina/OpenAeroStruct


text:
# (C) British Crown Copyright 2013 - 2019, Met Office
#
# This file is part of cartopy.
#
# cartopy is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cartopy is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with cartopy.  If not, see <https://www.gnu.org/licenses/>.

from __future__ import (absolute_import, division, print_function)

import warnings

import matplotlib.path as mpath
import numpy as np

def intersection_point(p0, p1, p2, p3):
    """
    Returns
    -------
    x, y
        The intersection point of the two infinite lines that pass through
        point p0->p1 and p2->p3 respectively.

    """
    x_1, y_1 = p0
    x_2, y_2 = p1
    x_3, y_3 = p2
    x_4, y_4 = p3

    div = (x_1 - x_2) * (y_3 - y_4) - (y_1 - y_2) * (x_3 - x_4)

    if div == 0:
        raise ValueError('Lines are parallel and cannot '
                         'intersect at any one point.')

    x = ((x_1 * y_2 - y_1 * x_2) * (x_3 - x_4) - (x_1 - x_2) * (x_3 *
         y_4 - y_3 * x_4)) / div
    y = ((x_1 * y_2 - y_1 * x_2) * (y_3 - y_4) - (y_1 - y_2) * (x_3 *
         y_4 - y_3 * x_4)) / div

    return x, y

# Provide a clip_path function which clips the given path to the given Bbox.
# There is inbuilt mpl functionality with v1.2.1 and beyond, but we provide
# a shim here for older mpl versions.
def clip_path(subject, clip_bbox):
    """
    Clip the given path to the given bounding box.

    """
    warnings.warn("This method has been deprecated. "
                  "You can replace ``clip_path(subject, clip_bbox)`` by "
                  "``subject.clip_to_bbox(clip_bbox)``. "
                  "See the \"What's new\" section for v0.17.",
                  DeprecationWarning,
                  stacklevel=2)
    return subject.clip_to_bbox(clip_bbox)

def lines_intersect(p0, p1, p2, p3):
    """
    Returns
    -------
    bool
        Boolean indicating whether the two lines defined by p0->p1 and p2->p3
        intersect.
    """
    x_1, y_1 = p0
    x_2, y_2 = p1
    x_3, y_3 = p2
    x_4, y_4 = p3

    return (x_1 - x_2) * (y_3 - y_4) - (y_1 - y_2) * (x_3 - x_4) != 0
    cp1 = np.cross(p1 - p0, p2 - p0)
    cp2 = np.cross(p1 - p0, p3 - p0)
    return np.sign(cp1) == np.sign(cp2) and cp1 != 0

def bbox_to_path(bbox):
    """
    Turn the given :class:`matplotlib.transforms.Bbox` instance into
    a :class:`matplotlib.path.Path` instance.

    """
    verts = np.array([[bbox.x0, bbox.y0], [bbox.x1, bbox.y0],
                      [bbox.x1, bbox.y1], [bbox.x0, bbox.y1],
                      [bbox.x0, bbox.y0]])
    return mpath.Path(verts)

max_stars_repo_name: ushham/JournalTool

max_issues_repo_name: ushham/JournalTool

max_forks_repo_name: ushham/JournalTool


text:
import datetime
import time
import RPi.GPIO as GPIO
import numpy as np
import motor

class Location(object):
    initialIndex=None
    Name=None

def bitMapRowGenerator(Map):
    ObstacleLimit=250
    currentNumber=0;
    for j in range(0,17):
        currentNumber=(currentNumber<<1)|(Map[j]<ObstacleLimit)
    return currentNumber

def SetValues(LocationArray):
	LocationArray[0].initialIndex=0
	LocationArray[1].initialIndex=4
	LocationArray[2].initialIndex=10
	LocationArray[3].initialIndex=13
	LocationArray[4].initialIndex=20
	LocationArray[0].Name="Liberty's 1st Gate"
	LocationArray[1].Name="Corridor Gate"
	LocationArray[2].Name="Liberty's 2nd Gate"
	LocationArray[3].Name="Corridor Before First Turn"
	LocationArray[4].Name="1st Turn - Right"

def reading():
	# GPIO.setwarnings(False)
	# GPIO.setmode(GPIO.BCM)
	# point the software to the GPIO pins the sensor is using
	# change these values to the pins you are using
	# GPIO output = the pin that's connected to "Trig" on the sensor
	# GPIO input = the pin that's connected to "Echo" on the sensor
	GPIO.setup(17,GPIO.OUT)
	GPIO.setup(24,GPIO.IN)
	GPIO.output(17, GPIO.LOW)
	# found that the sensor can crash if there isn't a delay here
	# no idea why. If you have odd crashing issues, increase delay
	time.sleep(0.1)

	# sensor manual says a pulse ength of 10Us will trigger the
	# sensor to transmit 8 cycles of ultrasonic burst at 40kHz and
	# wait for the reflected ultrasonic burst to be received

	# to get a pulse length of 10Us we need to start the pulse, then
	# wait for 10 microseconds, then stop the pulse. This will
	# result in the pulse length being 10Us.

	# start the pulse on the GPIO pin
	# change this value to the pin you are using
	# GPIO output = the pin that's connected to "Trig" on the sensor
	GPIO.output(17, True)
	# wait 10 micro seconds (this is 0.00001 seconds) so the pulse
	# length is 10Us as the sensor expects
	time.sleep(0.00001)

	# stop the pulse after the time above has passed
	# change this value to the pin you are using
	# GPIO output = the pin that's connected to "Trig" on the sensor
	GPIO.output(17, False)

	# listen to the input pin. 0 means nothing is happening. Once a
	# signal is received the value will be 1 so the while loop
	# stops and has the last recorded time the signal was 0
	# change this value to the pin you are using
	# GPIO input = the pin that's connected to "Echo" on the sensor
	while GPIO.input(24) == 0:
	  signaloff = time.time()
	# listen to the input pin. Once a signal is received, record the
	# time the signal came through
	# change this value to the pin you are using
	# GPIO input = the pin that's connected to "Echo" on the sensor
	while GPIO.input(24) == 1:
	  signalon = time.time()
	# work out the difference in the two recorded times above to
	# calculate the distance of an object in front of the sensor
	timepassed = signalon - signaloff

	# we now have our distance but it's not in a useful unit of
	# measurement. So now we convert this distance into centimetres
	distance = timepassed * 17000

	# return the distance of an object in front of the sensor in cm
	return distance

	# we're no longer using the GPIO, so tell software we're done
	# GPIO.cleanup()

def Turn():
	#GPIO.setwarnings(False)
	#GPIO.setmode(GPIO.BCM)

	StepPins=[10,9,11,25]
	MapArray=np.zeros(17)
	DataArray=np.zeros(5)

	for pin in StepPins:
		GPIO.setup(pin,GPIO.OUT)
		GPIO.output(pin,False)

	#time.sleep(5)

	StepCounter=0
	WaitTime=0.0015

	StepCount2=8
	Seq2=[]
	Seq2=range(0,StepCount2)
	Seq2[0]=[1,0,0,0]
	Seq2[1]=[1,1,0,0]
	Seq2[2]=[0,1,0,0]
	Seq2[3]=[0,1,1,0]
	Seq2[4]=[0,0,1,0]
	Seq2[5]=[0,0,1,1]
	Seq2[6]=[0,0,0,1]
	Seq2[7]=[1,0,0,1]

	Seq=Seq2
	StepCount=StepCount2
	count=0
	i=0
	j=0

	while i<16:
		for ReadingCount in range(0,5):
			DataArray[ReadingCount]=reading()
		MapArray[i]=np.median(DataArray)
		time.sleep(1)
		while j<128:
			for pin in range(0,4):
				xpin=StepPins[pin]
				if Seq[StepCounter][pin]!=0:
					GPIO.output(xpin,True)
				else:
					GPIO.output(xpin,False)
			StepCounter+=1

			if (StepCounter==StepCount):
				StepCounter=0
			if (StepCounter<0):
				StepCounter=StepCount

			time.sleep(WaitTime)
			j+=1
		j=0
		i+=1

	for ReadinCount in range(0,5):
		DataArray[ReadingCount]=reading()
	MapArray[i]=np.median(DataArray)

	StepCounter=StepCount2-1
	i=0
	while i<512*4:
		for pin in range(0,4):
			xpin=StepPins[pin]
			if Seq[StepCounter][pin]!=0:
				GPIO.output(xpin,True)
			else:
				GPIO.output(xpin,False)
		StepCounter-=1

		if (StepCounter==-1):
			StepCounter=StepCount2-1
		time.sleep(WaitTime)
		i+=1

	for pin in StepPins:
		GPIO.output(pin,False)

	#GPIO.cleanup()
	return MapArray

def ExactSearch(Map,LocalArray,Diff,Result):
	MapDimensions=Map.shape
	for i in range(0,MapDimensions[0]):
		Check=True
		for j in range(0,MapDimensions[1]):
			if (not(LocalArray[j]>=Map[i][j]-Diff and LocalArray[j]<=Map[i][j]+Diff)):
				Check=False
				break
		if (Check):
			Result[i]+=(50/Diff)

def MiddleSearch(Map,LocalArray,Diff,Result):
	MapDimensions=Map.shape
	for i in range(0,MapDimensions[0]):
		for j in range(0,MapDimensions[1]):
			if (LocalArray[j]>=Map[i][j]-Diff and LocalArray[j]<=Map[i][j]+Diff):
				Result[i]+=(50/Diff)

def NotMiddleSearch(Map,LocalArray,Diff,Result):
	MapDimensions=Map.shape
	DoubleMiddle=np.zeros(9)
	for i in range(0,MapDimensions[0]):
		for k in range(0,9):
			DoubleMiddle[k]=(Map[i][k]+Map[i][16-k])/2
		for j in range(0,9):
			if (LocalArray[j]>=DoubleMiddle[j]-Diff and LocalArray[j]<=DoubleMiddle[j]+Diff):
				Result[i]+=1

def MapLocationFinder(LocationArray,FoundArray,FoundValue):
	Dim=FoundArray.shape
	for i in range(0,Dim[0]):
		check=False
		if (FoundArray[i]==FoundValue):
			print i
			for j in range(0,5):
				if (i<LocationArray[j].initialIndex):
					check=True
					break
			if (check):
				print LocationArray[j-1].Name
			else:
				print LocationArray[j].Name

def main():
	#LocationArray=[Location() for i in range(5)]
	#SetValues(LocationArray)

	GPIO.setwarnings(False)
	GPIO.setmode(GPIO.BCM)

	Map=np.loadtxt('ATOM_Map.txt')

	'''
	Result0=np.zeros(MapDimensions[0])
	ExactSearch(Map,LocalArray,50,Result0)
	ExactSearch(Map,LocalArray,20,Result0)
	ExactSearch(Map,LocalArray,5,Result0)
	print Result0
	Max0=np.amax(Result0)
	MaxIndex0=np.argmax(Result0)
	print Max0
	print MaxIndex0
	#if (Max0>0):
		#MapLocationFinder(LocationArray,Result0,Max0)
	'''

	#Loading All Map files
	BitMap=np.loadtxt("bitMap.txt")
	NNMap=np.loadtxt("neuralNetworkMap.txt")
	ActualMap=np.loadtxt("ATOM_Map.txt")

	#Peforming Localization Multi-Step Process
	MapDimensions=BitMap.shape
	indices=[i for i in range(MapDimensions[0]+1)]
	indices[MapDimensions[0]]=-1
	totalReps=3
	marginForBitError=totalReps-1
	index=0 #Only for readings array
	bitReading=np.zeros(totalReps)
	readings=np.zeros((totalReps,17))

	'''
	Result1=np.zeros(MapDimensions[0])
	MiddleSearch(ActualMap,readings[index],50,Result1)
	MiddleSearch(ActualMap,readings[index],20,Result1)
	MiddleSearch(ActualMap,readings[index],5,Result1)
	print Result1
	Max1=np.amax(Result1)
	MaxIndex1=np.argmax(Result1)
	print Max1
	print MaxIndex1
	#MapLocationFinder(LocationArray,Result1,Max1)
	'''
	while (marginForBitError>=0):
		readings[index]=np.copy(Turn())
		print readings[index]
		ComparedBitValues=np.zeros(MapDimensions[0])
		bitReading[index]=bitMapRowGenerator(readings[index])

		counter=0
		i=indices[counter]
		while (i!=-1):
			if (i<MapDimensions[0]):
				ComparedBitValues[i]=17-(bin(int(bitReading[index])^int(BitMap[i])).count('1'))
			counter=counter+1
			i=indices[counter]
		print ComparedBitValues

		counter=0
		localmax=np.max(ComparedBitValues)
		for i in range(0,MapDimensions[0]):
			if (ComparedBitValues[i]+marginForBitError>=localmax):
				indices[counter]=i+1
				counter=counter+1
		indices[counter]=-1

		marginForBitError=marginForBitError-1
		index=index+1
		time.sleep(0.5)
		motor.forward(0.75)
		time.sleep(0.5)

	indices[0]-=1
	print indices[0]

	GPIO.cleanup()

	'''
	DoubleLocal=np.zeros(9)
	for i in range(0,9):
		DoubleLocal[i]=(LocalArray[i]+LocalArray[16-i])/2
	Result2=np.zeros(MapDimensions[0])
	NotMiddleSearch(Map,DoubleLocal,50,Result2)
	NotMiddleSearch(Map,DoubleLocal,20,Result2)
	NotMiddleSearch(Map,DoubleLocal,5,Result2)
	print Result2
	Max2=np.amax(Result2)
	MaxIndex2=np.argmax(Result2)
	print Max2
	print MaxIndex2
	#MapLocationFinder(LocationArray,Result2,Max2)
	'''
main()


text:
# Copyright (C) 2022. Huawei Technologies Co., Ltd. All rights reserved.

# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable
from transformer import (
    make_model,
    Batch,
    LabelSmoothing,
    NoamOpt,
    run_epoch,
    greedy_decode,
    SimpleLossCompute,
)

def data_gen(V, batch, nbatches):
    "Generate random data for a src-tgt copy task."
    for i in range(nbatches):
        data = torch.from_numpy(np.random.randint(1, V, size=(batch, 10)))
        print(data)
        data = data.type(torch.LongTensor)
        data[:, 0] = 1
        src = Variable(data, requires_grad=False)
        tgt = Variable(data, requires_grad=False)
        yield Batch(src, tgt, 0)

# Train the simple copy task.
V = 11
MODEL_SIZE = 10
Heads = 2
criterion = LabelSmoothing(size=V, padding_idx=0, smoothing=0.0)
model = make_model(V, V, N=2, h=Heads, d_model=MODEL_SIZE, dropout=0.0)

for p in model.parameters():
    nn.init.ones_(p)

model_opt = NoamOpt(
    model.src_embed[0].d_model,
    1,
    400,
    torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9),
)

nepoch = 5  # 10
batch_size = 2  # 30
nbatches = 2  # 20
nbatches_eval = 5

np.random.seed(0)

for epoch in range(nepoch):
    model.train()
    run_epoch(
        data_gen(V, batch_size, nbatches),
        model,
        SimpleLossCompute(model.generator, criterion, model_opt),
    )
    model.eval()
    print(
        run_epoch(
            data_gen(V, batch_size, nbatches_eval),
            model,
            SimpleLossCompute(model.generator, criterion, None),
        )
    )

model.eval()
src = Variable(torch.LongTensor([[i + 1 for i in range(V - 1)]]))
src_mask = Variable(torch.ones(1, 1, V - 1))
print("before decode")
print(greedy_decode(model, src, src_mask, max_len=V - 1, start_symbol=1))

max_stars_repo_name: steelONIONknight/bolt

max_issues_repo_name: steelONIONknight/bolt

max_forks_repo_name: steelONIONknight/bolt


text:
import numpy as np
from poseutils.datasets.unprocessed import TDPWDataset
from poseutils.props import get_edm

dpath = "/home/saad/Personal/Research/Dataset/CrossDataset/3dpw_wo_invalid.npz"

dataset = TDPWDataset(dpath)

d3d_train = dataset.get_3d_train()

edm = get_edm(d3d_train[:1, :, :], True)

for i in range(14):
    for j in range(i, 14):

        dist = np.sqrt(np.sum((d3d_train[0, i, :] - d3d_train[0, j, :])**2))

        assert np.abs(dist - edm[0, i, j]) < 1e-5

print(edm[0, :3, :3])

edm = get_edm(d3d_train[:1, :, :])

print(edm.shape)

max_stars_repo_name: SaadManzur/PoseUtils

max_issues_repo_name: SaadManzur/PoseUtils

max_forks_repo_name: SaadManzur/PoseUtils


text:
import unittest
import numpy as np
import matplotlib.pyplot as plt
from dasy.synthesizers.gaussian import GaussianSynth
from dasy.synthesizers.uniform import UniformSynth
from dasy.labelers.classification.centroids import CentroidsLabeler
from dasy.labelers.regression.linear import LinearRegressionLabeler

class TestQuickstart(unittest.TestCase):
    def test_quickstart(self):
        plt.clf()
### dasy for classification
# 1. Define the problem
        dim = 2 # dimension of each input
        classes = 2 # number of classes
        n = 100 # number of data points
# 2. Create synthetic input data X
        synth = GaussianSynth(dim=dim)
        X = synth.sample(n=n) # sample n dim-dimensional data points from Gaussian
# 3. Assign labels y
        labeler = CentroidsLabeler(classes=classes, dim=dim) # for 2 classes, this creates linearly separable labels
        y = labeler.assign(X)
# 4. Plot
        plt.scatter(X.T[0], X.T[1], c=y)
        plt.title('Synthetic Classification Problem')
        plt.tight_layout()
        plt.savefig('synthetic_classification')
        plt.clf()

### dasy for regression
# 1. Define the problem
        dim = 1 # dimension of each input
        n = 50 # number of data points
# 2. Create synthetic input data X
        synth = UniformSynth(dim=dim)
        X = synth.sample(n=n) # sample n dim-dimensional data points from Uniform distribution
# 3. Assign continuous targets y
        labeler = LinearRegressionLabeler(dim=dim)
        y = labeler.assign(X)
# 4. Plot
        plt.scatter(X.T[0], y)
        plt.axline((0, labeler.b), slope=labeler.w, color='r') # plot the underlying line which generated the targets
        plt.title('Synthetic Regression Problem')
        plt.tight_layout()
        plt.savefig('synthetic_regression')

max_stars_repo_name: bkestelman/synthetic-data

max_issues_repo_name: bkestelman/synthetic-data

max_forks_repo_name: bkestelman/synthetic-data


text:
import numpy as np
from sklearn.metrics import roc_auc_score,roc_curve
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_recall_curve
from sklearn import metrics

# load the ouput from gkmSVM
path_output = './saved_data/output.txt'
data = np.loadtxt(path_output,dtype=str,delimiter="\t")

# predict probability
prob = data[:,1]
prob = prob.astype(float)
num = len(prob)

# generate label
label = np.zeros((num,1))
label[0:int(num/2)] = 1

# calculate ROAUC and PRAUC
fpr,tpr,thresholds=roc_curve(label,prob)
precision, recall, thresholds = precision_recall_curve(label, prob)
auc=roc_auc_score(label,prob)
print("AUC(ROC):" + str(auc))
area = metrics.auc(recall, precision)
print("AUC(PRC):" + str(area))

max_stars_repo_name: simonlevine/DeepSilencer

max_issues_repo_name: simonlevine/DeepSilencer

max_forks_repo_name: simonlevine/DeepSilencer


text:
from itertools import count
# from msilib.schema import RadioButton
from pyexpat import ExpatError
import numpy as np
from torch.utils.data import DataLoader, Dataset, TensorDataset
import torch
from matplotlib import pyplot as plt

'''
MAKE DATASET
'''

def rmse(a,b):
    return np.sqrt(((a-b)**2).mean())
def mse(a,b):
    return ((a-b)**2).mean()
def mae(a,b):
    return np.abs(a-b).mean()
def atan(a,b):
    return (np.arctan(np.abs(a-b))).mean()
def quan(a:"numpyArray",q=10):
    return np.round(a/q)

class makeDataset(Dataset):

    def __init__(self):
        self.tx=torch.Tensor(0)
        self.ty=torch.Tensor(0)

    def tkzw(self,low,high,ori,res,width=6,height=3):
        xs=[]
        label=[]
        ny,nx=ori.shape #256,512
        for j in range(ny-height):
            for i in range(0,nx-width+1,2): #0,2,...,502,504  Total 253
                ind1=(j+height,i+width//2-1)
                ind2=(j+height,slice(i//2,i//2+(1+width)//2))
                ind3=(slice(j,j+height),slice(i,i+width))
                label.append(ori[ind1])
                xs.extend(np.ravel(ori[ind3]))
                xs.extend(np.repeat(np.ravel(low[ind2]),2)[:]/2)

        label=np.array(label).reshape((-1,1))
        xs=np.array(xs).reshape((-1,1,height+1,width))
#         xs=np.array(xs).reshape((-1,1,width,height+1))
        print("data increase",label.size)
        self.tx = torch.cat((self.tx,torch.Tensor(xs)),0)
        self.ty = torch.cat((self.ty,torch.Tensor(label)),0)

    def tkzw2(self,low,high,ori,res,width=6,height=3):
        xs=[]
        label=[]
        ny,nx=ori.shape #256,512
        for j in range(ny-height):
            for i in range(0,nx-width+1,2): #0,2,...,502,504  Total 253
                ind1=(j+height,slice(i+width//2-1,i+width//2+1))
                ind2=(j+height,slice(i//2,i//2+(1+width)//2))
                ind3=(slice(j,j+height),slice(i,i+width))
                label.extend(ori[ind1])
                xs.extend(np.ravel(res[ind3]))
                xs.extend(np.repeat(np.ravel(low[ind2]),2)[:]/np.sqrt(2))
                # xs[-4:-2]=label[-2:]
        label=np.array(label).reshape((-1,2))
        xs=np.array(xs).reshape((-1,1,height+1,width))
#         xs=np.array(xs).reshape((-1,1,width,height+1))

        # print("data increase",label.shape[0])
        self.tx = torch.cat((self.tx,torch.Tensor(xs)),0)
        self.ty = torch.cat((self.ty,torch.Tensor(label)),0)

    def my(self,low,high,ori,res,width=7,height=5):
        xs=[]
        label=[]
        ny,nx=ori.shape #256,512
        for j in range(ny-height): # 0~252  Total 253
            for i in range(0,nx-width+1,2): #0,2,...,502,504  Total 253
                ind1=(j+height,(i+width)//2-2)
                ind2=(j+height,slice(i//2,i//2+(1+width)//2))
                ind3=(slice(j,j+height),slice(i,i+width))
                label.append(high[ind1])
                xs.extend(np.ravel(res[ind3]))
                xs.extend(np.repeat(np.ravel(low[ind2]),2)[:-1])

        label=np.array(label).reshape((-1,1))
        xs=np.array(xs).reshape((-1,1,height+1,width))
        # xs=np.array(xs).reshape((-1,1,width,height+1))
        # print("- size append ",label.size)
        self.tx = torch.cat((self.tx,torch.Tensor(xs)),0)
        self.ty = torch.cat((self.ty,torch.Tensor(label)),0)

    def myN(self,low,high,ori,res,width=7,height=5):
        xs=[]
        label=[]
        RATIO=np.max(np.abs(low))
        ny,nx=ori.shape #256,512
        temp=np.empty((height+1,width))
        # low,high,ori,res=self.plus0001(low,high,ori,res)
        for j in range(ny-height): # 0~252  Total 253
            for i in range(0,nx-width+1,2): #0,2,...,502,504  Total 253
                ind1=(j+height,(i+width)//2-2)
                ind2=(j+height,slice(i//2,i//2+(1+width)//2))
                ind3=(slice(j,j+height),slice(i,i+width))

                label.append(high[ind1]/RATIO)

                temp[:height,:]=res[ind3]/RATIO*2**0.5
                temp[height:height+1,:]=np.repeat(np.ravel(low[ind2]),2)[:-1]/RATIO
                temp=temp
                xs.extend(np.ravel(temp))

        print("RATIO=np.max(np.abs(low)) =",RATIO)
        label=np.array(label).reshape((-1,1))
        xs=np.array(xs).reshape((-1,1,height+1,width))
        self.tx = torch.cat((self.tx,torch.Tensor(xs)),0)
        self.ty = torch.cat((self.ty,torch.Tensor(label)),0)

    def myN2(self,low,high,ori,res,width=7,height=3,normType=2):
        xs=[]
        label=[]
        norm=[]
        ny,nx=ori.shape #256,512
        RATIO=np.max(low)

        # normType=1
        for j in range(ny-height): # 0~252  Total 253
            for i in range(0,nx-width+1,2): #0,2,...,502,504  Total 253
                temp=np.empty((height+1,width))
                ind1=(j+height,(i+width)//2-2)
                ind2=(j+height,slice(i//2,i//2+(1+width)//2))
                ind3=(slice(j,j+height),slice(i,i+width))

                temp[:height,:]=res[ind3]
                temp[height:height+1,:]=(np.repeat(np.ravel(low[ind2]),2)[:-1])/2**0.5
                if normType==0:
                    RATIO= np.sum(np.abs(temp))
                    norm.append(RATIO)
                elif normType==1:
                    temp = (temp-np.min(temp))/(np.max(temp)-np.min(temp))
                    high[ind1] = high[ind1]/(np.max(temp)-np.min(temp))
                    RATIO=1
                elif normType==2:
                    pass
                elif normType==3:
                    RATIO=1

                # print("target=%03f, Rato=%03f"%(high[ind1]/RATIO,RATIO))

                label.append(high[ind1]/RATIO)
                xs.extend(np.ravel(temp/RATIO))

        label=np.array(label).reshape((-1,1))
        xs=np.array(xs).reshape((-1,1,height+1,width))
        self.tx = torch.cat((self.tx,torch.Tensor(xs)),0)
        self.ty = torch.cat((self.ty,torch.Tensor(label)),0)
        return np.array(norm)

    def plus0001(self,*arg):
        out=[]
        for array in arg:
            out.append(array+0.001)
        return out
    def minus0001(self,*arg):
        for array in arg:
            return array-0.001

    def plot(self,low,high,ori,res,s=10):
        print("ori.shape =",ori.shape)
        plt.figure(figsize=(s*2,s))
        plt.subplot(231)
        plt.title("low")
        plt.imshow(low,"gray")
        plt.subplot(232)
        plt.title("high")
        plt.imshow(high,"gray")
        plt.subplot(233)
        plt.title("res")
        plt.imshow(res,"gray")
        plt.subplot(234)
        plt.title("ori")
        plt.imshow(ori,"gray")
        plt.subplot(235)
        plt.title("ori-res")
        plt.imshow(ori-res,"gray")

    def out(self):
        print("Data amount =",self.ty.size()[0])
        return TensorDataset(self.tx,self.ty)

max_stars_repo_name: kamomehz/waveletCodingCNN

max_issues_repo_name: kamomehz/waveletCodingCNN

max_forks_repo_name: kamomehz/waveletCodingCNN