text:
import os
import numpy as np
import cv2
from calibrate_camera import calibrate_camera, get_calibration
from color_gradient_threshold import color_gradient_threshold
from perspective_transform import corners_unwarp

# define a class to store the characteristics of the lane detection
class Lanes:
    def __init__(self):
        self.frame = None
        self.frame_count = 0
        self.undist = None
        self.binary = None
        self.binary_warped = None
        self.Minv = None
        self.out_img = None
        self.color_warp = None
        self.color_warp2 = None
        self.result = None
        self.FLAG = True

        # was both lines detected in the last iteration
        self.detected = False
        self.invisible_count = 0

        # the left and right lane curve points
        self.left_x = None
        self.left_y = None
        self.right_x = None
        self.right_y = None

        # the coefficients of the fitted polynomials
        self.left_fit = None
        self.right_fit = None

        # radii of curvature
        self.left_rad_curve_metre = None
        self.left_rad_curve_pixel = None
        self.right_rad_curve_metre = None
        self.right_rad_curve_pixel = None

        # distance in meters of vehicle center from the line
        self.line_base_pos = None

        # store all left and right coefficients of the fitted polynomial
        self.all_left = [[],[],[]]
        self.all_right = [[],[],[]]

        # confidence levels
        self.left_conf = None
        self.right_conf = None

        # get objectpoints and imgpoints for camera calibration
        self.dist_pickle = None

    def get_calibration(self):
        self.dist_pickle = get_calibration(os.path.join(os.getcwd(), "advanced-lane-detection/camera_cal"))

    def get_binary_warped(self, img):
        self.frame = img
        # get undistorted img
        self.undist = calibrate_camera(self.frame, self.dist_pickle)
        # get binary threshold img
        self.binary = color_gradient_threshold(self.undist)
        # get perspective transformation img
        self.binary_warped, self.Minv = corners_unwarp(self.binary)

    def fit_polynomial(self):
        """
        main pipeline to fit polynomial into frames
        """
        # Find our lane pixels first
        if self.detected:
            self.invisible_count = 0
            self.search_around_poly()

        else:
            if self.frame_count != 0:
                self.invisible_count += 1

            if self.invisible_count > 5:
                self.all_left = [[],[],[]]
                self.all_right = [[],[],[]]
                self.invisible_count = 0

            self.find_lane_pixels()

        try:
            # Fit a second order polynomial to each using 'np.polyfit'
            self.left_fit = np.polyfit(self.left_y, self.left_x, 2)
            self.right_fit = np.polyfit(self.right_y, self.right_x, 2)
        except:
            self.FLAG = False
            pass

        if self.FLAG:
            # store fits
            self.all_left[0].append(self.left_fit[0])
            self.all_left[1].append(self.left_fit[1])
            self.all_left[2].append(self.left_fit[2])
            self.all_right[0].append(self.right_fit[0])
            self.all_right[1].append(self.right_fit[1])
            self.all_right[2].append(self.right_fit[2])

            # average the final past 10 accepted fits to produce best fit
            if len(self.all_right) > 10 and len(self.all_left) > 10:
                self.left_fit[0] = np.mean(self.all_left[0][-10:])
                self.left_fit[1] = np.mean(self.all_left[1][-10:])
                self.left_fit[2] = np.mean(self.all_left[2][-10:])
                self.right_fit[0] = np.mean(self.all_right[0][-10:])
                self.right_fit[1] = np.mean(self.all_right[1][-10:])
                self.right_fit[2] = np.mean(self.all_right[2][-10:])

            # Generate x and y values for plotting
            ploty = np.linspace(0, self.binary_warped.shape[0]-1, self.binary_warped.shape[0])
            left_fitx = self.left_fit[0]*ploty**2 + self.left_fit[1]*ploty + self.left_fit[2]
            right_fitx = self.right_fit[0]*ploty**2 + self.right_fit[1]*ploty + self.right_fit[2]

            # Create an image to draw the lines on
            warp_zero = np.zeros_like(self.binary_warped).astype(np.uint8)
            self.color_warp = np.dstack((warp_zero, warp_zero, warp_zero))
            self.color_warp2 = np.dstack((warp_zero, warp_zero, warp_zero))

            # Recast the x and y points into usable format for cv2.fillPoly()
            pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
            pts_right = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
            pts = np.hstack((pts_left, pts_right))

            # Generate a polygon to illustrate the search window area
            # And recast the x and y points into usable format for cv2.fillPoly()
            margin = 10
            margin_c = 3

            left_line_window1 = np.array([np.transpose(np.vstack([left_fitx-margin, ploty]))])
            left_line_window2 = np.array([np.flipud(np.transpose(np.vstack([left_fitx+margin, ploty])))])
            left_line_pts = np.hstack((left_line_window1, left_line_window2))

            right_line_window1 = np.array([np.transpose(np.vstack([right_fitx-margin, ploty]))])
            right_line_window2 = np.array([np.flipud(np.transpose(np.vstack([right_fitx+margin, ploty])))])
            right_line_pts = np.hstack((right_line_window1, right_line_window2))

            center_line_window1 = np.array([np.transpose(np.vstack([(left_fitx + right_fitx)/2-margin_c, ploty]))])
            center_line_window2 = np.array([np.flipud(np.transpose(np.vstack([(left_fitx + right_fitx)/2 + margin_c, ploty])))])
            center_line_pts = np.hstack((center_line_window1, center_line_window2))

            self.out_img[self.left_y, self.left_x] = [255, 0, 0]
            self.out_img[self.right_y, self.right_x] = [0, 0, 255]

            # Draw the lane onto the warped blank image
            cv2.fillPoly(self.color_warp, np.int_([pts]), (0, 255, 0))
            cv2.fillPoly(self.color_warp2, np.int_([left_line_pts]), (255, 255, 255))
            cv2.fillPoly(self.color_warp2, np.int_([right_line_pts]), (255, 255, 255))
            cv2.fillPoly(self.color_warp2, np.int_([center_line_pts]), (255, 255, 0))
            self.color_warp = cv2.addWeighted(self.color_warp, 1.0, self.color_warp2, 1.0, 0)

            # Warp the blank back to original image space using inverse perspective matrix (Minv)
            img = self.undist
            img_size = (img.shape[1], img.shape[0])
            new_out = cv2.warpPerspective(self.color_warp, self.Minv, img_size)
            # Combine the result with the original image
            self.result = cv2.addWeighted(self.undist, 1, new_out, 1.0, 0.0)
            # get radius of curvature
            self.measure_curvature(ploty)
        else:
            self.result = self.undist
            self.FLAG = True

    def find_lane_pixels(self):
        """
        function to find line using sliding windows
        """
        # Take a histogram of the bottom half of the image
        histogram = np.sum(self.binary_warped[self.binary_warped.shape[0]//2:,:], axis=0)
        # Create an output image to draw on and visualize the result
        self.out_img = np.dstack((self.binary_warped, self.binary_warped, self.binary_warped))
        # Find the peak of the left and right halves of the histogram
        # These will be the starting point for the left and right lines
        midpoint = np.int(histogram.shape[0]//2)
        leftx_base = np.argmax(histogram[:midpoint])
        rightx_base = np.argmax(histogram[midpoint:]) + midpoint

        # HYPERPARAMETERS
        # Choose the number of sliding windows
        nwindows = 9
        # Set the width of the windows +/- margin
        margin = 120
        # Set minimum number of pixels found to recenter window
        minpix = 60

        # Set height of windows - based on nwindows above and image shape
        window_height = np.int(self.binary_warped.shape[0]//nwindows)
        # Identify the x and y positions of all nonzero pixels in the image
        nonzero = self.binary_warped.nonzero()
        nonzeroy = np.array(nonzero[0])
        nonzerox = np.array(nonzero[1])
        # Current positions to be updated later for each window in nwindows
        leftx_current = leftx_base
        rightx_current = rightx_base

        # Create empty lists to receive left and right lane pixel indices
        left_lane_inds = []
        right_lane_inds = []

        # Step through the windows one by one
        for window in range(nwindows):
            # Identify window boundaries in x and y (and right and left)
            win_y_low = self.binary_warped.shape[0] - (window+1)*window_height
            win_y_high = self.binary_warped.shape[0] - window*window_height
            win_xleft_low = leftx_current - margin
            win_xleft_high = leftx_current + margin
            win_xright_low = rightx_current - margin
            win_xright_high = rightx_current + margin

            # Draw the windows on the visualization image
            cv2.rectangle(self.out_img,(win_xleft_low,win_y_low), (win_xleft_high,win_y_high),(0,255,0), 2)
            cv2.rectangle(self.out_img,(win_xright_low,win_y_low), (win_xright_high,win_y_high),(0,255,0), 2)

            # Identify the nonzero pixels in x and y within the window #
            good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) &
                (nonzerox >= win_xleft_low) &  (nonzerox < win_xleft_high)).nonzero()[0]
            good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) &
                (nonzerox >= win_xright_low) &  (nonzerox < win_xright_high)).nonzero()[0]

            # Append these indices to the lists
            left_lane_inds.append(good_left_inds)
            right_lane_inds.append(good_right_inds)

            # If you found > minpix pixels, recenter next window on their mean position
            if len(good_left_inds) > minpix:
                leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
            if len(good_right_inds) > minpix:
                rightx_current = np.int(np.mean(nonzerox[good_right_inds]))

        # Concatenate the arrays of indices (previously was a list of lists of pixels)

        left_lane_inds = np.concatenate(left_lane_inds)
        right_lane_inds = np.concatenate(right_lane_inds)

        # Extract left and right line pixel positions
        self.left_x = nonzerox[left_lane_inds]
        self.left_y = nonzeroy[left_lane_inds]
        self.right_x = nonzerox[right_lane_inds]
        self.right_y = nonzeroy[right_lane_inds]
        self.detected = True

        # evaluate polynomial
        self.evaluate_polynomial(left_lane_inds, right_lane_inds)

    def search_around_poly(self):
        # HYPERPARAMETER
        # Choose the width of the margin around the previous polynomial to search
        # The quiz grader expects 100 here, but feel free to tune on your own!
        margin = 100

        # Grab activated pixels
        nonzero = self.binary_warped.nonzero()
        nonzeroy = np.array(nonzero[0])
        nonzerox = np.array(nonzero[1])

        ### TO-DO: Set the area of search based on activated x-values ###
        ### within the +/- margin of our polynomial function ###
        ### Hint: consider the window areas for the similarly named variables ###
        ### in the previous quiz, but change the windows to our new search area ###
        left_lane_inds = ((nonzerox > (self.left_fit[0]*(nonzeroy**2) + self.left_fit[1]*nonzeroy +
                        self.left_fit[2] - margin)) & (nonzerox < (self.left_fit[0]*(nonzeroy**2) +
                        self.left_fit[1]*nonzeroy + self.left_fit[2] + margin)))
        right_lane_inds = ((nonzerox > (self.right_fit[0]*(nonzeroy**2) + self.right_fit[1]*nonzeroy +
                        self.right_fit[2] - margin)) & (nonzerox < (self.right_fit[0]*(nonzeroy**2) +
                        self.right_fit[1]*nonzeroy + self.right_fit[2] + margin)))

        # Again, extract left and right line pixel positions
        self.left_x = nonzerox[left_lane_inds]
        self.left_y = nonzeroy[left_lane_inds]
        self.right_x = nonzerox[right_lane_inds]
        self.right_y = nonzeroy[right_lane_inds]
        self.detected = True

        # evaluate polynomial
        self.evaluate_polynomial(left_lane_inds, right_lane_inds)

    def measure_curvature(self, ploty):
        '''
        Calculates the curvature of polynomial functions in meters.
        '''
        # Define conversions in x and y from pixels space to meters
        ym_per_pix = 30/720 # meters per pixel in y dimension
        xm_per_pix = 3.7/700 # meters per pixel in x dimension

        # Define y-value where we want radius of curvature
        # We'll choose the maximum y-value, corresponding to the bottom of the image
        y_eval = np.max(ploty)

        # Calculation of R_curve (radius of curvature pixel)
        self.left_rad_curve_pixel = ((1 + (2*self.left_fit[0]*y_eval + self.left_fit[1])**2)**1.5) / np.absolute(2*self.left_fit[0])
        self.right_rad_curve_pixel = ((1 + (2*self.right_fit[0]*y_eval + self.right_fit[1])**2)**1.5) / np.absolute(2*self.right_fit[0])

        # Fit new polynomials to x,y in world space
        left_fit_cr = np.polyfit(self.left_y*ym_per_pix, self.left_x*xm_per_pix, 2)
        right_fit_cr = np.polyfit(self.right_y*ym_per_pix, self.right_x*xm_per_pix, 2)

        # Calculation of R_curve (radius of curvature metres)
        self.left_rad_curve_metre = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
        self.right_rad_curve_metre = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])

        # Distance from center
        car_y = self.result.shape[0]
        left_line_location = self.left_fit[0]*car_y**2 + self.left_fit[1]*car_y + self.left_fit[2]
        right_line_location = self.right_fit[0]*car_y**2 + self.right_fit[1]*car_y + self.right_fit[2]
        car_center = xm_per_pix*(self.result.shape[1] / 2)
        lane_center = xm_per_pix*((left_line_location + right_line_location) / 2)
        self.line_base_pos = car_center - lane_center

        # Add to image
        font = cv2.FONT_HERSHEY_DUPLEX
        cv2.putText(self.result,'Left Curvature: %.2f m'%(self.left_rad_curve_metre),(60,60), font, 1,(255,255,255),2,cv2.LINE_AA)
        cv2.putText(self.result,'Right Curvature: %.2f m'%(self.right_rad_curve_metre),(60,100), font, 1,(255,255,255),2,cv2.LINE_AA)
        cv2.putText(self.result,'Distance from Center : %.2f m'%(self.line_base_pos),(60,140), font, 1,(255,255,255),2,cv2.LINE_AA)

    def evaluate_polynomial(self, left_lane_inds, right_lane_inds):
        # lane prediction confidence
        self.left_conf = 1. - abs(11000. - len(left_lane_inds))/ 15000.
        self.right_conf = 1. - abs(3000. - len(right_lane_inds))/ 8000.
        if len(left_lane_inds) < 11000:
            self.left_conf = 1.

        if self.left_conf < 0:
            self.left_conf = 0
            self.detected = False

        if len(right_lane_inds) < 3000:
            self.right_conf = 1.

        if self.right_conf < 0:
            self.right_conf = 0
            self.detected = False

        # check lane width
        if np.mean(self.right_x) - np.mean(self.left_x) > 1000 or np.mean(self.right_x) - np.mean(self.left_x) < 720:
            self.detected = False

        # check if pixels exist for each lane
        if self.right_x.size == 0 or self.right_y.size == 0:
            self.detected = False

        if self.left_x.size == 0 or self.left_y.size == 0:
            self.detected = False

max_stars_repo_name: sieniven/self-driving-project

max_issues_repo_name: sieniven/self-driving-project

max_forks_repo_name: sieniven/self-driving-project


text:
#!/usr/bin/env python3

import pandas as pd
import numpy as np
from typing import Generator
from sklearn.linear_model import SGDClassifier

def _create_train_test_idx(
    n: int,
    frac_training: float=2./3.,
    shuffle: bool=False,
    seed: int=1234
    ) -> tuple[float]:
    """Create training and test indeces for a dataset
    with 'n' samples.

    The indeces indicate which of the 'n' data samples
    should be dedicated as trainig and test data.

    Args:
        n (int): How many samples are in the dataset?
        frac_training (float, optional): Fraction of
            samples used for training. Defaults to 2./3..
        shuffle (bool, optional): Whether to shuffle
            index before creating the training / test split.
            Defaults to False.
        seed (int, optional): Random seed for shuffling.
            Defaults to 1234.

    Returns:
        tuple of training and test indeces
    """
    idx = np.arange(n)
    if shuffle:
        np.random.seed(seed)
        np.random.shuffle(idx)
    split = int(frac_training * idx.size)
    return (idx[:split], idx[split:])

def _yield_cv_train_val_idx(
    n: int,
    folds: int=3,
    shuffle: bool=False,
    seed: int=1234
    ) -> Generator[int, tuple[float], None]:
    """Yield cross-validation indeces for training and validation
    datasets for each cross-validation fold, given a dataset
    with 'n' samples.

    Args:
        n (int): How many samples are in the dataset?
        folds (int, optional): Number of cross-validation folds.
            Defaults to 3.
        shuffle (bool, optional): Whether to shuffle
            index before creating the training / test split.
            Otherwise data are separated into chunks in sequential order.
            Defaults to False.
        seed (int, optional): Random seed for shuffling.
            Defaults to 1234.

    Raises:
        ValueError: 'n' needs to larger than 'folds'

    Yields:
        Generator[int, tuple[float], None]: training and validation
            index for each fold
    """
    idx = np.arange(n)
    if idx.size < folds:
        raise ValueError(
            'n needs to be larger than folds.'
        )
    if shuffle:
        np.random.seed(seed)
        np.random.shuffle(idx)
    n_val = int(n / folds)
    for fold in range(folds):
        idx_val = idx[n_val*fold:n_val*(fold+1)]
        idx_train = np.array([i for i in idx if i not in idx_val])
        yield (idx_train, idx_val)

def cross_validate(
    X: np.ndarray,
    y: np.array,
    clf: SGDClassifier,
    steps: int,
    folds: int=3,
    ) -> pd.DataFrame:
    """Cross-validate classifier performance.

    Args:
        X (np.ndarray): Input data with shape (samples x features)
        y (np.array): Data labels with shape (samples)
        clf (sklearn classifier): Classifier to use. Classifier needs
            to be  compatible with sklearn'S classifier specification
            and implement {.score, .partial_fit} methods.
            Defaults to sklearns 'SGDClassifier'.
        steps (int): How many training steps?
            Each step represents one call to .partial_fit
        folds (int, optional): Number of cross-validation
            folds. Defaults to 3.

    Returns:
        pd.DataFrame: Cross-validation results
    """

    cv_results = []
    cv_iterator = _yield_cv_train_val_idx(
        n=y.size,
        folds=folds
    )
    for i, (idx_cv_train, idx_cv_val) in enumerate(cv_iterator):

        for _ in range(steps):
            clf.partial_fit(
                X=X[idx_cv_train],
                y=y[idx_cv_train],
                classes=np.unique(y)
            )

        score = clf.score(
            X=X[idx_cv_val],
            y=y[idx_cv_val],
        )

        res = {
            'CV-Fold': i,
            'Regularisation (Alpha)': clf.get_params()['alpha'],
            'Learning Rate': clf.get_params()['eta0'],
            'Steps': steps,
            'Random State': clf.get_params()['random_state'],
            'Accuracy' : score
        }

        cv_results.append(
            pd.DataFrame(res, index=[i])
        )

    cv_results = pd.concat(cv_results)
    cv_results['Mean Accuracy'] = cv_results['Accuracy'].mean()

    return cv_results

def evaluate_hyper_params_effects(
    X: np.ndarray,
    y: np.array,
    classifier: SGDClassifier,
    hyper_params: dict,
    folds: int=3,
    n_jobs=2,
    default_classifier_args: dict={
        'alpha': 0.0001,
        'eta0': 0.1,
        'steps': 50,
        'random_state': 999
        }
    ) -> pd.DataFrame:
    """Evaluate effect of idividually varying {alpha, eta0, steps}
    (while holding everything else constant ) on classifier
    performance in {X, y}.

    Classifier performance is estimated by the use of cross-validation.

    Args:
        X (np.ndarray): Input data with shape (samples x features)
        y (np.array): Data labels with shape (samples)
        classifier (sklearn classifier): Classifier to use. Classifier needs to be
            compatible with sklearn's classifier specification and
            implement {.score, .partial_fit} methods.
            Defaults to sklearns 'SGDClassifier'.
        hyper_params (dict): Hyper-parameter values to
            evaluate; can be created by calling
            hyper.sample_hyper_params()
        folds (int, optional): Number of cross-validation folds.
            Defaults to 3.
        n_jobs (int, optional): How many parallel jobs for fitting?
            Defaults to 2.
        default_classifier_args (dict, optional): Default classifier arguments
            that are used while individual factors are varied.
            Defaults to { 'alpha': 0.0001,
                          'eta0': 0.1,
                          'steps': 50,
                          'random_state': 999 }.

    Returns:
        pd.DataFrame: Cross-validation results
    """

    i, results = 0, []
    for par, values in hyper_params.items():

        for val in values:

            clf_args = dict(default_classifier_args)
            clf_args[par] = val

            clf = classifier(
                **{k: clf_args[k] for k in ["alpha", "eta0", "random_state"]},
                shuffle=True,
                learning_rate='constant',
                max_iter=clf_args["steps"],
                n_jobs=n_jobs
            )

            cv_results = cross_validate(
                X=X,
                y=y,
                clf=clf,
                steps=clf_args["steps"],
                folds=folds
            )

            cv_results = cv_results.set_index(np.arange(i,i+folds))
            i += 1

            cv_results["Varied"] = ' '.join(par.capitalize().split('_')) if '_' in par else par.capitalize()
            if par == 'eta0':
                cv_results["Varied"] = "Learning rate"
            elif par == 'alpha':
                cv_results["Varied"] = "Regularisation"

            results.append(cv_results)

    return pd.concat(results)

ext: py

max_stars_repo_name: athms/reproducible-modelling

max_issues_repo_name: athms/reproducible-modelling

max_forks_repo_name: athms/reproducible-modelling

reason: import numpy


text:
import numpy as np
import os

from optimization_problems.smr_problem import SMR_optProblem

import logging
import os.path

import numpy as np
from poap.controller import BasicWorkerThread, ThreadController

from pySOT.experimental_design import SymmetricLatinHypercube
from pySOT.optimization_problems import Hartmann6
from pySOT.strategy import LCBStrategy, SOPStrategy
from pySOT.surrogate import GPRegressor
from pySOT.surrogate import CubicKernel, LinearTail, RBFInterpolant

def main():
    if not os.path.exists("./logfiles"):
        os.makedirs("logfiles")
    if os.path.exists("./logfiles/example_lower_confidence_bounds.log"):
        os.remove("./logfiles/example_lower_confidence_bounds.log")
    logging.basicConfig(filename="./logfiles/example_lower_confidence_bounds.log", level=logging.INFO)

    num_threads = 1
    max_evals = 80

    smr1 = SMR_optProblem()
    gp = GPRegressor(dim=smr1.dim, lb=smr1.lb, ub=smr1.ub)
    rbf = RBFInterpolant(dim=smr1.dim, lb=smr1.lb, ub=smr1.ub, kernel=CubicKernel(), tail=LinearTail(smr1.dim))
    slhd = SymmetricLatinHypercube(dim=smr1.dim, num_pts=10 * (smr1.dim + 1))

    # Create a strategy and a controller
    controller = ThreadController()
    # controller.strategy = LCBStrategy(
    #     max_evals=max_evals, opt_prob=smr1, exp_design=slhd, surrogate=gp, asynchronous=True
    # )
    controller.strategy = SOPStrategy(
        max_evals=max_evals,
        opt_prob=smr1,
        exp_design=slhd,
        surrogate=rbf,
        asynchronous=False,
        ncenters=num_threads,
        batch_size=num_threads,
    )

    print("Number of threads: {}".format(num_threads))
    print("Maximum number of evaluations: {}".format(max_evals))
    print("Strategy: {}".format(controller.strategy.__class__.__name__))
    print("Experimental design: {}".format(slhd.__class__.__name__))
    print("Surrogate: {}".format(gp.__class__.__name__))

    # Launch the threads and give them access to the objective function
    for _ in range(num_threads):
        worker = BasicWorkerThread(controller, smr1.eval)
        controller.launch_worker(worker)

    # Run the optimization strategy
    result = controller.run()

    print("Best value found: {0}".format(result.value))
    print(
        "Best solution found: {0}\n".format(
            np.array_str(result.params[0], max_line_width=np.inf, precision=5, suppress_small=True)
        )
    )

def main2():
    smr1 = SMR_optProblem()
    x0 = np.array( [0.25/3600, 0.70/3600, 1.0/3600, 1.10/3600, 1.80/3600, 2.50e5, 50.00e5, -60+273.15] )*smr1.reg
    print(smr1.eval(1.0*x0))

if __name__ == "__main__":
    main2()

max_stars_repo_name: lf-santos/cosbo

max_issues_repo_name: lf-santos/cosbo

max_forks_repo_name: lf-santos/cosbo


text:
#!/usr/bin/env python3

import numpy as np

THICK = 4 # Membrane thickness
CONST = 2.3999632297286531

# This goes for any ..[PO][PO] lipid... Just change the topology
POPC =       "NC3 PO4 GL1 GL2 C1A D2A C3A C4A C1B C2B C3B C4B".split()
X = []
Y = []
Z = np.array([  6,  5,  4,  4,  3,  2,  1,  0,  3,  2,  1,  0]) * 20.0 / 7

PDBLINE = "ATOM  {:5}  {:3} POPC {:4}    {:8.3f}{:8.3f}{:8.3f}  1.00  0.00"

TRUNC4 = int(1e4)
TRUNC5 = int(1e5)

def _point(y, phi):
    r = np.sqrt(1-y*y)
    return np.cos(phi)*r, y, np.sin(phi)*r

def points_on_sphere(n):
    return np.array([_point((2.*k+1)/n-1, k*CONST) for k in range(n)])

def vesicle(radius, inner, outer):

    for leaflet in (-1, 1):
        pos = radius + leaflet*(0.1 + Z) + np.random.random(12)/10
        lip = list(zip(POPC, pos))

        atom = 1
        res = 1
        for xyz in points_on_sphere(inner):
            for a, r in lip:
                x,y,z = r*xyz
                line = PDBLINE.format(atom%TRUNC5, a, res%TRUNC4, x, y, z)
                print(line)
                atom += 1
            res += 1

def main(argv=None):

    radius = float(argv[1])
    inner = int(argv[2])
    outer = int(argv[3])

    vesicle(radius, inner, outer)

    return 0

if __name__ == "__main__":
    import sys
    sys.exit(main(sys.argv))

max_stars_repo_name: Tsjerk/vesicle

max_issues_repo_name: Tsjerk/vesicle

max_forks_repo_name: Tsjerk/vesicle


text:
import os.path as osp
import numpy as np
import pandas as pd
from collections import OrderedDict
from cytokit.ops import op as cytokit_op
from cytokit import io as cytokit_io
from cytokit.cytometry.cytometer import DEFAULT_CELL_INTENSITY_PREFIX
from cytokit.function import data as function_data
from sklearn.linear_model import HuberRegressor, Ridge, Lasso, LinearRegression
from sklearn.neighbors import KNeighborsRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.preprocessing import PolynomialFeatures, StandardScaler
from sklearn.pipeline import Pipeline
from skimage import transform
from cytokit import SEED
import logging
logger = logging.getLogger(__name__)

DEFAULT_FILTER_PARAMS = dict(percentile_range=[.01, .99], features=['cell_size'], max_cells=250000)
DEFAULT_FEATURE_PARAMS = dict(
    tile=dict(type='polynomial', degree=2),
    region=dict(type='polynomial', degree=2),
    index=dict(type='polynomial', degree=2)
)
DEFAULT_MODEL_PARAMS = dict(type='ls')
DEFAULT_PREDICTION_PARAMS = dict(step_size=10)
MODELS = {
    'ls': {'factory': lambda args: LinearRegression(**{**dict(normalize=True), **args})},
    'huber': {'factory': lambda args: HuberRegressor(**args)},
    'ridge': {'factory': lambda args: Ridge(**{**dict(normalize=True), **args})},
    'gbr': {'factory': lambda args: GradientBoostingRegressor(random_state=SEED, **args)},
    'knn': {'factory': lambda args: KNeighborsRegressor(**args)},
    'mlp': {'factory': lambda args: Pipeline([('scale', StandardScaler()), ('est', MLPRegressor(**args))])}
}

def _get_params(key, params, defaults):
    # Copy default values
    res = dict(defaults)
    # If params present, overwrite default values
    if key in params:
        res.update(params[key])
    return res

def _get_coordinate_features(region_shape, tile_shape, step=1):
    ri, ci = np.arange(0, region_shape[0], step), np.arange(0, region_shape[1], step)
    nr, nc = len(ri), len(ci)

    # Produce 2D array with 2 cols, ry and rx
    X_region = np.transpose([np.repeat(ri, nc), np.tile(ci, nr)])

    # Create tile index features by dividing region y/x coordinates by tile shape
    X_index = np.floor_divide(X_region, tile_shape)

    # Mod region coordinates by tile height and width to give coordinates within tile
    X_tile = np.mod(X_region, tile_shape)

    # Return all 4 features as a data frame as well as number of (potentially sampled
    # coordinate points along each dimension)
    # NOTE: All field names here must match fields in cytometry data frames
    X = pd.DataFrame(np.hstack((X_region, X_index, X_tile)), columns=['ry', 'rx', 'tile_y', 'tile_x', 'y', 'x'])

    return X, (nr, nc)

class IlluminationCorrection(cytokit_op.CytokitOp):

    def __init__(
            self, config,
            filter_params=DEFAULT_FILTER_PARAMS,
            feature_params=DEFAULT_FEATURE_PARAMS,
            model_params=DEFAULT_MODEL_PARAMS,
            prediction_params=DEFAULT_PREDICTION_PARAMS,
            intensity_prefix=DEFAULT_CELL_INTENSITY_PREFIX):
        super().__init__(config)

        params = config.illumination_correction_params

        # Get mapping of source to target channel names, and sort by source channel
        self.channel_mapping = pd.Series(params['channel_mapping']).sort_index()

        # Extra parameters relevant to each step
        self.filter_params = _get_params('filter_params', params, filter_params)
        self.feature_params = _get_params('feature_params', params, feature_params)
        self.model_params = _get_params('model_params', params, model_params)
        self.prediction_params = _get_params('prediction_params', params, prediction_params)

        # Extract prefix of intensity signals within cytometry data to be used for correction
        # (these correspond to cell or nucleus intensities, though either prefix is configurable)
        self.intensity_prefix = params.get('intensity_prefix', intensity_prefix)

        # Extra commonly used shape information from experiment configuration
        self.region_shape = (
            self.config.region_height * self.config.tile_height,
            self.config.region_width * self.config.tile_width
        )
        self.tile_shape = self.config.tile_height, self.config.tile_width

        ############################
        # Configuration Validation #
        ############################

        if len(self.filter_params['percentile_range']) != 2:
            raise ValueError(
                '`percentile_range` in filter_params must be 2 item list (given = {})'
                .format(self.filter_params['percentile_range'])
            )
        for v in self.filter_params['percentile_range']:
            if not 0 <= v <= 1:
                raise ValueError(
                    '`percentile_range` value in filter_params values must be in [0, 1] (given = {})'
                    .format(v)
                )

        if self.prediction_params['step_size'] < 1:
            raise ValueError(
                '`step_size` in prediction_params must be > 1 (given = {})'
                .format(self.prediction_params['step_size'])
            )
        if self.model_params['type'] not in MODELS:
            raise ValueError(
                '`type` in model_params type must be one of {} (given = "{}")'
                .format(list(MODELS.keys()), self.model_params['type'])
            )

        self.data = None
        self.data_saved = False

    def _get_filter_masks(self, df, features):
        """Get masks for each filtered feature

        Args:
            df: Cytometry dataframe
            features: List of features to perform percentile filtering on
        Returns:
            DataFrame with column for each filter feature and values equal to boolean mask
                (where true means that the record is WITHIN the desired range)
        """
        # Compute low/high thresholds as dataframe like:
        #     feat_1 feat_2
        # p_lo    .1     10
        # p_hi   5.3    100
        ranges = df[features].quantile(q=self.filter_params['percentile_range'])

        # Stack masks horizontally as dataframe
        return pd.concat([
            df[c].between(ranges[c].iloc[0], ranges[c].iloc[1])
            for c in ranges
        ], axis=1)

    def _get_filter_features(self, df):
        feats = []
        for feat in self.filter_params['features']:
            # Test membership for feature as provided
            if feat in df:
                feats.append(feat)
                continue
            # Test membership when prefixed by channel markers
            ch_feat = self.intensity_prefix + feat
            if ch_feat in df:
                feats.append(ch_feat)
                continue
            raise ValueError('Feature "{}" not present in cytometry data'.format(feat))
        return feats

    def _prepare_prediction_features(self, df):

        # Feature group preparation function
        def prep(X, group):
            params = self.feature_params[group]
            if params is None:
                return None
            type = params.get('type')
            if not type or type == 'none':
                return None
            if type != 'polynomial':
                raise ValueError(
                    'Feature preparation type "{}" is not valid (must be None or "polynomial")'.format(type))
            return PolynomialFeatures(degree=params['degree'], include_bias=False).fit_transform(X)

        # Get features for each group (region and tile) or None, if either is disabled
        features = [x for x in [
            prep(df[['ry', 'rx']], 'region'),
            prep(df[['tile_y', 'tile_x']], 'index'),
            prep(df[['y', 'x']], 'tile')
        ] if x is not None]
        if len(features) == 0:
            raise ValueError('At least one of region or tile features must be enabled')

        # Concatenate features, if necessary
        features = features[0] if len(features) == 1 else np.hstack(tuple(features))
        return features

    def _estimate_model(self, X, y):
        # Fit regression model used to represent illumination surface
        args = self.model_params.get('args', {})
        est = MODELS[self.model_params['type']]['factory'](args)
        return est.fit(X, y)

    def get_illumination_models(self, region_index, df):
        n = len(df)
        ests = OrderedDict()
        for channel in self.channel_mapping.index:

            # Set list of features to filter on
            features = self._get_filter_features(df)

            # Restrict cell data to only records matching the given filters
            dfm = df[self._get_filter_masks(df, features).all(axis=1).values].copy()

            # If necessary, downsample modeling data to improve performance
            if len(dfm) > self.filter_params['max_cells']:
                dfm = dfm.sample(n=self.filter_params['max_cells'], random_state=SEED)

            if len(dfm) == 0:
                raise ValueError(
                    'Cytometry data empty after application of feature filters for channel {}'.format(channel))

            # Extract spatial cell features and prediction target
            X = self._prepare_prediction_features(dfm)
            y = dfm[self.intensity_prefix + channel]

            logger.debug(
                'Building illumination model for region %s, channel "%s" using %s cells (%s originally) [feature '
                'array shape = %s, response shape = %s]',
                region_index, channel, len(dfm), n, X.shape, y.shape
            )

            if np.isclose(y.mean(), 0):
                raise ValueError(
                    'Average {} channel intensity for region {} (across {} cells) is ~0, '
                    'making illumination correction impossible'
                    .format(region_index, channel, len(df))
                )
            y = y / y.mean()

            # Fit regression model used to represent illumination surface
            ests[channel] = self._estimate_model(X, y)
        return ests

    def _estimate_image(self, est):
        # Get prepared coordinate features for entire region and run predictions
        step_size = self.prediction_params['step_size']
        df, image_shape = _get_coordinate_features(self.region_shape, self.tile_shape, step_size)
        y = est.predict(self._prepare_prediction_features(df))

        # Some models may intentionally produce nans for pixels too far away from foreground
        # objects so set those values to 1 as this will lead to no change in correction
        y = np.where(np.isnan(y), 1., y)

        # Reshape from 1D to 2D image shape (which will be shape of entire region or a downsampled version of it)
        y = y.reshape(image_shape)

        # Upsample if necessary
        if step_size > 1:
            y = transform.resize(
                y, self.region_shape, mode='constant', order=1,
                anti_aliasing=False, preserve_range=False
            )

        assert y.shape == self.region_shape, \
            'Expecting result with shape {} but was {}'.format(self.region_shape, y.shape)
        return y.astype(np.float32)

    def get_illumination_images(self, ests):
        """Get an illumination image by predicting the intensity at each pixel across a region

        Args:
            ests: Illumination models
        Returns:
            Dictionary mapping source channels to a 3D float32 array with shape equal to
            (region height * tile height, region width * tile width)
        """
        imgs = OrderedDict()
        for channel, est in ests.items():
            imgs[channel] = self._estimate_image(est)

        if len(imgs) > 0:
            img = list(imgs.values())[0]
            logger.debug('Resulting illumination image array shape = %s (dtype = %s)', img.shape, img.dtype)

        return imgs

    def prepare_region_data(self, output_dir):
        if self.data is not None:
            return
        logger.info('Preparing illumination correction models')

        # Use whatever cytometry data was generated, whether it was for best
        # z planes, all planes, or a specific one
        df = function_data.get_cytometry_data(output_dir, self.config, mode='all')
        n = len(df)
        if df is None or n == 0:
            raise ValueError('Cytometry data cannot be empty in order to use it for illumination correction')

        self.data = {}
        for region_index, region_data in df.groupby('region_index'):
            ests = self.get_illumination_models(region_index, region_data)
            imgs = self.get_illumination_images(ests)
            self.data[region_index] = (imgs, ests)

    def save_region_data(self, output_dir):
        if self.data is None:
            raise ValueError('Region data cannot be saved until `prepare_region_data` is called')
        if self.data_saved:
            return None
        path = None
        for region_index, (imgs, ests) in self.data.items():
            # Stack 2D images on first axis to give 3D array
            img = np.stack(list(imgs.values()), 0)
            assert img.ndim == 3, 'Expecting 3D array, got shape {}'.format(img.shape)
            path = osp.join(output_dir, cytokit_io.get_illumination_function_path(region_index))
            cytokit_io.save_image(path, img)

        self.data_saved = True
        return osp.dirname(path or '')

    def _run(self, tile, tile_indices):
        # Get illumination image for this region
        imgs = self.data[tile_indices.region_index][0]

        # Get tile type information prior to multiplication
        dtype = tile.dtype
        dinfo = np.iinfo(dtype)

        # Determine starting offsets for tile slicing
        r, c = tile_indices.tile_y * self.config.tile_height, tile_indices.tile_x * self.config.tile_width

        # Loop through channel mapping and apply each adjustment
        tile = tile.astype(np.float32)
        for source_channel, target_channel in self.channel_mapping.to_dict().items():

            logger.debug('Applying correction from source channel "%s" to target "%s"', source_channel, target_channel)

            # Extract matching patch in illumination image
            img = imgs[source_channel][r:(r + self.config.tile_height), c:(c + self.config.tile_width)]

            # If application is across all channels, do single matrix division
            if target_channel == 'all':
                # Divide correction across HW dimensions of tile (must first add leading dims to match 5D tile)
                tile = tile / img[np.newaxis, np.newaxis, np.newaxis, :, :]

            # Otherwise, find target channel's coordinates within
            # tile and apply adjustment only to that slice
            else:
                cyc, ch = self.config.get_channel_coordinates(target_channel)
                tile[cyc, :, ch, :, :] = tile[cyc, :, ch, :, :] / img[np.newaxis, :, :]

        # Clip to range of original data type and convert
        tile = tile.clip(dinfo.min, dinfo.max).astype(dtype)

        return tile

    def save(self, tile_indices, output_dir, tile):
        # Overwrite the original preprocessed tile with corrected version
        path = cytokit_io.get_processor_img_path(
            tile_indices.region_index, tile_indices.tile_x, tile_indices.tile_y)
        cytokit_io.save_tile(osp.join(output_dir, path), tile, config=self.config)
        return path

max_stars_repo_name: lcontento/cytokit

max_issues_repo_name: lcontento/cytokit

max_forks_repo_name: lcontento/cytokit


text:
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
# pylint: disable=no-else-return
"""The Python interface to the Relay reference interpreter."""
from __future__ import absolute_import

import numpy as np

import tvm._ffi
from tvm.runtime import container, Object

from . import _backend
from .. import _make, analysis
from ... import nd
from ..expr import Tuple, RefCreate, Call, Constant, GlobalVar, const
from ..function import Function
from ..scope_builder import ScopeBuilder

@tvm._ffi.register_object("relay.ConstructorValue")
class ConstructorValue(Object):
    def __init__(self, tag, fields, constructor):
        self.__init_handle_by_constructor__(_make.ConstructorValue, tag, fields, constructor)

@tvm._ffi.register_object("relay.RefValue")
class RefValue(Object):
    def __init__(self, value):
        self.__init_handle_by_constructor__(_make.RefValue, value)

def _arg_to_ast(mod, arg):
    if isinstance(arg, nd.NDArray):
        return Constant(arg.copyto(nd.cpu(0)))
    elif isinstance(arg, container.ADT):
        return Tuple([_arg_to_ast(mod, field) for field in arg])
    elif isinstance(arg, tuple):
        return Tuple([_arg_to_ast(mod, field) for field in arg])
    elif isinstance(arg, RefValue):
        return RefCreate(_arg_to_ast(mod, arg.value))
    elif isinstance(arg, ConstructorValue):
        return Call(mod.get_constructor(arg.tag), [_arg_to_ast(mod, field) for field in arg.fields])
    elif isinstance(arg, np.ndarray):
        return Constant(nd.array(arg))
    elif isinstance(arg, Constant):
        return arg
    else:
        return const(arg)

class Executor(object):
    """An abstract interface for executing Relay programs."""

    def _convert_args(self, expr, args, kwargs):
        """
        Convert the combination of arguments and keyword arguments
        into a sequence of arguments that may be passed to
        a Relay evaluator.

        We first provide all positional arguments, and then attempt
        to fill in the remaining arguments using the keyword arguments. We
        map the keyword arguments to the corresponding parameters, if there
        is an ambiguity between positional and keyword arguments this
        procedure will raise an error.

        Parameters
        ----------
        expr: relay.Expr
            The expression to evaluate

        args: List[tvm.nd.NDArray]
            The arguments to pass to the evaluator.

        kwargs: Dict[str, tvm.NDArrray]
            The keyword arguments to pass to the evaluator.

        Returns:
            args: List[tvm.nd.NDArray]
                The new arguments with all keyword arguments placed in the correct slot.
        """
        assert expr is not None

        if not kwargs:
            return args

        if kwargs and not isinstance(expr, Function):
            raise Exception(
                "can only supply keyword parameters for a " "relay.Function, found {0}".format(expr)
            )

        params = expr.params
        param_names = [p.name_hint for p in params]
        num_of_args = len(args)

        cargs = list(args)[:]
        for i, name in enumerate(param_names):
            if i < num_of_args:
                if kwargs.get(name):
                    raise Exception(
                        "duplicate argument supplied in "
                        "both positional args (at position: {0}), "
                        "and keyword argument (with name: {1})".format(i, name)
                    )
            else:
                cargs.append(kwargs[name])

        if len(cargs) != len(params):
            raise Exception(
                "insufficient arguments, expected "
                "{0}, provided {1}".format(len(cargs), len(params))
            )

        return tuple(cargs)

    def _make_executor(self, expr=None):
        """
        Construct a Python function that implements the evaluation
        of expression.

        Parameters
        ----------
        expr: Optional[relay.Expr]
            The Relay expression to execute.

        Returns
        -------
        executor: function,
            A Python function which implements the behavior of `expr`.
        """
        raise NotImplementedError()

    def evaluate(self, expr=None, binds=None):
        """
        Evaluate a Relay expression on the executor.

        Parameters
        ----------
        expr: Optional[tvm.relay.Expr]
            The expression to evaluate.

        binds: Optional[Map[tvm.relay.Var, tvm.relay.Expr]]
            Additional binding of free variable.

        Returns
        -------
        val : Union[function, Object]
            The evaluation result.
        """
        if binds:
            scope_builder = ScopeBuilder()
            for key, value in binds.items():
                scope_builder.let(key, _arg_to_ast(self.mod, value))
            scope_builder.ret(expr)
            expr = scope_builder.get()

        if not expr:
            return self._make_executor()

        if isinstance(expr, Function):
            assert not analysis.free_vars(expr)

        if isinstance(expr, (Function, GlobalVar)):
            return self._make_executor(expr)

        # normal expression evaluated by running a function.
        # TODO(mbs): This should really be type rather than syntax driven.
        func = Function([], expr)
        return self._make_executor(func)()

class Interpreter(Executor):
    """
    Simple interpreter interface.

    Parameters
    ----------
    mod : tvm.IRModule
        The module to support the execution.

    device : Device
        The runtime device to run the code on.

    target : tvm.Target
        The target option to build the function.

    CAUTION: Despite the API the module is prepared upon each call to evaluate
    rather than once in create_executor.
    That is:
    .. code-block:: python

        executor = relay.create_executor(kind="debug", mod=module)
        a = executor.evaluate(expr)(args1)
        b = executor.evaluate(expr)(args2)

    will prepare all the bindings in module twice. For efficiency, try to hoist
    calls to evaluate as high as possible, preferably immediately after create_executor:
    .. code-block:: python

        func = relay.create_executor(kind="debug", mod=module).evaluate(expr)
        a = func(args1)
        b = func(args2)
    """

    def __init__(self, mod, device, target):
        self.mod = mod
        self.device = device
        self.target = target

    def _make_executor(self, expr=None):
        if expr is None or isinstance(expr, GlobalVar):
            assert self.mod is not None

        if expr is None:
            # A missing expr denotes 'main' in the given module.
            expr = self.mod.get_global_var("main")

        # Evaluate expr to a packed function we can efficiently re-apply
        # to Relay arguments.
        func = _backend.EvalFunction(self.mod, expr, self.device, self.target)

        def _apply_args(*args, **kwargs):
            if isinstance(expr, GlobalVar):
                # When expanding args, look inside the actual global definition so kwargs
                # can be matched.
                args = self._convert_args(self.mod[expr.name_hint], args, kwargs)
            else:
                args = self._convert_args(expr, args, kwargs)
            # Reflect python arguments up into Relay.
            relay_args = []
            for arg in args:
                relay_args.append(_arg_to_ast(self.mod, arg))
            # Apply func to Relay args
            return func(relay_args)

        return _apply_args


text:
from  keras_models_factory.utils4 import hash_array_sum as hash_array
import numpy as np
import pandas as pd
from  keras_models_factory.utils4 import number_of_digits_post_decimal
class TestUtils4(object):

  def test_hash_array_np(self):
    nb_samples = 5

    np.random.seed(0)
    x1 = np.random.randn(nb_samples)
    np.random.seed(0)
    x2 = np.random.randn(nb_samples)
    assert (x1==x2).all()

    h1 = hash_array(x1)
    h2 = hash_array(x2)
    assert h1==h2

    x3 = x2.copy()
    h3 = hash_array(x3)
    assert h3==h2

  def test_hash_array_pd(self):
    nb_samples = 5

    np.random.seed(0)
    S1 = pd.Series(np.random.randn(nb_samples))
    S2 = pd.Series(np.random.randn(nb_samples))
    x1 = pd.concat({'main': S1, 'new': S2}, axis=1)
    x2 = x1.copy()
    assert (x1==x2).all().all()

    h1 = hash_array(x1)
    h2 = hash_array(x2)
    assert h1==h2

  def test_number_of_digits_post_decimal(self):
    assert number_of_digits_post_decimal(0.7576) == 4
    assert number_of_digits_post_decimal(3.14159) == 5
    assert number_of_digits_post_decimal(0.0169) == 18 # should have been 4!


text:
"""
If payment has not been received within 45 days of the close of a deal, the vendor sends a reminder to the customer.  By region, what is the current (where "today" is December 11, 2018) total value of contracts to be collected that are more than 45 days past close?  More than 90 days?  More than 135 days?  How does this compare to contracts closed in 2017?

More than 135 days past close:
APAC 13134.0
Africa 4167.0
EMEA 37223.0
Latin America 3823.0
North America 31025.0
"""

import pandas as pd
import numpy as np

import datetime
from datetime import datetime, timedelta

contracts = pd.DataFrame(pd.read_csv('contracts.csv'))
accounts = pd.DataFrame(pd.read_csv('accounts.csv'))

N = 135
date_N_days_ago = pd.to_datetime('12/11/2018') - timedelta(days=N)

contracts['closingDate'] = pd.to_datetime(contracts['closingDate'])
contracts['paymentDate'] = pd.to_datetime(contracts['paymentDate'])

regionGrouped = accounts.groupby(by = 'region')
# print(regionGrouped.size())
# region
# APAC              596
# Africa            190
# EMEA             1031
# Latin America     202
# North America     981
# dtype: int64

group_APAC = regionGrouped.get_group("APAC")
# # # print(group_APAC.head())
   # # # accountID region partnerInvolved
# # # 5     3oy2wf   APAC             Yes
# # # 8     0w3ynj   APAC              No
# # # 11    kt2n1f   APAC              No
# # # 15    ta2z0d   APAC              No
# # # 19    g8rycf   APAC              No

group_Africa = regionGrouped.get_group("Africa")
group_EMEA = regionGrouped.get_group("EMEA")
group_LatinAmerica = regionGrouped.get_group("Latin America")
group_NorthAmerica = regionGrouped.get_group("North America")

lst = group_APAC['accountID']
temporary = pd.DataFrame()
result = pd.Series()
val_Size = 0

for an_item in lst:
	previous = val_Size
	contracts_index = contracts['contractID'].str.contains('DKU-'+an_item)
	found_indices = contracts[contracts_index]
	temporary = temporary.append(found_indices)

temporary.paymentDate = temporary.paymentDate.replace('', np.nan)
temp_blankDate = temporary[temporary.paymentDate.isnull()]
fortyFive = temp_blankDate[temp_blankDate['closingDate']>= pd.to_datetime(date_N_days_ago)]
val_Size = fortyFive.contractSize * fortyFive.contractLength
total_Value = sum(val_Size)
print('APAC', total_Value)

###############################################################

lst = group_Africa['accountID']
temporary = pd.DataFrame()
result = pd.Series()
val_Size = 0

for an_item in lst:
	previous = val_Size
	contracts_index = contracts['contractID'].str.contains('DKU-'+an_item)
	found_indices = contracts[contracts_index]
	temporary = temporary.append(found_indices)

temporary.paymentDate = temporary.paymentDate.replace('', np.nan)
temp_blankDate = temporary[temporary.paymentDate.isnull()]
fortyFive = temp_blankDate[temp_blankDate['closingDate']>= pd.to_datetime(date_N_days_ago)]
val_Size = fortyFive.contractSize * fortyFive.contractLength
total_Value = sum(val_Size)
print('Africa', total_Value)

###############################################################
lst = group_EMEA['accountID']
temporary = pd.DataFrame()
result = pd.Series()
val_Size = 0

for an_item in lst:
	previous = val_Size
	contracts_index = contracts['contractID'].str.contains('DKU-'+an_item)
	found_indices = contracts[contracts_index]
	temporary = temporary.append(found_indices)

temporary.paymentDate = temporary.paymentDate.replace('', np.nan)
temp_blankDate = temporary[temporary.paymentDate.isnull()]
fortyFive = temp_blankDate[temp_blankDate['closingDate']>= pd.to_datetime(date_N_days_ago)]
val_Size = fortyFive.contractSize * fortyFive.contractLength
total_Value = sum(val_Size)
print('EMEA', total_Value)

###############################################################

lst = group_LatinAmerica['accountID']
temporary = pd.DataFrame()
result = pd.Series()
val_Size = 0

for an_item in lst:
	previous = val_Size
	contracts_index = contracts['contractID'].str.contains('DKU-'+an_item)
	found_indices = contracts[contracts_index]
	temporary = temporary.append(found_indices)

temporary.paymentDate = temporary.paymentDate.replace('', np.nan)
temp_blankDate = temporary[temporary.paymentDate.isnull()]
fortyFive = temp_blankDate[temp_blankDate['closingDate']>= pd.to_datetime(date_N_days_ago)]
val_Size = fortyFive.contractSize * fortyFive.contractLength
total_Value = sum(val_Size)
print('Latin America', total_Value)

###############################################################

lst = group_NorthAmerica['accountID']
temporary = pd.DataFrame()
result = pd.Series()
val_Size = 0

for an_item in lst:
	previous = val_Size
	contracts_index = contracts['contractID'].str.contains('DKU-'+an_item)
	found_indices = contracts[contracts_index]
	temporary = temporary.append(found_indices)

temporary.paymentDate = temporary.paymentDate.replace('', np.nan)
temp_blankDate = temporary[temporary.paymentDate.isnull()]
fortyFive = temp_blankDate[temp_blankDate['closingDate']>= pd.to_datetime(date_N_days_ago)]
val_Size = fortyFive.contractSize * fortyFive.contractLength
total_Value = sum(val_Size)
print('North America', total_Value)

max_stars_repo_name: aanahahn/cautious-succotash

max_issues_repo_name: aanahahn/cautious-succotash

max_forks_repo_name: aanahahn/cautious-succotash


text:
import numpy as np
import cv2, sys

# SPARSE OPTICAL FLOW
def sparseOpticalFlow():
    # use 0 for webcam capturing
    # cap = cv2.VideoCapture(0)

    cap = cv2.VideoCapture('test/Pedestrian overpass.mp4')

    # params for ShiTomasi corner detection
    feature_params = dict( maxCorners = 100,
                           qualityLevel = 0.3,
                           minDistance = 7,
                           blockSize = 7 )
    # Parameters for lucas kanade optical flow
    lk_params = dict( winSize  = (15,15),
                      maxLevel = 2,
                      criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03))
    # Create some random colors
    color = np.random.randint(0,255,(100,3))
    # Take first frame and find corners in it
    ret, old_frame = cap.read()
    old_gray = cv2.cvtColor(old_frame, cv2.COLOR_BGR2GRAY)
    p0 = cv2.goodFeaturesToTrack(old_gray, mask = None, **feature_params)
    # Create a mask image for drawing purposes
    mask = np.zeros_like(old_frame)
    while(1):
        ret,frame = cap.read()
        if ret is True:
            frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        else:
            break
        # calculate optical flow
        p1, st, err = cv2.calcOpticalFlowPyrLK(old_gray, frame_gray, p0, None, **lk_params)
        # Select good points
        good_new = p1[st==1]
        good_old = p0[st==1]
        # draw the tracks
        for i,(new,old) in enumerate(zip(good_new,good_old)):
            a,b = new.ravel()
            c,d = old.ravel()
            mask = cv2.line(mask, (a,b),(c,d), color[i].tolist(), 2)
            frame = cv2.circle(frame,(a,b),5,color[i].tolist(),-1)
        img = cv2.add(frame,mask)
        cv2.imshow('frame',img)
        k = cv2.waitKey(30) & 0xff
        if k == 27:
            break
        # Now update the previous frame and previous points
        old_gray = frame_gray.copy()
        p0 = good_new.reshape(-1,1,2)
    cv2.destroyAllWindows()
    cap.release()

# DENSE OPTICAL FLOW
def denseOpticalFlow():
    # use 0 for webcam capturing
    # cap = cv2.VideoCapture(0)

    cap = cv2.VideoCapture('test/Pedestrian overpass.mp4')
    ret, frame1 = cap.read()
    prvs = cv2.cvtColor(frame1,cv2.COLOR_BGR2GRAY)
    hsv = np.zeros_like(frame1)
    hsv[...,1] = 255

    while(1):
        ret, frame2 = cap.read()
        next = cv2.cvtColor(frame2,cv2.COLOR_BGR2GRAY)
        flow = cv2.calcOpticalFlowFarneback(prvs,next, None, 0.5, 3, 15, 3, 5, 1.2, 0)
        mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])

        hsv[...,0] = ang*180/np.pi/2
        hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX)

        # print(np.sum(mag[100:300, 100:300]))
        if (np.sum(mag)> 100000):
            print('motion detected')

        bgr = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR)
        cv2.imshow('frame2',bgr)

        k = cv2.waitKey(30) & 0xff
        if k == 27:
            break
        elif k == ord('s'):
            cv2.imwrite('opticalfb.png',frame2)
            cv2.imwrite('opticalhsv.png',bgr)
        prvs = next

    cap.release()
    cv2.destroyAllWindows()

if __name__=='__main__':
    if sys.argv[1] == 'dense':
        denseOpticalFlow()
    elif sys.argv[1] == 'sparse':
        sparseOpticalFlow()
    else:
        print('Enter sparse for sparse optical flow, else dense')


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
from audioop import avg

import sys

# sys.path.append('add you root path') # Root path

import os
import argparse
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim.lr_scheduler import CosineAnnealingLR
from model import PointNet, DGCNN
from PRNet.UMichPathology.pathology_classifier import PathologyClassifier
from PRNet.UMichPathology.train_process import get_batch_preds
from SAMCNet import SpatialDGCNN
import numpy as np
from torch.utils.data import DataLoader
from util import cal_loss, IOStream
import sklearn.metrics as metrics
import pandas as pd
import time as time
from datetime import datetime
import uuid # For filename hashing
import dataset, transforms
import torchvision.transforms
import pickle
import warnings
warnings.filterwarnings("ignore")

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

def _init_():
    if not os.path.exists('checkpoints'):
        os.makedirs('checkpoints')
    if not os.path.exists('checkpoints/'+args.exp_name):
        os.makedirs('checkpoints/'+args.exp_name)
    if not os.path.exists('checkpoints/'+args.exp_name+'/'+'models'):
        os.makedirs('checkpoints/'+args.exp_name+'/'+'models')
    # os.system('cp main.py checkpoints'+'/'+args.exp_name+'/'+'main.py.backup')
    # os.system('cp model.py checkpoints' + '/' + args.exp_name + '/' + 'model.py.backup')
    # os.system('cp util.py checkpoints' + '/' + args.exp_name + '/' + 'util.py.backup')
    # os.system('cp data.py checkpoints' + '/' + args.exp_name + '/' + 'data.py.backup')

def train(args, io):
    '''
    See __main__ for args parser
    '''
    args.transformed_samples = 5
    print(f'Using {args.dataset} dataset')
    if args.dataset == 'region':
        in_file = 'datasets/BestClassification_July2021_14Samples.tsv'
        df = dataset.read_tsv(in_file, use_margins=False)
        label_name = 'Pathology'
        output_channels = 3

    elif args.dataset == 'margin':
        in_file = 'datasets/BestClassification_July2021_14Samples.tsv'
        train_path = 'datasets/Interface/train.csv'
        df = dataset.read_dataset(in_file, train_path)
        label_name = 'Status'
        output_channels = 2

    elif args.dataset == 'tumor_core':
        in_file = 'datasets/BestClassification_July2021_14Samples.tsv'
        train_path = 'datasets/Tumor/train.csv'
        df = dataset.read_dataset(in_file, train_path)
        label_name = 'Status'
        output_channels = 2

    elif args.dataset == 'diseases':
        in_file = 'datasets/disease.csv'
        train_path = 'datasets/Disease/train.csv'
        df = dataset.read_dataset(in_file, train_path, dataset='disease')
        label_name = 'Status'
        output_channels = 2

    class_labels = list(df[label_name].cat.categories)
    num_classes = len(df.Phenotype.cat.categories)

    train_set = dataset.PathologyDataset(dataset=df, label_name=label_name, num_points=args.num_points,
                transforms=torchvision.transforms.Compose([transforms.RotateData(n_rotations=30)]), transformed_samples=args.transformed_samples)

    sampler = dataset.get_sampler(train_set.dataset, label_name, class_labels, args.transformed_samples)
    train_loader = torch.utils.data.DataLoader(train_set, batch_size=args.batch_size, collate_fn=dataset.collate_fn_pad, drop_last=True, sampler=sampler)

    #Try to load models
    if args.model == 'pointnet':
        model = PointNet(args, output_channels=output_channels).to(device)
    elif args.model == 'dgcnn':
        model = DGCNN(args, output_channels=output_channels).to(device)
    elif args.model == 'sdgcnn':
        model = SpatialDGCNN(args, num_classes=num_classes, output_channels=output_channels).to(device)
    elif args.model == 'srnet':
        args.min_grid_scale=1
        args.max_grid_scale=100
        args.grid_scale_count=10
        args.neighborhood_distance=50
        args.feature_type_count=12
        args.lr=0.0001 # the learning rate is different from the others
        args.regularization_weight=100
        args.diff_weight=1e-3
        args.sampling_ratio=1
        args.output_classes=output_channels
        model = PathologyClassifier(args.feature_type_count, args.grid_scale_count, output_channels).to(device)
    else:
        raise Exception("Not implemented")
    print(str(model))
    if args.model != 'srnet': # srnet doesn't support multi-GPUs
        model = nn.DataParallel(model)

    if args.use_sgd:
        opt = optim.SGD(model.parameters(), lr=args.lr*100, momentum=args.momentum, weight_decay=1e-4)
    else:
        opt = optim.Adam(model.parameters(), lr=args.lr, weight_decay=1e-4) # NOTE: GAT uses lr=0.005, weight_decay=5e-4

    scheduler = CosineAnnealingLR(opt, args.epochs, eta_min=args.lr)

    criterion = cal_loss

    best_train_acc = 0
    best_valid_acc = 0
    columns = ["train_average", "train_overall", "train_loss", "train_times", "train_pred", "train_true"]
    '''Output result to CSV'''
    df = pd.DataFrame(columns=columns)

    for epoch in range(args.epochs):
        train_average, train_losses, train_overall, train_times = [], [], [], []
        start_time = time.time()
        ####################
        # Train
        ####################
        train_loss = 0.0
        count = 0.0
        model.train()
        if args.model == 'srnet':
            model.epoch = epoch
        else:
            model.module.epoch = epoch
        train_pred = []
        train_true = []

        valid_loss = 0.0
        valid_pred = []
        valid_true = []

        for data, label, ls in train_loader:
            data, label = data.to(device), label.to(device).squeeze()
            batch_size = data.size()[1] #
            if args.model == 'srnet': # SRNet has unique get-pred calculation as well as customized loss function.
                opt.zero_grad()
                batch_pred_ys, batch_pred_prs, batch_true_ys = get_batch_preds(model, args, data, label, ls)
                pr_diff_sum = 0
                for pred_pr in batch_pred_prs:
                    pr_diff_sum += 1 / (torch.norm(pred_pr, 1) + 1e-5)
                paras = torch.cat([x.view(-1) for x in model.parameters()])
                regularization = torch.norm(paras, 1) / (paras.shape[0] + 1)

                ce = criterion(batch_pred_ys, batch_true_ys)
                loss = ce + args.regularization_weight * regularization + args.diff_weight * pr_diff_sum
                loss.backward()
                opt.step()

                preds = batch_pred_ys.max(dim=1)[1]
                count += batch_size
                train_loss += loss.item() * batch_size
                train_true.append(batch_true_ys.cpu().numpy())
                train_pred.append(preds.detach().cpu().numpy())
            else:
                data = data.permute(1, 2, 0)
                opt.zero_grad()
                logits, _ , _ = model(data)
                loss = criterion(logits, label)
                loss.backward()
                opt.step()

                preds = logits.max(dim=1)[1]
                count += batch_size
                train_loss += loss.item() * batch_size
                train_true.append(label.cpu().numpy())
                train_pred.append(preds.detach().cpu().numpy())
        train_true = np.concatenate(train_true)
        train_pred = np.concatenate(train_pred)
        train_acc = metrics.accuracy_score(train_true, train_pred)
        scheduler.step()
        if train_acc > best_train_acc:
            best_train_acc = train_acc
            torch.save(model.state_dict(), f'checkpoints/{args.exp_name}/models/{args.exp_name}.t7')

        avg_per_class_train_acc = metrics.balanced_accuracy_score(train_true, train_pred)
        train_times.append(time.time()-start_time)
        train_overall.append(train_acc)
        train_average.append(avg_per_class_train_acc)
        train_losses.append(train_loss*1.0/count)

        io.cprint(f'{datetime.now().strftime("%H:%M:%S")}: Epoch {epoch}')
        outstr = 'Train loss: %.6f, train acc: %.6f, train avg acc: %.6f' % (
                  train_loss*1.0/count, train_acc, avg_per_class_train_acc)
        io.cprint(outstr)
        torch.cuda.empty_cache()

        csv = {
            'train_average':  train_average,
            'train_overall':  train_overall,
            'train_loss':  train_losses,
            'train_times': train_times,
            'train_pred': [train_pred],
            'train_true': [train_true]
        }

        df = df.append(csv, ignore_index=True)
        # saving the dataframe
        df.to_csv(args.save_train_results + args.exp_name + "_results.csv") #
        valid_df = pd.DataFrame()
        model.eval()
        ####################
        # Validation
        ####################
        if args.dataset == 'tumor_core':
            in_file = 'datasets/BestClassification_July2021_14Samples.tsv'
            sub_path = 'datasets/Tumor/validation.csv'
            valid_df = dataset.read_dataset(in_file, sub_path)
            label_name = 'Status'
            output_channels = 2
            num_points = 1024

        elif args.dataset == 'margin':
            in_file = 'datasets/BestClassification_July2021_14Samples.tsv'
            sub_path = 'datasets/Interface/validation.csv'
            valid_df = dataset.read_dataset(in_file, sub_path)
            label_name = 'Status'
            output_channels = 2
            num_points = 1024

        elif args.dataset == 'diseases':
            in_file = 'datasets/disease.csv'
            sub_path = 'datasets/Disease/validation.csv'
            valid_df = dataset.read_dataset(in_file, sub_path, dataset="disease")
            label_name = 'Status'
            output_channels = 2
            num_points = 1024

        class_labels = list(valid_df[label_name].cat.categories)
        num_classes = len(valid_df.Phenotype.cat.categories)

        validation_set = dataset.PathologyDataset(dataset=valid_df, label_name=label_name, num_points=args.num_points,
        transforms=torchvision.transforms.Compose([transforms.RotateData(n_rotations=10)]), transformed_samples=args.transformed_samples)

        sampler = dataset.get_sampler(validation_set.dataset, label_name, class_labels, args.transformed_samples)
        valid_loader = torch.utils.data.DataLoader(validation_set, batch_size=args.batch_size, collate_fn=dataset.collate_fn_pad, drop_last=True, sampler=sampler)

        for data, label, ls in valid_loader:
            data, label = data.to(device), label.to(device).squeeze()
            if args.model == 'srnet':
                batch_pred_ys, batch_pred_prs, batch_true_ys = get_batch_preds(model, args, data, label, ls)
                preds = batch_pred_ys.max(dim=1)[1]
                valid_true.append(batch_true_ys.cpu().numpy())
                valid_pred.append(preds.detach().cpu().numpy())
            else:
                data = data.permute(1, 2, 0)
                logits, _, _ = model(data)
                preds = logits.max(dim=1)[1]
                valid_true.append(label.cpu().numpy())
                valid_pred.append(preds.detach().cpu().numpy())

        valid_true = np.concatenate(valid_true)
        valid_pred = np.concatenate(valid_pred)
        valid_acc = metrics.accuracy_score(valid_true, valid_pred)
        avg_per_class_acc = metrics.balanced_accuracy_score(valid_true, valid_pred)
        outstr = 'Validation :: valid acc: %.6f, valid avg acc: %.6f'%(valid_acc, avg_per_class_acc)
        if valid_acc>best_valid_acc:
            best_valid_acc = valid_acc
            torch.save(model.state_dict(), f'checkpoints/{args.exp_name}/models/validation_{args.exp_name}.t7')

        io.cprint(outstr)
        torch.cuda.empty_cache()

def test(args, io):

    if args.dataset == 'tumor_core':
        in_file = 'datasets/BestClassification_July2021_14Samples.tsv'
        tumor_core_file = 'datasets/Tumor/train.csv'
        df = dataset.read_dataset(in_file, tumor_core_file)
        # label_name = 'Tumor_Core'
        label_name = 'Status'
        output_channels = 2
        num_points = 1024

    elif args.dataset == 'margin':
        in_file = 'datasets/BestClassification_July2021_14Samples.tsv'
        sub_path = 'datasets/Interface/train.csv'
        df = dataset.read_dataset(in_file, sub_path)
        label_name = 'Status'
        output_channels = 2
        num_points = 1024

    elif args.dataset == 'disease':
        in_file = 'datasets/disease.csv'
        sub_path = 'datasets/Disease/train.csv'
        df = dataset.read_dataset(in_file, sub_path, dataset='disease')
        label_name = 'Status'
        output_channels = 2
        num_points = 1024

    num_classes = len(df.Phenotype.cat.categories)

    test_set = dataset.PathologyDataset(dataset=df, label_name=label_name, num_points=args.num_points)
    test_loader = torch.utils.data.DataLoader(test_set, batch_size=args.test_batch_size, collate_fn=dataset.collate_fn_pad, drop_last=True)
    device = torch.device("cuda" if args.cuda else "cpu")

    #Try to load models
    model = SpatialDGCNN(args, num_classes=num_classes, output_channels=output_channels).to(device)
    model = nn.DataParallel(model)

    model.load_state_dict(torch.load(args.model_path))
    model = model.eval()
    test_acc = 0.0
    test_true = []
    test_pred = []
    attention_1 =  {}
    attention_2 =  {}
    attention_3 =  {}
    attention_4 =  {}
    conv_layer_5 = {}
    point_pairs_add = {}
    mini_batch = {}
    count = 0
    for data, label, _ in test_loader:
        data, label = data.to(device), label.to(device).squeeze()
        data = data.permute(1, 2, 0)
        logits, a_x_4, point_pairs = model(data)
        ############################################################################
        '''
        Extracting features for other classifiers
        '''
        # attention_1[count], attention_2[count], attention_3[count], conv_layer_5[count] =  a_x_1, a_x_2, a_x_3, x_5
        attention_4[count] =  a_x_4
        mini_batch[count] = data
        point_pairs_add[count] =  point_pairs
        ############################################################################
        print(torch.cuda.memory_allocated()/10**9, torch.cuda.memory_reserved()/10**9)
        count+=1
        preds = logits.max(dim=1)[1]
        test_true.append(label.cpu().numpy())
        test_pred.append(preds.detach().cpu().numpy())
    test_true = np.concatenate(test_true)
    test_pred = np.concatenate(test_pred)
    test_acc = metrics.accuracy_score(test_true, test_pred)
    avg_per_class_acc = metrics.balanced_accuracy_score(test_true, test_pred)
    outstr = 'Test :: test acc: %.6f, test avg acc: %.6f'%(test_acc, avg_per_class_acc)

    pickle.dump(attention_4, open( args.save_features + "att4_train_2.p", "wb" ))
    pickle.dump(point_pairs_add, open( args.save_features + "stack_add_train_2.p", "wb" ))
    pickle.dump(mini_batch, open( args.save_features + "mini_batches_2.p", "wb" ))

    io.cprint(outstr)

if __name__ == "__main__":
    # Training settings
    parser = argparse.ArgumentParser(description='Spatial DGCNN')

    parser.add_argument('--exp_name', type=str, default='exp', metavar='N',
                        help='Name of the experiment')
    parser.add_argument('--dataset', type=str, default='tumor_core', metavar='N',
                        choices=['region', 'margin', 'tumor_core', 'diseases'],
                        help='Dataset to use, [region, margin, tumor_core, diseases]')
    parser.add_argument('--model', type=str, default='sdgcnn', metavar='N',
                        choices=['pointnet', 'dgcnn', 'sdgcnn', 'srnet'],
                        help='Model to use, [pointnet, dgcnn, sdgcnn]')
    parser.add_argument('--train_model', type=str, default='entire_model', metavar='N',
                        choices=['PE', 'neighbor_att', 'self_att', 'PE_self_att', 'PE_neighbor_att', 'self_att_neighbor_att', 'entire_model'],
                        help='the model needs to be trained for ablation studies')
    parser.add_argument('--self_neighbor', type=bool,  default=True,
                        help='use self as first neighbor in top k')

    parser.add_argument('--neighbor', type=bool,  default=True,
                        help='If consider neighbor attention')

    parser.add_argument('--use_pe', type=bool,  default=True,
                        help='use positional encoding')
    parser.add_argument('--batch_size', type=int, default=8, metavar='batch_size',
                        help='Size of batch)')
    parser.add_argument('--test_batch_size', type=int, default=7, metavar='batch_size',
                        help='Size of batch)')
    parser.add_argument('--epochs', type=int, default=1, metavar='N',
                        help='number of epochs to train ')
    parser.add_argument('--use_sgd', type=bool, default=False,
                        help='Use SGD')
    parser.add_argument('--lr', type=float, default=0.001, metavar='LR',
                        help='learning rate (default: 0.001, 0.1 if using sgd)')
    parser.add_argument('--momentum', type=float, default=0.9, metavar='M',
                        help='SGD momentum (default: 0.9)')
    parser.add_argument('--no_cuda', type=bool, default=False,
                        help='enables CUDA training')
    parser.add_argument('--seed', type=int, default=1, metavar='S',
                        help='random seed (default: 1)')
    parser.add_argument('--eval', type=bool,  default=False,
                        help='evaluate the model')
    parser.add_argument('--num_points', type=int, default=1024,
                        help='num of points to use')
    parser.add_argument('--num_heads', type=int, default=1,
                        help='num of attn heads to use. Set to 0 for no attn')
    parser.add_argument('--dropout', type=float, default=0.5,
                        help='dropout rate')
    parser.add_argument('--emb_dims', type=int, default=1024, metavar='N',
                        help='Dimension of embeddings')
    parser.add_argument('--k', type=int, default=10, metavar='N',
                        help='Num of nearest neighbors to use')
    parser.add_argument('--model_path', type=str, default='', metavar='N',
                        help='Pretrained model path')
    parser.add_argument('--PE_dim', type=float, default=32, metavar='N',
                        help='output dimmension of positional encoding (if use_pe fasle) this should be 4')
    parser.add_argument('--save_features', type=str, default='', metavar='N',
                        help='Save extracted features path')
    parser.add_argument('--save_train_results', type=str, default='/home/luo00042/M2SSD/SAMCNet/Results/', metavar='N',
                        help='save training results (e.g., loss, acc, ... )')

    args = parser.parse_args()

    _init_()

    io = IOStream('checkpoints/' + args.exp_name + '/run.log')
    io.cprint(str(args))

    args.cuda = not args.no_cuda and torch.cuda.is_available()
    torch.manual_seed(args.seed)
    if args.cuda:
        io.cprint(
            'Using GPU : ' + str(torch.cuda.current_device()) + ' from ' + str(torch.cuda.device_count()) + ' devices')
        torch.cuda.manual_seed(args.seed)
    else:
        io.cprint('Using CPU')

    if not args.eval:
        train(args, io)
    else:
        test(args, io) # Not implemented (yet)


text:
from abc import ABCMeta, abstractmethod
import numpy as np
import tensorflow as tf
import tqdm

from neural_nets import ActorCriticNetwork
from losses import actor_critic_loss
from mixin import AgentMixin

class BaseA2CAgent(AgentMixin, metaclass=ABCMeta):

    @abstractmethod
    def __init__(self, name, model, loss_fn,
                 optimizer, learning_rate,
                 gamma, critic_discount,
                 entropy_beta, input_shape,
                 output_dims, pretrained_weights,
                 tf_summary_writer=True,):

        self.name = name

        if tf_summary_writer:
            self.create_tf_summary_writer()

        self.model = model(output_dims=output_dims)
        self.model.build(input_shape=input_shape)

        if pretrained_weights:
            self.model.load_weights(pretrained_weights)

        # optimizer and loss function
        learning_rate_fn = tf.keras.optimizers.schedules.PolynomialDecay(
           initial_learning_rate=learning_rate,
           decay_steps=1024,
           end_learning_rate=learning_rate * 0.1,
           power=1,
           cycle=False,
        )
        self.loss_fn = loss_fn
        self.optimizer = optimizer(learning_rate)

        self.gamma = gamma
        self.critic_discount = critic_discount
        self.entropy_beta = entropy_beta

    def _train(self, env, num_episodes, num_steps):

        # initialize accumulators
        self.errors = []
        self.num_actions = []
        self.rewards = []

        best_reward = float('-inf')

        state = env.reset()

        pbar = tqdm.tqdm(range(num_episodes), desc=' episodes')
        while 1:

            actions, rewards, values, dones = np.empty((4, num_steps), dtype=np.float32)
            states = np.empty((num_steps, len(env.compound)), dtype=np.float32)

            for step in range(num_steps):
                policy_dist, value = self.model(state[np.newaxis], training=True)
                policy_dist, value = policy_dist.numpy()[0], value.numpy()[0]

                action = np.random.choice(len(policy_dist), p=policy_dist)

                actions[step] = action
                values[step] = value

                next_state, reward, done = env.step(action)

                rewards[step] = reward
                dones[step] = int(done)
                states[step] = state

                if done:
                    if self.tf_summary_writer:
                        with self.tf_summary_writer.as_default():
                            tf.summary.scalar('errors', env.episode_error, step=pbar.n)
                            tf.summary.scalar('rewards', np.sum(env.episode_rewards), step=pbar.n)
                            tf.summary.scalar('num_actions', len(env.episode_actions), step=pbar.n)

                    self.errors.append(env.episode_error)
                    self.num_actions.append(len(env.episode_actions))
                    self.rewards.append(np.sum(env.episode_rewards))
                    state = env.reset()
                    pbar.update(1)
                    pbar.set_description(
                      "actions {:.3f} : rewards {:.3f}".format(
                        np.mean(self.num_actions[-100:])-1, np.mean(self.rewards[-100:])))
                else:
                    state = next_state.copy()

            _, value = self.model(state[np.newaxis], training=True)

            q_values = np.append(np.zeros_like(rewards), value.numpy()[0], axis=-1)
            # Qt = rt+1 + gamma * Vt+1
            for t in reversed(range(len(rewards))):
                q_values[t] = rewards[t] + self.gamma * q_values[t+1] * (1 - dones[t])
            q_values = q_values[:-1]

            advantages = q_values - values

            with tf.GradientTape() as tape:
                policy_dists, values = self.model(states, training=True)
                loss = self.loss_fn(
                    y_true=[(actions, advantages), q_values], y_pred=[policy_dists, values],
                    critic_discount=self.critic_discount, entropy_beta=self.entropy_beta)
            gradients = tape.gradient(loss, self.model.trainable_variables)
            self.optimizer.apply_gradients(zip(gradients, self.model.trainable_variables))

            if pbar.n >= num_episodes:
                break

class A2CAgent(BaseA2CAgent):

    def __init__(self,
                 name='A2C',
                 model=ActorCriticNetwork,
                 loss_fn=actor_critic_loss,
                 optimizer=tf.keras.optimizers.Adam,
                 learning_rate=1e-4,
                 gamma=0.9,
                 critic_discount=0.5,
                 entropy_beta=0.0001,
                 input_shape=(None, 10),
                 output_dims=[11, 1],
                 pretrained_weights=None):

        super().__init__(
            name=name,
            model=model,
            loss_fn=loss_fn,
            optimizer=optimizer,
            learning_rate=learning_rate,
            gamma=gamma,
            critic_discount=critic_discount,
            entropy_beta=entropy_beta,
            input_shape=input_shape,
            output_dims=output_dims,
            pretrained_weights=pretrained_weights,)

    def train(self, env, num_episodes, num_steps):
        self._train(env, num_episodes, num_steps)

max_stars_repo_name: akensert/ddqn-isocratic-scouting-runs

max_issues_repo_name: akensert/ddqn-isocratic-scouting-runs

max_forks_repo_name: akensert/ddqn-isocratic-scouting-runs


text:
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import linear_kernel
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from ast import literal_eval

# Get citation data
fileobj = 'savedrecs-440.xls'
metadata = pd.read_excel(fileobj)
indices = pd.Series(metadata.index, index=metadata['Article Title']).drop_duplicates()

def get_recommendations(title, cosine_sim):
    # Get the index of the article that matches the title
    idx = indices[title]

    # Get the pairwsie similarity scores of all titles with that article
    sim_scores = list(enumerate(cosine_sim[idx]))

    # Sort the articles based on the similarity scores
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)

    # Get the scores of the 10 most similar articles
    sim_scores = sim_scores[0:11]

    # Get the article indices
    article_indices = [i[0] for i in sim_scores]

    # Return the top 10 most similar movies
    return metadata['Article Title'].iloc[article_indices]

### Get corresponding author ###
def get_last_author(x):
    authors = x.split(';')
    return authors[-1]

### Get Keywords ###
def get_keywords_list(x):
    if isinstance(x, str):
        names = [i for i in x.split(';')]
            #Check if more than 3 elements exist. If yes, return only first three. If no, return entire list.
        if len(names) > 3:
            names = names[:3]
        return names
    return []

### Clean data ###
def clean_data(x):
    if isinstance(x, list):
        return [str.lower(i.replace(" ", "")) for i in x]
    else:
        #Check if author exists. If not, return empty string
        if isinstance(x, str):
            return str.lower(x.replace(" ", ""))
        else:
            return ''

### for selected features ###
#features = ['Article Title', 'Authors', 'Author Keywords', 'Keywords Plus', 'Research Areas'] #'WoS Categories', '180 Day Usage Count', 'Times Cited, WoS Core'
features = ['Author Keywords', 'Author Full Names', 'Research Areas']

#for feature in features:
#    metadata[feature] = metadata[feature].apply(get_keywords_list)
#metadata['Author'] = metadata['Authors'].apply(get_last_author)

for feature in features:
    metadata[feature] = metadata[feature].apply(clean_data)
#print(metadata[features].head(3))

def create_soup(x):
    return ' '.join(x['Author Keywords']) + ' ' + ' '.join(x['Article Title']) + ' ' + x['Authors'] + ' ' + ' '.join(x['Research Areas'])

## Create a new soup feature
metadata['soup'] = metadata.apply(create_soup, axis=1)
#print(metadata['soup'].head(2))

count = CountVectorizer(stop_words='english')
count_matrix = count.fit_transform(metadata['soup'])
#print(count_matrix.shape)

cosine_sim2 = cosine_similarity(count_matrix, count_matrix)
# Reset index of the main DataFrame and construct reverse mapping
metadata = metadata.reset_index()
indices = pd.Series(metadata.index, index=metadata['Article Title'])

#print(get_recommendations('Enabling microbial syringol conversion through structure-guided protein engineering', cosine_sim2))
print(get_recommendations('A promiscuous cytochrome P450 aromatic O-demethylase for lignin bioconversion', cosine_sim2))