text:
import os
import sys
import csv
import cv2
import argparse
import numpy as np

def main():
    # Determine the correct calibration file to use
    calibration_path = 'calibration.csv'
    bit_offset_mm = 47
    axis_offset = 168

    parser = argparse.ArgumentParser()
    parser.add_argument('-f', '--file', help ='Image file to process')
    parser.add_argument('-c', '--calibrationFile', help ='Use path to supplied calibration file')
    parser.add_argument('-b', '--bitOffset', help ='Offset from the collet to the bit')
    args = parser.parse_args()

    # check for input file
    if args.file:
        if not os.path.exists(args.file):
            sys.exit('ERROR: Input file %s does not exist' % args.file)
    else:
        sys.exit('ERROR: Input file not provided')

    # load in calibration file
    if args.calibrationFile:
        print('Using calibration file at: %s' % args.calibrationFile)
        calibration_path = args.calibrationFile
    else:
        print('Using default calibration file at: %s' % calibration_path)

    calibration, x_max, x_min = ingest_calibration(calibration_path)
    x_height = x_max - x_min
    print('Calibration loaded sucessfully')

    # update the bit offset if needed
    if args.bitOffset:
        bit_offset_mm = int(args.bitOffset)
        print('Using supplied bit offset of %d' % bit_offset_mm)
    else:
        print('Using default bit offset of %d' % bit_offset_mm)

    # load input image
    img = cv2.imread(args.file)
    IMG_WIDTH = img.shape[1]
    IMG_HEIGHT = img.shape[0]

    # generate the pruned contours
    pruned_contours, contours = detect_contours(img)
    img_cont = draw_pruned_contours(img, pruned_contours, contours)

    # convert paths to pumkin coordinates
    paths = []
    for path in pruned_contours:
        path_mm = []
        for point in path:
            # convert to pumpkin space
            x_mm = ((IMG_HEIGHT - point[1]) * (x_height/IMG_HEIGHT)) + x_min
            y_mm = (IMG_WIDTH - point[0]) / (IMG_WIDTH/360)
            path_mm.append([x_mm, y_mm, 0])
        path_mm.append(path_mm[0]) # close the loop
        paths.append(path_mm)

    # generate toolpath from paths
    toolpath = generate_gcode(paths, calibration, bit_offset_mm, axis_offset)
    #TODO: use the name of the input file for the gcode

    # write gcode to file
    gcode_filename = 'output.gcode'
    f = open(gcode_filename, 'w')
    for line in toolpath:
        f.write(line)
    f.close()

    # show image
    cv2.imshow('Detected contours', img_cont)
    while True:
        if cv2.waitKey(33) == ord('q'):
            sys.exit('Closing...')

def ingest_calibration(filename):
    # TODO: return full calibration array with interpolated points
    raw = np.genfromtxt(filename, delimiter=',')

    # convert to mm/deg
    x_max = raw.max(axis=0)[0]
    x_min = raw.min(axis=0)[0]
    cal = np.full(shape = (int(x_max - x_min) + 1, 360), fill_value=-1 )

    for point in raw:
        cal[int(point[0] - x_min), int(point[1])] = point[2]
        # TODO: Fill out interpolated points

    return raw, x_max, x_min

def detect_contours(img):
    # color transform
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    ret, thresh = cv2.threshold(gray, 150, 255, cv2.THRESH_BINARY_INV)

    # detect contours
    contours, hierarchy = cv2.findContours(image=thresh, mode=cv2.RETR_EXTERNAL, method=cv2.CHAIN_APPROX_SIMPLE)
    print('Detected %d contours' % len(contours))

    # Iterate through the contours to prune
    contour_num = 0
    pruned_contours = []
    for contour in contours:
        pruned_contour = contour.copy().squeeze()

        i = 0
        while i < len(pruned_contour) -2:
            p0 = pruned_contour[i]
            p1 = pruned_contour[i+1]
            p2 = pruned_contour[i+2]

            if collinearity_check( p0, p1, p2, 10):
                pruned_contour = np.delete(pruned_contour, i+1, 0)
            else:
                i += 1

        print('Pruned contour %d from %d points to %d' % (contour_num, len(contour), len(pruned_contour)))
        pruned_contours.append(pruned_contour)
        contour_num += 1

    return pruned_contours, contours

def draw_pruned_contours(img, pruned_contours, contours):
    image_copy = img.copy()
    cv2.drawContours(image=image_copy, contours=contours, contourIdx=-1, color=(0, 255, 0), thickness=1, lineType=cv2.LINE_AA)
    cv2.drawContours(image=image_copy, contours=pruned_contours, contourIdx=-1, color=(255, 0, 0), thickness=1, lineType=cv2.LINE_AA)

    # interate through and draw the pruned verticies in red
    for contour in pruned_contours:
        for point in contour:
            cv2.circle( image_copy, (point[0], point[1]), 2, (0,0,255), 1)

    return image_copy

def collinearity_check(p0, p1, p2, epsilon = 1e-3):
    p0_ = np.insert(p0, 0, 1, axis =0)
    p1_ = np.insert(p1, 0, 1, axis =0)
    p2_ = np.insert(p2, 0, 1, axis =0)

    m = np.stack((p0_, p1_, p2_), 0)
    det = np.linalg.det(m)

    return abs(det) < epsilon

def generate_gcode(paths, cal, bit_offset, axis_offset):
    #TODO: Remove dependence on nominal cylinder and use actual calibration
    #TODO: add mechanism to pass parameters as an arguement efficiently

    # cut params
    cut_depth = 30
    cut_feed_x = 50
    cut_feed_y = 0.5
    cut_feed_diag = 50
    fudge = 14 #TODO: calibrate ToF sensor and remove this value

    # find the z value that puts the bit in contact with the nominal cylinder
    r_nom = cal.max(axis=0)[2] + fudge
    z_surface = int(axis_offset - (bit_offset + r_nom))
    print('Nominal cylinder surface Z value: %d' % z_surface)

    gcode = []
    gcode.append('S10000\n') # set the spindle speed

    # process each contour
    for path in paths:

        # Add gcode to setup the contour
        gcode.append('G0 X%.3f Y%.3f Z%.3f\n' % (path[0][0], path[0][1], 0)) # Go to starting point at full speed
        gcode.append('M3\n') # Turn on spindle
        gcode.append('G0 Z%.3f\n' % z_surface) #plunge to surface

        # iterate over path at each depth
        for depth in range(z_surface, z_surface + cut_depth + 2, 2):
            prev_point = None
            for point in path:
                # determine the cut speed based on the direction
                feed = cut_feed_x

                if prev_point:
                    y_motion = prev_point[1] != point[1]
                    x_motion = prev_point[0] != point[0]

                    if y_motion and x_motion:
                        feed = cut_feed_diag
                    elif y_motion:
                        feed = cut_feed_y
                    elif x_motion:
                        feed = cut_feed_x

                # close the loop if needed and return to the first point
                gcode.append('G1 X%.3f Y%.3f Z%.3f F%.3f\n' % (point[0], point[1], depth, feed))

                prev_point = point

            # close the loop and head back to the first point
            gcode.append('G1 X%.3f Y%.3f Z%.3f F%.3f\n' % (path[0][0], path[0][1], depth, cut_feed_y))

        # add the gcode to finish the contour
        gcode.append('G0 Z0\n') # retract spindle for travel
        gcode.append('M5\n') # disable spindle for travel

    # Finish by returning to the home position
    gcode.append('G0 X0 Y0 Z0\n')

    return gcode

if __name__ == '__main__':
    main()

max_stars_repo_name: chrisdalke/pumpkin-carving-robot

max_issues_repo_name: chrisdalke/pumpkin-carving-robot

max_forks_repo_name: chrisdalke/pumpkin-carving-robot


text:
from numpy.testing._private.utils import suppress_warnings
import streamlit as st

from PIL import Image
import numpy as np
import cv2
import tensorflow as tf
from tensorflow.keras.preprocessing import image as SImage
from keras.preprocessing.image import ImageDataGenerator
from keras.preprocessing.image import img_to_array
from tensorflow.keras.models import load_model
import tempfile

DEMO_IMAGE_YES = tf.keras.preprocessing.image.load_img(
    'Images\itsyes.jpg', grayscale=False, target_size=(64,64)
)
DEMO_IMAGE = 'Images\itsyes.jpg'
demo_image_hd = DEMO_IMAGE
image2 = np.array(Image.open(demo_image_hd))

classifier = load_model('BrainTumorModel.h5')
class_labels=['No','Yes']
@st.cache
def detect_tumor(image):
    # image=tf.keras.preprocessing.image.array_to_img(image)
    image = image.reshape((1,) + image.shape)
    labels=[]
    preds=classifier.predict(image)
    label=class_labels[preds.argmax()]
    return label

st.title('Brain Tumor Detection Application')

st.markdown('''
            * This model is trained to Detect Brain tumor when images of MRI are provided\n
            * Just upload the Image of your MRI report and see the results immediately
            ''')

img_file_buffer = st.file_uploader("Upload an image", type=[ "jpg", "jpeg",'png'])

if img_file_buffer is not None:
    image = np.array(Image.open(img_file_buffer))
    image2=image
    image = cv2.resize(image, (64,64))

else:
    demo_image = DEMO_IMAGE_YES
    image = np.array(demo_image)

st.subheader('Original Image')

st.image(image2, caption=f"Original Image",use_column_width= True)

tumor_detection = detect_tumor(image)
if tumor_detection=='Yes':
    st.subheader('Brain tumor is detected in this image, kindly contact a doctor as soon as possible')
else:
    st.subheader('No Brain tumor detected')

ext: py

reason: import numpy,from numpy


text:
"""
Code originates from: https://machinelearningmastery.com/a-gentle-introduction-to-normality-tests-in-python/

"""

from scipy.stats import shapiro, normaltest, anderson

"""
Shapiro-Wilk Test of Normality
The Shapiro-Wilk Test is more appropriate for small sample sizes (< 50 samples), but can also handle sample sizes as large as 2000.
The Shapiro-Wilk test is used as a numerical means of assessing normality.
"""
def run_shapiro_wilk_normality_test(data, alpha=0.05, print_results=True):
    stat, p = shapiro(data)
    if print_results:
        print('Statistics=%.3f, p=%.3f' % (stat, p))
        if p > alpha:
            print('Sample looks Gaussian (fail to reject H0) at significance level ', alpha)
        else:
            print('Sample does not look Gaussian (reject H0) at significance level ', alpha)
    return stat, p

def run_dagostino_pearson_test(data, alpha, print_results=True):
    stat, p = normaltest(data)
    if print_results:
        print('Statistics=%.3f, p=%.3f' % (stat, p))
        if p > alpha:
            print('Sample looks Gaussian (fail to reject H0) at significance level ', alpha)
        else:
            print('Sample does not look Gaussian (reject H0) at significance level ', alpha)
    return stat, p

def run_anderson_darling(data, print_results=True):
    result = anderson(data)
    print('Statistic: %.3f' % result.statistic)
    if print_results:
        for i in range(len(result.critical_values)):
            sl, cv = result.significance_level[i], result.critical_values[i]
            if result.statistic < result.critical_values[i]:
                print('%.3f: %.3f, data looks normal (fail to reject H0)' % (sl, cv))
            else:
                print('%.3f: %.3f, data does not look normal (reject H0)' % (sl, cv))
    return result

max_stars_repo_name: Julian-Theis/stat-kiste

max_issues_repo_name: Julian-Theis/stat-kiste

max_forks_repo_name: Julian-Theis/stat-kiste

reason: from scipy


text:
from  preprocess import Preprocess
from keras.models import load_model
import numpy as np
import cv2
import config

class FaceRecognition:
    def __init__(self):
        """initialize dataset and load model"""
        self.model = load_model(config.model_path)
        print("[Log] Pretrained model was loaded.")

        self.preprocess = Preprocess(database_path=config.database_path)
        print("[Log] Preprocess object was created.")

        self.database = self.init_database()

    def init_database(self):
        """ initilize face database"""
        return self.preprocess.load_images()
        print("[Log] Database initialized.",flush=True)

    def recognize_faces_in_video(self, videopath):

        vc = cv2.VideoCapture(videopath)
        while vc.isOpened():
            _, frame = vc.read()
            height, width, channels = frame.shape

            faces, coordinates = self.preprocess.getFace(frame)
            if faces == None:
                continue
            # Loop through all the faces
            for i, face in enumerate(faces):

                embeded_face = self.preprocess.embedding(face)
                name, similarity =  self.findFaceInDB(embeded_face)
                #print(name,"-",similarity)
                if similarity <= config.threshold:
                    x1, y1, x2, y2 = coordinates[i]
                    cv2.rectangle(frame,(x1,y1),(x2,y2),(255,0,0),2)
                    cv2.putText(frame,name, (x1-10, y1-10), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 0, 0), 2, cv2.LINE_AA)

            key = cv2.waitKey(100)
            cv2.imshow("Face Recognizer", frame)
            if key == 27: # exit on ESC
                break

        vc.release()
        cv2.destroyAllWindows()

    def findFaceInDB(self, embedded_face):
        """ This method finds person in person database."""
        who, similarity = None, None
        min_similarity = 200

        persons = list(self.database.keys())

        for person in persons:
            for vec in self.database[person]:
                sim = self.preprocess.euclid_distance(embedded_face,vec)
                if sim < min_similarity:
                    min_similarity = sim
                    who = person

        return who, min_similarity

if __name__ == "__main__":
    FR = FaceRecognition()
    FR.recognize_faces_in_video(config.video_path)

reason: import numpy


text:
from __future__ import division
'''
Author      : Uervirojnangkoorn, M.
Created     : 8/17/2016
Description : Handy class to do K-means clustering.
'''
import random
import numpy as np
import matplotlib.pyplot as plt
from random import randint

class kmeans_handler(object):
  """
  From a given N-dimensional array, find K clusters.
  Algorithms taken from Stanford CS221 pseudo-python code.
  See http://stanford.edu/~cpiech/cs221/handouts/kmeans.html
  """
  def __init__(self):
    """
    Constructor
    """
    self.MAX_ITERS = 1000

  def run(self, dataset, k, flag_plot=False):
    #define color for plotting
    colors = ['#%06X' % randint(0, 0xFFFFFF) for i in range(k)]
    #initialize controids randomly
    n_data, n_features = dataset.shape
    centroids = np.array([random.random() for i in range(k*n_features)]).reshape((k,n_features))
    scale = np.max(dataset, axis=0)
    centroids = centroids * scale
    if flag_plot:
      plt.scatter(dataset[:,0], dataset[:,1], s=10, marker='x', c='b')
      plt.scatter(centroids[:,0], centroids[:,1], s=20, marker='o', c='k')
      plt.title('Initial step (k=%3.0f)'%(k))
      plt.show()
    #initialize book keeping vars
    n_iters = 0
    old_centroids = None
    #run the k-means algorithm
    while not self.should_stop(old_centroids, centroids, n_iters):
      #save old centroids for convergenence test. book keeping.
      old_centroids = centroids[:]
      n_iters += 1
      #assign labels to each datapoint based on centroids
      labels = self.get_labels(dataset, centroids)
      #assign centroids based on datapoint labels
      centroids = self.get_centroids(dataset, labels, k)
      if flag_plot:
        for i in range(k):
          s = dataset[labels==i]
          plt.scatter(s[:,0], s[:,1], s=10, marker='x', c=colors[i])
          plt.scatter(centroids[i,0], centroids[i,1], s=20, marker='o', c='k')
        plt.title('Cycle %3.0f (k=%3.0f)'%(n_iters, k))
        plt.show()
    return centroids, labels

  def should_stop(self, old_centroids, centroids, n_iters):
    if n_iters > self.MAX_ITERS: return True
    return np.all(old_centroids == centroids)

  def get_labels(self, dataset, centroids):
    #For each element in the dataset, choose the closest centroid.
    #Make that centroid the element's label.
    n_data, n_features = dataset.shape
    n_centroids = len(centroids)
    dist = np.zeros((n_data, n_centroids))
    for c,i in zip(centroids, range(n_centroids)):
      dist[:,i] = np.sqrt(((dataset-c)**2).sum(axis=1))
    labels = np.argmin(dist, axis=1)
    return labels

  def get_centroids(self, dataset, labels, k):
    #For each label group, calculate a new geometric mean as a new centroid.
    #For empty group, assign a new random centroid.
    n_data, n_features = dataset.shape
    centroids = np.zeros((k, n_features))
    for i in range(k):
      s = dataset[labels==i]
      if len(s) > 0:
        centroids[i] = np.power(np.sqrt(np.prod(s, axis=0)**2), 1/len(s))
      else:
        scale = np.max(dataset, axis=0)
        centroids[i] = np.array([random.random() for i in range(n_features)]) * scale
    return centroids

if __name__=="__main__":
  #unit test
  points = np.vstack([np.random.multivariate_normal(mean, \
                 0.03 * np.diag([1,1]), 20) \
                 for mean in [(1, 1), (2, 4), (3, 2)]])
  k = 3
  kmh = kmeans_handler()
  centroids, labels = kmh.run(points, k, flag_plot=True)


text:
import os
from pathlib import Path

import numpy as np
import torch
from tqdm import tqdm
from tfrecord.tools.tfrecord2idx import create_index
from tfrecord.torch.dataset import TFRecordDataset

from vp_suite.base.base_dataset import VPDataset, VPData
import vp_suite.constants as constants

class BAIRPushingDataset(VPDataset):
    r"""
    Dataset class for the dataset "BAIR Robotic Pushing", as firstly encountered in
    "Self-Supervised Visual Planning with Temporal Skip Connections" by Ebert et al. (https://arxiv.org/abs/1710.05268).

    Each sequence depicts a robotic gripper arm that randomly moves its end effector inside a bin filled with objects,
    occasionally stirring them around. This dataset is split into training and test data by default and provides RGB
    images of size 64x64 as well as 4D action vectors describing the robot gripper movement per-frame.
    """
    NAME = "BAIR robot pushing"
    REFERENCE = "https://arxiv.org/abs/1710.05268"
    IS_DOWNLOADABLE = "Yes"
    DEFAULT_DATA_DIR = constants.DATA_PATH / "bair_robot_pushing"
    MIN_SEQ_LEN = 30
    ACTION_SIZE = 4
    DATASET_FRAME_SHAPE = (64, 64, 3)

    train_to_val_ratio = 0.96

    def __init__(self, split, **dataset_kwargs):
        super(BAIRPushingDataset, self).__init__(split, **dataset_kwargs)
        self.NON_CONFIG_VARS.extend(["obs_ids", "actions_ids", "obs_fps", "actions_fps"])

        self.data_dir = str((Path(self.data_dir) / "softmotion30_44k" / split).resolve())
        self.obs_ids = [fn for fn in sorted(os.listdir(self.data_dir)) if str(fn).endswith("obs.npy")]
        self.actions_ids = [fn for fn in sorted(os.listdir(self.data_dir)) if str(fn).endswith("actions.npy")]

        if len(self.obs_ids) != len(self.actions_ids):
            raise ValueError("Different number of obs and action files found -> Delete dataset and prepare again!")
        elif len(self.obs_ids) == 0:
            raise ValueError("No trajectory files (.npy) found! Maybe you forgot to prepare the dataset?")

        self.obs_fps = [os.path.join(self.data_dir, i) for i in self.obs_ids]
        self.actions_fps = [os.path.join(self.data_dir, i) for i in self.actions_ids]

    def __len__(self):
        return len(self.obs_fps)

    def __getitem__(self, i) -> VPData:
        if not self.ready_for_usage:
            raise RuntimeError("Dataset is not yet ready for usage (maybe you forgot to call set_seq_len()).")

        obs_fp = self.obs_fps[i]
        rgb_raw = np.load(obs_fp)
        rgb_raw = rgb_raw[:self.seq_len:self.seq_step]
        rgb = self.preprocess(rgb_raw)  # [t, c, h, w]

        actions = torch.from_numpy(np.load(self.actions_fps[i])).float()
        actions = actions[:self.seq_len:self.seq_step]  # [t, a]

        data = {"frames": rgb, "actions": actions, "origin": obs_fp}
        return data

    @classmethod
    def download_and_prepare_dataset(cls):
        d_path = cls.DEFAULT_DATA_DIR
        d_path.mkdir(parents=True, exist_ok=True)
        ds_path = d_path / "softmotion30_44k"
        if not os.path.exists(str(ds_path)):
            download_and_extract_bair(d_path)
        print("splitting trajectory files...")
        split_bair_traj_files(ds_path / "train", True)
        split_bair_traj_files(ds_path / "test", True)

# === BAIR data preparation tools ==============================================

def download_and_extract_bair(d_path : Path):
    r"""
    Download the BAIR dataset and unpack the downloaded archives into the specified target folder.

    Args:
        d_path (Path): Specified path to dataset.
    """
    tar_fname = "bair_robot_pushing_dataset_v0.tar"
    tar_path = str(d_path / tar_fname)
    if not os.path.exists(tar_path):
        URL = f"http://rail.eecs.berkeley.edu/datasets/{tar_fname}"
        from vp_suite.utils.utils import download_from_url
        download_from_url(URL, tar_path)

    print("Extracting data...")
    import tarfile
    tar = tarfile.open(tar_path)
    tar.extractall(d_path)
    tar.close()
    os.remove(tar_path)

def split_bair_traj_files(data_dir: Path, delete_tfrecords: bool):
    r"""
    Pre-processes downloaded BAIR pushing data by extracting the per-frame image and action information
    from the provided .tfrecord files and store the information in numpy arrays.

    Args:
        data_dir (Path): Specified path to dataset.
        delete_tfrecords (bool): If True, delete the .tfrecord files after extraction.

    Returns:

    """
    bair_ep_length = 30

    data_files = [fn for fn in sorted(os.listdir(str(data_dir.resolve()))) if str(fn).endswith(".tfrecords")]
    ep_number = 0
    for df in tqdm(data_files):
        tfr_fp = str((data_dir / df).resolve())
        index_fp = tfr_fp + ".index"
        if not os.path.isfile(index_fp):
            create_index(tfr_fp, index_fp)
        data_list = list(TFRecordDataset(tfr_fp, index_fp))
        for ep_i in range(len(data_list)):
            observations, actions = [], []
            for step_i in range(bair_ep_length):
                obs = np.array(data_list[ep_i][str(step_i) + '/image_aux1/encoded']).reshape(1, 64, 64, 3)
                # obs = np.array(data_list[ep][str(step) + '/image_main/encoded']).reshape(1, 64, 64, 3)
                action = np.array(data_list[ep_i][str(step_i) + '/action'])[np.newaxis, ...]

                observations.append(obs)
                actions.append(action)

            observations = np.concatenate(observations, axis=0)
            out_obs_fp = data_dir / f"seq_{ep_number:05d}_obs.npy"
            np.save(out_obs_fp, observations)

            actions = np.concatenate(actions, axis=0)
            out_actions_fp = data_dir / f"seq_{ep_number:05d}_actions.npy"
            np.save(out_actions_fp, actions)

            ep_number += 1

        if delete_tfrecords:
            os.remove(tfr_fp)
            os.remove(index_fp)


text:
import fnmatch
import os
from collections import Counter
from pathlib import Path
import sys
import matplotlib.pyplot as plt
import numpy as np
import torch
from scipy.io import loadmat
import h5py
# from scipy.optimize import curve_fit

my_path = os.path.join('./')
sys.path.append(my_path)
from data_loader.data_loaders import Bin_energy_data

project_path = Path(__file__).parent.parent
fig_path = os.path.join('./', 'saved', 'figs')
res_path = os.path.join('./', 'saved', 'diff_run_res')

def merge_and_split_data(path, relation, moment, min_shower_num, max_shower_num, file):
    """
    Merge the raw data files into concatenated dataset then split by the relation to train and test set and save in the
     folders.
    Notice that the train is split into train\valid sets in the train function later.
    """
    dl = []
    for i in file:
        # edep_file = path / "raw" / f"signal.al.elaser.IP0{i}.edeplist.mat"
        # en_file = path / "raw" / f"signal.al.elaser.IP0{i}.energy.mat"

        edep_file = os.path.join('./', 'data', 'raw', f'signal.al.elaser.IP0{i}.edeplist.mat')
        en_file = os.path.join('./', 'data', 'raw', f'signal.al.elaser.IP0{i}.energy.mat')

        dataset = Bin_energy_data(edep_file, en_file, moment=moment,
                                  min_shower_num=min_shower_num, max_shower_num=max_shower_num, file=i)
        dl.append(dataset)

    dataset = torch.utils.data.ConcatDataset(dl)

    # mean, std = get_mean_and_std(dataset)
    # print(mean, std)

    train_d, test_d = torch.utils.data.random_split(dataset, [int(relation * len(dataset))+2,
                                                              int((1 - relation) * len(dataset))-1])

    # torch.save(train_d, path / "train//train.pt")
    # torch.save(test_d, path / "test//test.pt")

    torch.save(train_d, os.path.join('./', 'data', 'train', 'train.pt'))
    torch.save(test_d, os.path.join('./', 'data', 'test', 'test.pt'))

def test_bins(output, target, nums, bin_num=10, name=None, run_num='0', config=0):
    """Analysis of the bin results from test"""

    # Generate the bins

    hf = h5py.File(os.path.join('./', 'num_classes.h5'), 'r')
    num_classes = hf.get('dataset_1')
    num_classes = int(np.array(num_classes))
    hf.close()

    bin_num = num_classes

    total_out = [0] * bin_num
    total_target = [0] * bin_num
    bars = np.linspace(0, 13, bin_num)

    # Sum over al of the bin results for each bin.
    for i in range(bin_num):
        out_sum = sum(t[i] for t in output)
        target_sum = sum(t[i] for t in target)
        total_out[i] = out_sum
        total_target[i] = target_sum

    # Calculate the entropy of the bin lists for output and the truelabel bins.

    out_entropy = -sum([f * np.log(f) if f > 0 else 0 for f in total_out])
    tar_entropy = -sum([f * np.log(f) if f > 0 else 0 for f in total_target])
    b = out_entropy - tar_entropy

    print(f'Entopies: True distribution: {tar_entropy:.3f}, Predicted distribution: {out_entropy:.3f}, Bias: {b:.3f}')

    KL_1 = sum(
        [f * np.log((f + 0.0000001) / (total_target[i] + 0.0000001)) for i, f in enumerate(total_out)])
    KL_2 = sum(
        [f * np.log((f + 0.0000001) / (total_out[i] + 0.0000001)) for i, f in enumerate(total_target)])
    D = KL_1 + KL_2

    print(f'KL dist - KL1(q=target): {KL_1:.3f}, KL2(q=output): {KL_2:.3f}, Symmetric: {D:.3f}')
    print(f'total out: {["%.3f" % item for item in total_out]}')
    print(f'total target: {["%.3f" % item for item in total_target]}')
    print(f'total N: {sum(total_out)}, target N: {sum(total_target)}')
    bars = [float(f'{i:.2f}') for i in bars]

    # Text generation for bin legend
    text = 'Bin Energy range [GeV]: \n'
    for i in range(bin_num - 1):
        text += f'{i}: {bars[i]:.1f} - {bars[i + 1]:.1f} \n'
    # print(text)

    tot_mean_en_pred = []
    tot_mean_en_true = []
    for i in range(bin_num - 1):
        me = bars[i] + 0.35
        e_p = me * total_out[i]
        e_t = me * total_target[i]
        tot_mean_en_pred.append(e_p)
        tot_mean_en_true.append(e_t)

    mean_en_p = sum(tot_mean_en_pred) / sum(total_out)
    mean_en_t = sum(tot_mean_en_true) / sum(total_target)

    print(f'Mean E: {mean_en_p}, target E: {mean_en_t}')

    rng = [i + 1 for i in range(num_classes)]

    plt.figure(figsize=(28,20))
    plt.bar(rng, total_out, label='output', alpha=0.5)
    plt.errorbar(rng, total_out, yerr=(1 / np.sqrt(np.abs(total_out))), fmt="+", color="b")
    plt.bar(rng, total_target, label='true_val', alpha=0.3)
    plt.xlabel('bins number for energies')
    plt.ylabel('number of particles')
    # plt.text(15.5, 0.015, text,
    plt.text(15.5, 0.015, text,
             bbox={'facecolor': 'white', 'alpha': 0.5, 'pad': 3}, fontsize='x-small')

    plt.xticks(rng, rotation=65)
    plt.title(f'{len(output)} samples')
    plt.legend()
    plt.savefig(os.path.join(res_path, f'binsgraph_run_{run_num}.png'))
    flag = -1
    try:
        epoch_num = int(str(config.resume)[-7:-4])
        flag = 0
    except:
        pass
    if flag == -1:
        try:
            epoch_num = int(str(config.resume)[-6:-4])
            flag = 0
        except:
            pass
    if flag == -1:
        epoch_num = 0
    csv_path = os.path.join('./csv_files', f'epoch_{epoch_num}')
    plt.savefig(os.path.join(csv_path, f'binsgraph_run_{run_num}.png'))
    # plt.show()
    plt.clf()
    save = {'output bins': total_out, 'output entropy': out_entropy,
            'target bins': total_target, 'target entropy': tar_entropy,
            'KL': KL_2}

    res_file = open(os.path.join(res_path, f'bin_results_run_{run_num}.txt'), 'wt')
    res_file.write(str(save))
    res_file.close()
    return

def evaluate_test(output, target, incdices, shower_nums, config):
    """
        This function evaluates the test results. The first part relates to the N prediction - assuming we have 1 class.
        The second part handles the 20 bin prediction.
    """
    file_tag = "x_pred"

    # evaluate_xy(output=output[:, 0], target=target[:, 0], run_num=file_tag)
    # np.savetxt(res_path / f'output_{file_tag}.txt', output[:, 0], delimiter="\n", fmt='%1.2f')

    # For each sample evaluate N and compare with Ntrue - get Nbias. Produce halina graphs with them and our graphs:
    # N_pred = np.sum(output, axis=1)  # Total number of showers in the test set
    # H_graphs(N_true=shower_nums, N_pred=N_pred, run_num=file_tag)  # Generate relevant graph images

    # Bins - sum over output bins and target bins. Compare graphs. produce PNG. Save the bins as text to calculate
    test_bins(output, target, shower_nums, bin_num=20, run_num=file_tag, config=config)

    ################################################################
    ######### Save idx list as txt file with the file tag ##########
    # np.savetxt(res_path / f'idx_run_{file_tag}.csv', [int(i) for i in incdices], delimiter=",", fmt='%i')
    ################################################################

    return

if __name__ == '__main__':
    # EDA("C:\\Users\\elihu\\PycharmProjects\\LUXE\\nongitdata\\Multiple Energies\\")
    # data_path = Path("C:\\Users\\elihu\\PycharmProjects\\LUXE\\LUXE-project-master\\data\\")

    data_path = Path(my_path = os.path.join('./', 'data'))
    merge_and_split_data(data_path, 0.8, moment=3, min_shower_num=1, max_shower_num=50000, file=[3])
    exit()

reason: import numpy,from scipy


text:
"""
Created:  2018-08-08
Modified: 2019-03-07
Author:   Christopher Albert <albert@alumni.tugraz.at>
"""
from numpy import cos, sin, empty_like, linspace, roll
from matplotlib.pyplot import figure, plot, xlabel, ylabel, subplot, grid, legend

def plot_orbit(z):
    figure()
    plot(z[0,:]*cos(z[1,:]), z[0,:]*sin(z[1,:]),',')
    xlabel(r'$R-R_0$')
    ylabel(r'$Z$')

def plot_cost_function(F, zlast, zold, pthold):
    """Plot the cost function"""
    r = linspace(-0.2,0.2,100)
    fres = empty_like(r)
    for kr in range(len(r)):
        fres[kr] = F([r[kr]], zold[1:], pthold)

    figure()
    subplot(2,1,1)
    plot(r,fres)
    out_conv = F([zlast[0]], zold[1:], pthold)
    plot(zlast[0], out_conv, 'rx')
    xlabel('r')
    ylabel('F')
    grid()

def plot_cost_function_jac(F, zlast, zold, pthold):
    """Plot the cost function with Jacobian"""

    r = linspace(-0.2,0.2,100)
    fres = empty_like(r)
    jac = empty_like(r)
    for kr in range(len(r)):
        out = F([r[kr]], zold[1:], pthold)
        fres[kr] = out[0]
        jac[kr] = out[1][0]

    jacnum = (roll(fres,-1)-roll(fres,1))/(2.*(r[1]-r[0]))
    jacnum[0] = jacnum[1]
    jacnum[-1] = jacnum[-2]

    figure()
    subplot(2,1,1)
    plot(r,fres)
    out_conv = F([zlast[0]],zold[1:],pthold)
    plot(zlast[0],out_conv[0],'rx')
    xlabel('r')
    ylabel('F')
    grid()

    subplot(2,1,2)
    plot(r,jac)
    plot(r,jacnum,'r--')
    plot(zlast[0],out_conv[1],'rx')
    xlabel('r')
    ylabel('dF/dr')
    legend(['analytical', 'numerical'])
    grid()

max_stars_repo_name: itpplasma/SIMPLE

max_issues_repo_name: landreman/SIMPLE

max_forks_repo_name: landreman/SIMPLE

reason: from numpy


text:
import itertools
from typing import Dict

import numpy as np
import pandas as pd
from sklearn import preprocessing

from carla.models.catalog import MLModelCatalog
from carla.recourse_methods.api import RecourseMethod

from .action_set import get_discretized_action_sets
from .cost import action_set_cost

def _series_plus_dict(x: pd.Series, y: dict):
    """Helper function to implemention addition for a Series object and a dict with overlapping keys

    Parameters
    ----------
    x: pd.Series
    y: dict

    Returns
    -------
    pd.Series analogous to x + y
    """
    y = pd.Series(y)

    result = x + y
    nan_cols = result.index[result.isna()].tolist()
    result = result.drop(index=nan_cols)
    result = pd.concat([result, x[nan_cols]])

    return result

# https://stackoverflow.com/a/1482316/2759976
def powerset(iterable):
    """powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)"""
    s = list(iterable)
    return itertools.chain.from_iterable(
        itertools.combinations(s, r) for r in range(len(s) + 1)
    )

class CausalRecourse(RecourseMethod):
    def __init__(self, mlmodel: MLModelCatalog, hyperparams: Dict):

        self._mlmodel = mlmodel
        self._dataset = mlmodel.data

        self._optimization_approach = hyperparams["optimization_approach"]
        self._num_samples = hyperparams["num_samples"]
        self._scm = hyperparams["scm"]

        self._constraint_handle = hyperparams["constraint_handle"]
        self._sampler_handle = hyperparams["sampler_handle"]

    def get_intervenable_nodes(self) -> dict:
        intervenable_nodes = {
            "continuous": np.setdiff1d(
                self._dataset.continous, self._dataset.immutables
            ),
            "categorical": np.setdiff1d(
                self._dataset.categoricals, self._dataset.immutables
            ),
        }
        return intervenable_nodes

    def _get_original_df(self):
        normalized = self._mlmodel.use_pipeline
        if normalized:
            data_df = self.encode_normalize_order_factuals(self._dataset.raw)
        else:
            data_df = self._dataset.raw
        return data_df

    def _get_range_values(self):
        data_df = self._get_original_df()
        min_values = data_df.min()
        max_values = data_df.max()
        return min_values, max_values

    def _get_mean_values(self):
        data_df = self._get_original_df()
        mean_values = data_df.mean()
        return mean_values

    def compute_optimal_action_set(
        self, factual_instance, constraint_handle, sampling_handle
    ):

        intervenables_nodes = self.get_intervenable_nodes()
        min_values, max_values = self._get_range_values()
        mean_values = self._get_mean_values()

        min_cost = np.infty
        min_action_set = {}
        if self._optimization_approach == "brute_force":
            valid_action_sets = get_discretized_action_sets(
                intervenables_nodes, min_values, max_values, mean_values
            )

            # we need to make sure that actions don't go out of bounds [0, 1]
            if self._mlmodel.use_pipeline and isinstance(
                self._mlmodel.scaler, preprocessing.MinMaxScaler
            ):
                out_of_bounds_idx = []
                for i, action_set in enumerate(valid_action_sets):
                    instance = _series_plus_dict(factual_instance, action_set)
                    if not np.all((1 > instance.values) & (instance.values > 0)):
                        out_of_bounds_idx.append(i)
                valid_action_sets = [
                    action_set
                    for i, action_set in enumerate(valid_action_sets)
                    if i not in set(out_of_bounds_idx)
                ]

            for action_set in valid_action_sets:
                if constraint_handle(
                    self._scm,
                    factual_instance,
                    action_set,
                    sampling_handle,
                    self._mlmodel,
                ):
                    cost = action_set_cost(
                        factual_instance, action_set, max_values - min_values
                    )
                    if cost < min_cost:
                        min_cost = cost
                        min_action_set = action_set

        elif self._optimization_approach == "gradient_descent":
            raise NotImplementedError
        else:
            raise ValueError("optimization approach not recognized")

        # print("MIN COST", min_cost, "ACTION SET", min_action_set)

        return min_action_set, min_cost

    def get_counterfactuals(self, factuals: pd.DataFrame):

        normalized = self._mlmodel.use_pipeline
        if normalized:
            factual_df = self.encode_normalize_order_factuals(factuals)
        else:
            factual_df = factuals.drop(columns=self._dataset.target)

        cfs = []
        # actions = []
        for index, factual_instance in factual_df.iterrows():
            min_action_set, _ = self.compute_optimal_action_set(
                factual_instance, self._constraint_handle, self._sampler_handle
            )
            cf = _series_plus_dict(factual_instance, min_action_set)
            # min_action_set["cost"] = min_cost
            # actions.append(min_action_set)
            cfs.append(cf)

        # convert to dataframe
        cfs = pd.DataFrame(cfs)
        # action_df = pd.DataFrame(actions)
        return cfs

max_stars_repo_name: jayanthyetukuri/CARLA

max_issues_repo_name: jayanthyetukuri/CARLA

max_forks_repo_name: jayanthyetukuri/CARLA


text:
import warnings
from collections import Counter

import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import xarray as xr
from scipy import interpolate

try:
    # reduce moved in py3
    from functools import reduce
except ImportError:
    pass

def _find_bin_on_percentile(q, bin_counts, debug_plot=False):
    """
    Find value for percentile by ranking all bins and through linear
    interpolation on the cumulative sum of points finding the bin
    count which would match the percentile requested
    """
    bc = np.sort(bin_counts.flatten())

    y = np.cumsum(bc) / np.sum(bc)
    x = np.arange(len(bc)) / len(bc)

    y, idxs_unique = np.unique(y, return_index=True)
    x = x[idxs_unique]
    bc = bc[idxs_unique]

    idxs_nearest = np.sort(np.argsort(np.abs(y - q / 100.0))[:2])

    # y = ax+b
    # y1 - y0 = (x1 - x0)a
    # a = (y1 - y0)/(x1 - x0)
    x0, x1 = x[idxs_nearest]
    y0, y1 = y[idxs_nearest]
    a = (y1 - y0) / (x1 - x0)

    # y1 = a*x1 + b
    # b = y1 - a*x1
    b = y1 - a * x1

    # y2 = a*x2 + b
    # x2 = (y2 - b)/a
    x_fit = (q / 100.0 - b) / a

    fn = interpolate.interp1d(x, bc, fill_value="extrapolate")
    v = fn(x_fit)

    if debug_plot:
        ax = plt.gcf().axes[0]
        ax.plot(x, y, marker=".")
        ax.plot(x[idxs_nearest], y[idxs_nearest], marker="x")
        ax.plot(x, a * x + b)
        ax.axvline(x_fit)

        ax.set_ylim(0.0, 1.1)
        ax.set_xlim(0.5, None)

        ax = plt.gcf().axes[1]
        ax.plot(x, bc, marker=".")
        ax.axhline(v)
        ax.axvline(x_fit)
        ax.set_xlim(0.5, None)

    return v

class PlotGrid:
    def __init__(
        self,
        height=6,
        ratio=5,
        space=0.4,
        dropna=True,
        width=None,
        xlim=None,
        ylim=None,
        size=None,
        extra_x_marg=False,
        extra_y_marg=False,
    ):
        # Set up the subplot grid
        try:
            ratio_x, ratio_y = ratio
        except TypeError:
            ratio_x = ratio_y = ratio
        if width is None:
            width = height
        N_x_marg = 1 if not extra_x_marg else 2
        N_y_marg = 1 if not extra_y_marg else 2
        f = plt.figure(
            figsize=(
                width * (ratio_x + N_x_marg) / ratio_x,
                height * (ratio_y + N_y_marg) / ratio_y,
            )
        )
        gs = plt.GridSpec(ratio_y + N_y_marg, ratio_x + N_x_marg)

        ax_joint = f.add_subplot(gs[N_x_marg:, :-N_y_marg])
        ax_marg_x = f.add_subplot(gs[0, :-N_y_marg], sharex=ax_joint)
        ax_marg_y = f.add_subplot(gs[N_x_marg:, -N_y_marg], sharey=ax_joint)

        if extra_x_marg:
            ax_marg_x2 = f.add_subplot(gs[1, :-N_y_marg], sharex=ax_joint)

        if extra_y_marg:
            ax_marg_y2 = f.add_subplot(gs[N_x_marg:, -1], sharey=ax_joint)

        self.fig = f
        self.ax_joint = ax_joint
        self.ax_marg_x = ax_marg_x
        self.ax_marg_y = ax_marg_y

        if extra_x_marg:
            self.ax_marg_x2 = ax_marg_x2

        if extra_y_marg:
            self.ax_marg_y2 = ax_marg_y2

        # Turn off tick visibility for the measure axis on the marginal plots
        plt.setp(ax_marg_x.get_xticklabels(), visible=False)
        plt.setp(ax_marg_y.get_yticklabels(), visible=False)
        plt.setp(ax_marg_y.get_xticklabels(), rotation=30.0)
        ax_marg_y.get_xaxis().set_major_locator(plt.MaxNLocator(2))

        if extra_x_marg:
            plt.setp(ax_marg_x2.get_xticklabels(), visible=False)

        if extra_y_marg:
            plt.setp(ax_marg_y2.get_yticklabels(), visible=False)
            plt.setp(ax_marg_y2.get_xticklabels(), rotation=30.0)

        # Turn off the ticks on the density axis for the marginal plots
        # plt.setp(ax_marg_x.yaxis.get_majorticklines(), visible=False)
        # plt.setp(ax_marg_x.yaxis.get_minorticklines(), visible=False)
        # plt.setp(ax_marg_y.xaxis.get_majorticklines(), visible=False)
        # plt.setp(ax_marg_y.xaxis.get_minorticklines(), visible=False)
        # plt.setp(ax_marg_x.get_yticklabels(), visible=False)
        # plt.setp(ax_marg_y.get_xticklabels(), visible=False)
        # ax_marg_x.yaxis.grid(False)
        # ax_marg_y.xaxis.grid(False)

        sns.despine(f)
        sns.despine(ax=ax_marg_x, left=False)
        sns.despine(ax=ax_marg_y, bottom=False)
        if extra_x_marg:
            sns.despine(ax=ax_marg_x2, left=False)
        if extra_y_marg:
            sns.despine(ax=ax_marg_y2, left=False)

        f.tight_layout()
        f.subplots_adjust(hspace=space, wspace=space)

class JointHistPlotError(Exception):
    pass

def _estimate_bin_count(xd):
    import operator

    n = reduce(operator.mul, xd.shape, 1)
    bins = int((float(n)) ** (1.0 / 4.0))
    return bins

def _raw_calc_joint_hist(xd, yd, bins=None):
    x_range = (np.nanmin(xd), np.nanmax(xd))
    y_range = (np.nanmin(yd), np.nanmax(yd))

    if np.any([np.isnan(x_range), np.isnan(y_range)]):
        raise JointHistPlotError

    if bins is None:
        bins = _estimate_bin_count(xd=xd)

    # add bins at edges we are certain will have zero counts, this will ensure
    # that when we contour around bins later that there are enough sample
    # points that the contouring will produce closed contours
    dx = (x_range[1] - x_range[0]) / bins
    dy = (y_range[1] - y_range[0]) / bins
    x_range = (x_range[0] - dx, x_range[1] + dx)
    y_range = (y_range[0] - dy, y_range[1] + dy)
    bins += 2

    bin_counts, x_bins, y_bins = np.histogram2d(
        xd, yd, bins=bins, range=(x_range, y_range)
    )

    x_c = 0.5 * (x_bins[1:] + x_bins[:-1])
    y_c = 0.5 * (y_bins[1:] + y_bins[:-1])

    return (x_c, y_c), bin_counts

def calc_joint_hist(xd, yd, bins=None):
    if isinstance(xd, xr.DataArray):
        (x_c, y_c), bin_counts = _raw_calc_joint_hist(
            xd=xd.values.flatten(), yd=yd.values.flatten(), bins=bins
        )

        da__x_c = xr.DataArray(name=xd.name, dims=(xd.name,), data=x_c, attrs=xd.attrs)
        da__y_c = xr.DataArray(name=yd.name, dims=(yd.name,), data=y_c, attrs=yd.attrs)

        da = xr.DataArray(
            data=bin_counts,
            name="bin_counts",
            dims=(xd.name, yd.name),
            coords={xd.name: da__x_c, yd.name: da__y_c},
        )

        return da
    else:
        return _raw_calc_joint_hist(xd=xd, yd=yd, bins=bins)

def joint_hist_contoured(xd, yd, bins=None, normed_levels=None, ax=None, **kwargs):
    """
    Create joint histogram with contour levels at `normed_levels` percentiles
    """
    if ax is None:
        import matplotlib.pyplot as plt

        ax = plt.gca()

    (x_c, y_c), bin_counts = _raw_calc_joint_hist(xd=xd, yd=yd, bins=bins)
    x_, y_ = np.meshgrid(x_c, y_c, indexing="ij")

    if "plot_hist2d" in kwargs:
        ax.pcolormesh(x_c, y_c, bin_counts)
        cnt = None
    elif normed_levels is not None:
        levels = [
            _find_bin_on_percentile(bin_counts=bin_counts, q=q) for q in normed_levels
        ]

        linestyles = [":", "--", "-"]
        if len(normed_levels) > len(linestyles):
            raise NotImplementedError
        else:
            linestyles = linestyles[-len(normed_levels) :]

        cnt = ax.contour(
            x_, y_, bin_counts, levels=levels, linestyles=linestyles, **kwargs
        )

    else:
        cnt = ax.contour(x_, y_, bin_counts, **kwargs)

    return (x_, y_), bin_counts, cnt

def _get_counts(x, y):
    c = Counter([tuple(v) for v in zip(x, y)])
    x_, y_ = zip(*c.keys())
    vals = np.array(list(c.values())).astype(int)

    return (np.array(x_), np.array(y_)), vals

def make_marker_plot(x, y, scale=100.0, ax=None, marker="o", **kwargs):
    """
    Like a scatter point only instead of overplotting points at the same
    location the marker size is increased.
    """
    if ax is None:
        ax = plt.gca()

    if not len(x.shape) == 1:
        raise Exception("Only pass in 1D arrays")

    pts, vals = _get_counts(x, y)
    x_, y_ = pts
    z_ = vals

    s = np.argsort(z_)
    x_, y_, z_ = x_[s], y_[s], z_[s]

    s = z_ / np.nanmax(z_) * scale

    sc = ax.scatter(x_, y_, s=s, marker=marker, edgecolors="none", **kwargs)
    color = sc.get_facecolor()

    # idxs = [np.argmin(np.abs(np.percentile(z_, q) - z_)) for q in [5, 50, 95]]

    idxs = [np.argmax(z_), np.argmin(np.abs(z_ - np.median(z_))), np.argmin(z_)]
    idxs = set(idxs)

    for i in idxs:
        ax.scatter(
            x_[i],
            y_[i],
            s=s[i],
            vmin=0,
            vmax=1,
            edgecolors="none",
            label="{:d}".format(z_[i]),
            marker=marker,
            color=color,
            **kwargs
        )
    # add the legend
    ax.legend(
        scatterpoints=1,
        handletextpad=1.0,
        labelspacing=scale / 500.0,
        loc="upper right",
    )

    return ax

cmaps_for_colors = {
    "r": "Reds",
    "g": "Greens",
    "b": "Blues",
    "tab:purple": "Purples",
    "tab:orange": "Oranges",
}

def get_color_cmap(color):
    if color not in cmaps_for_colors:
        if color[0] in cmaps_for_colors:
            # red => r
            color = color[0]
        else:
            raise NotImplementedError(color)
    return cmaps_for_colors[color]

def multi_jointplot(
    x,
    y,
    z,
    ds,
    joint_type="pointhist",
    lgd_ncols=1,
    joint_kwargs=dict(alpha=0.5, marker="."),
    colors="default",
    **kwargs
):
    """
    like seaborn jointplot but making it possible to plot for multiple (up to
    four) datasets
    """
    if colors == "default":
        colors = ["b", "r", "g", "orange"]

    cmaps = [get_color_cmap(color=c) for c in colors]

    if np.any(np.isinf(ds[x])) or np.any(np.isinf(ds[y])):
        ds = ds.copy()
        ds = ds.where(~np.logical_or(np.isinf(ds[x]), np.isinf(ds[y])))
        warnings.warn("inf values will be filtered out")

    xlim = kwargs.get("xlim", np.array([ds[x].min(), ds[x].max()]))
    ylim = kwargs.get("ylim", np.array([ds[y].min(), ds[y].max()]))
    bins_x = 20
    bins_y = 10

    z_values = ds[z].values

    if len(z_values) > len(cmaps):
        raise NotImplementedError(
            "Need to add some more colourmaps to handle" " this number of datasets"
        )

    g = sns.JointGrid(x=x, y=y, data=ds.sel(**{z: z_values}), **kwargs)
    for c, cmap, z_ in zip(colors, cmaps, z_values):
        ds_ = ds.sel(**{z: z_}).dropna(dim="object_id")
        if joint_type == "pointhist":

            def point_hist():
                bin_counts, pts = np.histogramdd(
                    [ds_[x], ds_[y]], bins=(bins_x, bins_y), range=(xlim, ylim)
                )
                x_e, y_e = pts
                s = 1500.0 / np.sum(bin_counts)
                center = lambda x_: 0.5 * (x_[1:] + x_[:-1])  # noqa
                x_c, y_c = np.meshgrid(center(pts[0]), center(pts[1]), indexing="ij")
                assert bin_counts.shape == x_c.shape
                g.ax_joint.scatter(
                    x_c, y_c, s=s * bin_counts, marker="o", color=c, alpha=0.6
                )

            point_hist()
        elif joint_type == "scatter":
            _ = g.ax_joint.scatter(ds_[x], ds_[y], color=c, **joint_kwargs)
        elif joint_type == "kde":
            _ = g.ax_joint.scatter(ds_[x], ds_[y], color=c, **joint_kwargs)
            sns.kdeplot(ds_[x], ds_[y], cmap=cmap, ax=g.ax_joint, n_levels=5)
        else:
            raise NotImplementedError(joint_type)
        _ = g.ax_marg_x.hist(
            ds_[x], alpha=0.6, color=c, range=xlim, density=True, bins=bins_x
        )
        _ = g.ax_marg_y.hist(
            ds_[y],
            alpha=0.6,
            color=c,
            orientation="horizontal",
            range=ylim,
            density=True,
            bins=bins_y,
        )

    g.ax_joint.set_xlabel(xr.plot.utils.label_from_attrs(ds[x]))
    g.ax_joint.set_ylabel(xr.plot.utils.label_from_attrs(ds[y]))

    LABEL_FORMAT = "{name}: {count} objects"
    g.ax_joint.legend(
        labels=[
            LABEL_FORMAT.format(
                name=z_,
                count=int(ds.sel(**{z: z_}).dropna(dim="object_id").object_id.count()),
            )
            for z_ in z_values
        ],
        bbox_to_anchor=[0.5, -0.25],
        loc="lower center",
        ncol=lgd_ncols,
    )

    return g

def fixed_bin_hist(v, dv, ax, **kwargs):
    if dv is None:
        nbins = None
        vrange = None
    else:
        vmin = np.floor(v.min() / dv) * dv
        vmax = np.ceil(v.max() / dv) * dv
        nbins = int((vmax - vmin) / dv)
        vrange = (vmin, vmax)
    ax.hist(v, range=vrange, bins=nbins, **kwargs)

def adjust_fig_to_fit_figlegend(fig, figlegend, direction="bottom"):
    # Draw the plot to set the bounding boxes correctly
    fig.draw(fig.canvas.get_renderer())

    if direction == "right":
        # Calculate and set the new width of the figure so the legend fits
        legend_width = figlegend.get_window_extent().width / fig.dpi
        figure_width = fig.get_figwidth()
        fig.set_figwidth(figure_width + legend_width)

        # Draw the plot again to get the new transformations
        fig.draw(fig.canvas.get_renderer())

        # Now calculate how much space we need on the right side
        legend_width = figlegend.get_window_extent().width / fig.dpi
        space_needed = legend_width / (figure_width + legend_width) + 0.02
        # margin = .01
        # _space_needed = margin + space_needed
        right = 1 - space_needed

        # Place the subplot axes to give space for the legend
        fig.subplots_adjust(right=right)
    elif direction == "top":
        # Calculate and set the new width of the figure so the legend fits
        legend_height = figlegend.get_window_extent().height / fig.dpi
        figure_height = fig.get_figheight()
        fig.set_figheight(figure_height + legend_height)

        # Draw the plot again to get the new transformations
        fig.draw(fig.canvas.get_renderer())

        # Now calculate how much space we need on the right side
        legend_height = figlegend.get_window_extent().height / fig.dpi
        space_needed = legend_height / (figure_height + legend_height) + 0.02
        # margin = .01
        top = 1 - space_needed

        # Place the subplot axes to give space for the legend
        fig.subplots_adjust(top=top)
    elif direction == "bottom":
        # Calculate and set the new width of the figure so the legend fits
        legend_height = figlegend.get_window_extent().height / fig.dpi
        figure_height = fig.get_figheight()
        fig.set_figheight(figure_height + legend_height)

        # Draw the plot again to get the new transformations
        fig.draw(fig.canvas.get_renderer())

        # Now calculate how much space we need on the right side
        legend_height = figlegend.get_window_extent().height / fig.dpi
        space_needed = legend_height / (figure_height + legend_height) + 0.02
        # margin = .01
        bottom = 2 * space_needed

        # Place the subplot axes to give space for the legend
        fig.subplots_adjust(bottom=bottom)
    else:
        raise NotImplementedError(direction)

max_stars_repo_name: leifdenby/genesis

max_issues_repo_name: leifdenby/genesis

max_forks_repo_name: leifdenby/genesis


text:
"""
Pyqtgraph widget for import as a custom widget in Qt Designer

Adapted from: https://stackoverflow.com/questions/45872255/embed-a-pyqtgraph-plot-into-a-qt-ui

"""

import sys
import pyqtgraph as pg
from pyqtgraph.Qt import QtCore, QtGui
import numpy as np

class Custom_pyqtgraph_widget(pg.ImageView):
    """
    Custom pyqtgraph widget used for the camera view
    """
    def __init__(self, parent=None, **kargs):
        super().__init__()
#        import pdb; pdb.set_trace()
        self.ui.graphicsView.setAntialiasing = False
        self.ui.graphicsView.background = 'None'
        self.ui.histogram.hide()
        self.ui.roiPlot.autoPixelRange

if __name__ == '__main__':
    app = QtGui.QApplication([])
    w = Custom_pyqtgraph_widget()
    w.show()
    QtGui.QApplication.instance().exec_()


text:
import os

import h5py
import pandas as pd
import numpy as np
import uuid
from astropy import units as u
from tqdm import tqdm
from scipy import ndimage as nd
from scipy.interpolate import interp1d
import starkit

class BaseProcessGrid(object):

    def __init__(self, index, input_wavelength, meta, wavelength_start=0*u.angstrom, wavelength_stop=np.inf*u.angstrom,
                 R=None, R_sampling=4):

        self.index = index
        self.meta = meta
        self.R = float(R)
        self.R_sampling = int(R_sampling)

        wavelength_start = np.max(u.Quantity([wavelength_start, np.min(input_wavelength)]))
        wavelength_stop = np.min(u.Quantity([wavelength_stop, np.max(input_wavelength)]))
        self.wavelength_start = wavelength_start
        self.wavelength_stop = wavelength_stop

        input_wavelength = input_wavelength.to('angstrom')

        self.start_idx, self.stop_idx = self._get_wavelength_bounds(
            input_wavelength, wavelength_start, wavelength_stop)
        self.cut_wavelength = input_wavelength[
                              self.start_idx:self.stop_idx]

        self.output_wavelength = self._logrange(
            wavelength_start, wavelength_stop, R, R_sampling)

    @staticmethod
    def _get_wavelength_bounds(wavelength, wavelength_start, wavelength_stop):
        start_idx = np.max([wavelength.searchsorted(wavelength_start) - 5, 0])
        stop_idx = np.min([wavelength.searchsorted(wavelength_stop) + 5, len(wavelength)])
        return start_idx, stop_idx

    @staticmethod
    def _logrange(wavelength_start, wavelength_stop, R, sampling):
        return np.exp(
            np.arange(np.log(u.Quantity(wavelength_start, 'angstrom').value),
                      np.log(u.Quantity(wavelength_stop, 'angstrom').value),
                      1 / (sampling * float(R))))

    def get_fluxes(self):
        """
        Load the fluxes from the files given in the raw index, process them, interpolate them and return them.
        This return float64 arrays
        Returns
        -------
            fluxes : numpy.ndarray
        """
        fluxes = np.empty((len(self.index), len(self.output_wavelength)), dtype=np.float64)

        for i, fname in tqdm(enumerate(self.index.filename),
                             total=len(self.index)):
            flux = self.load_flux(fname)
            fluxes[i] = self.interp_wavelength(flux)
        return fluxes

    def interp_wavelength(self, flux):
        cut_flux = flux[self.start_idx:self.stop_idx]
        rescaled_R = 1 / np.sqrt((1/self.R)**2 - (1/self.R_initial)**2)
        sigma = ((self.R_initial / rescaled_R) * self.R_initial_sampling
                 / (2 * np.sqrt(2 * np.log(2))))

        processed_flux = nd.gaussian_filter1d(cut_flux, sigma)
        output_flux = interp1d(self.cut_wavelength, processed_flux)(
            self.output_wavelength)
        return output_flux

    def get_meta(self):
        """
        Get the meta data and add processing information on

        Returns
        -------
            : pandas.Series
        """
        meta = self.meta.copy()
        parameters = []

        for param in meta['parameters']:
            if len(self.index[param].unique()) > 1:
                parameters.append(param)
        meta['parameters'] = parameters
        meta['flux_unit'] = 'erg/s/angstrom'
        meta['R'] = self.R
        meta['R_sampling'] = self.R_sampling
        meta['grid_type'] = 'log'
        meta['uuid'] = str(uuid.uuid4())
        meta['format_version'] = starkit.gridkit.FORMAT_VERSION
        return meta

    def get_index(self):
        """
        Remove the filenames from the index
        Returns
        -------
            : pandas DataFrame
        """

        index = self.index.copy()
        return index.drop('filename', axis=1)

    def to_hdf(self, fname, overwrite=False):
        """

        Parameters
        ----------
        fname
        overwrite : bool

        Returns
        -------

        """

        if os.path.exists(fname) and not overwrite:
            raise IOError('File {0} exists and overwrite=False ... aborting'.format(fname))
        if os.path.exists(fname) and overwrite:
            os.system('rm -f {0}'.format(fname))

        fluxes = self.get_fluxes()
        meta = self.get_meta()
        index = self.get_index()

        with h5py.File(fname, mode='w') as fh:
            fh['fluxes'] = fluxes

        meta.to_hdf(fname, 'meta')
        index.to_hdf(fname, 'index')
        pd.DataFrame(self.output_wavelength).to_hdf(fname, 'wavelength')

reason: import numpy,from scipy,from astropy


text:
import os
import tensorflow as tf
from flask import Flask, request, render_template, jsonify
from keras.models import load_model
from keras.preprocessing import image
from keras.applications.resnet50 import preprocess_input, decode_predictions
import numpy as np

app = Flask(__name__)
app.config['MODEL_PATH'] = os.path.join('static/model.h5')
app.config['UPLOAD_FOLDER'] = os.path.join('static/images')

model = load_model(app.config['MODEL_PATH'])
graph = tf.get_default_graph()

@app.route('/', methods=['GET', 'POST'])
def upload_and_predict():
    if request.method == 'POST':
        file = request.files['file']
        #filename = file.filename
        filename='uploaded_image'
        file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
        file.save(file_path)
        img = image.load_img(file_path, target_size=(224, 224))
        x = image.img_to_array(img)
        x = np.expand_dims(x, axis=0)
        x = preprocess_input(x)

        global graph
        with graph.as_default():
            preds = model.predict(x)
        pred = str(decode_predictions(preds)[0][0][1])
        prob = str(decode_predictions(preds)[0][0][2])
        return render_template('prediction.html', prediction = pred,
                                                  probability = prob,
                                                  image_path = '../' + file_path)

    return render_template('upload.html')

if __name__ == '__main__':
    app.run(host='0.0.0.0', debug=False, port=5000)


text:
# pomp dp
# https://arxiv.org/pdf/1902.05795.pdf

import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
import PIL
import numpy as np
import matplotlib.pyplot as plt
from models.parser import parser
from models.common import sumtree, memory

class actorCritic:

    def __init__(self, env):
        # we need two network
        # actor
        # critic
        self.env = env
        self.time_step = 4
        self.action_size = self.env.action_space.n
        self.discount = 0.99
        with tf.variable_scope(name_or_scope="placeholders",reuse=tf.AUTO_REUSE):
            self.x_input = tf.placeholder(tf.float32,shape=(None,7,7,1),name="input")
            self.target  = tf.placeholder(tf.float32,shape=(None,1),name="target")
            self.actions  = tf.placeholder(tf.int32,shape=(None,self.action_size),name="action")

        self.act_value,self.policyparam,self.trainable_param = self.architecture("actor_critic_network")
        print(self.trainable_param)

    def weightVariable(self,name):
        with tf.variable_scope(name_or_scope=name,reuse=tf.AUTO_REUSE):

        	self.conv1 = tf.get_variable(name="conv1",shape=(2,2,1,16),initializer=tf.initializers.glorot_uniform)

        	self.conv2 = tf.get_variable(name="conv2",shape=(2,2,16,32),initializer=tf.initializers.glorot_uniform)

        	self.conv3 = tf.get_variable(name="conv3",shape=(2,2,32,64),initializer=tf.initializers.glorot_uniform)

        	weight  = tf.random_normal(mean=1,stddev=0,shape=(64,self.action_size))
        	weight  *=  1/tf.sqrt(tf.reduce_sum(tf.pow(weight,2),axis=1,keepdims=True))
        	self.action_value  = tf.get_variable(name="action",initializer=weight)
        	weight  = tf.random_normal(mean=1,stddev=0,shape=(64,1))
        	weight  *=  1/tf.sqrt(tf.reduce_sum(tf.pow(weight,2),axis=1,keepdims=True))
        	self.value_network = tf.get_variable(name="value",initializer=weight)

    def architecture(self,name):
        self.weightVariable(name)
        with tf.variable_scope(name_or_scope=name,reuse=tf.AUTO_REUSE):

            # we need to perform the convolution operation along with LSTM cell

            self.conv  =  tf.nn.conv2d(self.x_input,self.conv1,strides=[1,1,1,1],padding="VALID")
            self.conv  =  tf.nn.relu(self.conv)
            self.conv  =  tf.keras.layers.MaxPool2D((2,2))(self.conv)

            self.conv  = tf.nn.conv2d(self.conv,self.conv2,strides=[1,1,1,1],padding="VALID")
            self.conv  = tf.nn.relu(self.conv)

            self.conv  = tf.nn.conv2d(self.conv,self.conv3,strides=[1,1,1,1],padding="VALID")
            self.conv  = tf.nn.relu(self.conv)

            self.conv_flat  = tf.keras.layers.Flatten()(self.conv)

            # we will reshape it according to the requirement of LSTM
            # (batch,timestep,x)

            self.conv_flat = tf.stack([self.conv_flat]*4,axis=1)

            # now we can pass this to our LSTM network

            self.lstm  = tf.keras.layers.GRU(units=64,return_sequences=False)(self.conv_flat)
            # now we can use dense layers
            # but since we are using actor critic we need to get value and action values

            self.value = tf.matmul(self.lstm,self.value_network)
            self.act_value  =  tf.matmul(self.lstm,self.action_value)
            self.act_value  = tf.nn.softmax(self.act_value)

            # we have value
            trainable_param = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,"actor_critic_network")

            return self.value,self.act_value,trainable_param

    def huberLoss(self,a,b):
    	error  = a -b
    	result  = tf.cond(tf.reduce_mean(tf.math.abs(error)) > 1.0,lambda:tf.math.abs(error)-0.5,lambda:error*error/2)
    	return result

    def actor_critic_loss(self):

        # the loss is going to be critic and actor losses

        self.adv =  self.target - self.act_value

        self.adv  = (self.adv  - tf.keras.backend.mean(self.adv) )/(tf.keras.backend.std(self.adv) + 1e-8)
        self.critic_loss =  self.huberLoss(self.target,self.act_value)

        # now we gonna calculate the actor loss

        self.actor = tf.nn.softmax_cross_entropy_with_logits(logits=self.policyparam,labels=self.actions)
        self.actor_loss = tf.reduce_mean(self.actor)*tf.stop_gradient(self.adv)

        # we gonna calculate the entropy to encourage more exploration

        entropy =  tf.reduce_sum(tf.reduce_sum(self.policyparam* tf.log(tf.clip_by_value(self.policyparam, 1e-20, 1.0))))

        self.total_loss = 0.5*self.critic_loss + self.actor_loss - 0.0001*entropy
        self.grad  =  tf.gradients(self.total_loss,self.trainable_param)

        # clipped_grad  = []
        # for g in self.grad:
        # 	clipped_grad.append(tf.clip_by_value(g,clip_value_min =1,clip_value_max=-1))

        # we will clip by global norm
        max_grad_norm = 0.5
        norm_grad,grad_ = tf.clip_by_global_norm(self.grad,max_grad_norm)

        self.grad  = zip(norm_grad,self.trainable_param)

        #self.total_optimizer = tf.train.GradientDescentOptimizer(learning_rate=1e-5).apply_gradients(self.grad)
        self.total_optimizer = tf.train.RMSPropOptimizer(learning_rate=1e-5).apply_gradients(self.grad)
        # self.critic_optimizer= tf.train.AdamOptimizer(learning_rate=0.0001).minimize(self.critic_loss)

    def chooseAction(self, state, sess):
        # here we choose the action
        action_prob = self.policyparam.eval(session = sess, feed_dict={self.x_input:[state]})[0]
        return np.random.choice(self.action_size,p=action_prob)

    def chooseActionTest(self, state, sess):
        # here we choose the action for testing phase
        action_prob = self.policyparam.eval(session = sess, feed_dict={self.x_input:[state]})[0]
        return np.argmax(action_prob)

def preprocessing(image):
    img = PIL.Image.fromarray(image).convert("L")
    img = np.array(img).reshape(7,7,1)
    return img

# def run():

#     # tf.reset_default_graph()
#     # RA = parser()
#     # ac_network = actorCritic()
#     # ac_network.actor_critic_loss()
#     # init = tf.global_variables_initializer()

#     total_reward = 0

#     for each_episode in tqdm(range(episode_to_restore, episode_to_restore+n_episodes), total=n_episodes):

#         # we will use monte carlo approach
#         # that is we first record the data
#         # than use it

#         done = False
#         in_trinsic  = 0
#         state = env.reset()["image"]
#         state  = preprocessing(state)

#         # these are temporary data storage lists
#         save_states, save_nextStates, save_actions, save_rewards, save_dones = [np.empty([0]) for i in range(5)]

#         c_loss = 0
#         total_loss_actor_critic = 0
#         count_batch = 0

#         while not done:

#             # first thing we need to choose the action
#             action = ac_network.chooseAction(state)

#             # now i can perform the step in the environment

#             next_state,reward,done,_ = env.step(action)
#             next_state = next_state["image"]

#             open_door = env.door.is_open

#             key = False if env.carrying == None else True

#             intrinsic_reward,failDfa = RA.trace(key,open_door)
#             in_trinsic += intrinsic_reward
#             t_reward  = reward +intrinsic_reward

#             # now we gonna store the trajectory

#             save_states = np.append(save_states, state)
#             state = preprocessing(next_state)
#             save_nextStates = np.append(save_nextStates, state)
#             save_actions = np.append(save_actions, action)
#             save_rewards = np.append(save_rewards, t_reward)
#             save_dones = np.append(save_dones, done)

#             total_reward += reward

#             # done = failDfa

#             if count_batch % 32 ==0:
#                 value_predicted  = ac_network.act_value.eval(feed_dict={ac_network.x_input:save_nextStates})
#                 target  = []
#                 save_rewards = (save_rewards  - save_rewards.mean())/(save_rewards.std() + np.finfo(np.float32).eps)

#                 for i in range(len(value_predicted)):
#                     target.append(save_rewards[i] + ac_network.discount*value_predicted[i]*(1-save_dones[i]))

#                 target = np.array(target)
#                 action_convert = save_actions.reshape(-1)
#                 onehotEncodeAction = np.eye(ac_network.action_size)[action_convert]
#                 total_loss_actor_critic,_ = sess.run([ac_network.total_loss,ac_network.total_optimizer],feed_dict  = {ac_network.x_input:save_states,ac_network.target:target,ac_network.actions:onehotEncodeAction})

#                 save_states, save_nextStates, save_actions, save_rewards, save_dones = [], [], [], []
#                 count_batch = 0

#             count_batch+=1

#         if episode % 20 == 0 and episode!=0:

#             print("After episode ",str(episode)," the total loss  ",str(np.sum(total_loss_actor_critic))," And reward ",str(total_reward))
#             print("Intrinsic reward after episode ",str(episode), "  is ",str(in_trinsic))
#             total_reward = 0


text:
"""

    Copyright (c) 2014-2015-2015, The University of Texas at Austin.
    All rights reserved.

    This file is part of BLASpy and is available under the 3-Clause
    BSD License, which can be found in the LICENSE file at the top-level
    directory or at http://opensource.org/licenses/BSD-3-Clause

"""

from ..helpers import random_vector, random_triangular_matrix
from blaspy import trmv
from numpy import allclose, copy, dot
from itertools import product
from random import randint

N_MIN, N_MAX = 2, 1e3           # matrix/vector sizes
SCAL_MIN, SCAL_MAX = -100, 100  # scalar values
STRIDE_MAX = 1e2                # max vector stride
RTOL, ATOL = 5e-01, 5e-02       # margin of error

def acceptance_test_trmv():
    """
    Test triangular matrix-vector multiplication.

    Returns:
        A list of strings representing the failed tests.
    """

    tests_failed = []

    # values to test
    dtypes = ('float64', 'float32')
    bools = (True, False)
    strides = (1, None)  # None indicates random stride
    uplos = ('u', 'l')
    trans_tuple = ('n', 't')
    diags = ('n', 'u')

    # test all combinations of all possible values
    for (dtype, as_matrix, x_is_row, stride, uplo, trans, diag) \
            in product(dtypes, bools, bools, strides, uplos, trans_tuple, diags):

        # if a test fails, create a string representation of its name and append it to the list
        # of failed tests
        if not passed_test(dtype, as_matrix, x_is_row, stride, uplo, trans, diag):
            variables = (dtype,
                         "_matrix" if as_matrix else "_ndarray",
                         "_row" if x_is_row else "_col",
                         "_rand_stride_" if stride is None else "_",
                         uplo, "_",
                         trans, "_",
                         diag)
            test_name = "".join(variables)
            tests_failed.append(test_name)

    return tests_failed

def passed_test(dtype, as_matrix, x_is_row, stride, uplo, trans_a, diag):
    """
    Run one triangular matrix-vector multiplication test.

    Arguments:
        dtype:        either 'float64' or 'float32', the NumPy dtype to test
        as_matrix:    True to test a NumPy matrix, False to test a NumPy ndarray
        x_is_row:     True to test a row vector as parameter x, False to test a column vector
        stride:       stride of x and y to test; if None, a random stride is assigned
        uplo:         BLASpy uplo parameter to test
        trans:        BLASpy trans parameter to test
        diag:         BLASpy diag parameter to test

    Returns:
        True if the expected result is within the margin of error of the actual result,
        False otherwise.
    """

    # generate random sizes for matrix/vector dimensions and vector stride (if necessary)
    n = randint(N_MIN, N_MAX)
    stride = randint(N_MIN, STRIDE_MAX) if stride is None else stride
    n_A = n / stride + (n % stride > 0)

    # create random vectors and matrices to test
    x = random_vector(n, x_is_row, dtype, as_matrix)
    A = random_triangular_matrix(n_A, dtype, as_matrix, uplo, diag)

    # create copies/views of A, x, and y that can be used to calculate the expected result
    A_2 = A if trans_a == 'n' else A.T
    x_2 = copy(x.T) if x_is_row else copy(x)
    x_3 = copy(x.T) if x_is_row else copy(x)

    # compute the expected result
    if stride == 1:
        x_2 = dot(A_2, x_2)
    else:
        for i in range(0, n, stride):
            A_partition = A_2[i / stride, :]
            x_partition = x_3[:: stride, :]
            x_2[i, 0] = dot(A_partition, x_partition)

    # get the actual result
    trmv(A, x, uplo, trans_a, diag, inc_x=stride,)

    # if x is a row vector, make x_2 a row vector as well
    if x_is_row:
        x_2 = x_2.T

    # compare the actual result to the expected result and return result of the test
    return allclose(x, x_2, RTOL, ATOL)

max_stars_repo_name: nicholas-moreles/blaspy

max_issues_repo_name: nicholas-moreles/blaspy

max_forks_repo_name: nicholas-moreles/blaspy


text:
import numpy as np
import pandas as pd
import networkx as nx
from collections import Counter

def m_time(pos1,pos2): #计算rgv在两点间运动的最短时间
    d=abs(pos1-pos2)
    if d==1:
        return m1
    elif d==2:
        return m2
    elif d==3:
        return m3
    elif d==0:
        return 0
#m1,m2,m3,p1,p21,p22,to,te,w=20,33,46,400,400,378,10,50,50
#m1,m2,m3,p1,p21,p22,to,te,w=20,33,46,560,400,378,28,31,25
#m1,m2,m3,p1,p21,p22,to,te,w=23,41,59,580,280,500,30,35,30
m1,m2,m3,p1,p21,p22,to,te,w=18,32,46,545,455,182,27,32,25
t=0
v_pos = 1  # 初始位置为1，右侧依次为2、3、4
v_sit = 0  # 0停止，1移动，2上料，3下料、上料、清洗
rgv=np.array([[t,v_pos,v_sit]])

c_sit=0 #表示cnc的作业状态，0为空闲且无孰料，1为空闲且有孰料，2为工作中
cnc=[]
cnc_inf=[]
c_pos=0 #表示初始位置，无实际意义
for num in range(1,9):
    if num==1 or num==2:
        c_pos=1
    elif num==3 or num==4:
        c_pos=2
    elif num==5 or num==6:
        c_pos=3
    elif num==7 or num==8:
        c_pos=4
    if num==1 or num==3 or num==5 or num==7:
        tc=to
    else:
        tc=te
    cnc.append(np.array([[t, num, c_pos, c_sit]]))
    cnc_inf.append(np.array([num,p1,p21,p22,tc]))

    #储存了cnc的加工、上下料时间，是固定的信息

judge_cnc=np.array([[30000,30000,30000,30000,30000,30000,300000,300000]]) #初始cnc加工→休息判断矩阵
judge_move=30000
judge_load=np.array([[30000,30000,30000,30000,30000,30000,300000,300000]])
judge_layoff=np.array([[30000,30000,30000,30000,30000,30000,300000,300000]])
t_wait=0
load_num=0
layoff_num=0
result=np.array([[0,0,0,0]])

print("stop")

for t in range(1,28800): #从第1秒到第5秒
    #print(t)
    #################################
    rgv_new =np.array([t, rgv[t-1,1], rgv[t-1,2]])# 0停止，1移动，2上料，3下熟料、上半熟料、清洗
    rgv=np.vstack((rgv, rgv_new))#t改变，其他rgv参数复制上一秒的信息
    for num in range(1, 9):
        cnc_new=np.array([t, cnc[num-1][t-1,1],cnc[num-1][t-1,2],cnc[num-1][t-1,3]])
        cnc[num-1]=np.vstack((cnc[num-1], cnc_new))#t改变，其他cnc参数复制上一秒的信息
    #################################
    for num in range(1, 9):
        if cnc[num-1][t,3]==2 and t>=judge_cnc[0,num-1] and t<judge_cnc[0,num-1]+1: #cnc加工完成，给rgv发信号
            cnc[num - 1][t, 3] = 1   #cnc从2（工作状态）到1（空闲，上有孰料状态）
    #################################
    if rgv[t,2]==1:    #如果rgv上一秒在移动
        if t>=judge_move and t <judge_move+1:     #判断rgb移动是否完成
            rgv[t,2]=0     #rgv瞬间进入移动后的等待状态，按照算法，这里应当立即对cnc进行操作
            for num in range(1, 9):
                if cnc[num - 1][t, 3] == 1 and num==mov_pro:  # 如果cnc是空闲,上有孰料状态，目标机器匹配
                    rgv[t, 2] = 3
                    judge_layoff[0, num - 1] = t + cnc_inf[num - 1][4]
                    load_num = load_num + 1
                    layoff_num = layoff_num + 1
                    print("第%d个CNC在第%d秒开始下第%d个料" % (num, t, layoff_num))
                    print("第%d个CNC在第%d秒开始上第%d个料" % (num, t + cnc_inf[num - 1][4] / 2, load_num))
                    result_new = np.array([load_num, num, t + cnc_inf[num - 1][4] / 2, 0])
                    result = np.vstack((result, result_new))
                    result[layoff_num, 3] = t
                    break  # 一个时间rgv只能进入一种状态，这里进入了下料、上料、清洗状态
                if cnc[num - 1][t, 3] == 0 and num==mov_pro:  # 如果cnc也是空闲，无料状态（初始状态）,且目标机器匹配
                    rgv[t, 2] = 2  # rgv进入上料状态
                    judge_load[0, num - 1] = t + cnc_inf[num - 1][4] / 2  # 假设空cnc上料时间为上下料时间的一半
                    load_num = load_num + 1
                    print("第%d个CNC在第%d秒开始上第%d个料" % (num, t, load_num))
                    result_new = np.array([load_num, num, t, 0])
                    result = np.vstack((result, result_new))
                    break  # 一个时间rgv只能进入一种状态，这里进入了上料状态
            continue     #移动结束连着上（下）料，继续循环,等待着在judge上下料跳出循环
        else:
            continue       #上一秒在移动，还未停止，继续循环，一直在移动，直到judge移动停止
    if rgv[t,2]==2 or rgv[t,2]==3:
        for num in range(1, 9):
            if t>=judge_load[0,num-1] and t <judge_load[0,num-1]+1:       #这个情况只有在最开始的时候情况
                cnc[num-1][t,3]=2   #cnc从0（空闲，上无料状态）到2（工作状态）
                judge_cnc[0, num - 1] = t + cnc_inf[num - 1][1]  #当前时间加cnc工作时间，为cnc的预测空闲时间
                rgv[t,2]=0
            if t>=judge_layoff[0,num-1] and t <judge_layoff[0,num-1]+1:     #上下料同时进行
                cnc[num-1][t,3]=2 #cnc从1（空闲，上有孰料状态）到2（工作状态）
                judge_cnc[0, num - 1] = t + cnc_inf[num - 1][1]  #当前时间加cnc工作时间，为cnc的预测空闲时间
            if t >= judge_layoff[0, num - 1] +w and t <judge_layoff[0, num - 1]+w+1:  # 上下料同时进行
                rgv[t,2] = 0  #rgv清洗后再进入等待状态
                # rgv瞬间进入等待状态,因为我们判断下料后一定会上料，这里要不要直接跳过搜索空闲cnc的过程？
###################################################################
    if rgv[t,2]==0:  #如果rgv空闲
        cnc_stop_num=[]
        cnc_count_zeros=[]
        for num in range(1, 9):
            if cnc[num-1][t,3]==0 or cnc[num-1][t,3]==1:   #如果cnc也是空闲状态
                cnc_stop_num.append(cnc[num-1][t,1])   #调出空闲状态的cnc机号
            cnc_count_zeros.append(cnc[num-1][t,3])
##########################################################################################################################################################
        if cnc_stop_num == []:
            t_wait = t_wait + 1
            continue  # 跳出大循环，表示rgv处于等候状态
        else:
            est_t = []
            for i in cnc_stop_num:  # 1-8中的几个数
                est_t.append(m_time(cnc[i - 1][t, 2], rgv[t, 1]) + cnc_inf[i - 1][4])
            mov_pro = cnc_stop_num[est_t.index(min(est_t))]  # 索引最小的估计时间对应的cnc的号码
            ##################################################################################
        mov_pos = 0
        if mov_pro == 1 or mov_pro == 2:
            mov_pos = 1
        elif mov_pro == 3 or mov_pro == 4:
            mov_pos = 2
        elif mov_pro == 5 or mov_pro == 6:
            mov_pos = 3
        elif mov_pro == 7 or mov_pro == 8:
            mov_pos = 4  # 计算目标机位（mov_pro 1-8）所在的位置（mov_pos 1-4）
        if mov_pos != rgv[t, 1]:  # 如果判断的移动位置和rgv所在位置不同，则立即进入移动状态
            rgv[t, 2] = 1  # 0停止，1移动，2上料，3下料、上料、清洗
            judge_move = t + m_time(mov_pos, rgv[t, 1])
            rgv[t, 1] = mov_pos  # 移动开始，未知参数就变为目标位置
        elif mov_pos == rgv[t, 1]:  # 如果判断的移动位置和rgv所在位置相同，则立即进入 上料 或者 下料、上料、清洗 状态
            for num in range(1, 9):
                if cnc[num - 1][t, 3] == 1 and mov_pro == num:  # 如果cnc是空闲,上有孰料状态
                    rgv[t, 2] = 3
                    judge_layoff[0, num - 1] = t + cnc_inf[num - 1][4]
                    load_num = load_num + 1
                    layoff_num = layoff_num + 1
                    print("第%d个CNC在第%d秒开始下第%d个料" % (num, t, layoff_num))
                    print("第%d个CNC在第%d秒开始上第%d个料" % (num, t + cnc_inf[num - 1][4] / 2, load_num))
                    result_new = np.array([load_num, num, t + cnc_inf[num - 1][4] / 2, 0])
                    result = np.vstack((result, result_new))
                    result[layoff_num, 3] = t
                    break  # 一个时间rgv只能进入一种状态，这里进入了下料、上料、清洗状态
                if cnc[num - 1][t, 3] == 0 and mov_pro == num:  # 如果cnc也是空闲，无料状态（初始状态）
                    rgv[t, 2] = 2  # rgv进入上料状态
                    judge_load[0, num - 1] = t + cnc_inf[num - 1][4] / 2  # 假设空cnc上料时间为上下料时间的一半
                    load_num = load_num + 1
                    print("第%d个CNC在第%d秒开始上第%d个料" % (num, t, load_num))
                    result_new = np.array([load_num, num, t, 0])
                    result = np.vstack((result, result_new))
                    break  # 一个时间rgv只能进入一种状态，这里进入了上料状态
print(t_wait)
# result_df = pd.DataFrame(result)
# writer = pd.ExcelWriter('test_GA3.xlsx')
# result_df.to_excel(writer,'page_1',float_format='%.5f') # float_format 控制精度
# writer.save()

max_stars_repo_name: CarlossShi/mcm-shit-mountain

max_issues_repo_name: CarlossShi/MCM-shit-mountain

max_forks_repo_name: CarlossShi/MCM-shit-mountain

reason: import numpy,import networkx


text:
from __future__ import print_function, division
import sys, petsc4py
petsc4py.init(sys.argv)
from mpi4py import MPI
from petsc4py import PETSc
import numpy as np

from PETScGMsFEM import *

comm = MPI.COMM_WORLD

if(comm.Get_rank() == 0):
    print("Running Grid Test.py")

class DarcyEx1:

    def __init__(self, n, L, overlap, comm):

        # Build Grid

        self.dim = len(n)

        self.L = L

        self.comm = comm

        self.isBnd = lambda x: self.isBoundary(x)

        self.da = PETSc.DMDA().create(n, dof=1, stencil_width=overlap)
        self.da.setUniformCoordinates(xmax=L[0], ymax=L[1], zmax=L[2])
        self.da.setMatType(PETSc.Mat.Type.AIJ)

        self.numSub = comm.Get_size()

        self.sub2proc = [ [] for i in range(self.numSub) ]
        for i in range(self.numSub):
            self.sub2proc[i] = [i, 0]

        self.M = 1

        # Define Finite Element space

        self.fe = DarcyQ1(self.dim)

        # Setup global and local matrices + communicators

        self.A = self.da.createMatrix()
        r, _ = self.A.getLGMap() # Get local to global mapping
        self.is_A = PETSc.IS().createGeneral(r.indices) # Create Index Set for local indices
        A_local = self.A.createSubMatrices(self.is_A)[0] # Construct local submatrix on domain
        vglobal = self.da.createGlobalVec()
        vlocal = self.da.createLocalVec()
        self.scatter_l2g = PETSc.Scatter().create(vlocal, None, vglobal, self.is_A)

        self.A_local = A_local

        # Identify boundary nodes

        nnodes = int(self.da.getCoordinatesLocal()[ :].size/self.dim)
        coords = np.transpose(self.da.getCoordinatesLocal()[:].reshape((nnodes,self.dim)))
        Dirich, Neumann, P2P = checkFaces(self.da, self.isBnd, coords)

        # Construct Partition of Unity

        self.cS = coarseSpace(self.da, self.A, self.comm, self.scatter_l2g)

        self.cS.buildPOU(True)

    def isBoundary(self, x):
        val = 0.0
        if(x[0] < 1e-6):
            output = True
            val = 0.0
            type = 1
        elif(x[0] > self.L[0] - 1e-6):
            output = True
            val = 1.0
            type = 1
        elif((x[1] < 1e-6) or (x[2] < 1e-6) or (x[1] > L[1] - 1e-6) or (x[2] > L[2] - 1e-6)):
            type = 2 # Neumann Boundary
        else:
            type = 0 # Either internal or processor to processor boundary

        return val, type

    def solvePDE(self, plotSolution = False, isCoarse = False, filename = "Solution"):

        # Solve A * x = b

        # Assemble Global Stifness Matrix

        self.A = self.da.createMatrix()

        b = self.da.createGlobalVec()
        b_local = self.da.createLocalVec()

        elem = self.da.getElements()
        nnodes = int(self.da.getCoordinatesLocal()[ :].size/self.dim)
        coords = np.transpose(self.da.getCoordinatesLocal()[:].reshape((nnodes,self.dim)))
        for ie, e in enumerate(elem,0): # Loop over all local elements
            Ke = self.fe.getLocalStiffness(coords[:,e], 1.0)
            self.A.setValuesLocal(e, e, Ke, PETSc.InsertMode.ADD_VALUES)
            b_local[e] = self.fe.getLoadVec(coords[:,e])
        self.A.assemble()
        self.comm.barrier()

        if(isCoarse == False): # This is the fine solve

            # Implement Boundary Conditions
            rows = []
            for i in range(nnodes):
                val, type = self.isBoundary(coords[:,i])
                if(type == 1): # It's Dirichlet
                    rows.append(i)
                    b_local[i] = val
            rows = np.asarray(rows,dtype=np.int32)
            self.A.zeroRowsLocal(rows, diag = 1.0)

        self.scatter_l2g(b_local, b, PETSc.InsertMode.INSERT_VALUES)

        # Solve

        # Setup linear system vectors
        x = self.da.createGlobalVec()
        x.set(0.0)

        if(isCoarse == False):

            # Setup Krylov solver - currently using AMG
            ksp = PETSc.KSP().create()
            pc = ksp.getPC()
            ksp.setType('cg')
            pc.setType('gamg')

            # Iteratively solve linear system of equations A*x=b
            ksp.setOperators(self.A)
            ksp.setInitialGuessNonzero(True)
            ksp.setFromOptions()
            ksp.solve(b, x)

        else:

            self.buildCoarseSpace(self.A)

            x = self.cS.coarseSolve(b)

        if(plotSolution): # Plot solution to vtk file
            viewer = PETSc.Viewer().createVTK(filename + ".vts", 'w', comm = comm)
            x.view(viewer)
            viewer.destroy()

        return x

    def addtoBasis(self, x):
        # x is global soltuion vector from a previous solve. Need to obtain solution on subdomain
        x_loc = self.da.createLocalVec()
        self.scatter_l2g(x, x_loc, PETSc.InsertMode.INSERT_VALUES, PETSc.ScatterMode.SCATTER_REVERSE)
        self.cS.addBasisElement(x_loc) # Note that 0 since in this case 1 proc = 1 subdomain

    def buildCoarseSpace(self, A):

        self.cS.A = A

        self.cS.getCoarseVecs() # Builds Coarse Vectos on Each subdomain

        self.cS.compute_Av()

        self.cS.assembleCoarseMatrix()

L = [1.0, 1.0, 1.0]

n = [40, 40, 40]

dim = 3

overlap = 1

plotSolution = True

myModel = DarcyEx1(n, L, overlap, comm)

myModel.cS.getSharedProcessors()

x = myModel.solvePDE(True, False)

myModel.addtoBasis(x) # Add solution to Coarse basis.

x = myModel.solvePDE(True, True, filename = "coarseSolution")

#myModel.buildCoarseSpace()

max_stars_repo_name: tjdodwell/PETSc-GMsFEM

max_issues_repo_name: tjdodwell/PETSc-GMsFEM

max_forks_repo_name: tjdodwell/PETSc-GMsFEM


text:
# Built-in imports
import os
import argparse

# External imports
import mlflow
import pickle
import numpy as np
from hyperopt.pyll import scope
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import RandomForestRegressor
from hyperopt import STATUS_OK, Trials, fmin, hp, tpe

mlflow.set_tracking_uri("http://127.0.0.1:5000")
mlflow.set_experiment("random-forest-hyperopt")

def load_pickle(filename):
    with open(filename, "rb") as f_in:
        return pickle.load(f_in)

def run(data_path, num_trials):

    X_train, y_train = load_pickle(os.path.join(data_path, "train.pkl"))
    X_valid, y_valid = load_pickle(os.path.join(data_path, "valid.pkl"))

    def objective(params):
        with mlflow.start_run():
            mlflow.set_tag("model", "random-forest")
            mlflow.log_params(params)
            rf = RandomForestRegressor(**params)
            rf.fit(X_train, y_train)
            y_pred = rf.predict(X_valid)
            rmse = mean_squared_error(y_valid, y_pred, squared=False)
            mlflow.log_metric("rmse", rmse)

        return {'loss': rmse, 'status': STATUS_OK}

    search_space = {
        'max_depth': scope.int(hp.quniform('max_depth', 1, 20, 1)),
        'n_estimators': scope.int(hp.quniform('n_estimators', 10, 50, 1)),
        'min_samples_split': scope.int(hp.quniform('min_samples_split', 2, 10, 1)),
        'min_samples_leaf': scope.int(hp.quniform('min_samples_leaf', 1, 4, 1)),
        'random_state': 42
    }

    rstate = np.random.default_rng(42)  # for reproducible results
    fmin(
        fn=objective,
        space=search_space,
        algo=tpe.suggest,
        max_evals=num_trials,
        trials=Trials(),
        rstate=rstate
    )

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--data_path",
        default="./output",
        help="the location where the processed NYC taxi trip data was saved."
    )
    parser.add_argument(
        "--max_evals",
        default=50,
        help="the number of parameter evaluations for the optimizer to explore."
    )
    args = parser.parse_args()

    run(args.data_path, args.max_evals)


text:
import numpy as np
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torchvision
import torchvision.transforms as transforms
import torch.utils.data as dt
import pickle as cPickle
import gzip
import cv2

drawing = False
pt_arr = []
l = 0

class ConvNet(nn.Module):
    def __init__(self):
        super(ConvNet, self).__init__()

        self.layer1 = nn.Sequential(
            nn.Conv2d(1, 32, 5, 1, padding=2),
            torch.nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2)
        )

        self.layer2 = nn.Sequential(
            nn.Conv2d(32, 64, 5, 1, padding=2),
            torch.nn.ReLU(),
            nn.MaxPool2d(2, 2)
        )

        self.dropout = nn.Dropout()

        self.fc1 = nn.Linear(7 * 7 * 64, 1000)
        self.fc2 = nn.Linear(1000, 10)

    def forward(self, x):
        out = self.layer1(x)
        out = self.layer2(out)
        # print(out.shape)
        out = out.reshape((-1, 64 * 7 * 7))
        # print(out.shape)
        out = self.dropout(out)
        out = self.fc1(out)
        out = self.fc2(out)

        return out

MODEL_STORE_PATH = "/media/khurshed2504/Data/PycharmProjects/ML_temp0/PyTorch Models/MNIST 0"
model = ConvNet()
model.load_state_dict(torch.load(MODEL_STORE_PATH))
model.eval()

def draw(event, x, y, flags, param):
    global l, pt_arr, drawing

    if event == cv2.EVENT_LBUTTONDOWN:
        drawing = True
        pt_arr.append((x, y))
        l = l + 1

    elif event == cv2.EVENT_MOUSEMOVE:
        if drawing:
            pt_arr.append((x, y))
            cv2.line(pad, pt_arr[l - 1], pt_arr[l], color=0, thickness=1)
            l = l + 1

    elif event == cv2.EVENT_LBUTTONUP:
        drawing = False

pad = np.ones((28, 28))*255

cv2.namedWindow('Pad')
ans = np.ones((50,250))*255
cv2.setMouseCallback('Pad', draw)
font = cv2.FONT_HERSHEY_SIMPLEX
while(1):
    cv2.imshow('Pad', pad)
    k = cv2.waitKey(1) & 0xFF
    if k == ord('q'):
        break
    if k == 27:
        ans = np.ones((50,250))*255
        x = torch.from_numpy(255 - pad)
        x = x.reshape(1, 1, 28, 28)
        x = x.type(torch.float32)
        test_out = model(x)
        _, predicted = torch.max(test_out.data, 1)
        cv2.putText(ans, 'I think its a : ' + str(predicted.item()), (25,25), font, 0.5, 0, thickness=1,lineType=cv2.LINE_AA)
        pad = np.ones((28,28))*250

    cv2.imshow('Pad', pad)
    cv2.imshow('Answer', ans)

max_stars_repo_name: IvLabs/Real-Time-Digit-Classifier

max_issues_repo_path: FInal Code/PyTorch/Write_Pad_0.py

max_forks_repo_path: FInal Code/PyTorch/Write_Pad_0.py


text:
'''Backtest Moving Average (MA) crossover strategies
'''

import math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import talib
from backtester import Backtester

class RSIBacktester(Backtester):
    '''Backtest a RSI strategy

    Parameters:
    series: (Panda Series) a list of CLOSE prices by date
    lookback: (int) lookback period
    long_only: (boolean) True if the strategy can only go long
    '''

    def __init__(self, series, lookback=14, buy_on=70, sell_on=30, long_only=False, slippage=0):
        self._lookback=lookback
        self._buy_on=buy_on
        self._sell_on=sell_on
        super(RSIBacktester,self).__init__(series,long_only=long_only,slippage=slippage)

    def __str__(self):
        return "RSI Backtest Strategy (lookback=%d, buy_on=%d, sell_on=%d, long_only=%s, start=%s, end=%s)" % (
            self._lookback, self._buy_on, self._sell_on, str(self._long_only), str(self._start_date), str(self._end_date))

    def plot(self, start_date=None, end_date=None, figsize=None):
        sns.set_style("dark")
        fig, (ax1, ax2) = plt.subplots(nrows=2, sharex=True, figsize=figsize, gridspec_kw = {'height_ratios':[3, 1]})
        fig.suptitle(self.__str__(), size=13)

        Backtester.plot(self,start_date=start_date,end_date=end_date, ax=ax1)
        temp = self._df.loc[start_date:end_date]
        ax1.legend()

        ax2.plot(temp['RSI'])
        ax2.hlines(self._buy_on,temp.index[0],temp.index[-1],colors="silver",linestyles="dashed")
        ax2.hlines(self._sell_on,temp.index[0],temp.index[-1],colors="silver",linestyles="dashed")
        ax2.set_ylim(0,100)
        ax2.set_ylabel('RSI')

        plt.tight_layout()
        plt.show()

    def _trade_logic(self):
        '''Implements the trade logic in order to come up with
        a set of stances
        '''

        # delta = self._df['last'].diff()
        # dUp, dDown = delta.copy(), delta.copy()
        # dUp[dUp < 0] = 0
        # dDown[dDown > 0] = 0

        # RolUp = dUp.rolling(window=self._lookback).mean()
        # RolDown = dDown.abs().rolling(window=self._lookback).mean()

        # RS = RolUp / RolDown
        # RSI = 100.0 - (100.0 / (1.0 + RS))
        # self._df['RSI'] = RSI

        self._df['RSI'] = talib.RSI(self._df['last'],timeperiod=self._lookback)

        # long when RSI is high, short when it is low, in cash otherwise
        # there are other strategies
        # https://tradingsim.com/blog/rsi-relative-strength-index/

        self._df['stance'] = np.where(self._df['RSI'] >= self._buy_on, 1, 0)

        if not self._long_only:
            self._df['stance'] = np.where(self._df['RSI'] <= self._sell_on, -1, self._df['stance'])

max_issues_repo_name: dcimring/crypto-price-analysis

max_forks_repo_name: dcimring/crypto-price-analysis