text:
"""This module contains evaluation utilities for measuring pose estimation accuracy."""

import os
import numpy as np

def replace_path(video_list, new_paths):
    if isinstance(new_paths, str):
        new_paths = [new_paths] * len(video_list)

    for video, new_path in zip(video_list, new_paths):
        video["backend"]["filename"] = new_path

def find_frame_pairs(labels_gt, labels_pr):
    frame_pairs = []
    for video_gt in labels_gt.videos:

        # Find matching video instance in predictions.
        video_pr = None
        for video in labels_pr.videos:
            if isinstance(video.backend, type(video_gt.backend)) and video.matches(video_gt):
                video_pr = video
                break

        if video_pr is None:
            continue

        # Find user labeled frames in this video.
        labeled_frames_gt = [
            lf for lf in labels_gt.find(video_gt) if lf.has_user_instances
        ]

        # Attempt to match each labeled frame in the ground truth.
        for labeled_frame_gt in labeled_frames_gt:
            labeled_frames_pr = labels_pr.find(
                video_pr, frame_idx=labeled_frame_gt.frame_idx
            )

            if not labeled_frames_pr:
                # No match
                continue
            elif len(labeled_frames_pr) == 1:
                # Match!
                frame_pairs.append((labeled_frame_gt, labeled_frames_pr[0]))
            else:
                # Too many matches.
                raise ValueError("More than one labeled frame found in predictions.")

    return frame_pairs

def compute_instance_area(points):
    """Computes the area of the bounding box of a set of keypoints."""

    if points.ndim == 2:
        points = np.expand_dims(points, axis=0)

    min_pt = np.nanmin(points, axis=-2)
    max_pt = np.nanmax(points, axis=-2)

    return np.prod(max_pt - min_pt, axis=-1)

def compute_oks(points_gt, points_pr, scale=None, stddev=0.025):
    """Computes the object keypoints similarity between sets of points.

    Args:
        points_gt: Ground truth instances of shape (n_gt, n_nodes, n_ed),
            where n_nodes is the number of body parts/keypoint types, and n_ed
            is the number of Euclidean dimensions (typically 2 or 3). Keypoints
            that are missing/not visible should be represented as NaNs.
        points_pr: Predicted instance of shape (n_pr, n_nodes, n_ed).
        scale: Size scaling factor to use when weighing the scores, typically
            the area of the bounding box of the instance (in pixels). This
            should be of the length n_gt. If a scalar is provided, the same
            number is used for all ground truth instances. If set to None, the
            bounding box area of the ground truth instances will be calculated.
        stddev: The standard deviation associated with the spread in the
            localization accuracy of each node/keypoint type. This should be of
            the length n_nodes. "Easier" keypoint types will have lower values
            to reflect the smaller spread expected in localizing it.

    Returns:
        The object keypoints similarity between every pair of ground truth and
        predicted instance, a numpy array of of shape (n_gt, n_pr) in the range
        of [0, 1.0], with 1.0 denoting a perfect match.

    Notes:
        It's important to set the stddev appropriately when accounting for the
        difficulty of each keypoint type. For reference, the median value for
        all keypoint types in COCO is 0.072. The "easiest" keypoint is the left
        eye, with stddev of 0.025, since it is easy to precisely locate the
        eyes when labeling. The "hardest" keypoint is the left hip, with stddev
        of 0.107, since it's hard to locate the left hip bone without external
        anatomical features and since it is often occluded by clothing.

        The implementation here is based off of the descriptions in:
        Ronch & Perona. "Benchmarking and Error Diagnosis in Multi-Instance Pose
        Estimation." ICCV (2017).
    """

    if points_gt.ndim != 3 or points_pr.ndim != 3:
        raise ValueError(
            "Points must be rank-3 with shape (n_instances, n_nodes, n_ed)."
        )

    if scale is None:
        scale = compute_instance_area(points_gt)

    n_gt, n_nodes, n_ed = points_gt.shape  # n_ed = 2 or 3 (euclidean dimensions)
    n_pr = points_pr.shape[0]

    # If scalar scale was provided, use the same for each ground truth instance.
    if np.isscalar(scale):
        scale = np.full(n_gt, scale)

    # If scalar standard deviation was provided, use the same for each node.
    if np.isscalar(stddev):
        stddev = np.full(n_nodes, stddev)

    # Compute displacement between each pair.
    displacement = np.reshape(points_gt, (n_gt, 1, n_nodes, n_ed)) - np.reshape(
        points_pr, (1, n_pr, n_nodes, n_ed)
    )
    assert displacement.shape == (n_gt, n_pr, n_nodes, n_ed)

    # Convert to pairwise Euclidean distances.
    distance = (displacement ** 2).sum(axis=-1)  # (n_gt, n_pr, n_nodes)
    assert distance.shape == (n_gt, n_pr, n_nodes)

    # Compute the normalization factor per keypoint.
    spread_factor = (2 * stddev) ** 2
    scale_factor = 2 * (scale + np.spacing(1))
    normalization_factor = np.reshape(spread_factor, (1, 1, n_nodes)) * np.reshape(
        scale_factor, (n_gt, 1, 1)
    )
    assert normalization_factor.shape == (n_gt, 1, n_nodes)

    # Since a "miss" is considered as KS < 0.5, we'll set the
    # distances for predicted points that are missing to inf.
    missing_pr = np.any(np.isnan(points_pr), axis=-1)  # (n_pr, n_nodes)
    assert missing_pr.shape == (n_pr, n_nodes)
    distance[:, missing_pr] = np.inf

    # Compute the keypoint similarity as per the top of Eq. 1.
    ks = np.exp(-(distance / normalization_factor))  # (n_gt, n_pr, n_nodes)
    assert ks.shape == (n_gt, n_pr, n_nodes)

    # Set the KS for missing ground truth points to 0.
    # This is equivalent to the visibility delta function of the bottom
    # of Eq. 1.
    missing_gt = np.any(np.isnan(points_gt), axis=-1)  # (n_gt, n_nodes)
    assert missing_gt.shape == (n_gt, n_nodes)
    ks[np.expand_dims(missing_gt, axis=1)] = 0

    # Compute the OKS.
    n_visible_gt = np.sum(
        (~missing_gt).astype("float64"), axis=-1, keepdims=True
    )  # (n_gt, 1)
    oks = np.sum(ks, axis=-1) / n_visible_gt
    assert oks.shape == (n_gt, n_pr)

    return oks

def match_instances(frame_gt, frame_pr, stddev=0.025, scale=None, threshold=0):
    # Sort predicted instances by score.
    scores_pr = np.array(
        [
            instance.score
            for instance in frame_pr.instances
            if hasattr(instance, "score")
        ]
    )
    idxs_pr = np.argsort(-scores_pr, kind="mergesort")  # descending
    scores_pr = scores_pr[idxs_pr]

    available_instances_gt = frame_gt.user_instances
    positive_pairs = []
    for idx_pr in idxs_pr:
        # Pull out predicted instance.
        instance_pr = frame_pr.instances[idx_pr]

        # Convert instances to point arrays.
        points_pr = np.expand_dims(instance_pr.points_array, axis=0)
        points_gt = np.stack(
            [instance.points_array for instance in available_instances_gt], axis=0
        )

        # Find the best match by computing OKS.
        oks = compute_oks(points_gt, points_pr, stddev=stddev, scale=scale)
        oks = np.squeeze(oks, axis=1)
        assert oks.shape == (len(points_gt),)

        oks[oks <= threshold] = np.nan
        best_match_gt_idx = np.argsort(-oks, kind="mergesort")[0]
        best_match_oks = oks[best_match_gt_idx]
        if np.isnan(best_match_oks):
            continue

        # Remove matched ground truth instance and add as a positive pair.
        instance_gt = available_instances_gt.pop(best_match_gt_idx)
        positive_pairs.append((instance_gt, instance_pr, best_match_oks))

        # Stop matching lower scoring instances if we run out of
        # candidates in the ground truth.
        if not available_instances_gt:
            break

    # Any remaining ground truth instances are considered false negatives.
    false_negatives = available_instances_gt

    return positive_pairs, false_negatives

def match_frame_pairs(frame_pairs, stddev=0.025, scale=None, threshold=0):
    # Match instances within each frame pair.
    positive_pairs = []
    false_negatives = []
    for frame_gt, frame_pr in frame_pairs:
        positive_pairs_frame, false_negatives_frame = match_instances(
            frame_gt, frame_pr, stddev=stddev, scale=scale, threshold=threshold
        )
        positive_pairs.extend(positive_pairs_frame)
        false_negatives.extend(false_negatives_frame)

    return positive_pairs, false_negatives

def compute_generalized_voc_metrics(
    positive_pairs,
    false_negatives,
    match_scores,
    match_score_thresholds=np.linspace(0.5, 0.95, 10),  # 0.5:0.05:0.95
    recall_thresholds=np.linspace(0, 1, 101),  # 0.0:0.01:1.00
    name="gvoc",
):

    detection_scores = np.array([pp[1].score for pp in positive_pairs])

    inds = np.argsort(-detection_scores, kind="mergesort")
    detection_scores = detection_scores[inds]
    match_scores = match_scores[inds]

    precisions = []
    recalls = []

    npig = len(positive_pairs) + len(false_negatives)  # total number of GT instances

    for match_score_threshold in match_score_thresholds:

        tp = np.cumsum(match_scores >= match_score_threshold)
        fp = np.cumsum(match_scores < match_score_threshold)

        rc = tp / npig
        pr = tp / (fp + tp + np.spacing(1))

        recall = rc[-1]  # best recall at this OKS threshold

        # Ensure strictly decreasing precisions.
        for i in range(len(pr) - 1, 0, -1):
            if pr[i] > pr[i - 1]:
                pr[i - 1] = pr[i]

        # Find best precision at each recall threshold.
        rc_inds = np.searchsorted(rc, recall_thresholds, side="left")
        precision = np.zeros(rc_inds.shape)
        is_valid_rc_ind = rc_inds < len(pr)
        precision[is_valid_rc_ind] = pr[rc_inds[is_valid_rc_ind]]

        precisions.append(precision)
        recalls.append(recall)

    precisions = np.array(precisions)
    recalls = np.array(recalls)

    AP = precisions.mean(
        axis=1
    )  # AP = average precision over fixed set of recall thresholds
    AR = recalls  # AR = max recall given a fixed number of detections per image

    mAP = precisions.mean()  # mAP = mean over all OKS thresholds
    mAR = recalls.mean()  # mAR = mean over all OKS thresholds

    return {
        name + ".match_score_thresholds": match_score_thresholds,
        name + ".recall_thresholds": recall_thresholds,
        name + ".match_scores": match_scores,
        name + ".precisions": precisions,
        name + ".recalls": recalls,
        name + ".AP": AP,
        name + ".AR": AR,
        name + ".mAP": mAP,
        name + ".mAR": mAR,
    }

def compute_dists(positive_pairs):
    dists = []
    for instance_gt, instance_pr, _ in positive_pairs:
        points_gt = instance_gt.points_array
        points_pr = instance_pr.points_array

        dists.append(np.linalg.norm(points_pr - points_gt, axis=-1))
    dists = np.array(dists)

    return dists

def compute_dist_metrics(dists):
    return {
        "dist.dists": dists,
        "dist.avg": np.nanmean(dists),
        "dist.p50": np.percentile(dists[~np.isnan(dists)], 50),
        "dist.p75": np.percentile(dists[~np.isnan(dists)], 75),
        "dist.p90": np.percentile(dists[~np.isnan(dists)], 90),
        "dist.p95": np.percentile(dists[~np.isnan(dists)], 95),
        "dist.p99": np.percentile(dists[~np.isnan(dists)], 99),
    }

def compute_pck_metrics(dists, thresholds=np.linspace(1, 10, 10)):

    dists = np.copy(dists)
    dists[np.isnan(dists)] = np.inf
    pcks = np.expand_dims(dists, -1) < np.reshape(thresholds, (1, 1, -1))
    mPCK_parts = pcks.mean(axis=0).mean(axis=-1)
    mPCK = mPCK_parts.mean()

    return {
        "pck.thresholds": thresholds,
        "pck.pcks": pcks,
        "pck.mPCK_parts": mPCK_parts,
        "pck.mPCK": mPCK,
    }

def compute_visibility_conf(positive_pairs):

    vis_tp = 0
    vis_fn = 0
    vis_fp = 0
    vis_tn = 0

    for instance_gt, instance_pr, _ in positive_pairs:
        missing_nodes_gt = np.isnan(instance_gt.points_array).any(axis=-1)
        missing_nodes_pr = np.isnan(instance_pr.points_array).any(axis=-1)

        vis_tn += ((missing_nodes_gt) & (missing_nodes_pr)).sum()
        vis_fn += ((~missing_nodes_gt) & (missing_nodes_pr)).sum()
        vis_fp += ((missing_nodes_gt) & (~missing_nodes_pr)).sum()
        vis_tp += ((~missing_nodes_gt) & (~missing_nodes_pr)).sum()

    return {
        "vis.tp": vis_tp,
        "vis.fp": vis_fp,
        "vis.tn": vis_tn,
        "vis.fn": vis_fn,
        "vis.precision": vis_tp / (vis_tp + vis_fp),
        "vis.recall": vis_tp / (vis_tp + vis_fn),
    }

def evaluate(labels_gt, labels_pr, oks_stddev=0.025, oks_scale=None, match_threshold=0):

    frame_pairs = find_frame_pairs(labels_gt, labels_pr)
    positive_pairs, false_negatives = match_frame_pairs(
        frame_pairs, stddev=oks_stddev, scale=oks_scale, threshold=match_threshold
    )
    dists = compute_dists(positive_pairs)

    metrics = {}
    metrics.update(compute_visibility_conf(positive_pairs))
    metrics.update(compute_dist_metrics(dists))
    metrics.update(compute_pck_metrics(dists))

    pair_oks = np.array([oks for _, _, oks in positive_pairs])
    pair_pck = metrics["pck.pcks"].mean(axis=-1).mean(axis=-1)

    metrics["oks.mOKS"] = pair_oks.mean()
    metrics.update(
        compute_generalized_voc_metrics(
            positive_pairs, false_negatives, match_scores=pair_oks, name="oks_voc"
        )
    )
    metrics.update(
        compute_generalized_voc_metrics(
            positive_pairs, false_negatives, match_scores=pair_pck, name="pck_voc"
        )
    )

    return metrics

max_stars_repo_name: jens-k/sleap

max_issues_repo_name: jens-k/sleap

max_forks_repo_name: jens-k/sleap


text:
from sklearn.model_selection import train_test_split
import numpy as np
import matplotlib.pyplot as plt
import csv
import pandas as pd
from tqdm import tqdm
from copy import deepcopy

class DimensionValueError(ValueError):
    pass

class TypeError(ValueError):
    pass

class IterError(ValueError):
    pass

class DataProcess:
    # 读入数据
    def loadCSV(self, path):
        f = csv.reader(open(path))
        l = []
        for row in f:
            l.append(row)
        l = np.array(l[1:], dtype = np.float64)
        return l

    #预处理
    def Predata(self, path):
        csvf = self.loadCSV(path)
        data = csvf[:, :2]
        labels = csvf[:, 2:]

        # labels.resize((labels.shape[0], ))
        labels = labels.reshape((labels.shape[0]))
        return data, labels

    #划分数据集
    def Split_Data(self, path, t_s=0.3):
        data, labels = self.Predata(path)
        #为w和b和统一表示，加一维全为1的值
        # data = np.hstack((np.ones((data.shape[0], 1)), data))
        return train_test_split(data, labels, test_size = t_s, random_state = 0)

class KMeans():
    def __init__(self):
        pass

    def generate_krandom_center(self, data):
        mean = np.mean(data, axis = 0)
        std = np.std(data, axis = 0)

        self.center = np.random.randn(self.k, data.shape[1]) * std + mean
        pass

    def cal_dis(self, data):
        for i in range(self.k):
            self.dis[:, i] = np.linalg.norm(data - self.center[i], axis=1)

        #得到最小值的下标
        self.clusters = np.argmin(self.dis, axis=1)
        #重新计算中心点
        for i in range(self.k):
            self.center[i] = np.mean(data[self.clusters == i], axis=0)
        pass

    def fit(self, data, k = 3, epoch = 500, eps = 1e-8):
        self.k = k
        self.generate_krandom_center(data)
        self.dis = np.zeros((data.shape[0], self.k))

        for i in tqdm(range(epoch)):
            pre_center = deepcopy(self.center)
            self.cal_dis(data)
            sep = np.linalg.norm(pre_center - self.center)
            loss = self.loss(data)
            print('sep = ', sep, 'loss = ', loss)
            if sep < eps or loss < .5:
                break
        self.visual(data)
        return self

    def visual(self, data):
        plt.clf()
        plt.scatter()
        plt.scatter(data[:, 0], data[:, 1], alpha=0.5, c=self.clusters)
        plt.scatter(self.center[:, 0], self.center[:, 1], marker='*', c='k')
        plt.show()

    def predict(self, data):
        self.dis = np.zeros((data.shape[0], self.k))
        self.cal_dis(data)
        loss = self.loss(data)
        print('loss = ', loss)
        self.visual(data)

    def loss(self, data):
        loss = .0
        # self.visual(data)
        for i in range(self.k):
            loss += np.sum(np.linalg.norm(data[self.clusters == i] - self.center[i]))
        # print('loss = ', loss)
        return loss

if __name__ == "__main__":

    Train_data, Validation_data, Train_labels, Validation_labels = DataProcess().Split_Data('data.csv')

    KM = KMeans().fit(Train_data)
    KM.predict(Validation_data)

ext: py

max_issues_repo_name: K-ona/ML-Tutorial

max_forks_repo_name: K-ona/ML-Tutorial

reason: import numpy


text:
import os
import numpy as np
import cv2

def getFiles(path, suffix):
    return [os.path.join(root, file) for root, dirs, files in os.walk(path) for file in files if file.endswith(suffix)]

files = getFiles('results','.npy')

for _,name in enumerate(files):
    data =np.load(name)
    data = data.transpose(1,2,0)#CHW to HWC
    data*=255.0
    cv2.imwrite('results_picture/'+(((name.split('/'))[-1]).split('.'))[0]+'.png',data)


text:
import matplotlib.pyplot as plt
import pandas as pd
import sqlalchemy
import numpy as np
import matplotlib.units as munits
import matplotlib.dates as mdates
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()
from matplotlib.offsetbox import AnchoredText

converter = mdates.ConciseDateConverter()

engine = sqlalchemy.create_engine('mysql+pymysql://jcz:yub.miha@127.0.0.1:3306/weather')

my_query = ''' SELECT `NORM_DATE`, `kwh_per_dollars`, `NORM_DATE`, `shades`, `monthly_elec_use` FROM `v_monthly_utilities` WHERE kwh_per_dollars IS NOT NULL; '''
df1 = pd.read_sql_query(my_query,engine)
cost = df1['monthly_elec_use']
shades = df1['shades']
xaxis = df1['kwh_per_dollars']

fig, ax = plt.subplots(figsize=(12, 6))

ax.scatter(xaxis, cost, c=shades, label="Use:Cost Ratio Colored by Season. Red-Summer, Brown-Fall, Blue-Winter, Green-Spring")

ax.set_xlabel('Monthly kwh per dollars (monthly kwh used / electric bill)',fontsize=15,fontweight='bold')
ax.set_ylabel('Sum kwh per month',fontsize=15,fontweight='bold')
ax.set_title('Cost of Electrons by Usage', color="black",fontsize=15,fontweight='bold')

ax.grid(True)
fig.tight_layout()
ax.legend(loc='lower center', ncol=2, fancybox=True, framealpha=1, shadow=True, borderpad=1)
plt.grid(visible=True, which='major', color='#CCCCCC', linestyle='--')
plt.grid(visible=True, which='minor', color='#CCCCCC', linestyle=':')

plt.savefig('monthly_elec_cost_use.png', dpi=200)
plt.show()
plt.close()

max_stars_repo_name: johnzastrow/weather

max_issues_repo_name: johnzastrow/weather

max_forks_repo_name: johnzastrow/weather


text:
"""
copyright Kwangho Heo
2016-07-04
"""

# XML parsing
from xml.etree.ElementTree import *

# matrix math
import numpy as np

class OptimalPolicy:

    def __init__(self, filename):
        tree = parse(filename)
        root = tree.getroot()
        avec = list(root)[0]
        alphas = list(avec)
        self.action_nums = []
        val_arrs = []
        for alpha in alphas:
            self.action_nums.append(int(alpha.attrib['action']))
            vals = []
            for val in alpha.text.split():
                vals.append(float(val))
            val_arrs.append(vals)
        self.pMatrix = np.array(val_arrs)

    def get_best_action(self, belief):
        """
        Returns tuple:
            (best-action-num, expected-reward-for-this-action).
        """
        res = self.pMatrix.dot(belief)
        highest_expected_reward = res.max()
        best_action = self.action_nums[res.argmax()]
        # return best_action, highest_expected_reward
        return best_action

    def end(self):
        print 'end debug here'

max_stars_repo_name: hkhpub/gp-voicemail

max_issues_repo_name: hkhpub/gp-voicemail

max_forks_repo_name: hkhpub/gp-voicemail


text:
#  Copyright (c) 2019, CRS4
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy of
#  this software and associated documentation files (the "Software"), to deal in
#  the Software without restriction, including without limitation the rights to
#  use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
#  the Software, and to permit persons to whom the Software is furnished to do so,
#  subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in all
#  copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
#  FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
#  COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
#  IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
#  CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import sys, argparse, logging, cv2, pickle
import numpy as np
from math import sqrt, log
try:
    import simplejson as json
except ImportError:
    import json

# TODO: install.py for odin lib and remove this abomination
sys.path.append('../../')

from odin.libs.regions_of_interest.shapes_manager import Shape

LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']

class AutomaticCoresExtractor(object):

    def __init__(self, log_level='INFO', log_file=None):
        self.logger = self._get_logger(log_level, log_file)

    def _get_logger(self, log_level='INFO', log_file=None, mode='a'):
        LOG_FORMAT = '%(asctime)s|%(levelname)-8s|%(message)s'
        LOG_DATEFMT = '%Y-%m-%d %H:%M:%S'

        logger = logging.getLogger('mask_applier')
        if not isinstance(log_level, int):
            try:
                log_level = getattr(logging, log_level)
            except AttributeError:
                raise ValueError('Unsupported literal log level: %s' % log_level)
        logger.setLevel(log_level)
        logger.handlers = []
        if log_file:
            handler = logging.FileHandler(log_file, mode=mode)
        else:
            handler = logging.StreamHandler()
        formatter = logging.Formatter(LOG_FORMAT, datefmt=LOG_DATEFMT)
        handler.setFormatter(formatter)
        logger.addHandler(handler)
        return logger

    def _load_mask(self, tissue_mask_file, format='json'):
        if format == 'json':
            with open(tissue_mask_file) as f:
                data = json.loads(f.read())
                data['mask'] = np.uint8(np.array(data['mask']))
        elif format == 'pkl':
            with open(tissue_mask_file, 'rb') as f:
                data = pickle.load(f)
        else:
            raise ValueError('Unsupported file format %s' % format)
        return data['mask'], data['dimensions']

    def _contour_to_shape(self, contour):
        normalize_contour = list()
        for x in contour:
            normalize_contour.append(tuple(x[0]))
        try:
            return Shape(normalize_contour)
        except ValueError:
            return None

    def _get_cores(self, tissue_mask):
        contours, _ = cv2.findContours(tissue_mask, mode=cv2.RETR_EXTERNAL, method=cv2.CHAIN_APPROX_SIMPLE)
        contours = filter(None, [self._contour_to_shape(c) for c in contours])
        return contours

    def _filter_cores(self, cores, slide_area, core_min_area=0.02):
        accepted_cores = list()
        for core in cores:
            if (core.get_area()*100 / slide_area) >= core_min_area:
                accepted_cores.append(core)
        return accepted_cores

    def _get_scale_factor(self, slide_resolution, mask_resolution):
        scale_factor = sqrt((slide_resolution[0]*slide_resolution[1]) / (mask_resolution[0]*mask_resolution[1]))
        self.logger.info('Scale factor is %r', scale_factor)
        return scale_factor

    def _get_sorted_cores_map(self, cores):
        cores_map = dict()
        for c in cores:
            bounds = c.get_bounds()
            cores_map.setdefault((bounds['y_min'], bounds['y_max']), []).append(c)
        sorted_y_coords = cores_map.keys()
        sorted_y_coords.sort(key=lambda x: x[0])
        return cores_map, sorted_y_coords

    def _group_nearest_cores(self, cores, slide_height, height_tolerance=0.01):
        cores_map, sorted_y_coords = self._get_sorted_cores_map(cores)
        cores_groups = list()
        tolerance = slide_height*height_tolerance
        current_group = cores_map[sorted_y_coords[0]]
        for i, yc in enumerate(sorted_y_coords[1:]):
            if yc[0] <= sorted_y_coords[i][1] + tolerance:
                current_group.extend(cores_map[yc])
            else:
                cores_groups.append(current_group)
                current_group = cores_map[yc]
        cores_groups.append(current_group)
        return cores_groups

    def _get_slice(self, cores_group):
        x_min = min([c.get_bounds()['x_min'] for c in cores_group])
        y_min = min([c.get_bounds()['y_min'] for c in cores_group])
        x_max = max([c.get_bounds()['x_max'] for c in cores_group])
        y_max = max([c.get_bounds()['y_max'] for c in cores_group])
        return Shape([(x_min, y_min), (x_max, y_min), (x_max, y_max), (x_min, y_max)])

    def _build_slide_json(self, cores_group, scale_factor):
        scale_factor = log(scale_factor, 2)
        slide_shapes = list()
        for group in cores_group:
            slice = self._get_slice(group)
            slice_map = {
                'coordinates': slice.get_coordinates(scale_factor),
                'cores': [
                    {'coordinates': c.get_coordinates(scale_factor),
                     'length': c.get_length(scale_factor),
                     'area': c.get_area(scale_factor)}
                    for c in group]
            }
            slide_shapes.append(slice_map)
        return slide_shapes

    def run(self, mask_file, output_file):
        img_mask, original_resolution = self._load_mask(mask_file)
        cores = self._filter_cores(self._get_cores(img_mask), img_mask.size)
        grouped_cores = self._group_nearest_cores(cores, img_mask.shape[0])
        scale_factor = self._get_scale_factor(original_resolution, img_mask.shape)
        slide_json = self._build_slide_json(grouped_cores, scale_factor)
        with open(output_file, 'w') as ofile:
            ofile.write(json.dumps(slide_json))

def get_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument('--tissue-mask', type=str, required=True,
                        help='the file containing the tissue mask')
    parser.add_argument('--output-file', type=str, required=True, help='output JSON file')
    parser.add_argument('--log-level', type=str, default='INFO', help='log level (default=INFO)')
    parser.add_argument('--log-file', type=str, default=None, help='log file (default=stderr)')
    return parser

def main(argv):
    parser = get_parser()
    args = parser.parse_args(argv)
    cores_extractor = AutomaticCoresExtractor(args.log_level, args.log_file)
    cores_extractor.run(args.tissue_mask, args.output_file)

if __name__ == '__main__':
    main(sys.argv[1:])

max_stars_repo_name: lucalianas/odin

max_issues_repo_name: lucalianas/odin

max_forks_repo_name: lucalianas/odin


text:
# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
from nose.tools import with_setup

from nvidia.dali import pipeline_def
import nvidia.dali.fn as fn

from test_external_source_parallel_utils import setup_function, teardown_function, capture_processes

def large_sample_cb(sample_info):
    return np.full((512, 1024, 1024), sample_info.idx_in_epoch, dtype=np.int32)

@with_setup(setup_function, teardown_function)
def _test_large_sample(start_method):
    batch_size = 2

    @pipeline_def
    def create_pipeline():
        large = fn.external_source(
            large_sample_cb, batch=False, parallel=True, prefetch_queue_depth=1)
        # iteration over array in Python is too slow, so reduce the number of elements to iterate over
        reduced = fn.reductions.sum(large, axes=(1, 2))
        return reduced

    pipe = create_pipeline(batch_size=batch_size, py_num_workers=2, py_start_method=start_method,
                           prefetch_queue_depth=1, num_threads=2, device_id=0)
    pipe.build()
    capture_processes(pipe._py_pool)
    for batch_idx in range(8):
        (out,) = pipe.run()
        for idx_in_batch in range(batch_size):
            idx_in_epoch = batch_size * batch_idx + idx_in_batch
            expected_val = idx_in_epoch * 1024 * 1024
            a = np.array(out[idx_in_batch])
            assert a.shape == (512,), "Expected shape (512,) but got {}".format(a.shape)
            for val in a.flat:
                assert val == expected_val, \
                    "Unexpected value in batch: got {}, expected {}, for batch {}, sample {}".format(
                        val, expected_val, batch_idx, idx_in_batch)

def test_large_sample():
    for start_method in ("fork", "spawn"):
        yield _test_large_sample, start_method


text:
#!/usr/bin/python3
"""
script for making PCA map based on precomputed design matrix
"""

import argparse
import sys
import os
import numpy as np
from matplotlib import pyplot as plt

from asaplib.data import ASAPXYZ
from asaplib.io import str2bool
from asaplib.reducedim import Dimension_Reducers
from asaplib.plot import Plotters, set_color_function

def main(fmat, fxyz, ftags, fcolor, colorscol, prefix, output, peratom, keepraw, scale, pca_d, pc1, pc2, projectatomic, plotatomic,
         adtext):
    """

    Parameters
    ----------
    fmat: Location of descriptor matrix file or name of the tags in ase xyz file. You can use gen_descriptors.py to compute it.
    fxyz: Location of xyz file for reading the properties.
    ftags: Location of tags for the first M samples. Plot the tags on the PCA map.
    fcolor: Location of a file or name of the tags in ase xyz file. It should contain properties for all samples (N floats) used to color the scatterplot'
    colorscol: The column number of the properties used for the coloring. Starts from 0.
    prefix: Filename prefix, default is ASAP
    output: The format for output files ([xyz], [matrix]). Default is xyz.
    peratom: Whether to output per atom pca coordinates (True/False)
    keepraw: Whether to keep the high dimensional descriptor when output is an xyz file (True/False)
    scale: Scale the coordinates (True/False). Scaling highly recommanded.
    pca_d: Number of the principle components to keep
    pc1: Plot the projection along which principle axes
    pc2: Plot the projection along which principle axes
    projectatomic: build the projection using the (big) atomic descriptor matrix
    plotatomic: Plot the PCA coordinates of all atomic environments (True/False)
    adtext: Whether to adjust the texts (True/False)

    Returns
    -------

    """

    foutput = prefix + "-pca-d" + str(pca_d)
    use_atomic_desc = (peratom or plotatomic or projectatomic)

    # try to read the xyz file
    if fxyz != 'none':
        asapxyz = ASAPXYZ(fxyz)
        desc, desc_atomic = asapxyz.get_descriptors(fmat, use_atomic_desc)
        if projectatomic: desc = desc_atomic.copy()
    else:
        asapxyz = None
        print("Did not provide the xyz file. We can only output descriptor matrix.")
        output = 'matrix'
    # we can also load the descriptor matrix from a standalone file
    if os.path.isfile(fmat[0]):
        try:
            desc = np.genfromtxt(fmat[0], dtype=float)
            print("loaded the descriptor matrix from file: ", fmat)
        except:
            raise ValueError('Cannot load the descriptor matrix from file')
    # sanity check
    if len(desc) == 0:
        raise ValueError('Please supply descriptor in a xyz file or a standlone descriptor matrix')
    print("shape of the descriptor matrix: ", np.shape(desc), "number of descriptors: ", np.shape(desc[0]))

    if ftags != 'none':
        tags = np.loadtxt(ftags, dtype="str")[:]
        ndict = len(tags)
    else:
        tags = []

    reduce_dict = { "pca":
                   {"type": 'PCA', 'parameter':{"n_components": pca_d, "scalecenter": scale}}
                  }
    """
    reduce_dict = { "umap":
                   {"type": 'UMAP', 'parameter':{"n_components": pca_d, "n_neighbors": 10}}
                  }

    reduce_dict = {
        "reduce1_pca": {"type": 'PCA', 'parameter':{"n_components": 20, "scalecenter":True}},
        "reduce2_tsne": {"type": 'TSNE', 'parameter': {"n_components": 2, "perplexity":20}}
        }
    """
    dreducer = Dimension_Reducers(reduce_dict)

    proj = dreducer.fit_transform(desc)
    if peratom or plotatomic and not projectatomic:
        proj_atomic_all = dreducer.transform(desc_atomic)

    # save
    if output == 'matrix':
        np.savetxt(foutput + ".coord", proj, fmt='%4.8f', header='low D coordinates of samples')
        if peratom:
            np.savetxt(foutput + "-atomic.coord", proj_atomic_all, fmt='%4.8f', header='low D coordinates of samples')
    if output == 'xyz':
        asapxyz.set_descriptors(proj, 'pca_coord')
        if peratom:
            asapxyz.set_atomic_descriptors(proj_atomic_all, 'pca_coord')
        # remove the raw descriptors
        if not keepraw:
            asapxyz.remove_descriptors(fmat)
            asapxyz.remove_atomic_descriptors(fmat)
        asapxyz.write(foutput)

    # color scheme
    plotcolor, plotcolor_peratom, colorlabel, colorscale = set_color_function(fcolor, asapxyz, colorscol, 0, (peratom or plotatomic), projectatomic)

    if plotatomic:
        outfile = 'PCA_4_' + prefix + '-c-' + fcolor + '-plotatomic.png'
    else:
        outfile = 'PCA_4_' + prefix + '-c-' + fcolor + '.png'

    fig_spec_dict = {
        'outfile': outfile,
        'show': False,
        'title': None,
        'xlabel': 'Principal Axis 1',
        'ylabel': 'Principal Axis 2',
        'xaxis': True,  'yaxis': True,
        'remove_tick': False,
        'rasterized': True,
        'fontsize': 16,
        'components':{
            "first_p": {"type": 'scatter', 'clabel': colorlabel},
            "second_p": {"type": 'annotate', 'adtext': adtext}
             }
        }
    asap_plot = Plotters(fig_spec_dict)
    asap_plot.plot(proj[::-1, [pc1, pc2]], plotcolor[::-1], [], tags)
    if peratom or plotatomic and not projectatomic:
        asap_plot.plot(proj_atomic_all[::-1, [pc1, pc2]], plotcolor_peratom[::-1],[],[])
    plt.show()

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('-fmat', nargs='+', type=str, required=True,
                        help='Location of descriptor matrix file or name of the tags in ase xyz file. You can use gen_descriptors.py to compute it.')
    parser.add_argument('-fxyz', type=str, default='none', help='Location of xyz file for reading the properties.')
    parser.add_argument('-tags', type=str, default='none',
                        help='Location of tags for the first M samples. Plot the tags on the PCA map.')
    parser.add_argument('-colors', type=str, default='none',
                        help='Location of a file or name of the tags in ase xyz file. It should contain properties for all samples (N floats) used to color the scatter plot')
    parser.add_argument('--colorscolumn', type=int, default=0,
                        help='The column number of the properties used for the coloring. Starts from 0.')
    parser.add_argument('--prefix', type=str, default='ASAP', help='Filename prefix')
    parser.add_argument('--output', type=str, default='matrix', help='The format for output files ([xyz], [matrix])')
    parser.add_argument('--peratom', type=str2bool, nargs='?', const=True, default=False,
                        help='Do you want to output per atom pca coordinates (True/False)?')
    parser.add_argument('--keepraw', type=str2bool, nargs='?', const=True, default=False,
                        help='Do you want to keep the high dimensional descriptor when output xyz file (True/False)?')
    parser.add_argument('--scale', type=str2bool, nargs='?', const=True, default=True,
                        help='Scale the coordinates (True/False). Scaling highly recommended.')
    parser.add_argument('--d', type=int, default=10, help='number of the principle components to keep')
    parser.add_argument('--pc1', type=int, default=0, help='Plot the projection along which principle axes')
    parser.add_argument('--pc2', type=int, default=1, help='Plot the projection along which principle axes')
    parser.add_argument('--projectatomic', type=str2bool, nargs='?', const=True, default=False,
                        help='Building the KPCA projection based on atomic descriptors instead of global ones (True/False)')
    parser.add_argument('--plotatomic', type=str2bool, nargs='?', const=True, default=False,
                        help='Plot the PCA coordinates of all atomic environments (True/False)')
    parser.add_argument('--adjusttext', type=str2bool, nargs='?', const=True, default=False,
                        help='Do you want to adjust the texts (True/False)?')

    if len(sys.argv) == 1:
        parser.print_help(sys.stderr)
        sys.exit(1)
    args = parser.parse_args()

    main(args.fmat, args.fxyz, args.tags, args.colors, args.colorscolumn, args.prefix, args.output, args.peratom,
         args.keepraw, args.scale, args.d, args.pc1, args.pc2, args.projectatomic, args.plotatomic, args.adjusttext)

max_stars_repo_name: BingqingCheng/ASAP

max_issues_repo_name: FelixWodaczek/ASAP

max_forks_repo_name: FelixWodaczek/ASAP


text:
import traceback
import argparse
import numpy
import sys
import os

from ctypes import *

if __name__ == "__main__":
    from ladspa.dtypes import LADSPA_Descriptor, LADSPA_PortRangeHint, LadspaConsts
    from ladspa.routines import calc_default_port_value
    from utils.common import find_plugins_in_lib
else:
    from pyladspa.ladspa.dtypes import LADSPA_Descriptor, LADSPA_PortRangeHint, LadspaConsts
    from pyladspa.ladspa.routines import calc_default_port_value
    from pyladspa.utils.common import find_plugins_in_lib

def setup_cli(arg_parser):
    arg_parser.add_argument("lib", metavar="LADSPA_LIBRARY", help="Path to LADSPA library in which to search for plugin.")
    arg_parser.add_argument("plug", metavar="PLUGIN_LABEL", help="Label (string id) of the plugin to analyse.")
    arg_parser.add_argument("--classic_fmt", "-f", required=False, default=False, action="store_true", help="Set this flag to enable 'classic' output formatting as in LADSPA SDK tools.")
    return arg_parser

def print_port_description(port_descr, port_name, port_range_hints):

    """
    This abomination is slightly improved version of the code that you can actually find
    in LADSPA SDK. Did not have time to make a proper re-factoring, sorry :(
    """

    def port_type_str(port_descr):
        res = ""
        if LadspaConsts.is_port_input(port_descr) and LadspaConsts.is_port_output(port_descr):
            res += "ERROR: INPUT AND OUTPUT, "
        elif LadspaConsts.is_port_input(port_descr):
            res += "input, "
        elif LadspaConsts.is_port_output(port_descr):
            res += "output, "
        else:
            res += "ERROR: NEITHER INPUT NOR OUTPUT, "

        if LadspaConsts.is_port_audio(port_descr) and LadspaConsts.is_port_control(port_descr):
            res += "ERROR: CONTROL AND AUDIO"
        elif LadspaConsts.is_port_audio(port_descr):
            res += "audio"
        elif LadspaConsts.is_port_control(port_descr):
            res += "control"
        else:
            res += "ERROR: NEITHER CONTROL NOR AUDIO"
        return res

    def range_hint_str(hint):
        props = hint.HintDescriptor
        lbnd = hint.LowerBound
        ubnd = hint.UpperBound
        res = ""

        # step 1 : deal with boundaries
        if LadspaConsts.is_hint_bounded_above(props) or LadspaConsts.is_hint_bounded_bellow(props):
            res += ", "
            if LadspaConsts.is_hint_bounded_bellow(props):
                res += "{}*srate".format(lbnd) if LadspaConsts.is_hint_samplerate(props) else "{}".format(lbnd)
            else:
                res += "..."
            res += " to "
            if LadspaConsts.is_hint_bounded_above(props):
                res += "{}*srate".format(ubnd) if LadspaConsts.is_hint_samplerate(props) else "{}".format(ubnd)
            else:
                res += "..."

        # step 2 : toggled
        if LadspaConsts.is_hint_toggled(props):
            if LadspaConsts.is_hint_properly_toggled(props):
                res += ", toggled"
            else:
                res += ", ERROR: TOGGLED INCOMPATIBLE WITH OTHER HINT"

        # step 3 : default values ...
        if LadspaConsts.hint_has_default(props):
            if (LadspaConsts.is_hint_default_min(props) or
                LadspaConsts.is_hint_default_low(props) or
                LadspaConsts.is_hint_default_middle(props) or
                LadspaConsts.is_hint_default_high(props) or
                LadspaConsts.is_hint_default_max(props)
            ):
                d = calc_default_port_value(hint)
                if LadspaConsts.is_hint_samplerate(props) and d != 0:
                    res += ", default {}*srate".format(d)
                else:
                    res += ", default {}".format(d)
            elif LadspaConsts.is_hint_default_0(props):
                res += ", default 0"
            elif LadspaConsts.is_hint_default_1(props):
                res += ", default 1"
            elif LadspaConsts.is_hint_default_100(props):
                res += ", default 100"
            elif LadspaConsts.is_hint_default_440(props):
                res += ", default 440"
            else:
                res += ", UNKNOWN DEFAULT CODE"

        # step 4 : leftovers ...
        if LadspaConsts.is_hint_logarithmic(props):
            res += ", logarithmic"
        if LadspaConsts.is_hint_integer(props):
            res += ", integer"

        return res

    print("\t\"{}\", {}{}".format(port_name.decode("utf-8"), port_type_str(port_descr), range_hint_str(port_range_hints)))

def print_plugin_contents(plugin, classif_fmt=False):
    if classif_fmt:
        print("\nPlugin Name: {}".format(plugin.Name.decode("utf-8")))
        print("Plugin Label: {}".format(plugin.Label.decode("utf-8")))
        print("Plugin Unique ID: {}".format(plugin.UniqueID))
        print("Maker: {}".format(plugin.Maker.decode("utf-8")))
        print("Copyright: {}".format(plugin.Copyright.decode("utf-8")))
        print("Must Run Real-Time: {}".format("Yes" if LadspaConsts.is_realtime(plugin.Properties) else "No"))
        print("Has activate() Function: {}".format("Yes" if plugin.activate else "No"))
        print("Has deactivate() Function: {}".format("Yes" if plugin.deactivate else "No"))
        print("Has run_adding() Function: {}".format("Yes" if plugin.run_adding else "No"))

        if not plugin.instantiate:
	        print("ERROR: PLUGIN HAS NO INSTANTIATE FUNCTION.")
        if not plugin.connect_port:
	        print("ERROR: PLUGIN HAS NO CONNECT_PORT FUNCTION.")
        if not plugin.run:
	        print("ERROR: PLUGIN HAS NO RUN FUNCTION.")
        if plugin.run_adding and not plugin.set_run_adding_gain:
	        print("ERROR: PLUGIN HAS RUN_ADDING FUNCTION BUT NOT SET_RUN_ADDING_GAIN.")
        if not plugin.run_adding and plugin.set_run_adding_gain:
	        print("ERROR: PLUGIN HAS SET_RUN_ADDING_GAIN FUNCTION BUT NOT RUN_ADDING.")
        if not plugin.cleanup:
	        print("ERROR: PLUGIN HAS NO CLEANUP FUNCTION.")

        print("Environment: {}".format("Normal or Hard Real-Time" if LadspaConsts.is_hard_rt_capable(plugin.Properties) else "Normal"))
        if LadspaConsts.is_inplace_broken(plugin.Properties):
            print("This plugin cannot use in-place processing. It will not work with all hosts.")

        print("Ports:")
        if not plugin.PortCount:
            print("\tERROR: PLUGIN HAS NO PORTS.")
        pdescrs = pointer(cast(plugin.PortDescriptors, POINTER(c_int)))

        pnames = pointer(cast(plugin.PortNames, POINTER(c_char_p)))

        prange_hints = pointer(plugin.PortRangeHints)
        # prange_hints = plugin.PortRangeHints

        for idx in range(plugin.PortCount):
            print_port_description(pdescrs[0][idx], pnames[0][idx], prange_hints[0][idx])

    else:
        print("{}".format(plugin.Name.decode("utf-8")))

def main_cli(input_args = None):
    arg_parser = argparse.ArgumentParser("Print properties of specified LADSPA plugin in specified LADSPA library")
    setup_cli(arg_parser)
    args = arg_parser.parse_args(input_args) # when called with 'None' will go through sys.args

    assert os.path.exists(args.lib) and os.path.isfile(args.lib), "ERROR: cannot locate specified LADSPA library {}".format(args.lib)
    plugs = find_plugins_in_lib(args.lib)
    the_one = [p for p in filter(lambda x: x.Label.decode("utf-8") == args.plug, plugs)]
    assert len(the_one), "ERROR: could not find plugin with label {} in specified LADSPA library!".format(args.plug)
    assert len(the_one) == 1, "ERROR: found several plugins with label {}, non-standard compliant LADSPA library!".format(args.plug)
    print_plugin_contents(the_one[0], classif_fmt=args.classic_fmt)

if __name__ == "__main__":
    main_cli()

max_stars_repo_name: pglushkov/pyladspa

max_issues_repo_name: pglushkov/pyladspa

max_forks_repo_name: pglushkov/pyladspa


text:
import os, sys
import numpy as np
import itertools
import glob
import random
import time

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages

import tensorflow as tf

import data

'''
data transformations
'''

class Identity:
    def __call__(self, x, y):
        return x, y

class Mat2Gray:
    def __call__(self, x, y):
        x = tf.expand_dims(x, -1)
        return x, y

class Mat2RGB:
    def __call__(self, x, y):
        x = tf.expand_dims(x, -1)
        x = tf.repeat(x, 3, -1)
        return x, y

class RGB2Gray:
    def __call__(self, x, y):
        x = tf.image.rgb_to_grayscale(x)
        return x, y

class Float:
    def __call__(self, x, y):
        x = tf.cast(x, tf.float32)/255.
        return x, y

class Cast:
    def __init__(self, x_type, y_type):
        self.x_type = x_type
        self.y_type = y_type

    def __call__(self, x, y):
        if self.x_type is not None:
            x = tf.cast(x, self.x_type)
        if self.y_type is not None:
            y = tf.cast(y, self.y_type)
        return x, y

class Scaling:
    def __init__(self, scaling_factor):
        self.scaling_factor = scaling_factor

    def __call__(self, x, y):
        x = x * self.scaling_factor
        return x, y

class Resize:
    def __init__(self, size):
        self.hw = size

    def __call__(self, x, y):
        # try:
        #     x = tf.image.resize_with_pad(x, self.hw[0], self.hw[1])
        # except tf.python.framework.errors_impl.InvalidArgumentError:
        x = tf.image.resize(x, self.hw)
        return x, y

class Pad:
    def __init__(self, pad_size):
        assert(pad_size[0]%2==0 and pad_size[1]%2==0)
        self.pad_size = pad_size

    def __call__(self, x, y):
        padding = tf.constant([[self.pad_size[0]//2, self.pad_size[0]//2], [self.pad_size[1]//2, self.pad_size[1]//2], [0, 0]])
        x = tf.pad(x, padding)
        return x, y

class Map2Tuple:
    def __call__(self, m, x_name='image', y_name='label'):
        return m[x_name], m[y_name]

class File2Tensor:
    def __init__(self, n_channels):
        self.n_channels = n_channels

    def _get_image(self, fn):
        img = tf.io.read_file(fn)
        img = tf.image.decode_jpeg(img, channels=self.n_channels)
        return img

    def _get_label(self, fn):
        ##TODO: assume a numeric label in the file path
        label = tf.strings.to_number(tf.strings.split(fn, '/')[-2], tf.int32)
        return label

    def __call__(self, fn):
        x = self._get_image(fn)
        y = self._get_label(fn)
        return x, y

class PerImageStd:
    def __call__(self, x, y):
        x = tf.image.per_image_standardization(x)
        return x, y

class DomainLabel:
    def __init__(self, domain_id):
        self.domain_id = tf.constant(domain_id)

    def __call__(self, x, y):
        y = self.domain_id
        return x, y

class Squeeze:
    def __call__(self, x, y):
        x = tf.squeeze(x, axis=0)
        y = tf.squeeze(y, axis=0)
        return x, y

class Normalize:
    def __init__(self, mean, std):
        self.mean = mean
        self.std = std

    def __call__(self, x, y):
        x = (x - self.mean) / self.std
        return x, y

class PNGFile2LabeledExample:
    def __init__(self, str2label, n_channels):
        self.str2label = str2label
        self.n_channels = n_channels

    def __call__(self, fn, y):
        x = tf.io.read_file(fn)
        x = tf.image.decode_png(x, channels=self.n_channels)
        return x, y

class JPGFile2LabeledExample:
    def __init__(self, str2label, n_channels):
        self.str2label = str2label
        self.n_channels = n_channels

    def __call__(self, fn, y):
        x = tf.io.read_file(fn)
        x = tf.image.decode_jpeg(x, channels=self.n_channels)
        return x, y

class RandomCrop:
    def __init__(self, size):
        self.size = size

    def __call__(self, x, y):
        x = tf.image.random_crop(x, self.size)
        return x, y

class RandomHorizontalFlip:
    def __call__(self, x, y):
        x = tf.image.random_flip_left_right(x)
        return x, y

class CenterCrop:
    def __init__(self, size, size_ori):
        self.size = size
        self.size_ori = size_ori

    def __call__(self, x, y):
        x = tf.image.central_crop(x, float(self.size)/float(self.size_ori))
        return x, y

class DuplicateX:
    def __call__(self, x, y):
        x1 = tf.identity(x)
        x2 = tf.identity(x)
        return [x1, x2], y

"""
loaders
"""
class DataLoader:

    def _init_loader(self, dataset, n, shuffle, batch_size, tforms=[]):
        AUTOTUNE = tf.data.experimental.AUTOTUNE
        dataset = dataset.cache()
        if shuffle:
            dataset = dataset.shuffle(n, reshuffle_each_iteration=True)
        for tform in tforms:
            dataset = dataset.map(tform, num_parallel_calls=AUTOTUNE)
        dataset = dataset.batch(batch_size)
        dataset = dataset.prefetch(AUTOTUNE)
        return dataset

    def _split_data(self, x, y, ratio, random=True, seed=0):
        assert(x.shape[0] == y.shape[0])
        idx = [i for i in range(y.shape[0])]
        if random:
            np.random.seed(seed)
            np.random.shuffle(idx)
        n1 = round(y.shape[0]*ratio)
        x1, y1 = x[idx[:n1]], y[idx[:n1]]
        x2, y2 = x[idx[n1:]], y[idx[n1:]]
        return (x1, y1), (x2, y2)

class DataFolderLoader(DataLoader):

    def __init__(self, root, batch_size, shuffle, tforms=[], ext='png', n_channels=3, seed=0):
        self.root = root
        self.fns = glob.glob(os.path.join(self.root, '**', '*.'+ext))
        random.seed(seed)
        random.shuffle(self.fns)

        label_str = list(set([os.path.split(os.path.split(fn)[0])[1] for fn in self.fns]))
        str2label = {k: i for i, k in enumerate(label_str)}
        labels = [str2label[os.path.split(os.path.split(fn)[0])[1]] for fn in self.fns]

        AUTOTUNE = tf.data.experimental.AUTOTUNE
        self.dataset = tf.data.Dataset.from_tensor_slices((tf.constant(self.fns), labels))
        self.dataset = self.dataset.cache()
        if shuffle:
            self.dataset = self.dataset.shuffle(len(self.fns), reshuffle_each_iteration=True)
        if ext == 'png':
            self.dataset = self.dataset.map(PNGFile2LabeledExample(str2label, n_channels), num_parallel_calls=AUTOTUNE)
        else:
            assert(ext == 'jpg')
            self.dataset = self.dataset.map(JPGFile2LabeledExample(str2label, n_channels), num_parallel_calls=AUTOTUNE)

        for tform in tforms:
            self.dataset = self.dataset.map(tform, num_parallel_calls=AUTOTUNE)
        self.dataset = self.dataset.batch(batch_size)
        self.dataset = self.dataset.prefetch(AUTOTUNE)

    def __len__(self):
        return len(self.fns)

    def __iter__(self):
        return self.dataset.__iter__()

    def __next__(self):
        return self.datset.__next__()

if __name__ == '__main__':
    ld = DataFolderLoader('Imagenet32/val', 100, True)

    for x, y in ld:
        print(x.shape, y.shape)
        print(y)

class MultiSourceDataset(DataLoader):
    def __init__(self, src_names, aug_params, batch_size, **kwargs):

        ## init source datasets
        ds_list = []
        for s, a in itertools.product(src_names, aug_params):
            if a is None:
                print("src: %s + none"%(s)) ##TODO: simplify
            else:
                print("src: %s + %s"%(s, " + ".join(a_param[0] for a_param in a)))
            ds_list.append(
                getattr(data, s)(
                    root=os.path.join('data', s.lower()),
                    batch_size=batch_size, # draw batch from each source
                    aug_list=a,
                    **kwargs)
            )

        ## convert to multi-source loaders
        self.train = tf.data.experimental.sample_from_datasets([d.train for d in ds_list])
        self.val = tf.data.experimental.sample_from_datasets([d.val for d in ds_list])
        self.test = tf.data.experimental.sample_from_datasets([d.test for d in ds_list])
        if 'domain_id' in kwargs.keys() and kwargs['domain_id'] is not None:
            self.train_dom = tf.data.experimental.sample_from_datasets([d.train_dom for d in ds_list])
            self.val_dom = tf.data.experimental.sample_from_datasets([d.val_dom for d in ds_list])
            self.test_dom = tf.data.experimental.sample_from_datasets([d.test_dom for d in ds_list])

# class MultiSourceDataset(DataLoader):
#     def __init__(self, ds_list, batch_size, train_shuffle=True, val_shuffle=False, test_shuffle=False, buffer_size=1000):

#         # self.train = self._init_loader(tf.data.experimental.sample_from_datasets([d.train for d in ds_list]), buffer_size, train_shuffle, batch_size, [Squeeze()])
#         # self.val = self._init_loader(tf.data.experimental.sample_from_datasets([d.val for d in ds_list]), buffer_size, val_shuffle, batch_size, [Squeeze()])
#         # self.test = self._init_loader(tf.data.experimental.sample_from_datasets([d.test for d in ds_list]), buffer_size, test_shuffle, batch_size, [Squeeze()])

#         # ##TODO
#         # sys.exit()
#         self.train = tf.data.experimental.sample_from_datasets([d.train for d in ds_list])
#         self.val = tf.data.experimental.sample_from_datasets([d.val for d in ds_list])
#         self.test = tf.data.experimental.sample_from_datasets([d.test for d in ds_list])

class JointLoader:
    def __init__(self, lds):
        self.lds = lds

    def __iter__(self):
        self.iters = [iter(ld) for ld in self.lds]
        self.iter_end = [False for ld in self.lds]
        return self

    def __next__(self):
        x_list, y_list = [], []
        for i, it in enumerate(self.iters):
            try:
                x, y = next(it)
            except StopIteration:
                self.iter_end[i] = True

                if all(self.iter_end):
                    raise StopIteration
                else:
                    self.iters[i] = iter(self.lds[i])
                    x, y = next(self.iters[i])
            x_list.append(x)
            y_list.append(y)

        # maintain the same batch size
        bs_min = min([o.shape[0] for o in x_list])
        x_list = [o[:bs_min] for o in x_list]
        x_list = tf.concat(x_list, 0)
        y_list = [o[:bs_min] for o in y_list]
        y_list = tf.concat(y_list, 0)
        return x_list, y_list

class ChainLoader:
    def __init__(self, ld1, ld2):
        self.ld1 = ld1
        self.ld2 = ld2

    def __iter__(self):
        self.iter = itertools.chain(self.ld1, self.ld2)
        return self

    def __next__(self):
        return next(self.iter)

class DomainDataset(DataLoader):
    def __init__(self, dsld_src, dsld_tar):
        self.train = JointLoader([dsld_src.train_dom, dsld_tar.train_dom])
        self.val = JointLoader([dsld_src.val_dom, dsld_tar.val_dom])
        self.test = JointLoader([dsld_src.test_dom, dsld_tar.test_dom])

class DomainDataset_old(DataLoader):
    def __init__(self, src_names, aug_params, tar, batch_size, buffer_size=1000,
                 train_shuffle=True, val_shuffle=False, test_shuffle=False,
                 train_aug=True, val_aug=False, test_aug=False,
                 **kwargs
    ):

        ## init source datasets
        src_ds_list = []
        for s, a in itertools.product(src_names, aug_params):
            if a is None:
                print("[dom] src: %s + none"%(s)) ##TODO: simplify
            else:
                print("[dom] src: %s + %s"%(s, " + ".join(a_param[0] for a_param in a)))
            src_ds_list.append(
                getattr(data, s)(
                    root=os.path.join('data', s.lower()),
                    batch_size=1, # draw one sample from each source
                    aug_list=a,
                    domain_id=1,
                    train_shuffle=train_shuffle, val_shuffle=val_shuffle, test_shuffle=test_shuffle,
                    train_aug=train_aug, val_aug=val_aug, test_aug=test_aug,
                    **kwargs
                )
            )

        ## init the target dataset
        tar_ds = getattr(data, tar)(
            root=os.path.join('data', tar.lower()),
            batch_size=1,
            domain_id=0,
            train_shuffle=train_shuffle, val_shuffle=val_shuffle, test_shuffle=test_shuffle,
            train_aug=False, val_aug=False, test_aug=False,
            **kwargs
        )

        ## init loaders
        train = [d.train_dom for d in src_ds_list] + [tar_ds.train_dom]
        val = [d.val_dom for d in src_ds_list] + [tar_ds.val_dom]
        test = [d.test_dom for d in src_ds_list] + [tar_ds.test_dom]
        self.train = self._init_loader(tf.data.experimental.sample_from_datasets(train),
                                       buffer_size, train_shuffle, batch_size, [Squeeze()])
        self.val = self._init_loader(tf.data.experimental.sample_from_datasets(val),
                                     buffer_size, val_shuffle, batch_size, [Squeeze()])
        self.test = self._init_loader(tf.data.experimental.sample_from_datasets(test),
                                      buffer_size, test_shuffle, batch_size, [Squeeze()])

    # class DomainDataset(DataLoader):
    # def __init__(self, src_list, tar, batch_size, train_shuffle=True, val_shuffle=False, test_shuffle=False, buffer_size=1000):
    #     raise NotImplementedError

    #     train = [d.train_dom for d in src_list] + [tar.train_dom]
    #     val = [d.val_dom for d in src_list] + [tar.val_dom]
    #     test = [d.test_dom for d in src_list] + [tar.test_dom]
    #     weights = [0.5/len(src_list)]*len(src_list) + [0.5]

    #     self.train = self._init_loader(tf.data.experimental.sample_from_datasets(train, weights=weights),
    #                                    buffer_size, train_shuffle, batch_size, [Squeeze()])
    #     self.val = self._init_loader(tf.data.experimental.sample_from_datasets(val, weights=weights),
    #                                  buffer_size, val_shuffle, batch_size, [Squeeze()])
    #     self.test = self._init_loader(tf.data.experimental.sample_from_datasets(test, weights=weights),
    #                                   buffer_size, test_shuffle, batch_size, [Squeeze()])

def rot_gaussian(rot, mu, cov):
    rot_rad = np.deg2rad(rot)
    R = np.array([[np.cos(rot_rad), -np.sin(rot_rad)], [np.sin(rot_rad), np.cos(rot_rad)]])
    mu_rot = np.transpose(np.matmul(R, np.transpose(mu)))
    cov_rot = np.matmul(np.matmul(R, cov), np.transpose(R))
    return mu_rot, cov_rot

"""
plot
"""
def plot_data(x, y, markers, colors, alphas, labels, facecolors=None, fn=None, markersize=2, linewidth=1, w=None, classifier=None):
    #y_id = np.unique(y)
    y_id = np.arange(0, y.max()+1)
    assert(len(y_id) == len(markers) == len(alphas))
    if facecolors is None:
        facecolors = colors
    plt.figure(1)
    plt.clf()

    ## plot data
    hs = []
    for y_i, m_i, c_i, fc_i, a_i, l_i in zip(y_id, markers, colors, facecolors, alphas, labels):
        x_i = x[y==y_i]
        h = plt.plot(x_i[:, 0], x_i[:, 1], m_i, alpha=a_i, markerfacecolor=fc_i, markeredgecolor=c_i, markersize=markersize, linewidth=linewidth, label=l_i)
        hs.append(h[0])

    ## plot decision
    if classifier is not None:
        X, Y = np.meshgrid(
            np.linspace(-2.5, 2.5, 100),
            np.linspace(-2.5, 2.5, 100))
        XY = np.concatenate((X.flatten()[..., np.newaxis], Y.flatten()[..., np.newaxis]), 1)
        P = classifier(XY)
        Z = P[:, 1]
        Z = np.reshape(Z, X.shape)

        plt.contourf(X, Y, Z, alpha=0.3, cmap='RdYlGn')
        plt.colorbar()

    ## plot targetness
    if w is not None:
        X, Y = np.meshgrid(
            np.linspace(-2.5, 2.5, 100),
            np.linspace(-2.5, 2.5, 100))
        XY = np.concatenate((X.flatten()[..., np.newaxis], Y.flatten()[..., np.newaxis]), 1)
        W = w(tf.constant(XY, dtype=tf.float32))
        W = np.reshape(W.numpy(), X.shape)
        Z = W / (1+W)

        Z = ((1 - 2*np.abs(Z-0.5)) > 1e-3).astype(np.float32)

        #fig, ax = plt.subplots(1,1)
        plt.contourf(X, Y, Z, alpha=0.5, zorder=5)
        plt.colorbar()

    ## beautify
    plt.grid('on')
    plt.gca().set_aspect('equal')
    plt.xlim((-2.5, 2.5))
    plt.ylim((-2.5, 2.5))
    plt.legend(handles=hs)
    if fn is not None:
        plt.savefig(fn+'.png', bbox_inches='tight')

    plt.close()

def plot_targetness(w, alpha=0.7, fn=None, fig_id=None, close=True):

    overlay = True if fig_id is not None else False

    X, Y = np.meshgrid(
        np.linspace(-2.5, 2.5, 100),
        np.linspace(-2.5, 2.5, 100))
    XY = np.concatenate((X.flatten()[..., np.newaxis], Y.flatten()[..., np.newaxis]), 1)
    W = w(tf.constant(XY, dtype=tf.float32))
    W = np.reshape(W.numpy(), X.shape)
    Z = W / (1+W)

    if overlay:
        plt.figure(fig_id)
    else:
        plt.figure(1)
        plt.clf()
    fig, ax = plt.subplots(1,1)
    cp = ax.contourf(X, Y, Z, alpha=alpha, zorder=10)
    fig.colorbar(cp)
    if fn is not None:
        plt.savefig(fn+'.png')
    if close:
        plt.close()

def shuffle_labeled_examples(x, y, seed=None):
    assert(x.shape[0] == y.shape[0])
    n = y.shape[0]
    if seed is None:
        seed = int(time.time())
    np.random.seed(seed)

    i_rnd = np.random.permutation(n)
    x, y = x[i_rnd], y[i_rnd]
    return x, y


text:
# This code is part of Qiskit.
#
# (C) Copyright IBM 2018, 2021.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

"""Test Two Layer QNN."""

import unittest

from test import QiskitMachineLearningTestCase

import numpy as np
from ddt import ddt, data
from qiskit.providers.aer import StatevectorSimulator
from qiskit.circuit.library import RealAmplitudes, ZZFeatureMap
from qiskit.utils import QuantumInstance

from qiskit_machine_learning.neural_networks import TwoLayerQNN

@ddt
class TestTwoLayerQNN(QiskitMachineLearningTestCase):
    """Two Layer QNN Tests."""

    def setUp(self):
        super().setUp()

        # specify "run configuration"
        quantum_instance = QuantumInstance(StatevectorSimulator())

        # define QNN
        num_qubits = 2
        feature_map = ZZFeatureMap(num_qubits)
        ansatz = RealAmplitudes(num_qubits, reps=1)
        self.qnn = TwoLayerQNN(num_qubits, feature_map=feature_map,
                               ansatz=ansatz, quantum_instance=quantum_instance)

        self.qnn_no_qi = TwoLayerQNN(num_qubits, feature_map=feature_map,
                                     ansatz=ansatz)

    @data(
        "qi",
        "no_qi"
    )
    def test_qnn_simple_new(self, qnn_type: str):
        """Simple Opflow QNN Test for a specified neural network."""

        input_data = np.zeros(self.qnn.num_inputs)
        weights = np.zeros(self.qnn.num_weights)

        if qnn_type == "qi":
            qnn = self.qnn
        else:
            qnn = self.qnn_no_qi

        # test forward pass
        result = qnn.forward(input_data, weights)
        self.assertEqual(result.shape, (1, *self.qnn.output_shape))

        # test backward pass
        result = self.qnn.backward(input_data, weights)
        # batch dimension of 1
        self.assertEqual(result[0].shape, (1, *self.qnn.output_shape, self.qnn.num_inputs))
        self.assertEqual(result[1].shape, (1, *self.qnn.output_shape, self.qnn.num_weights))

    @data(
        "qi",
        "no_qi"
    )
    def _test_qnn_batch(self, qnn_type: str):
        """Batched Opflow QNN Test for the specified network."""
        batch_size = 10

        input_data = np.arange(batch_size * self.qnn.num_inputs)\
            .reshape((batch_size, self.qnn.num_inputs))
        weights = np.zeros(self.qnn.num_weights)

        if qnn_type == "qi":
            qnn = self.qnn
        else:
            qnn = self.qnn_no_qi

        # test forward pass
        result = qnn.forward(input_data, weights)
        self.assertEqual(result.shape, (batch_size, *self.qnn.output_shape))

        # test backward pass
        result = self.qnn.backward(input_data, weights)
        self.assertEqual(result[0].shape,
                         (batch_size, *self.qnn.output_shape, self.qnn.num_inputs))
        self.assertEqual(result[1].shape,
                         (batch_size, *self.qnn.output_shape, self.qnn.num_weights))

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: Zoufalc/qiskit-machine-learning

max_issues_repo_name: Zoufalc/qiskit-machine-learning

max_forks_repo_name: Zoufalc/qiskit-machine-learning


text:
r"""@package motsfinder.ipyutils.plotting3d

Functions for plotting in 3D, \eg 2-dimensional surfaces via plot_2d().
"""

from matplotlib import cm

import numpy as np

from .plotctx import plot_ctx_3d
from .plotting import plot_data
from ..utils import isiterable, insert_missing

__all__ = [
    "prepare_grid",
    "plot_2d",
    "plot_scatter",
    "plot_tri",
    "plot_tri_cut",
]

def prepare_grid(fun, space_x, space_y):
    """Evaluate `fun` to build X, Y, Z arrays usable for plotting with `Axes3D.plot_surface`."""
    X, Y = np.meshgrid(space_x, space_y)
    zs = np.array([fun(x, y) for x, y in zip(np.ravel(X), np.ravel(Y))])
    Z = zs.reshape(X.shape)
    return X, Y, Z

def plot_scatter(points, s=20, c='b', plot_kw=None, show=True, close=False,
                 **kw):
    r"""Create a scatter plot of 3D points.

    Options are similar to the ones of plot_1d() with the below additions.

    @param points
        Iterable of 3D points to plot.
    @param s
        Size of the individual point markers. Default is `20`.
    @param c
        Color of the markers. Default is blue, i.e. ``'b'``.
    @param zlim
        Optional interval of the z-axis to show. Default is automatic.
    @param zscilimits
        As the `xscilimits` parameter but for the z-axis.
    @param ztick_spacing
        As `xtick_spacing` but for the z-axis.
    @param zlabel
        Label for the z-axis
    @param azim
        Azimuthal angle used for the 3D projection. Default is `-60`.
    @param elev
        Elevation angle used for the 3D projection. Default is `32`.
    @param dist
        Custom "distance" to the plotted area. Modifies perspective and can
        help in cases where important parts are cropped off the image.
    """
    if plot_kw is None:
        plot_kw = dict()
    xs, ys, zs = zip(*points)
    with plot_ctx_3d(show=show, close=close, **kw) as ax:
        ax.scatter(xs, ys, zs, s=s, c=c, **plot_kw)
    if not show and not close:
        return ax

def plot_2d(f, domain=None, rel_range=((0, 1), (0, 1)), points=30, lw=.4,
            c='w', ec='k', cmap=None, absolute=False, zoffset=0, zlog=False,
            plot_kw=None, show=True, close=False, **kw):
    r"""Plot a surface parameterized as `z = f(x, y)`.

    Parameters not described here are described in plot_scatter() or plot_1d().

    @param f
        Function to plot. Should take a 2-tuple/list/array and return a
        floating type. Alternatively, this may be a (NumPy) matrix which will
        be plotted as ``f(i, j) = mat[i, j]``, where `i` and `j` run over the
        full shape (i.e. ranges and `points` arguments are ignored).
    @param domain
        Domain to plot `f` on as ``((x1, x2), (y1, y2))``. If not given, the
        function's `domainX` and `domainY` attributes are queried. If these
        don't exist, the interval `(0,1)` is used.
    @param rel_range
        Similar as `rel_range` in plot_1d(), but containing the relative range
        to plot for both axes.
    @param points
        Number of samples to take in each direction. The function will be
        evaluated on a grid of `points*points` points. May also be a
        2-tuple/list with a resolution for each axis. Default is `30`.
    @param lw
        Line width for drawing the edges of the 3D mesh. Default is `0.4`.
    @param c
        Color of the mesh surfaces. Default is white, i.e. ``'w'``.
    @param ec
        Edge color. Default is black, i.e. ``'k'``.
    @param cmap
        Optional color map to indicate height of the surface.
    @param absolute
        If `True`, uses the absolute of the values returned by `f`.
    @param zoffset
        Value to add to the values returned by `f`.
    @param zlog
        Whether to plot the base-10 logarithm of the function values.
    """
    if isinstance(f, (np.ndarray, list, tuple)):
        mat = np.asarray(f)
        f = lambda xy: mat[xy[0], xy[1]]
        points = range(mat.shape[0]), range(mat.shape[1])
    if plot_kw is None:
        plot_kw = dict()
    if 'abs_range' in kw:
        domain = kw.pop('abs_range')
    domain_x, domain_y = (None, None) if domain is None else domain
    x1, x2 = getattr(f, 'domainX', (0, 1)) if domain_x is None else domain_x
    y1, y2 = getattr(f, 'domainY', (0, 1)) if domain_y is None else domain_y
    x1, x2, y1, y2 = map(float, (x1, x2, y1, y2))
    (rx1, rx2), (ry1, ry2) = rel_range
    if not isiterable(points):
        points = [points, points]
    x_space, y_space = points
    if not isiterable(x_space):
        x_space = np.linspace(x1 + rx1*(x2-x1), x1 + rx2*(x2-x1), x_space)
    if not isiterable(y_space):
        y_space = np.linspace(y1 + ry1*(y2-y1), y1 + ry2*(y2-y1), y_space)
    with plot_ctx_3d(show=show, close=close, **kw) as ax:
        if absolute:
            f_with_sign = f
            f = lambda x: abs(f_with_sign(x))
        if zoffset != 0:
            f_no_offset = f
            f = lambda x: f_no_offset(x) + zoffset
        if zlog:
            f_linear = f
            def _f(x):
                fx = f_linear(x)
                if fx > 0:
                    return np.log10(fx)
                return np.nan
            f = _f
        X, Y, Z = prepare_grid(lambda x, y: float(f([x, y])), x_space, y_space)
        ax.plot_surface(
            X, Y, Z,
            **insert_missing(plot_kw, rstride=1, cstride=1,
                             cmap=cmap, color=c, edgecolor=ec,
                             linewidth=lw, antialiased=True)
        )
    if not show and not close:
        return ax

def plot_tri(points, triangles=None, lw=.4, c='w', ec='k', facecolors=None,
             cmap=cm.jet, colorbar=True, plot_kw=None, show=True, close=False,
             **kw):
    r"""Create a 3D surface plot with manual triangulation.

    Options not described here are similar to the ones of plot_2d().

    @param points
        Vertex points of the mesh
    @param triangles
        List of 3-tuples indicating which vertices form triangles. If not
        given, `points` will be interpreted as an object with a `surface()`
        method returning the actual points and triangles.
    @param lw (float, optional)
        Linewidth of the triangle edges. Default is `.4`.
    @param c (color, optional)
        Color of the triangle faces. May be overridden by `facecolors`.
    @param ec (color, optional)
        Edge color for the triangle edges. Set to `None` to not draw the wire.
    @param facecolors (iterable of floats, optional)
        A value assigned to each "face" (i.e. triangle). The values will be
        associated with colors of the colormap specified via `cmap`.
    @param cmap (colormap, optional)
        A colormap (e.g. one of the `matplotlib.cm` properties) to use for the
        face colors. Has no effect without `facecolors`.
    @param colorbar (boolean, optional)
        Whether to show a color bar indicating the values associated with
        colors of the `cmap`. Has no effect without `facecolors`.
    """
    if not isiterable(points):
        points, triangles = points.surface()
    if plot_kw is None:
        plot_kw = dict()
    xs, ys, zs = zip(*points)
    with plot_ctx_3d(show=show, close=close, **kw) as ax:
        pcollection = ax.plot_trisurf(
            xs, ys, zs, triangles=triangles, color=c,
            linewidth=lw, edgecolor=ec, **plot_kw
        )
        if facecolors is not None:
            pcollection.set_cmap(cmap)
            pcollection.set_array(np.array(facecolors))
            if colorbar:
                opts = colorbar if isinstance(colorbar, dict) else dict()
                ax.figure.colorbar(pcollection, **opts)
    if not show and not close:
        return ax

def _plane_intersection(p1, p2):
    r"""Return the point at which the line between two points crosses the x-y-plane.

    If the line does not cross the x-y-plane, `None` is returned.
    """
    z1, z2 = p1[2], p2[2]
    if z1 * z2 < 0.0 or (z1 * z2 == 0.0 and z1 + z2 != 0.0):
        t = - z1 / (z2-z1)
        return (p1 + t * (p2 - p1))[0:2]
    return None

def plot_tri_cut(points, triangles=None, figsize=(4, 4), **kw):
    r"""Plot the intersection curve of a triangle surface and the x-y-plane.

    The options are the same as for plot_data().
    """
    if not isiterable(points):
        points, triangles = points.surface()
    points = np.array(points)
    pts = []
    for tri in triangles:
        p1, p2, p3 = [points[t] for t in tri]
        for pa, pb in ((p1, p2), (p1, p3), (p2, p3)):
            pt = _plane_intersection(pa, pb)
            if pt is not None:
                pts.append(pt)
    # sort based on angle
    c = sum(pts) / len(pts)
    pts.sort(key=lambda pt: np.arctan2(*(pt - c)))
    if pts:
        pts.append(pts[0])
        xs, ys = zip(*pts)
    else:
        xs = ys = []
    return plot_data(xs, ys, figsize=figsize, **kw)

max_stars_repo_name: daniel-dpk/distorted-motsfinder-public

max_issues_repo_name: daniel-dpk/distorted-motsfinder-public

max_forks_repo_name: daniel-dpk/distorted-motsfinder-public


text:
from typing import Union

import tensorflow as tf
import numpy as np

ArrayLike = Union[np.ndarray, tf.python.framework.ops.EagerTensor]
TfTensor = tf.python.framework.ops.EagerTensor
FreeRV = ArrayLike

def stabilize(K, shift=1e-6):
    r"""Add a diagonal shift to a covarience matrix"""
    return tf.linalg.set_diag(K, tf.linalg.diag_part(K) + shift)


text:
# from sklearn.datasets import load_digits,load_boston
import numpy as np
from numpy import linalg as linalg
import math

# digits = load_digits()
# boston = load_boston()
# d_data = digits.data
# b_data = boston.data
# d_target = digits.target
# b_target = boston.target
#
# t0 = np.median(b_target)
# target = np.zeros(len(b_target))
# target[b_target <= t0 ] = 1
# b_target = target
#
# divider = 1797-100
#
#
#
# x_train = d_data[:divider,:] # N*D
# x_test = d_data[divider:,:]
#
# t_train = d_target[0:divider]
# t_test = d_target[divider:]
###############################

# b_divider =  406
#
# x_train = b_data[:b_divider,:] # N*D
# x_test = b_data[b_divider:,:]
#
# t_train = b_target[0:b_divider]
# t_test = b_target[b_divider:]

def GNB(train_data, train_target, test_data, test_target):
    #### Function begin:
    # x_train = x_train[None,:,:] # 1*N*D
    x_train = train_data
    t_train = train_target
    x_test = test_data
    t_test = test_target

    max_t = int(max(t_train))
    min_t = int(min(t_train))
    n_class = max_t-min_t+1
    N, D = (x_train.shape)
    # K = n_class

    classes = t_train ==0
    for idx in range(min_t+1,max_t+1,1):
        class_i = t_train==idx
        classes = np.vstack((classes,class_i))
    n_elem = np.sum(classes, 1) # classes: K*N*1

    # x_train_parted = x_train[classes[0]] # N*D
    # x_train_parted = x_train_parted[None,:,:] # K*N*D
    x_train_parted = dict()
    for idx in range(min_t, max_t+1, 1):
        x_train_parted[idx] = x_train[classes[idx]]
    # print('partitioned shape:', x_train_parted.shape)

    pi = n_elem/np.sum(n_elem)
    # pi = pi[:,None,None]

    mu = np.zeros((n_class,D))
    for cl in x_train_parted:
        mu[cl] = np.mean(x_train_parted[cl],0)
    # mu = np.mean(x_train_parted,1)
    mu.shape
    mu = mu[:,None,:] # K*N*D

    S = np.zeros((D,D ))
    for idx,number in enumerate(n_elem):
        Sk = np.dot( (x_train_parted[idx]-mu[idx]).transpose(),  (x_train_parted[idx]-mu[idx]) )
        S = S + Sk
    S = S/sum(n_elem)
    ## training ends

    ## begin testing:

    # y = np.zeros((n_class,N))
    # for idx,number in enumerate(n_elem):
    #     y[idx] = math.log(pi[idx]) - 0.5 * np.diag((x_train_parted[idx]-mu[idx]).dot(
    #         linalg.pinv(S).dot((x_train_parted[idx]-mu[idx]).transpose()))
    #     ) # -0.5*math.log(linalg.det(S))

    # x_test: N*D

    mu = mu[:,0,:] # K*D
    y = np.zeros((x_test.shape[0], n_class))
    logpi = np.zeros(len(pi))
    for i in range(len(pi)):
        logpi[i] = math.log(pi[i])
    for n, x in enumerate(x_test):
        # x : N=1*D (D,)
        y[n] = logpi - 0.5 * np.diag((x[None,:] - mu).dot(
            linalg.pinv(S).dot(
                (x[None, :] - mu).transpose()
            )
        ))
    result = np.argmax(y, 1)

    # for n,x in enumerate(x_test):
    #     for k in range(n_class):
    #         y[n,k] = logpi[k] - 0.5 * np.dot((x - mu[k]), np.dot(linalg.pinv(S),(x - mu[k]).transpose()))
    # result = np.argmax(y,1)

    err = 1 - np.mean( result == t_test )
    # print('err_rate',err)
    # print('result', result)

    return err

reason: import numpy,from numpy


text:
"""Provides the 'OffshoreSubstationDesign` class."""

__author__ = "Jake Nunemaker"
__copyright__ = "Copyright 2020, National Renewable Energy Laboratory"
__maintainer__ = "Jake Nunemaker"
__email__ = "Jake.Nunemaker@nrel.gov"

import numpy as np
from wisdem.orbit.phases.design import DesignPhase

class OffshoreSubstationDesign(DesignPhase):
    """Offshore Substation Design Class."""

    expected_config = {
        "site": {"depth": "m"},
        "plant": {"num_turbines": "int"},
        "turbine": {"turbine_rating": "MW"},
        "substation_design": {
            "mpt_cost_rate": "USD/MW (optional)",
            "topside_fab_cost_rate": "USD/t (optional)",
            "topside_design_cost": "USD (optional)",
            "shunt_cost_rate": "USD/MW (optional)",
            "switchgear_cost": "USD (optional)",
            "backup_gen_cost": "USD (optional)",
            "workspace_cost": "USD (optional)",
            "other_ancillary_cost": "USD (optional)",
            "topside_assembly_factor": "float (optional)",
            "oss_substructure_cost_rate": "USD/t (optional)",
            "oss_pile_cost_rate": "USD/t (optional)",
            "num_substations": "int (optional)",
        },
    }

    output_config = {
        "num_substations": "int",
        "offshore_substation_topside": "dict",
        "offshore_substation_substructure": "dict",
    }

    def __init__(self, config, **kwargs):
        """
        Creates an instance of OffshoreSubstationDesign.

        Parameters
        ----------
        config : dict
        """

        config = self.initialize_library(config, **kwargs)
        self.config = self.validate_config(config)
        self._outputs = {}

    def run(self):
        """Main run function."""

        self.calc_substructure_length()
        self.calc_substructure_deck_space()
        self.calc_topside_deck_space()

        self.calc_num_mpt_and_rating()
        self.calc_mpt_cost()
        self.calc_topside_mass_and_cost()
        self.calc_shunt_reactor_cost()
        self.calc_switchgear_cost()
        self.calc_ancillary_system_cost()
        self.calc_assembly_cost()
        self.calc_substructure_mass_and_cost()

        self._outputs["offshore_substation_substructure"] = {
            "type": "Monopile",  # Substation install only supports monopiles
            "deck_space": self.substructure_deck_space,
            "mass": self.substructure_mass,
            "length": self.substructure_length,
            "unit_cost": self.substructure_cost,
        }

        self._outputs["offshore_substation_topside"] = {
            "deck_space": self.topside_deck_space,
            "mass": self.topside_mass,
            "unit_cost": self.substation_cost,
        }

        self._outputs["num_substations"] = self.num_substations

    @property
    def substation_cost(self):
        """Returns total procuremet cost of the topside."""

        return (
            self.mpt_cost
            + self.topside_cost
            + self.shunt_reactor_cost
            + self.switchgear_costs
            + self.ancillary_system_costs
            + self.land_assembly_cost
        )

    @property
    def total_cost(self):
        """Returns total procurement cost of the substation(s)."""

        if not self._outputs:
            raise Exception("Has OffshoreSubstationDesign been ran yet?")

        return (self.substructure_cost + self.substation_cost) * self.num_substations

    def calc_substructure_length(self):
        """
        Calculates substructure length as the site depth + 10m
        """

        self.substructure_length = self.config["site"]["depth"] + 10

    def calc_substructure_deck_space(self):
        """
        Calculates required deck space for the substation substructure.

        Coming soon!
        """

        self.substructure_deck_space = 1

    def calc_topside_deck_space(self):
        """
        Calculates required deck space for the substation topside.

        Coming soon!
        """

        self.topside_deck_space = 1

    def calc_num_mpt_and_rating(self):
        """
        Calculates the number of main power transformers (MPTs) and their rating.

        Parameters
        ----------
        num_turbines : int
        turbine_rating : float
        """

        _design = self.config.get("substation_design", {})
        self.num_substations = _design.get("num_substations", 1)

        num_turbines = self.config["plant"]["num_turbines"]
        turbine_rating = self.config["turbine"]["turbine_rating"]

        self.num_mpt = np.ceil(num_turbines * turbine_rating / (250 * self.num_substations))
        self.mpt_rating = (
            round(((num_turbines * turbine_rating * 1.15) / (self.num_mpt * self.num_substations)) / 10.0) * 10.0
        )

    def calc_mpt_cost(self):
        """
        Calculates the total cost for all MPTs.

        Parameters
        ----------
        mpt_cost_rate : float
        """

        _design = self.config.get("substation_design", {})
        mpt_cost_rate = _design.get("mpt_cost_rate", 12500)

        self.mpt_cost = self.mpt_rating * self.num_mpt * mpt_cost_rate

    def calc_topside_mass_and_cost(self):
        """
        Calculates the mass and cost of the substation topsides.

        Parameters
        ----------
        topside_fab_cost_rate : int | float
        topside_design_cost: int | float
        """

        _design = self.config.get("substation_design", {})
        topside_fab_cost_rate = _design.get("topside_fab_cost_rate", 14500)
        topside_design_cost = _design.get("topside_design_cost", 4.5e6)

        self.topside_mass = 3.85 * self.mpt_rating * self.num_mpt + 285
        self.topside_cost = self.topside_mass * topside_fab_cost_rate + topside_design_cost

    def calc_shunt_reactor_cost(self):
        """
        Calculates the cost of the shunt reactor.

        Parameters
        ----------
        shunt_cost_rate : int | float
        """

        _design = self.config.get("substation_design", {})
        shunt_cost_rate = _design.get("shunt_cost_rate", 35000)

        self.shunt_reactor_cost = self.mpt_rating * self.num_mpt * shunt_cost_rate * 0.5

    def calc_switchgear_cost(self):
        """
        Calculates the cost of the switchgear.

        Parameters
        ----------
        switchgear_cost : int | float
        """

        _design = self.config.get("substation_design", {})
        switchgear_cost = _design.get("switchgear_cost", 14.5e5)

        self.switchgear_costs = self.num_mpt * switchgear_cost

    def calc_ancillary_system_cost(self):
        """
        Calculates cost of ancillary systems.

        Parameters
        ----------
        backup_gen_cost : int | float
        workspace_cost : int | float
        other_ancillary_cost : int | float
        """

        _design = self.config.get("substation_design", {})
        backup_gen_cost = _design.get("backup_gen_cost", 1e6)
        workspace_cost = _design.get("workspace_cost", 2e6)
        other_ancillary_cost = _design.get("other_ancillary_cost", 3e6)

        self.ancillary_system_costs = backup_gen_cost + workspace_cost + other_ancillary_cost

    def calc_assembly_cost(self):
        """
        Calculates the cost of assembly on land.

        Parameters
        ----------
        topside_assembly_factor : int | float
        """

        _design = self.config.get("substation_design", {})
        topside_assembly_factor = _design.get("topside_assembly_factor", 0.075)
        self.land_assembly_cost = (
            self.switchgear_costs + self.shunt_reactor_cost + self.ancillary_system_costs
        ) * topside_assembly_factor

    def calc_substructure_mass_and_cost(self):
        """
        Calculates the mass and associated cost of the substation substructure.

        Parameters
        ----------
        oss_substructure_cost_rate : int | float
        oss_pile_cost_rate : int | float
        """

        _design = self.config.get("substation_design", {})
        oss_substructure_cost_rate = _design.get("oss_substructure_cost_rate", 3000)
        oss_pile_cost_rate = _design.get("oss_pile_cost_rate", 0)

        substructure_mass = 0.4 * self.topside_mass
        substructure_pile_mass = 8 * substructure_mass ** 0.5574
        self.substructure_cost = (
            substructure_mass * oss_substructure_cost_rate + substructure_pile_mass * oss_pile_cost_rate
        )

        self.substructure_mass = substructure_mass + substructure_pile_mass

    @property
    def design_result(self):
        """
        Returns the results of self.run().
        """

        if not self._outputs:
            raise Exception("Has OffshoreSubstationDesign been ran yet?")

        return self._outputs

    @property
    def detailed_output(self):
        """Returns detailed phase information."""

        _outputs = {
            "num_substations": self.num_substations,
            "substation_mpt_rating": self.mpt_rating,
            "substation_topside_mass": self.topside_mass,
            "substation_topside_cost": self.topside_cost,
            "substation_substructure_mass": self.substructure_mass,
            "substation_substructure_cost": self.substructure_cost,
        }

        return _outputs

max_stars_repo_name: johnjasa/WISDEM

max_issues_repo_name: johnjasa/WISDEM

max_forks_repo_name: johnjasa/WISDEM


text:
import numpy as np
# # from iotdata import IOTData
# # from ShimmerBluetooth import ShimmerBluetooth

# # test = ShimmerBluetooth("COM9", 256)
# # test.getFrame()

# data = np.array([[4, 3, 2, 1],[3,2,4,5]])
# print data

# print data.shape, type(data.shape[0])

# print data.shape[0]/2

# a = np.reshape(data,(1,-1))
# a = a[0]
# print a

# tempdata = np.array(data)
# data2d = np.zeros([3, 1])
# data2d[0,0] = data[0]
# data2d[1,0] = data[1]
# data2d[2,0] = data[2]

# print data2d

# a = [[1,2,4],[1,4,7]]
# an = np.array(a)
# print an

# from ShimmerBluetooth import ShimmerBluetooth as sb
# from calibration import calibrateInertialSensorData

# some test code for calibration

# print calibrateInertialSensorData([2033,2035,1237],AMa,SMa,OVa)

# print calibrateInertialSensorData([-129,62,-13],AMg,SMg,OVg)

# from iotdata import IOTData
# from ShimmerBluetooth import ShimmerBluetooth

# sensor = ShimmerBluetooth("COM9", 256)
# df = sensor.getFrame()
# print df[0]

# print 'a' is not 'a'

# a = np.array([5, 4, 3, 5])

# c = np.argsort(a)
# print c
# # expected: c = np.array([2, 1, 0, 3])
# for i in range(len(a)):
#     print a[c[i]]

# loc = np.where(c==5)
# print loc[0]
# print loc[0].size is 0

# from peakdetect2 import locatePeaks as lpk
# import matplotlib.pyplot as plt
# testData = np.array([0,1,1,1,2,3,2,1,1,2,2,2,1,1,4,1,2,1])
# plt.plot(testData)
# print lpk(testData,2,1,2)
# plt.show()

a = np.array([1,1,2]).reshape(-1,1)
b = np.array([2,3,1]).reshape(-1,1)
c = np.square(a)+np.square(b)
print c

max_stars_repo_name: Healbadbad/InternetOfThingsClass

max_issues_repo_name: Healbadbad/InternetOfThingsClass

max_forks_repo_name: Healbadbad/InternetOfThingsClass


text:
# Imports modules
import argparse

import torch
from torchvision import transforms,datasets,models

from PIL import Image

import numpy as np

def get_input_args_train():

    parser = argparse.ArgumentParser()

    parser.add_argument('--data_dir', type = str, default = 'flowers',
                        help='dataset directory')
    parser.add_argument('--save_dir', type = str, default = '/home/workspace/ImageClassifier/',
                        help = 'path to the folder for saving checkpoints')
    parser.add_argument('--arch',type = str, default = 'densenet',
                        help = 'NN Model Architecture vgg or densenet. default = densenet')
    parser.add_argument('--learning_rate',type = float, default = 0.001,
                        help = 'value of learning rate')
    parser.add_argument('--hidden_units',type = int, default = 512,
                        help = 'number of hidden units')
    parser.add_argument('--epochs',type = int, default = 10,
                        help = 'number of iterations for training network')
    parser.add_argument('--gpu', type = bool, default = 'False',
                        help='device to run your model : gpu or cpu. Default = False i.e cpu')

    return parser.parse_args()

def get_input_args_predict():

    parser = argparse.ArgumentParser()

    parser.add_argument('--image_path', type = str, default = '/home/workspace/ImageClassifier/flowers/test/1/image_06743.jpg',
                        help = 'path to image')
    parser.add_argument('--checkpoint',type = str, default = 'checkpoint.pth',
                        help = 'trained model checkpoint')
    parser.add_argument('--top_k',type = int, default = 3,
                        help = 'number of classes with highest prob.')
    parser.add_argument('--category_names', default = 'cat_to_name.json',
                        help = 'mapping of categories to real names file')
    parser.add_argument('--gpu', type = bool, default = 'False',
                        help='device to run your model : gpu or cpu.Default = False i.e cpu')

    return parser.parse_args()

def process_data(train_dir, test_dir, valid_dir):

    train_transforms = transforms.Compose([transforms.RandomRotation(30),
                                           transforms.RandomResizedCrop(224),
                                           transforms.RandomHorizontalFlip(),
                                           transforms.ToTensor(),
                                           transforms.Normalize([0.485, 0.456, 0.406],
                                                                [0.229, 0.224, 0.225])])
    test_transforms = transforms.Compose([transforms.Resize(255),
                                          transforms.CenterCrop(224),
                                          transforms.ToTensor(),
                                          transforms.Normalize([0.485, 0.456, 0.406],
                                                               [0.229, 0.224, 0.225])])

    trainsets = datasets.ImageFolder(train_dir, transform = train_transforms)
    testsets = datasets.ImageFolder(test_dir, transform = test_transforms)
    validsets = datasets.ImageFolder(valid_dir, transform = test_transforms)

    trainloader = torch.utils.data.DataLoader(trainsets, batch_size=64, shuffle=True)
    testloader = torch.utils.data.DataLoader(testsets, batch_size=64)
    validloader = torch.utils.data.DataLoader(validsets, batch_size=64)

    return trainloader, testloader, validloader, trainsets

def process_image(image):
    ''' Scales, crops, and normalizes a PIL image for a PyTorch model,
        returns an Numpy array
    '''
    image = Image.open(image)

    if image.size[0] > image.size[1]:
        aspect = image.size[1] / 256
        new_size = (image.size[0] / aspect, 256)
    else:
        aspect = image.size[0] / 256
        new_size = (256, image.size[1] / aspect)
        image.thumbnail(new_size, Image.ANTIALIAS)

    # crop out center of image
    width, height = image.size # Get dimensions
    left = (width - 224) / 2
    top = (height - 224) / 2
    right = (width + 224) / 2
    bottom = (height + 224) / 2
    image = image.crop((left, top, right, bottom))

    np_image = np.array(image)
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])

    np_image = np_image / 255.0
    np_image = (np_image - mean)/std
    np_image = np.transpose(np_image, (2, 0, 1))

    return np_image

max_stars_repo_name: AyushAniket/ImageClassification

max_issues_repo_name: AyushAniket/ImageClassification

max_forks_repo_name: AyushAniket/ImageClassification