text:
"""
@file   graphs.py
@author Emil Roy, Joshua Tenorio

This file contains the Graphs and States widgets.
"""
from  PyQt5.QtWidgets import QLabel, QPlainTextEdit, QLineEdit, QGridLayout, QVBoxLayout
from PyQt5.QtCore import Qt
import pyqtgraph as pg
import states
import simulation as sim
import numpy as np

# used for stand in values
initial_array = [0] * 120 # create a List of 120 zero's
initial_cx = np.empty(120)
initial_sp1x = np.array(list(range(-120, 0)))
initial_sp2x = np.array(list(range(-120, 0)))

# graph the containers voltage
containerVoltageGraph = pg.GraphicsLayoutWidget()
containerVoltageData = np.array(initial_array).astype(float)
cvPlot = containerVoltageGraph.addPlot(title = "Container Voltage Data")
containerVoltageCurve = cvPlot.plot(containerVoltageData)
cvPlot.setLabel('left', "Volts(V)")
cvPlot.setLabel('bottom', "# of Packets")

#graph to check the containers temperature
containerTempGraph = pg.GraphicsLayoutWidget()
containerTempData = np.array(initial_array).astype(float)
ctPlot = containerTempGraph.addPlot(title = "Container Temperature Data")
containerTempCurve = ctPlot.plot(containerTempData)
ctPlot.setLabel('left', "Temperature(C°)")
ctPlot.setLabel('bottom', "# of Packets")

#graph to check the containers altitude
containerAltitudeGraph = pg.GraphicsLayoutWidget()
containerAltitudeData = np.array(initial_array).astype(float)
caPlot = containerAltitudeGraph.addPlot(title = "Container Altitude Data")
containerAltitudeCurve = caPlot.plot(containerAltitudeData)
caPlot.setLabel('left', "Altitude(m)")
caPlot.setLabel('bottom', "# of Packets")

#PAYLOAD 1 STUFF BELOW------------------------------------------------------------

#graph to check the payload temperatures
payload1TempGraph = pg.GraphicsLayoutWidget()
p1TempData = np.array(initial_array).astype(float)
p1tPlot = payload1TempGraph.addPlot(title = "Payload 1 Temperature Data")
p1TempCurve = p1tPlot.plot(p1TempData)
p1tPlot.setLabel('left', "Temperature(C°)")
p1tPlot.setLabel('bottom', "# of Packets")

#graph to check the payload altitude
payload1AltitudeGraph = pg.GraphicsLayoutWidget()
p1AltitudeData = np.array(initial_array).astype(float)
p1aPlot = payload1AltitudeGraph.addPlot(title = "Payload 1 Altitude Data")
p1AltitudeCurve = p1aPlot.plot(p1AltitudeData)
p1aPlot.setLabel('left', "Altitude(m)")
p1aPlot.setLabel('bottom', "# of Packets")

#graph to check the payload speed
payload1RotationRateGraph = pg.GraphicsLayoutWidget()
p1RPMdata = np.array(initial_array).astype(float)
p1rPlot = payload1RotationRateGraph.addPlot(title = "Payload 1 Rotation Rate Data")
p1RPMcurve = p1rPlot.plot(p1RPMdata)
p1rPlot.setLabel('left', "Rotations Per Minute(RPM)")
p1rPlot.setLabel('bottom', "# of Packets")

#PAYLOAD 2 Stuff Below-----------------------------------------------------

#graph to check the payload temperatures
payload2TempGraph = pg.GraphicsLayoutWidget()
p2TempData = np.array(initial_array).astype(float)
p2tPlot = payload2TempGraph.addPlot(title = "Payload 2 Temperature Data")
p2TempCurve = p2tPlot.plot(p2TempData)
p2tPlot.setLabel('left', "Temperature(C°)")
p2tPlot.setLabel('bottom', "# of Packets")

#graph to check the payload altitude
payload2AltitudeGraph = pg.GraphicsLayoutWidget()
p2AltitudeData = np.array(initial_array).astype(float)
p2aPlot = payload2AltitudeGraph.addPlot(title = "Payload 2 Altitude Data")
p2AltitudeCurve = p2aPlot.plot(p2AltitudeData)
p2aPlot.setLabel('left', "Altitude(m)")
p2aPlot.setLabel('bottom', "# of Packets")

#graph to check the payload speed
payload2RotationRateGraph = pg.GraphicsLayoutWidget()
p2RPMdata = np.array(initial_array).astype(float)
p2rPLot = payload2RotationRateGraph.addPlot(title = "Payload 2 Rotation Rate Data")
p2RPMcurve = p2rPLot.plot(p2RPMdata)
p2rPLot.setLabel('left', "Rotations Per Minute(RPM)")
p2rPLot.setLabel('bottom', "# of Packets")

def build():
    layout = QGridLayout()
    pay1Widget: QVBoxLayout = states.buildPay1Layout()
    pay2Widget: QVBoxLayout = states.buildPay2Layout()
    conWidget: QVBoxLayout = states.buildContainerLayout()
    layout.addLayout(pay1Widget, 0, 0)
    layout.addWidget(payload1TempGraph, 0, 1)
    layout.addWidget(payload1AltitudeGraph, 0, 2)
    layout.addWidget(payload1RotationRateGraph, 0, 3)
    #Adding payload 2 graphs now
    layout.addLayout(pay2Widget, 1, 0)
    layout.addWidget(payload2TempGraph, 1, 1)
    layout.addWidget(payload2AltitudeGraph, 1, 2)
    layout.addWidget(payload2RotationRateGraph, 1, 3)

    layout.setRowMinimumHeight(2, 10)#adds spacing between payloads and container
    #add container stuff
    layout.addLayout(conWidget, 3, 0)
    layout.addWidget(containerTempGraph, 3, 1)
    layout.addWidget(containerAltitudeGraph, 3, 2)
    layout.addWidget(containerVoltageGraph, 3, 3)

    return layout

# these variables are used for updating the graphs
containerPtr = 0
p1Ptr = 0
p2Ptr = 0

# TODO: implement update function
# ie setData setPos functions
def update():
    # update container
    if containerPtr > 119:
        containerVoltageCurve.setData(containerVoltageData)
        containerAltitudeCurve.setData(containerAltitudeData)
        containerTempCurve.setData(containerTempData)
        containerVoltageCurve.setPos(containerPtr-120, containerPtr)
        containerAltitudeCurve.setPos(containerPtr-120, containerPtr)
        containerTempCurve.setPos(containerPtr-120, containerPtr)
    else:
        containerVoltageCurve.setData(containerVoltageData[:containerPtr])
        containerAltitudeCurve.setData(containerAltitudeData[:containerPtr])
        containerTempCurve.setData(containerTempData[:containerPtr])

    #update payloads
    if p1Ptr > 119:
        p1RPMcurve.setData(p1RPMdata)
        p1AltitudeCurve.setData(p1AltitudeData)
        p1TempCurve.setData(p1TempData)
        p1RPMcurve.setPos(p1Ptr-120, p1Ptr)
        p1AltitudeCurve.setPos(p1Ptr-120, p1Ptr)
        p1TempCurve.setPos(p1Ptr-120, p1Ptr)
    else:
        p1RPMcurve.setData(p1RPMdata[:p1Ptr])
        p1AltitudeCurve.setData(p1AltitudeData[:p1Ptr])
        p1TempCurve.setData(p1TempData[:p1Ptr])

    if p2Ptr > 119:
        p2RPMcurve.setData(p2RPMdata)
        p2AltitudeCurve.setData(p2AltitudeData)
        p2TempCurve.setData(p2TempData)
        p2RPMcurve.setPos(p2Ptr-120, p2Ptr)
        p2AltitudeCurve.setPos(p2Ptr-120, p2Ptr)
        p2TempCurve.setPos(p2Ptr-120, p2Ptr)
    else:
        p2RPMcurve.setData(p2RPMdata[:p2Ptr])
        p2AltitudeCurve.setData(p2AltitudeData[:p2Ptr])
        p2TempCurve.setData(p2TempData[:p2Ptr])

# given a packet, update arrays
def update_data(packet):
    global containerAltitudeData, containerVoltageData, containerTempData
    global p1AltitudeData, p1RPMdata, p1TempData
    global p2AltitudeData, p2RPMdata, p2TempData
    global containerPtr, p1Ptr, p2Ptr

    # send packet to states widget
    states.update_state(packet)

    # parse packet for stuff to update
    packet_args = packet.split(",")

    if packet_args[3] == "C":
        if containerPtr > 119:
            containerPtr += 1
            containerAltitudeData[:-1] = containerAltitudeData[1:]
            containerAltitudeData[-1] = float(packet_args[7])

            containerTempData[:-1] = containerTempData[1:]
            containerTempData[-1] = float(packet_args[8])

            containerVoltageData[:-1] = containerVoltageData[1:]
            containerVoltageData[-1] = float(packet_args[9])
        else:
            containerAltitudeData[containerPtr] = float(packet_args[7])
            containerTempData[containerPtr] = float(packet_args[8])
            containerVoltageData[containerPtr] = float(packet_args[9])
            containerPtr += 1

    elif packet_args[3] == "S1":
        if p1Ptr > 119:
            p1Ptr += 1
            p1AltitudeData[:-1] = p1AltitudeData[1:]
            p1AltitudeData[-1] = float(packet_args[4])

            p1TempData[:-1] = p1TempData[1:]
            p1TempData[-1] = float(packet_args[5])

            p1RPMdata[:-1] = p1RPMdata[1:]
            p1RPMdata[-1] = float(packet_args[6])
        else:
            p1AltitudeData[p1Ptr] = float(packet_args[4])
            p1TempData[p1Ptr] = float(packet_args[5])
            p1RPMdata[p1Ptr] = float(packet_args[6])
            p1Ptr += 1

    elif packet_args[3] == "S2":
        if p2Ptr > 119:
            p2Ptr += 1
            p2AltitudeData[:-1] = p2AltitudeData[1:]
            p2AltitudeData[-1] = float(packet_args[4])

            p2TempData[:-1] = p2TempData[1:]
            p2TempData[-1] = float(packet_args[5])

            p2RPMdata[:-1] = p2RPMdata[1:]
            p2RPMdata[-1] = float(packet_args[6])
        else:
            p2AltitudeData[p1Ptr] = float(packet_args[4])
            p2TempData[p1Ptr] = float(packet_args[5])
            p2RPMdata[p1Ptr] = float(packet_args[6])
            p2Ptr += 1
    else:
        print("GRAPH ERR: invalid packet")


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""

   Copyright 2021 Recurve Analytics, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

"""
import numpy as np
import pandas as pd
import pytest
import os
import random
import sqlite3
from tempfile import mkdtemp

from flexvalue.calculations import FlexValueRun
from flexvalue.settings import ACC_COMPONENTS_ELECTRICITY, ACC_COMPONENTS_GAS

@pytest.fixture
def metered_ids():
    return [f"id_{i}" for i in range(5)]

@pytest.fixture
def deer_ids():
    return ["DEER_LS_1", "DEER_LS_2"]

@pytest.fixture
def metered_load_shape(metered_ids):

    random.seed(0)
    output = []
    for _id in metered_ids:
        for hour in range(8760):
            savings = random.random() * 0.1
            output.append(
                {"identifier": _id, "hour_of_year": hour, "hourly_mwh_savings": savings}
            )
    df = (
        pd.DataFrame(output)
        .pivot(index="hour_of_year", columns="identifier", values="hourly_mwh_savings")
        .reset_index()
        .set_index("hour_of_year")
    )
    df.columns.name = None
    return df

@pytest.fixture
def user_inputs(metered_ids, deer_ids):
    return pd.DataFrame(
        [
            {
                "ID": id_,
                "load_shape": id_,
                "start_year": 2021,
                "start_quarter": 1,
                "utility": "PGE",
                "climate_zone": "CZ1",
                "units": 1,
                "eul": 5,
                "ntg": 1.0,
                "discount_rate": 0.0766,
                "admin": 100,
                "measure": 2000,
                "incentive": 1000,
                "therms_profile": "winter",
                "therms_savings": 400,
                "mwh_savings": 1,
            }
            for id_ in metered_ids + deer_ids
        ]
    ).set_index("ID")

@pytest.fixture
def database_year(pytestconfig):
    database_year = pytestconfig.getoption("database_year")
    if not database_year:
        database_year = "1111"
        db_path = mkdtemp()
        os.environ["DATABASE_LOCATION"] = db_path
        con = sqlite3.connect(f"{db_path}/{database_year}.db")

        random.seed(1)

        acc_elec_cols = {
            col: random.random()
            for col in ACC_COMPONENTS_ELECTRICITY + ["marginal_ghg"]
        }
        df_acc_elec = pd.DataFrame(
            [
                {
                    "climate_zone": "CZ1",
                    "utility": "PGE",
                    "hour_of_year": hour,
                    "hour_of_day": hour % 24,
                    "year": year,
                    "month": (
                        pd.Timestamp("2020-01-01") + pd.Timedelta(hour, unit="H")
                    ).month,
                    **acc_elec_cols,
                }
                for hour in range(0, 8760)
                for year in range(2020, 2051)
            ]
        )
        df_acc_elec.to_sql("acc_electricity", con=con)

        acc_gas_cols = {col: random.random() for col in ACC_COMPONENTS_GAS}
        df_acc_gas = pd.DataFrame(
            [
                {
                    "climate_zone": "CZ1",
                    "utility": "PGE",
                    "year": year,
                    "month": month,
                    **acc_gas_cols,
                }
                for month in range(1, 13)
                for year in range(2020, 2051)
            ]
        )
        df_acc_gas.to_sql("acc_gas", con=con)

        df_deer_load_shapes = pd.DataFrame(
            [
                {
                    "DEER_LS_1": random.random(),
                    "DEER_LS_2": random.random(),
                    "hour_of_year": hour,
                }
                for hour in range(0, 8760)
            ]
        )
        df_deer_load_shapes.to_sql("deer_load_shapes", con=con)
    return database_year

@pytest.fixture
def flexvalue_run(metered_load_shape, database_year):
    return FlexValueRun(
        metered_load_shape=metered_load_shape, database_year=database_year
    )

def test_user_inputs_from_example_metered(
    snapshot, database_year, user_inputs, flexvalue_run
):

    (
        df_output_table,
        df_output_table_totals,
        elec_benefits,
        gas_benefits,
    ) = flexvalue_run.get_results(user_inputs)
    snapshot.assert_match(df_output_table_totals, "df_output_table_totals")

def test_electric_benefits_full_outputs(
    user_inputs, metered_load_shape, snapshot, flexvalue_run
):
    elec_ben = flexvalue_run.get_electric_benefits_full_outputs(user_inputs)
    snapshot.assert_match(elec_ben, "elec_ben")

def test_user_inputs_basic(user_inputs, metered_load_shape, snapshot, flexvalue_run):
    _, df_output_table_totals, _, _ = flexvalue_run.get_results(user_inputs)
    snapshot.assert_match(df_output_table_totals, "df_output_table_totals")

def test_user_inputs_full(user_inputs, metered_load_shape, snapshot, flexvalue_run):
    (
        df_output_table,
        df_output_table_totals,
        elec_ben,
        gas_ben,
    ) = flexvalue_run.get_results(user_inputs)
    snapshot.assert_match(df_output_table_totals, "df_output_table_totals")

def test_user_inputs_single_row(
    user_inputs, metered_load_shape, snapshot, flexvalue_run
):
    (
        df_output_table,
        df_output_table_totals,
        elec_ben,
        gas_ben,
    ) = flexvalue_run.get_results(user_inputs[-1:])
    snapshot.assert_match(df_output_table_totals, "df_output_table_totals")

ext: py

max_stars_repo_name: natphi/flexvalue

max_issues_repo_name: natphi/flexvalue

max_forks_repo_name: natphi/flexvalue

reason: import numpy


text:
import tensorflow as tf
import keras
import keras_contrib
from keras_contrib.layers.normalization.instancenormalization import InstanceNormalization

import torch

import numpy as np
import cv2
import six

from refinenet.resnet import rf_lw50, rf_lw101, rf_lw152
from refinenet.helpers import prepare_img
from style_transfer import layers
from style_transfer import utils

if __name__ == '__main__':

	has_cuda = torch.cuda.is_available()
	n_classes = 7
	model_inits = {
		'rf_lw50_person'   : rf_lw50,
		'rf_lw101_person'  : rf_lw101,
		'rf_lw152_person'  : rf_lw152,
		}
	models = dict()
	for key,fun in six.iteritems(model_inits):
		net = fun(n_classes, pretrained=True).eval()
		if has_cuda:
			net = net.cuda()
		models[key] = net
	rf = ['rf_lw50_person', 'rf_lw101_person', 'rf_lw152_person']
	mnets = [models.get(rf[0]), models.get(rf[1]), models.get(rf[2])]

	custom_objects = {
		'InstanceNormalization':
			InstanceNormalization,
		'DeprocessStylizedImage': layers.DeprocessStylizedImage
	}
	transfer_net = keras.models.load_model(
		'style_transfer/leaf.h5',
		custom_objects=custom_objects
	)
	inputs = [transfer_net.input, keras.backend.learning_phase()]
	outputs = [transfer_net.output]
	transfer_style = keras.backend.function(inputs, outputs)

	img = cv2.imread('./img/tmp.jpg')
	res = img.shape
	# img = cv2.resize(img, (512, 512), interpolation = cv2.INTER_LANCZOS4)
	seg_inp = torch.tensor(prepare_img(img).transpose(2, 0, 1)[None]).float()
	if has_cuda:
		seg_inp = seg_inp.cuda()

	segs = []
	for mnet in mnets:
		seg = mnet(seg_inp)[0].data.cpu().numpy().transpose(1, 2, 0).argmax(axis=2).astype(np.uint8)
		segs.append(seg)

	style_inp = np.expand_dims(img, axis=0)
	style = transfer_style([style_inp, 1])[0]
	style = np.uint8(style[0])

	for i in range(len(segs)):
		seg = segs[i]
		seg = cv2.resize(seg, (res[1], res[0]), interpolation = cv2.INTER_LANCZOS4)
		mask = np.uint8( np.ones(res) * np.expand_dims(seg, axis=2) * 255 ) / 255
		masked = np.uint8( (1 - mask) * style + mask * img )
		window12 = np.hstack((img, np.uint8(mask * 255) ) )
		window34 = np.hstack((style, masked))
		output = np.vstack((window12, window34))
		# output = cv2.resize(output, (res[1], res[0]), interpolation = cv2.INTER_LANCZOS4)
		cv2.imwrite('./img/' + rf[i] + '_result.jpg', output)

max_stars_repo_name: wzlyo/seg-style

max_issues_repo_name: wzlyo/seg-style

max_forks_repo_name: wzlyo/seg-style


text:
import pygplates
import numpy as np
import geopandas as gpd
from shapely.geometry import LineString, Polygon
from .sphere import healpix_mesh
import os, tempfile

def create_gpml_crustal_thickness(longitude_array,latitude_array,thickness,filename=None):

    multi_point = pygplates.MultiPointOnSphere(zip(latitude_array,longitude_array))

    scalar_coverages = {
        pygplates.ScalarType.create_gpml('CrustalThickness'): thickness}

    ct_feature = pygplates.Feature()
    ct_feature.set_geometry((multi_point,scalar_coverages))
    ct_feature.set_name('Crustal Thickness')

    output_feature_collection = pygplates.FeatureCollection(ct_feature)

    if filename is not None:
        output_feature_collection.write(filename)
    else:
        return output_feature_collection

def create_gpml_velocity_feature(longitude_array,latitude_array,filename=None,feature_type=None):
# function to make a velocity mesh nodes at an arbitrary set of points defined in Lat
# Long and Lat are assumed to be 1d arrays.

    multi_point = pygplates.MultiPointOnSphere(zip(latitude_array,longitude_array))

    # Create a feature containing the multipoint feature.
    # optionally, define as 'MeshNode' type, so that GPlates will recognise it as a velocity layer
    if feature_type=='MeshNode':
        meshnode_feature = pygplates.Feature(pygplates.FeatureType.create_from_qualified_string('gpml:MeshNode'))
        meshnode_feature.set_name('Velocity Mesh Nodes')
    else:
        meshnode_feature = pygplates.Feature()
        meshnode_feature.set_name('Multipoint Feature')

    meshnode_feature.set_geometry(multi_point)

    output_feature_collection = pygplates.FeatureCollection(meshnode_feature)

    if filename is not None:
        output_feature_collection.write(filename)
    else:
        return output_feature_collection

def create_gpml_healpix_mesh(nSide,filename=None,feature_type=None):

    # call the function to create a healpix array
    longitude_array,latitude_array = healpix_mesh(nSide)

    # call the function to create a multipoint feature, with user-defined type
    output_feature_collection = create_gpml_velocity_feature(longitude_array,latitude_array,filename,feature_type)

    if filename is not None:  # This is superfluous, since file has already been written in previous line???
        output_feature_collection.write(filename)
    else:
        return output_feature_collection

def create_gpml_regular_long_lat_mesh(Sampling=1,filename=None,feature_type=None):

    # call the function to create a healpix array
    longitude_array,latitude_array = np.meshgrid(np.arange(-180.,180.001,Sampling),np.arange(-90.,90.001,Sampling))
    longitude_array = longitude_array.flatten()
    latitude_array = latitude_array.flatten()

    # call the function to create a multipoint feature, with user-defined type
    output_feature_collection = create_gpml_velocity_feature(longitude_array,latitude_array,filename,feature_type)

    if filename is not None:
        output_feature_collection.write(filename)
    else:
        return output_feature_collection

def geometries_to_geodataframe(geometries, geometry_type='polygon'):
    gdf = gpd.GeoDataFrame()
    gdf['geometry'] = None
    for i,geometry in enumerate(geometries):
        if geometry_type in ['PolyLine','Polyline']:
            poly = LineString([tuple(coord) for coord in np.fliplr(geometry)])
        else:
            poly = Polygon([tuple(coord) for coord in np.fliplr(geometry)])
        gdf.loc[i, 'geometry'] = poly

    return gdf

def geodataframe_to_geometries(gdf):
# from a geopandas geodataframe, return a list of the geometries (as type pygplates.GeometryOnSphere)

    geometry_list = []
    gdf = gdf.explode()
    for i,row in gdf.iterrows():
        geometry_list.append([(lat,lon) for lat,lon in zip(row.geometry.xy[1], row.geometry.xy[0])])
    return geometry_list

def gdf2gpml(gdf):
    """
    Given a geopandas geodataframe, returns a gplates feature collection
    """

    temporary_file = tempfile.NamedTemporaryFile(delete=True, suffix='.geojson')
    temporary_file.close()

    gdf.to_file(temporary_file.name, driver='GeoJSON')
    feature_collection = pygplates.FeatureCollection(temporary_file.name)

    os.unlink(temporary_file.name)

    return feature_collection

def gpml2gdf(features):
    """
    Given a gplates feature collection, or a list of features, or a list
    of reconstructed features, returns a geopandas geodataframe containing
    the same features
    """

    if isinstance(features, pygplates.FeatureCollection):
        pass
    elif isinstance(features, list):
        if isinstance(features[0], pygplates.Feature):
            features = pygplates.FeatureCollection(features)
        elif isinstance(features[0], pygplates.ReconstructedFeatureGeometry):
            features = _reconstructed_features_to_features(features)
            features = pygplates.FeatureCollection(features)
        else:
            raise TypeError('Unexpected list item of type {:s} for gpml2gdf input'.format(type(features[0])))
    else:
        raise TypeError('Unexpected type {:s} for gpml2gdf input'.format(type(features)))

    temporary_file = tempfile.NamedTemporaryFile(delete=True, suffix='.geojson')
    temporary_file.close()

    features.write(temporary_file.name)
    gdf = gpd.read_file(temporary_file.name)

    gdf['NAME'] = gdf['NAME'].astype(str)

    os.unlink(temporary_file.name)

    return gdf

def _reconstructed_features_to_features(reconstructed_features):

    features = []
    for feature in reconstructed_features:
        f = feature.get_feature().clone()
        f.set_geometry(feature.get_reconstructed_geometry())
        features.append(f)

    return features


text:
"""
secondary processing: figure plotting

see main program in ../process_raw_data.py for information

version 2018-11-19
"""

import glob
import json
import math
import numpy as np
import os
import sys

import matplotlib.pyplot as plt

from tools.config import *
from tools.helpers import *

# --- plot main figures ----------------------------------------------------------------------------

def plot_numclients_vs_variable(ctx, datasets, app_name, variable_name):
    """
    plots numclients vs variable (throughput / response time) and expected variable from app data
    aggregated data over all clients
    stable windows average
    mean and SD
    read and write
    plot separate figures for read and write (except 210/220)
    combine expected values into the plots and create a *-* for each WT (see mn)
    """
    figures_path = ctx['figures_path']
    prefix = ctx['prefix']
    # access the aggregated data instance
    experiment = 'r_' + ctx['experiment_folder']
    run = create_or_get_dict(datasets, experiment)
    app = create_or_get_dict(run, 'app_' + app_name)

    # collect the data for plotting :: typical use is for set and get; but mixed is possible, too
    x = set()
    dicts_template = {
        'set': {},
        'set_expected': {},
        'get': {},
        'get_expected': {},
        'mixed': {},
        'mixed_expected': {}
    }
    y_dicts = {}
    sd_dicts = {}

    any_mn = -1
    for exp_name, exp_data in app.items():
        windows = exp_data['windows']
        metadata = exp_data['metadata']
        cn = metadata['cn']
        op = metadata['op']
        mapped_op = PLOT_LABELS_OP_MAPPING[op]
        mn = metadata['mn']
        any_mn = mn
        mc = metadata['mc']
        sn = metadata['sn']
        stable_avg = windows['stable_avg']
        variables = stable_avg['both']  # must be 'both' for all op types

        x.add(cn)
        y_dict = create_or_get_template(y_dicts, mn, dicts_template)
        sd_dict = create_or_get_template(sd_dicts, mn, dicts_template)
        y_dict[mapped_op][cn] = variables[variable_name]['all']['mean']
        sd_dict[mapped_op][cn] = variables[variable_name]['all']['sd']
        if 'Expected' + variable_name in variables:
            y_dict[mapped_op + '_expected'][cn] = variables['Expected' + variable_name]['all']['mean']
            sd_dict[mapped_op + '_expected'][cn] = variables['Expected' + variable_name]['all']['sd']

    # create sorted lists
    lists_template = {
        'set': [],
        'set_expected': [],
        'get': [],
        'get_expected': [],
        'mixed': [],
        'mixed_expected': []
    }
    y_lists = {}
    sd_lists = {}

    x = sorted(x)
    y_max = 0
    for mn, y_dict in y_dicts.items():
        y_list = create_or_get_template(y_lists, mn, lists_template)
        for k, v in y_dict.items():
            if len(v) == 0:
                continue
            for cn in x:
                #if cn not in v:
                #    y_list[k].append(np.NaN)
                #    ctx['warning'].append('missing cn {cn} in dict {k} for mn {mn}'
                #                          .format(cn=cn, k=k, mn=mn))
                #    continue
                value = v[cn]
                y_list[k].append(value)
                if value > y_max:
                    y_max = value
    for mn, sd_dict in sd_dicts.items():
        sd_list = create_or_get_template(sd_lists, mn, lists_template)
        for k, v in sd_dict.items():
            if len(v) == 0:
                continue
            for cn in x:
                if cn not in v:
                    sd_list[k].append(np.NaN)
                    continue
                sd_list[k].append(v[cn])

    # render the figure
    workers = sorted(y_lists.keys())
    ops = PLOT_LABELS_OP_MAPPING.values()
    if len(workers) == 1 and workers[0] == 0:
        # combine set and get for e210 and e220
        fig, ax = plt.subplots()
        color = 0
        for op in ['set', 'get']:
            if any_mn < 0:
                continue
            if len(y_lists[any_mn][op]) == 0:
                continue
            ax.errorbar(x, y_lists[0][op], sd_lists[0][op],
                        color=PLOT_COLORS[color], fmt='-o', capsize=3, elinewidth=1, markeredgewidth=1,
                        label=op)
            if len(y_lists[0][op + '_expected']) > 0:
                if variable_name in ['Throughput', 'ResponseTime']:
                    expected_label = 'expected by IL: '
                else:
                    expected_label = 'expected: '
                ax.errorbar(x, y_lists[0][op + '_expected'], sd_lists[0][op + '_expected'],
                            color=lighten_color(PLOT_COLORS[color], PLOT_ADJUST_LUMINOSITY_FOR_EXPECTED), fmt=':x', capsize=3, elinewidth=1, markeredgewidth=1,
                            label=expected_label + op)
            color += 1

        if y_max < 1.0:
            y_max = 1.0
        ax.set_ylim(ymin=0, ymax=y_max)
        ax.set_ylim(ymin=0)
        ax.set_xlim(xmin=0)
        # additional y axis tweaking
        y_tick_locs, y_tick_labels = plt.yticks()
        ax.set_ylim(ymin=0, ymax=y_tick_locs[-1] + 0.1)
        # more formatting
        ax.grid(which='major', axis='both')
        ax.legend(loc='best', prop={'size': 12})
        #if variable_name == 'ResponseTime':
        #    ax.legend(loc='lower right', prop={'size': 7})
        #else:
        #    ax.legend(loc='lower right', prop={'size': 8})
        # x axis labels
        if x[-1] > 200:
            plt.xticks(x[1:])
        else:
            plt.xticks(x)
        # labels
        plt.xlabel('Total number of clients')
        plt.ylabel(PLOT_LABELS_VARIABLE_NAME_MAPPING[variable_name] + ' [' + PLOT_LABELS_VARIABLE_UNITS_MAPPING[
            variable_name] + ']')
        plt.title(
            PLOT_LABELS_VARIABLE_NAME_MAPPING[variable_name] + ': set and get,' + plural_helper(sn, 'server'))
        filename = os.path.join(figures_path, prefix + experiment + '_app_' + app_name + '_x_numclients_y_' + variable_name
                                + '_op_set-and-get_sn_' + str(sn) + '.pdf')
        # aspect ratio tweaking
        # size = plt.gcf().get_size_inches()
        # size[0] = 2.5 * size[1]
        # size[0] = 1.3 * size[0]
        # plt.gcf().set_size_inches(size)
        # tight layout preferred for integration into the report
        plt.tight_layout()
        plt.savefig(filename)
        plt.close()
    else:
        # default: separate plots for get and set
        for op in ops:
            if any_mn < 0:
                continue
            if len(y_lists[any_mn][op]) == 0:
                continue
            fig, ax = plt.subplots()
            for color, mn in enumerate(workers):
                ax.errorbar(x, y_lists[mn][op], sd_lists[mn][op],
                            color=PLOT_COLORS[color], fmt='-o', capsize=3, elinewidth=1, markeredgewidth=1,
                            label=str(mn) + ' workers (WT=' + (str(int(mn / int(mc))) if int(mc) > 0 else '0') + ')')
                if len(y_lists[mn][op + '_expected']) > 0:
                    if variable_name in ['Throughput', 'ResponseTime']:
                        expected_label = 'expected by IL: '
                    else:
                        expected_label = 'expected: '
                    ax.errorbar(x, y_lists[mn][op + '_expected'], sd_lists[mn][op + '_expected'],
                                color=lighten_color(PLOT_COLORS[color], PLOT_ADJUST_LUMINOSITY_FOR_EXPECTED), fmt=':x', capsize=3, elinewidth=1, markeredgewidth=1,
                                label=expected_label + str(mn) + ' workers (WT=' + (str(int(mn / int(mc))) if int(mc) > 0 else '0') + ')')

            if y_max < 1.0:
                y_max = 1.0
            ax.set_ylim(ymin=0, ymax=y_max)
            ax.set_ylim(ymin=0)
            ax.set_xlim(xmin=0)
            # additional y axis tweaking
            y_tick_locs, y_tick_labels = plt.yticks()
            ax.set_ylim(ymin=0, ymax=y_tick_locs[-1] + 0.1)
            # more formatting
            ax.grid(which='major', axis='both')
            # legend tweaking for the main report plots
            if not ((app_name == 'mw') and (ctx['experiment_folder'] in ['e310', 'e320']) and (variable_name == 'ResponseTime' or op == 'write' or op == 'set')):
                ax.legend(loc='best')
            #if variable_name == 'ResponseTime':
            #    ax.legend(loc='lower right', prop={'size': 7})
            #else:
            #    ax.legend(loc='lower right', prop={'size': 8})
            # x axis labels
            if x[-1] > 200:
                plt.xticks(x[1:])
            else:
                plt.xticks(x)
            # labels
            plt.xlabel('Total number of clients')
            plt.ylabel(PLOT_LABELS_VARIABLE_NAME_MAPPING[variable_name] + ' [' + PLOT_LABELS_VARIABLE_UNITS_MAPPING[variable_name] + ']')
            plt.title(PLOT_LABELS_VARIABLE_NAME_MAPPING[variable_name] + ': ' + op + ', ' + plural_helper(sn, 'server'))
            filename = os.path.join(figures_path, prefix + experiment + '_app_' + app_name + '_x_numclients_y_' + variable_name
                                    + '_op_' + op + '_sn_' + str(sn) + '.pdf')
            # aspect ratio tweaking
            # size = plt.gcf().get_size_inches()
            # size[0] = 2.5 * size[1]
            # size[0] = 1.3 * size[0]
            # plt.gcf().set_size_inches(size)
            # tight layout preferred for integration into the report
            plt.tight_layout()
            plt.savefig(filename)
            plt.close()

def plot_throughput_vs_responsetime(ctx, datasets, app_name):
    """
    plots Throughput vs ResponseTime from app data
    aggregated data over all clients requesting
    stable windows average
    mean and SD
    read and write
    """
    figures_path = ctx['figures_path']
    prefix = ctx['prefix']
    # access the aggregated data instance
    experiment = 'r_' + ctx['experiment_folder']
    run = create_or_get_dict(datasets, experiment)
    app = create_or_get_dict(run, 'app_' + app_name)

    lists_template = {
        'set': [],
        'get': [],
        'mixed': [],
    }
    x_lists = {}
    x_sd_lists = {}
    y_lists = {}
    y_sd_lists = {}

    any_mn = -1
    x_max = 0
    y_max = 0
    for exp_name, exp_data in app.items():
        windows = exp_data['windows']
        metadata = exp_data['metadata']
        cn = metadata['cn']
        op = metadata['op']
        mapped_op = PLOT_LABELS_OP_MAPPING[op]
        mn = metadata['mn']
        any_mn = mn
        mc = metadata['mc']
        sn = metadata['sn']
        stable_avg = windows['stable_avg']
        variables = stable_avg['both']  # must be 'both' for all op types

        x = variables['Throughput']['all']['mean']
        x_sd = variables['Throughput']['all']['sd']
        y = variables['ResponseTime']['all']['mean']
        y_sd = variables['ResponseTime']['all']['sd']

        if x > x_max:
            x_max = x
        if y > y_max:
            y_max = y

        x_list = create_or_get_template(x_lists, mn, lists_template)
        x_sd_list = create_or_get_template(x_sd_lists, mn, lists_template)
        y_list = create_or_get_template(y_lists, mn, lists_template)
        y_sd_list = create_or_get_template(y_sd_lists, mn, lists_template)

        x_list[mapped_op].append(x)
        x_sd_list[mapped_op].append(x_sd)
        y_list[mapped_op].append(y)
        y_sd_list[mapped_op].append(y_sd)

    # render the figure
    workers = sorted(y_lists.keys())
    ops = PLOT_LABELS_OP_MAPPING.values()
    if len(workers) == 1 and workers[0] == 0:
        # combine set and get for e210 and e220
        fig, ax = plt.subplots()
        color = 0
        for op in ['set', 'get']:
            if any_mn < 0:
                continue
            if len(y_lists[any_mn][op]) == 0:
                continue
            ax.errorbar(x_lists[0][op], y_lists[0][op], xerr=x_sd_lists[0][op], yerr=y_sd_lists[0][op],
                        color=PLOT_COLORS[color], fmt='o', capsize=3, elinewidth=1, markeredgewidth=1,
                        label=op)
            color += 1

        if x_max < 1.0:
            x_max = 1.0
        if y_max < 1.0:
            y_max = 1.0
        ax.set_xlim(xmin=0, xmax=x_max)
        ax.set_ylim(ymin=0, ymax=y_max)
        # additional axis tweaking
        x_tick_locs, x_tick_labels = plt.xticks()
        ax.set_xlim(xmin=0, xmax=x_tick_locs[-1] + 0.1)
        y_tick_locs, y_tick_labels = plt.yticks()
        ax.set_ylim(ymin=0, ymax=y_tick_locs[-1] + 0.1)
        # more formatting
        ax.grid(which='major', axis='both')
        ax.legend(loc='upper left', prop={'size': 8})
        # labels
        plt.xlabel(PLOT_LABELS_VARIABLE_NAME_MAPPING['Throughput'] + ' [' + PLOT_LABELS_VARIABLE_UNITS_MAPPING['Throughput'] + ']')
        plt.ylabel(PLOT_LABELS_VARIABLE_NAME_MAPPING['ResponseTime'] + ' [' + PLOT_LABELS_VARIABLE_UNITS_MAPPING['ResponseTime'] + ']')
        plt.title('Throughput vs response time: set and get,' + plural_helper(sn, 'server'))
        filename = os.path.join(figures_path, prefix + experiment + '_app_' + app_name + '_x_Throughput_y_ResponseTime_op_set-and-get_sn_' + str(sn) + '.pdf')
        # aspect ratio tweaking
        #size = plt.gcf().get_size_inches()
        #size[0] = 2.5 * size[1]
        #plt.gcf().set_size_inches(size)
        # tight layout preferred for integration into the report
        plt.tight_layout()
        plt.savefig(filename)
        plt.close()
    else:
        # default: separate plots for get and set
        for op in ops:
            if any_mn < 0:
                continue
            if len(y_lists[any_mn][op]) == 0:
                continue
            fig, ax = plt.subplots()
            for color, mn in enumerate(workers):
                ax.errorbar(x_lists[mn][op], y_lists[mn][op], xerr=x_sd_lists[mn][op], yerr=y_sd_lists[mn][op],
                            color=PLOT_COLORS[color], fmt='o', capsize=3, elinewidth=1, markeredgewidth=1,
                            label=str(mn) + ' workers (WT=' + (str(int(mn / int(mc))) if int(mc) > 0 else '0') + ')')

            if x_max < 1.0:
                x_max = 1.0
            if y_max < 1.0:
                y_max = 1.0
            ax.set_xlim(xmin=0, xmax=x_max)
            ax.set_ylim(ymin=0, ymax=y_max)
            # additional axis tweaking
            x_tick_locs, x_tick_labels = plt.xticks()
            ax.set_xlim(xmin=0, xmax=x_tick_locs[-1] + 0.1)
            y_tick_locs, y_tick_labels = plt.yticks()
            ax.set_ylim(ymin=0, ymax=y_tick_locs[-1] + 0.1)
            # more formatting
            ax.grid(which='major', axis='both')
            ax.legend(loc='upper left', prop={'size': 8})
            # labels
            plt.xlabel(PLOT_LABELS_VARIABLE_NAME_MAPPING['Throughput'] + ' [' + PLOT_LABELS_VARIABLE_UNITS_MAPPING[
                'Throughput'] + ']')
            plt.ylabel(PLOT_LABELS_VARIABLE_NAME_MAPPING['ResponseTime'] + ' [' + PLOT_LABELS_VARIABLE_UNITS_MAPPING[
                'ResponseTime'] + ']')
            plt.title('Throughput vs response time: ' + op + ', ' + plural_helper(sn, 'server'))
            filename = os.path.join(figures_path, prefix + experiment + '_app_' + app_name + '_x_Throughput_y_ResponseTime_op_' + op + '_sn_' + str(sn) + '.pdf')
            # aspect ratio tweaking
            #size = plt.gcf().get_size_inches()
            #size[0] = 2.5 * size[1]
            #plt.gcf().set_size_inches(size)
            # tight layout preferred for integration into the report
            plt.tight_layout()
            plt.savefig(filename)
            plt.close()

def plot_memtier_numclients_vs_variables(ctx, datasets):
    plot_numclients_vs_variable(ctx, datasets, 'memtier', 'Throughput')
    plot_numclients_vs_variable(ctx, datasets, 'memtier', 'ThroughputPerClient')
    plot_numclients_vs_variable(ctx, datasets, 'memtier', 'ResponseTime')
    plot_numclients_vs_variable(ctx, datasets, 'memtier', 'ResponseTimePerClient')
    plot_numclients_vs_variable(ctx, datasets, 'memtier', 'Data')
    plot_throughput_vs_responsetime(ctx, datasets, 'memtier')

def plot_middleware_numclients_vs_variables(ctx, datasets):
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'Throughput')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'ThroughputPerClient')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'ResponseTime')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'ResponseTimePerClient')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'Data')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'QueueLen')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'QueueingTime')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'ServiceTime')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'ServersOverallResponseTime')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'ServersNettoResponseTime')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'ServerRttMax')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'ServerRepliesDelayTime')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'WorkerWaitTimeBetweenJobsPerRequest')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'WorkerWaitTimeBetweenJobsPerSecond')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'WorkerWaitTimeWhileProcessingJobPerRequest')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'WorkerWaitTimeWhileProcessingJobPerSecond')
    plot_numclients_vs_variable(ctx, datasets, 'mw', 'WorkerUtilization')
    plot_throughput_vs_responsetime(ctx, datasets, 'mw')

# --- plot detailed figures: time resolution for each experiment data point ------------------------

def plot_detailed_time_vs_variable(ctx, datasets, app_name, more_detailed=False):
    """
     plots time vs variable individually for each cn_mn_op configuration of app_name
     for the variables defined in FIGURE_DETAILED_PLOT_VARIABLE_IN_TIME
     for 'both' operations, which is exactly what op defines (read, write, mixed)
     """
    figures_path = ctx['detailed_figures_path']
    prefix = ctx['prefix']
    figures_dict = FIGURE_DETAILED_PLOT_VARIABLE_IN_TIME
    if more_detailed:
        if ctx['conserve_output_space']:
            print('    more detailed: skipped, see -x flag')
            return
        figures_path = ctx['more_detailed_figures_path']
        figures_dict = FIGURE_MORE_DETAILED_PLOT_VARIABLE_IN_TIME
    # access the aggregated data instance
    experiment = 'r_' + ctx['experiment_folder']
    run = create_or_get_dict(datasets, experiment)
    app = create_or_get_dict(run, 'app_' + app_name)

    # iterate over cn, mn, op and defined variable sets to create all plots
    for exp_name, exp_data in app.items():
        windows = exp_data['windows']
        metadata = exp_data['metadata']
        cn = metadata['cn']
        op = metadata['op']
        ck = metadata['ck']
        mn = metadata['mn']
        mc = metadata['mc']
        sn = metadata['sn']

        available_variables = set(windows['1']['both'])
        for plot_config in figures_dict:
            needed_variables = set(plot_config[3])
            variables = available_variables.intersection(needed_variables)
            if len(variables) == 0:
                continue
            # create a plot
            x = range(len(windows) - 2)
            y_max = 0
            fig, ax = plt.subplots()
            for color, var in enumerate(plot_config[3]):
                if var not in variables:
                    continue
                y = []
                sd = []
                for i in x:
                    var_all_instances = windows[str(i)]['both'][var]['all']
                    value = var_all_instances['mean']
                    y.append(value)
                    sd.append(var_all_instances['sd'])
                    if value > y_max:
                        y_max = value

                ax.errorbar(x, y, sd,
                            color=PLOT_COLORS[color], fmt='-o', capsize=3, elinewidth=1, markeredgewidth=1,
                            label=var)

            if y_max < 1.0:
                y_max = 1.0
            ax.set_ylim(ymin=0, ymax=y_max)
            ax.set_xlim(xmin=0)
            # additional y axis tweaking
            y_tick_locs, y_tick_labels = plt.yticks()
            ax.set_ylim(ymin=0, ymax=y_tick_locs[-1] + 0.1)
            # more formatting
            ax.grid(which='major', axis='both')
            ax.legend(loc='lower right', prop={'size': 8})
            plt.xlabel('Time (s)')
            plt.ylabel(plot_config[2])
            plt.title(plot_config[1] + ': ' + PLOT_LABELS_OP_MAPPING[op] + ', ' + plural_helper(cn, 'client') + ', '
                      + plural_helper(mn, ' worker') + ', ' + plural_helper(sn, 'server'))
            filename = os.path.join(figures_path, prefix + experiment + '_app_' + app_name + '_x_time_y_' + plot_config[0]
                                    + '_op_' + PLOT_LABELS_OP_MAPPING[op] + '_ck_' + ck + '_cn_' + str(cn) + '_mn_' + str(mn)
                                    + '_sn_' + str(sn) + '.pdf')
            plt.savefig(filename)
            plt.close()

    if not more_detailed:
        plot_detailed_time_vs_variable(ctx, datasets, app_name, True)

# --- plot system tools data -----------------------------------------------------------------------

def plot_system_tools_time_vs_variable(ctx, datasets, app_name, app_dict, window_duration, more_detailed=False):
    """
     plots time vs variable for system tools individually for each cn_mn_op configuration of app_name
     for the variables defined in app_dict
     note: time covers one iteration of one experiment run. mean ± SD are shown for the n iterations
     """
    figures_path = ctx['detailed_figures_path']
    if more_detailed:
        figures_path = ctx['more_detailed_figures_path']
    prefix = ctx['prefix']
    figures_dict = app_dict

    # access the aggregated data instance
    experiment = 'r_' + ctx['experiment_folder']
    run = create_or_get_dict(datasets, experiment)
    app = create_or_get_dict(run, 'app_' + app_name)

    # define specific configuration parts of the entire run
    configurations = get_experiment_configurations(ctx['experiment_folder'])
    configurations_count = len(configurations)

    # iterate over the data dicts (vm_types for dstat; connection for ping)
    for data_name, data_dict in app.items():
        available_variables = set(data_dict[1])
        for plot_config in figures_dict:
            needed_variables = set(plot_config[3])
            variables = available_variables.intersection(needed_variables)
            if len(variables) == 0:
                continue
            # 1) plot summary of the entire experiment run (time in minutes)
            access_x = range(len(data_dict))
            plot_x = [(float(x * window_duration) / 60.0) for x in access_x]
            y_max = 0
            fig, ax = plt.subplots()
            for color, var in enumerate(plot_config[3]):
                if var not in variables:
                    continue
                y = []
                sd = []
                for i in access_x:
                    if var not in data_dict[i]:
                        # - special case for ping data:
                        #   only for the very last window(s), either LongPing or DefaultPing may be missing
                        #   because the ping commands for either of them are running separately
                        #   and are stopped separately
                        # - similar special case for dstat data:
                        #   not all instances of e.g. clients may have been stoppd at the very same time
                        #   thus, aggregated data may not be available for the very last window(s) for all
                        # both cases: append 0 to have matching shape with list of x values.
                        y.append(0)
                        sd.append(0)
                        continue
                    var_all_instances = data_dict[i][var]['all']
                    value = var_all_instances['mean']
                    y.append(value)
                    sd.append(var_all_instances['sd'])
                    if value > y_max:
                        y_max = value

                ax.errorbar(plot_x, y, sd,
                            color=PLOT_COLORS[color], fmt='-o', capsize=3, elinewidth=1, markeredgewidth=1,
                            label=var)

            if y_max < 1.0:
                y_max = 1.0
            # special configuration
            if 'CPUusage' in plot_config[0]:
                y_max = 100.0
            ax.set_ylim(ymin=0, ymax=y_max)
            ax.set_xlim(xmin=0)
            # additional y axis tweaking
            y_tick_locs, y_tick_labels = plt.yticks()
            ax.set_ylim(ymin=0, ymax=y_tick_locs[-1] + 0.1)
            # more formatting
            ax.grid(which='major', axis='both')
            ax.legend(loc='lower right', prop={'size': 8})
            plt.xlabel('Time (minutes)')
            plt.ylabel(plot_config[2])
            plt.title(app_name + ': ' + data_name + ' ' + plot_config[1])
            filename = os.path.join(figures_path, prefix + experiment + '_app_' + app_name + '_' + data_name + '_x_time_y_' + plot_config[0] + '.pdf')
            # aspect ratio tweaking
            size = plt.gcf().get_size_inches()
            size[0] = 2.5 * size[1]
            plt.gcf().set_size_inches(size)
            # tight layout preferred for integration into the report
            plt.tight_layout()
            plt.savefig(filename)
            plt.close()

            # 2) plot per specific experiment configuration (time in seconds)
            delta = int(len(data_dict) / configurations_count)
            for config_nr, config_name in enumerate(configurations):
                base = config_nr * delta
                access_x = range(base, base + delta)
                plot_x = [float(x * window_duration) for x in range(delta)]
                y_max = 0
                fig, ax = plt.subplots()
                for color, var in enumerate(plot_config[3]):
                    if var not in variables:
                        continue
                    y = []
                    sd = []
                    for i in access_x:
                        if var not in data_dict[i]:
                            # - special case for ping data:
                            #   only for the very last window(s), either LongPing or DefaultPing may be missing
                            #   because the ping commands for either of them are running separately
                            #   and are stopped separately
                            # - similar special case for dstat data:
                            #   not all instances of e.g. clients may have been stoppd at the very same time
                            #   thus, aggregated data may not be available for the very last window(s) for all
                            # both cases: append 0 to have matching shape with list of x values.
                            y.append(0)
                            sd.append(0)
                            continue
                        var_all_instances = data_dict[i][var]['all']
                        value = var_all_instances['mean']
                        y.append(value)
                        sd.append(var_all_instances['sd'])
                        if value > y_max:
                            y_max = value

                    ax.errorbar(plot_x, y, sd,
                                color=PLOT_COLORS[color], fmt='-o', capsize=3, elinewidth=1, markeredgewidth=1,
                                label=var)

                if y_max < 1.0:
                    y_max = 1.0
                # special configuration
                if 'CPUusage' in plot_config[0]:
                    y_max = 100.0
                ax.set_ylim(ymin=0, ymax=y_max)
                ax.set_xlim(xmin=0)
                # additional y axis tweaking
                y_tick_locs, y_tick_labels = plt.yticks()
                ax.set_ylim(ymin=0, ymax=y_tick_locs[-1] + 0.1)
                # more formatting
                ax.grid(which='major', axis='both')
                ax.legend(loc='lower right', prop={'size': 8})
                plt.xlabel('Time (s)')
                plt.ylabel(plot_config[2])
                plt.title(app_name + ': ' + data_name + ' ' + plot_config[1])
                filename = os.path.join(figures_path, prefix + experiment + '_app_' + app_name + '_' + data_name +
                                        '_x_time_y_' + plot_config[0] + '_' + config_name + '.pdf')
                # aspect ratio tweaking
                size = plt.gcf().get_size_inches()
                size[0] = 2.5 * size[1]
                plt.gcf().set_size_inches(size)
                # tight layout preferred for integration into the report
                plt.tight_layout()
                plt.savefig(filename)
                plt.close()

def plot_dstat(ctx, datasets):
    plot_system_tools_time_vs_variable(ctx, datasets, 'dstat', DSTAT_FIGURE_DETAILED_PLOT_VARIABLE_IN_TIME, DSTAT_WINDOW_DURATION)

def plort_ping(ctx, datasets):
    plot_system_tools_time_vs_variable(ctx, datasets, 'ping', PING_FIGURE_DETAILED_PLOT_VARIABLE_IN_TIME, PING_WINDOW_DURATION)
    plot_system_tools_time_vs_variable(ctx, datasets, 'ping', PING_FIGURE_MORE_DETAILED_PLOT_VARIABLE_IN_TIME, PING_WINDOW_DURATION, True)

# --- plot histograms ------------------------------------------------------------------------------

def plot_histograms(ctx, datasets, app_name, variable_name, in_detailed_folder=False):
    """
    plots the histogram for the app and variable using the global configuration for bucket count
    and bucket size (to be consistent for all histograms) -> see aggregation
    aggregated data over all clients
    entire runtime
    mean and SD
    read (get) is shown above, write (set) is shown below x axis.
    iterates over parameter settings of interest to create all histograms:
    cn, ck, mn, ms, sn
    """
    figures_path = ctx['figures_path']
    if in_detailed_folder:
        figures_path = ctx['detailed_figures_path']
    prefix = ctx['prefix']

    # access the aggregated data instance
    experiment = 'r_' + ctx['experiment_folder']
    run = create_or_get_dict(datasets, experiment)
    app = create_or_get_dict(run, 'app_' + app_name)

    access_x = range(HISTOGRAM_MAX_BIN_NR + 1)
    bin_center = 0.5 * HISTOGRAM_TIME_RESOLUTION
    plot_x = [HISTOGRAM_TIME_RESOLUTION * float(x) + bin_center for x in access_x]
    plots_dict = {}  # dict of config_combinations -> dict of get/set -> arrays of y values (counts) and SD
    # iterate over cn, ck, mn, ms to collect all data
    for exp_name, exp_data in app.items():
        histograms = exp_data['histograms']['processed_bins']
        metadata = exp_data['metadata']
        cn = metadata['cn']
        ck = metadata['ck']
        mn = metadata['mn']
        mc = metadata['mc']
        ms = metadata['ms']
        sn = metadata['sn']
        config = variable_name + '_cn_' + str(cn) + '_ck_' + ck + '_mn_' + str(mn) + '_ms_' + ms + '_sn_' + str(sn)
        info_str = variable_name + ': ' + plural_helper(cn, 'client') + ' ' + plural_helper(int(ck), 'key')
        info_str += ' ' + plural_helper(mn, 'worker') + (' sharded' if ms == 'true' else ', non-sharded')
        info_str += ' ' + plural_helper(sn, 'server')
        config_template = {
            'info_str': info_str
        }

        # in e610 there are both settings used, 1 server and 3 servers -> decide based on sn
        if variable_name == 'ServerRtt2' or variable_name == 'ServerRtt3':
            if sn == 1:
                # no warning needed
                continue

        config_dict = create_or_get_template(plots_dict, config, config_template)
        for op_name, op_data in histograms.items():
            if op_name not in ['set', 'get']:
                continue
            if variable_name not in op_data:
                ctx['warning'].append('variable {variable} not found in histogram data of {app} op {op}'
                                      .format(variable=variable_name, app=app_name, op=op_name))
                continue

            if op_name in config_dict:
                ctx['error'].append('variable {variable} for op {op} already defined in {app}'
                                    .format(variable=variable_name, op=op_name, app=app_name))
                continue

            y_list = []
            sd_list = []
            instance = op_data[variable_name]['all']
            for x in access_x:
                bin_data = instance[x]
                y_list.append(bin_data['mean'])
                sd_list.append(bin_data['sd'])
            data_dict = {
                'mean': y_list,
                'sd': sd_list
            }
            config_dict[op_name] = data_dict

    # plotting
    for plot_name, plot_data in plots_dict.items():
        fig, ax = plt.subplots()
        info_str = plot_data['info_str']
        # stacked plots
        # print get and/or get dependent on what is available
        # if 'get' in plot_data:
        #    get_data = plot_data['get']
        #    ax.bar(plot_x, get_data['mean'], HISTOGRAM_BAR_WIDTH, yerr=get_data['sd'],
        #           label='get', color=PLOT_COLORS[1])
        #    if 'set' in plot_data:
        #        set_data = plot_data['set']
        #        ax.bar(plot_x, set_data['mean'], HISTOGRAM_BAR_WIDTH, bottom=get_data['mean'], yerr=set_data['sd'], label='set', color=PLOT_COLORS[0])
        # elif 'set' in plot_data:
        #    set_data = plot_data['set']
        #    ax.bar(plot_x, set_data['mean'], HISTOGRAM_BAR_WIDTH, yerr=set_data['sd'],
        #           label='set', color=PLOT_COLORS[0])

        # new design: read (get) up; write (set) down; bidirectional (bid)
        # max_y = 0
        # if 'get' in plot_data:
        #     get_data = plot_data['get']
        #     max_y = max(max_y, max(get_data['mean']))
        #     ax.bar(plot_x, scale_list(get_data['mean'], 0.001), HISTOGRAM_BAR_WIDTH, yerr=scale_list(get_data['sd'], 0.001),
        #            label='get', color=PLOT_COLORS[1])
        # if 'set' in plot_data:
        #     set_data = plot_data['set']
        #     max_y = max(max_y, max(set_data['mean']))
        #     adjusted_set_data = [-y for y in set_data['mean']]
        #     ax.bar(plot_x, scale_list(adjusted_set_data, 0.001), HISTOGRAM_BAR_WIDTH, yerr=scale_list(set_data['sd'], 0.001),
        #            label='set', color=PLOT_COLORS[0])

        # newer design: read (get) and write (set) next to each other to avoid lots of empty white space
        # note:
        max_y = 0
        if 'set' in plot_data and 'get' in plot_data:
            set_data = plot_data['set']
            max_y = max(max_y, max(set_data['mean']))
            ax.bar(add_offset_to_list(plot_x, -HISTOGRAM_BAR_OFFSET), scale_list(set_data['mean'], 0.001), HISTOGRAM_BAR_WIDTH, yerr=scale_list(set_data['sd'], 0.001),
                   label='set', color=PLOT_COLORS[0])

            get_data = plot_data['get']
            max_y = max(max_y, max(get_data['mean']))
            ax.bar(add_offset_to_list(plot_x, HISTOGRAM_BAR_OFFSET), scale_list(get_data['mean'], 0.001), HISTOGRAM_BAR_WIDTH, yerr=scale_list(get_data['sd'], 0.001),
                   label='get', color=PLOT_COLORS[1])

        elif 'get' in plot_data:
            get_data = plot_data['get']
            max_y = max(max_y, max(get_data['mean']))
            ax.bar(plot_x, scale_list(get_data['mean'], 0.001), 2 * HISTOGRAM_BAR_WIDTH, yerr=scale_list(get_data['sd'], 0.001),
                   label='get', color=PLOT_COLORS[1])

        elif 'set' in plot_data:
            set_data = plot_data['set']
            max_y = max(max_y, max(set_data['mean']))
            adjusted_set_data = [-y for y in set_data['mean']]
            ax.bar(plot_x, scale_list(adjusted_set_data, 0.001), 2 * HISTOGRAM_BAR_WIDTH, yerr=scale_list(set_data['sd'], 0.001),
                   label='set', color=PLOT_COLORS[0])

        if max_y == 0:
            ctx['warning'].append('plot_histograms() for ' + app_name + ' :: ' + variable_name + ': only empty histogram bins for ' + plot_name)
            return

        max_y *= 0.001

        ax.set_xlim(xmin=0, xmax=HISTOGRAM_TIME_RESOLUTION * HISTOGRAM_MAX_BIN_NR + 1)
        # for bidirectional (bid)
        # (bid) ax.set_ylim(ymin=-max_y, ymax=max_y)
        ax.set_ylim(ymin=0, ymax=max_y)
        # additional x axis tweaking
        # x_tick_locs, x_tick_labels = plt.xticks()
        # ax.set_xlim(xmin=0, xmax=x_tick_locs[-1] + 0.1)
        # adjust x axis tics to have 1 ms tics
        x_tick_locs, x_tick_labels = plt.xticks()
        max_x = x_tick_locs[-1]
        ax.set_xticks([x for x in range(int(max_x)+1)])
        # additional y axis tweaking
        y_tick_locs, y_tick_labels = plt.yticks()
        if ctx['experiment_folder'] in ['e510', 'e520'] and variable_name == 'ResponseTime':
            if y_tick_locs[-1] < 60.0:
                y_tick_locs[-1] = 60.0
        # (bid) ax.set_ylim(ymin=-(y_tick_locs[-1] + 0.1), ymax=y_tick_locs[-1] + 0.1)
        ax.set_ylim(ymin=0, ymax=y_tick_locs[-1] + 0.1)
        # adjust tick labels to show pos. values also for negative y axis parts
        ax.set_yticklabels([str(abs(y)) for y in ax.get_yticks()])
        # more formatting
        ax.grid(which='major', axis='both')
        #ax.legend(loc='upper right', prop={'size': 8})
        ax.legend(loc='upper right', prop={'size': 12})
        # labels
        plt.xlabel('Time [ms]')
        plt.ylabel('Number of requests [x 1000]')
        # tweaking for report figures
        if not (ctx['experiment_folder'] in ['e510', 'e520'] and variable_name == 'ResponseTime'):
            plt.title(info_str)
        filename = os.path.join(figures_path, prefix + experiment + '_app_' + app_name + '_histogram_' + plot_name + '.pdf')
        # aspect ratio tweaking
        # size = plt.gcf().get_size_inches()
        # size[0] = 2.5 * size[1]
        # plt.gcf().set_size_inches(size)
        # tight layout preferred for integration into the report
        plt.tight_layout()
        plt.savefig(filename)
        plt.close()

# --- plot percentiles -----------------------------------------------------------------------------

def plot_percentiles_for_e500(ctx, datasets, app_name):
    """
    plots numclients vs variable mean, median, p25, p75, p90, p95 of the variables with percentile info from app data
    note: due to technical limitations of memtier, the percentiles (incl. median) are from the entire run
    and the mean is from stable windows (to allow comparison with otherwise used data)
    no error bars here
    aggregated data over all clients
    read and write
    plot separate figures for read and write

    this generates the special plot format needed for experiments e510 and e520
    that need a different data organization
    """
    figures_path = ctx['figures_path']
    prefix = ctx['prefix']
    # access the aggregated data instance
    experiment = 'r_' + ctx['experiment_folder']
    run = create_or_get_dict(datasets, experiment)
    app = create_or_get_dict(run, 'app_' + app_name)

    # collect the data for plotting :: typical use is for set and get; but mixed is possible, too
    x = set()
    y_dicts = {
        'set': {},
        'get': {},
        'mixed': {}
    }

    any_mn = -1
    available_variables = set()
    for exp_name, exp_data in app.items():
        percentiles = exp_data['percentiles']
        metadata = exp_data['metadata']
        cn = metadata['cn']
        op = metadata['op']
        mapped_op = PLOT_LABELS_OP_MAPPING[op]
        mn = metadata['mn']
        any_mn = mn
        mc = metadata['mc']
        sn = metadata['sn']

        x.add(cn)
        models = get_experiment_models(ctx['experiment_folder'], op)
        for model_name in models:
            model_metadata = extract_metadata(model_name)
            if not metadata_matches_requested_config(metadata, model_metadata):
                continue
            for op_name, op_data in percentiles.items():
                if 'all' not in op_data:
                    continue
                if op_name == 'both':
                    op_name = 'mixed'
                if op_name not in y_dicts:
                    continue
                model_dict = create_or_get_dict(y_dicts[op_name], model_name)
                variables = op_data['all']['all']
                for variable_name, variable_data in variables.items():
                    available_variables.add(variable_name)
                    variable_dict = create_or_get_dict(model_dict, variable_name)
                    for p in FIGURES_MATRIX['plot_percentiles']:
                        percentile_dict = create_or_get_dict(variable_dict, p)
                        percentile_dict[cn] = variable_data[p]

    # render the figures
    cn = 12
    for op_name, op_data in y_dicts.items():
        for mn_str in ['64', '128']:
            for variable_name in available_variables:
                # plot one figure for each op, mn, variable combo
                # reorganize data to draw bars of different colors for each percentile with grouping by ck
                # -> need dict of percentile -> list mapping
                # each list holds the percentile associated with that ck in order
                y_dicts2 = {}
                for p in FIGURES_MATRIX['plot_percentiles']:
                    p_dict = create_or_get_dict(y_dicts2, p)

                for model_name, model_data in op_data.items():
                    model_metadata = extract_metadata(model_name)
                    if mn_str != model_metadata['mn']:
                        continue

                    ck = model_metadata['ck']
                    for p_name, p_data in model_data[variable_name].items():
                        y_dicts2[p_name][ck] = p_data[cn]

                y_lists = {}
                groups = None
                for p_name, p_data in y_dicts2.items():
                    if len(p_data) == 0:
                        continue
                    p_list = create_or_get_template(y_lists, p_name, [])
                    if groups is None:
                        groups = sorted(p_data)
                    for ck in groups:
                        p_list.append(p_data[ck])

                if groups is None:
                    continue

                # render plot
                fig, ax = plt.subplots()

                y_max = 0.0
                bar_width = 0.6 / (len(groups) + 1)
                index = [1 + i for i in range(len(groups))]

                for color, p_name in enumerate(reversed(FIGURES_MATRIX['plot_percentiles'])):
                    p_data = y_lists[p_name]
                    if len(p_data) == 0:
                        print('debug')
                    y_max = max(y_max, max(p_data))
                    ax.bar(add_offset_to_list(index, color * bar_width), p_data, bar_width, color=PLOT_COLORS[color], label=p_name)

                ax.set_xlim(xmin=0)
                ax.set_ylim(ymin=0, ymax=30.1)
                ax.set_xticks(add_offset_to_list(index, 0.5 * bar_width * len(FIGURES_MATRIX['plot_percentiles'])))
                ax.set_xticklabels(groups)
                # additional y axis tweaking
                #y_tick_locs, y_tick_labels = plt.yticks()
                #ax.set_ylim(ymin=0, ymax=y_tick_locs[-1] + 0.1)
                ax.grid(which='major', axis='both')
                ax.legend(loc='upper left', prop={'size': 12})

                plt.xlabel('Multi-get size')
                plt.ylabel(PLOT_LABELS_VARIABLE_NAME_MAPPING[variable_name] + ' [' + PLOT_LABELS_VARIABLE_UNITS_MAPPING[variable_name] + ']')
                # plt.title(PLOT_LABELS_VARIABLE_NAME_MAPPING[variable_name] + ': ' + op_name + ', ' + mn_str + ' workers')
                filename = os.path.join(figures_path, prefix + experiment + '_app_' + app_name + '_x_ck_y_' + variable_name
                                        + '_op_' + op_name + '_mn_' + mn_str + '_percentiles.pdf')
                # aspect ratio tweaking
                # size = plt.gcf().get_size_inches()
                # size[0] = 2.5 * size[1]
                # plt.gcf().set_size_inches(size)
                # tight layout preferred for integration into the report
                plt.tight_layout()
                plt.savefig(filename)
                plt.close()

def plot_percentiles(ctx, datasets, app_name):
    """
    plots numclients vs variable mean, median, p25, p75, p90, p95 of the variables with percentile info from app data
    note: due to technical limitations of memtier, the percentiles (incl. median) are from the entire run
    and the mean is from stable windows (to allow comparison with otherwise used data)
    no error bars here
    aggregated data over all clients
    read and write
    plot separate figures for read and write
    """
    if ctx['experiment_folder'] in ['e510', 'e520']:
        plot_percentiles_for_e500(ctx, datasets, app_name)
        return

    figures_path = ctx['figures_path']
    prefix = ctx['prefix']
    # access the aggregated data instance
    experiment = 'r_' + ctx['experiment_folder']
    run = create_or_get_dict(datasets, experiment)
    app = create_or_get_dict(run, 'app_' + app_name)

    # collect the data for plotting :: typical use is for set and get; but mixed is possible, too
    x = set()
    y_dicts = {
        'set': {},
        'get': {},
        'mixed': {}
    }

    any_mn = -1
    for exp_name, exp_data in app.items():
        percentiles = exp_data['percentiles']
        metadata = exp_data['metadata']
        cn = metadata['cn']
        op = metadata['op']
        mapped_op = PLOT_LABELS_OP_MAPPING[op]
        mn = metadata['mn']
        any_mn = mn
        mc = metadata['mc']
        sn = metadata['sn']

        x.add(cn)
        models = get_experiment_models(ctx['experiment_folder'], op)
        for model_name  in models:
            model_metadata = extract_metadata(model_name)
            if not metadata_matches_requested_config(metadata, model_metadata):
                continue
            for op_name, op_data in percentiles.items():
                if 'all' not in op_data:
                    continue
                if op_name == 'both':
                    op_name = 'mixed'
                if op_name not in y_dicts:
                    continue
                model_dict = create_or_get_dict(y_dicts[op_name], model_name)
                variables = op_data['all']['all']
                for variable_name, variable_data in variables.items():
                    variable_dict = create_or_get_dict(model_dict, variable_name)
                    for p in FIGURES_MATRIX['plot_percentiles']:
                        percentile_dict = create_or_get_dict(variable_dict, p)
                        percentile_dict[cn] = variable_data[p]

    # create sorted lists
    y_lists = {}

    x = sorted(x)
    y_max = 0
    for op_name, op_data in y_dicts.items():
        op_lists = create_or_get_dict(y_lists, op_name)
        for model_name, model_data in op_data.items():
            model_lists = create_or_get_dict(op_lists, model_name)
            for variable_name, variable_data in model_data.items():
                variable_lists = create_or_get_dict(model_lists, variable_name)
                for p_name, p_data in variable_data.items():
                    p_list = create_or_get_template(variable_lists, p_name, [])
                    for cn in x:
                        p_list.append(p_data[cn])

    # render the figures
    for op_name, op_data in y_lists.items():
        for model_name, model_data in op_data.items():
            for variable_name, variable_data in model_data.items():
                # one figure per op / model / variable combo
                fig, ax = plt.subplots()
                color = 0
                for p_name in FIGURES_MATRIX['plot_percentiles']:
                    p_data = variable_data[p_name]
                    ax.plot(x, p_data, '-o', color=PLOT_COLORS[color], label=p_name)
                    color += 1

                ax.set_xlim(xmin=0)
                ax.set_ylim(ymin=0)
                # additional y axis tweaking
                y_tick_locs, y_tick_labels = plt.yticks()
                ax.set_ylim(ymin=0, ymax=y_tick_locs[-1] + 0.1)
                ax.grid(which='major', axis='both')
                ax.legend(loc='upper left', prop={'size': 12})
                if x[-1] > 200:
                    plt.xticks(x[1:])
                else:
                    plt.xticks(x)
                plt.xlabel('Total number of clients')
                plt.ylabel(PLOT_LABELS_VARIABLE_NAME_MAPPING[variable_name] + ' [' + PLOT_LABELS_VARIABLE_UNITS_MAPPING[variable_name] + ']')
                plt.title(PLOT_LABELS_VARIABLE_NAME_MAPPING[variable_name] + ': ' + op_name + ', ' + model_name)
                filename = os.path.join(figures_path, prefix + experiment + '_app_' + app_name + '_x_numclients_y_' + variable_name
                                        + '_op_' + op_name + '_' + model_name + '_percentiles.pdf')
                # aspect ratio tweaking
                # size = plt.gcf().get_size_inches()
                # size[0] = 2.5 * size[1]
                # plt.gcf().set_size_inches(size)
                # tight layout preferred for integration into the report
                plt.tight_layout()
                plt.savefig(filename)
                plt.close()

# --- plot figures ---------------------------------------------------------------------------------

def plot_figures(ctx, datasets):
    """Creates all the plots for the experiments. Note: selection of plots is tailored to the experiment."""
    print('### plotting figures ###')
    figures_path = os.path.join(ctx['output_folder'], ctx['experiment_folder'], FIGURES_FOLDER)
    detailed_figures_path = os.path.join(figures_path, 'detailed')
    more_detailed_figures_path = os.path.join(detailed_figures_path, 'more_detailed')
    make_path(more_detailed_figures_path)
    ctx['figures_path'] = figures_path
    ctx['detailed_figures_path'] = detailed_figures_path
    ctx['more_detailed_figures_path'] = more_detailed_figures_path

    # follow the figure matrix to create the defined plots
    exp = ctx['experiment_folder']

    # system tools data: dstat, ping (see stats summary for iperf)
    print('    plotting dstat and ping detail figures')
    plot_dstat(ctx, datasets)
    plort_ping(ctx, datasets)

    # memtier
    print('    plotting memtier main figures')
    if exp in FIGURES_MATRIX['plot_memtier_numclients_vs_variables']:
        plot_memtier_numclients_vs_variables(ctx, datasets)
    plot_percentiles(ctx, datasets, 'memtier')
    plot_histograms(ctx, datasets, 'memtier', 'ResponseTime')

    print('    plotting memtier detail figures (time vs variables for each experiment key)')
    plot_detailed_time_vs_variable(ctx, datasets, 'memtier')

    if exp in FIGURES_MATRIX['no_middleware_involved']:
        print('    no middleware data in this experiment')
        return

    # middleware
    print('    plotting middleware main figures')
    if exp in FIGURES_MATRIX['plot_middleware_numclients_vs_variables']:
        plot_middleware_numclients_vs_variables(ctx, datasets)
    plot_percentiles(ctx, datasets, 'mw')
    plot_histograms(ctx, datasets, 'mw', 'ResponseTime')

    print('    plotting middleware detail figures (time vs variables for each experiment key)')
    plot_detailed_time_vs_variable(ctx, datasets, 'mw')
    plot_histograms(ctx, datasets, 'mw', 'QueueingTime', True)
    plot_histograms(ctx, datasets, 'mw', 'ServiceTime', True)
    plot_histograms(ctx, datasets, 'mw', 'ServerRtt1', True)
    if exp not in FIGURES_MATRIX['middleware_uses_only_one_server']:
        plot_histograms(ctx, datasets, 'mw', 'ServerRtt2', True)
        plot_histograms(ctx, datasets, 'mw', 'ServerRtt3', True)

max_stars_repo_name: pirminschmid/Queueing

max_issues_repo_name: pirminschmid/Queueing

max_forks_repo_name: pirminschmid/Queueing


text:
import os, sys
import torch
from tqdm import trange
import numpy as np
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
from loss_multi import compute_loss_color_warp
from core.utils.train_utils import params_to_mtrx

'''
This function performs inverse optimization over either shape code (optionally with a similarity transformations) on posed multi-view images.

Required parameters:
- sdfrenderer:          A differentiable sdf renderer,
- evaluator:            An instance of class <"Evaluator">
- shape_code:           class <torch.Tensor>
- shape_optimizer:      Ann instance of PyTorch optimizer, either for shape code or camera extrinsics, which is optional for the key "optimizer_type".
- images                List of image tensors, each has type class <torch.Tensor>.
- cameras               List of cameras, each has type class <common.geometry.Camera>.
- weight_dict:          A dictionary of weights. ['color', 'l2reg']

Optional parameters:
- num_views_per_round:  number of used views for each round.
- num_iters:            number of iterations.
- num_sample_points:    number of sampled points (10k for PMO evaluation)
- sep_dist:             sampling strategy (set sep_dist > 1 to skip frames and improve efficientcy).
- test_step:            evaluation interval.
- points_gt:            groundtruth shapes for evaluation.
- sim3:                 input similarity transform. If not None, the shape code is jointly optimized with sim3.
- full_flag:            To evaluate and update the best model.

Returns:
The optimized shape_code and the optimizer are returned.
'''

def optimize_multi_view(renderer, evaluator, shape_code, shape_optimizer, images, cameras, weight_list,
                        num_views_per_round=8, num_iters=20, num_sample_points=30000, sep_dist=1, test_step=5, points_gt=None, sim3=None, sim3_init=None, visualizer=None, vis_dir=None, vis_flag=None,  full_flag=True):
    counter = 0
    best_chamfer = 100
    best_epoch = 0

    # save initialized shape
    fname = os.path.join(vis_dir, 'mesh_initial.ply')
    evaluator.latent_vec_to_points(shape_code, num_points=num_sample_points, fname=fname, silent=True)

    # optimization start
    num_images = len(images)
    rot_freq = num_images / num_views_per_round
    for epoch in trange(num_iters):
        for idx in range(0, int(np.ceil(rot_freq)), sep_dist):
            loss_total = 0.0
            shape_optimizer.zero_grad()

            if sim3 is not None:
                sim_mtrx = params_to_mtrx(sim3).clone()
                sim_mtrx[:, 3] = torch.matmul(sim_mtrx[:3, :3].clone(), sim3_init[:, 3]) + sim_mtrx[:, 3].clone()
                sim_mtrx[:3, :3] = torch.matmul(sim_mtrx[:3, :3].clone(), sim3_init[:3, :3])
                sim3_scale = torch.norm(sim_mtrx[:3, :3].clone())/np.sqrt(3)
            else:
                sim_mtrx = None
                sim3_scale = None

            for i in range(0, num_views_per_round):
                if visualizer is not None:
                    visualizer.reset_data()
                idx1, idx2 = idx + int(np.floor(i * rot_freq)), idx + int(np.floor(i * rot_freq)) + sep_dist
                if idx2 >= num_images:
                    idx1 = num_images - 1
                    idx2 = idx1 - sep_dist
                loss, loss_pack = compute_loss_color_warp(renderer,
                                        shape_code,
                                        images,
                                        cameras,
                                        idx1, idx2,
                                        weight_list,
                                        sim3=sim_mtrx, sim3_scale=sim3_scale,
                                        visualizer=visualizer)
                loss_total += loss

            counter += 1
            loss_total.backward()
            shape_optimizer.step()
            loss_total = loss_total.detach()

        if vis_flag:
            loss_color = loss_pack['color']
            loss_l2reg = loss_pack['l2reg']
            print('[{0}] loss_color: {1:.4f}, loss_l2reg: {2:.4f}\n'.format(epoch, loss_color, loss_l2reg))
            visualizer.show_all_data_color_warp(os.path.join(vis_dir, 'vis_{}.png'.format(epoch)))

        if (epoch % test_step == 0):
            fname = os.path.join(vis_dir, 'mesh_{}.ply'.format(epoch))
            points_pred = evaluator.latent_vec_to_points(shape_code, num_points=num_sample_points, fname=fname, silent=True)

            if points_pred is None:
                print('The current latent code does not correspond to a valid shape.')
            elif (points_gt is not None) and full_flag: # evaluation and update the best model.
                dist1, dist2 = evaluator.compute_chamfer_distance(points_gt, points_pred, separate=True)
                dist = dist1 + dist2
                if dist * 1000 < best_chamfer:
                    best_chamfer = dist * 1000
                    best_epoch = epoch
                    fname = os.path.join(vis_dir, 'mesh_best.ply')
                    evaluator.latent_vec_to_points(shape_code, num_points=num_sample_points, fname=fname, silent=True)

                print('CHAMFER DISTANCE: {0:.3f} & {1:.3f} at epoch {2}'.format(dist1 * 1000, dist2 * 1000, epoch))
                print('BEST SUM CHAMFER DISTANCE: {0:.3f} at epoch {1}'.format(best_chamfer, best_epoch))
    return shape_code, shape_optimizer

max_stars_repo_name: hyunynim/DIST-Renderer

max_issues_repo_name: hyunynim/DIST-Renderer

max_forks_repo_name: hyunynim/DIST-Renderer


text:
import torch
import numpy as np
import sys
from model.Stacked_RNN import RNN

torch.set_num_threads(8)

data = open(sys.argv[1],"r").read()
vocab = list(set(data))
data_size,vocab_size = len(data),len(vocab)
print('data has %d characters, %d unique.' % (data_size, vocab_size))

char_to_idx = {ch:i for i,ch in enumerate(vocab)}
idx_to_char = {i:ch for i,ch in enumerate(vocab)}

hidden_size = 100
seq_length = 25
learning_rate = 1e-3
layers = 3
device = torch.device("cpu")
# device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") #Uncomment this line to make torch use GPU

rnn = RNN(layers=layers, hidden_size=hidden_size,vocab_size=vocab_size,seq_length=seq_length)

n,p = 0,0

smooth_loss = -np.log(1.0/vocab_size)*seq_length

while True:
    if p+seq_length+1 >= data_size or n==0:
        hprev = {}
        for layer in range(layers):
            hprev[layer] = torch.zeros(hidden_size,1,device=device)
        p = 0

    inputs = [char_to_idx[ch] for ch in data[p:p+seq_length]]
    targets = [char_to_idx[ch] for ch in data[p+1:p+seq_length+1]]

    if n % 1000 == 0:
        sample_ix = rnn.sampler(hprev, inputs[0], 200)
        txt = ''.join(idx_to_char[ix] for ix in sample_ix)
        print ('----\n %s \n----' % (txt, ))

    loss, hprev = rnn.forward(inputs,targets,hprev)
    smooth_loss = smooth_loss * 0.999 + loss * 0.001

    if n % 1000 == 0: print('iter %d, loss: %f' % (n, smooth_loss)) # print progress

    grads = rnn.backward()

    rnn.optimize(grads=grads,learning_rate=learning_rate)

    p += seq_length
    n += 1

max_stars_repo_name: iVishalr/Recurrent-Networks

max_issues_repo_name: iVishalr/Recurrent-Networks

max_forks_repo_name: iVishalr/Recurrent-Networks


text:
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
import os
import time
import numpy as np

from mindspore import context, Tensor, Parameter
from mindspore.context import ParallelMode
from mindspore.communication.management import init, get_rank
from mindspore.train import Model
from mindspore.train.callback import TimeMonitor, LossMonitor, ModelCheckpoint, CheckpointConfig
from mindspore.train.serialization import load_checkpoint, load_param_into_net
from mindspore.nn.optim import Adam
from mindspore.common import set_seed

from src.model import get_pose_net
from src.network_define import JointsMSELoss, WithLossCell
from src.dataset import keypoint_dataset
from src.model_utils.moxing_adapter import moxing_wrapper
from src.model_utils.config import config
from src.model_utils.device_adapter import get_device_id, get_device_num, get_rank_id

set_seed(1)

def get_lr(begin_epoch,
           total_epochs,
           steps_per_epoch,
           lr_init=0.1,
           factor=0.1,
           epoch_number_to_drop=(90, 120)
           ):
    """
    Generate learning rate array.

    Args:
        begin_epoch (int): Initial epoch of training.
        total_epochs (int): Total epoch of training.
        steps_per_epoch (float): Steps of one epoch.
        lr_init (float): Initial learning rate. Default: 0.316.
        factor:Factor of lr to drop.
        epoch_number_to_drop:Learing rate will drop after these epochs.
    Returns:
        np.array, learning rate array.
    """
    lr_each_step = []
    total_steps = steps_per_epoch * total_epochs
    step_number_to_drop = [steps_per_epoch * x for x in epoch_number_to_drop]
    for i in range(int(total_steps)):
        if i in step_number_to_drop:
            lr_init = lr_init * factor
        lr_each_step.append(lr_init)
    current_step = steps_per_epoch * begin_epoch
    lr_each_step = np.array(lr_each_step, dtype=np.float32)
    learning_rate = lr_each_step[current_step:]
    return learning_rate

def modelarts_pre_process():
    '''modelarts pre process function.'''
    def unzip(zip_file, save_dir):
        import zipfile
        s_time = time.time()
        if not os.path.exists(os.path.join(save_dir, config.modelarts_dataset_unzip_name)):
            zip_isexist = zipfile.is_zipfile(zip_file)
            if zip_isexist:
                fz = zipfile.ZipFile(zip_file, 'r')
                data_num = len(fz.namelist())
                print("Extract Start...")
                print("unzip file num: {}".format(data_num))
                data_print = int(data_num / 100) if data_num > 100 else 1
                i = 0
                for file in fz.namelist():
                    if i % data_print == 0:
                        print("unzip percent: {}%".format(int(i * 100 / data_num)), flush=True)
                    i += 1
                    fz.extract(file, save_dir)
                print("cost time: {}min:{}s.".format(int((time.time() - s_time) / 60),
                                                     int(int(time.time() - s_time) % 60)))
                print("Extract Done.")
            else:
                print("This is not zip.")
        else:
            print("Zip has been extracted.")

    if config.modelarts_dataset_unzip_name:
        zip_file_1 = os.path.join(config.data_path, config.modelarts_dataset_unzip_name + ".zip")
        save_dir_1 = os.path.join(config.data_path)

        sync_lock = "/tmp/unzip_sync.lock"

        # Each server contains 8 devices as most.
        if get_device_id() % min(get_device_num(), 8) == 0 and not os.path.exists(sync_lock):
            print("Zip file path: ", zip_file_1)
            print("Unzip file save dir: ", save_dir_1)
            unzip(zip_file_1, save_dir_1)
            print("===Finish extract data synchronization===")
            try:
                os.mknod(sync_lock)
            except IOError:
                pass

        while True:
            if os.path.exists(sync_lock):
                break
            time.sleep(1)

        print("Device: {}, Finish sync unzip data from {} to {}.".format(get_device_id(), zip_file_1, save_dir_1))
    config.ckpt_save_dir = os.path.join(config.output_path, config.ckpt_save_dir)
    config.DATASET.ROOT = config.data_path
    config.MODEL.PRETRAINED = os.path.join(os.path.abspath(os.path.dirname(__file__)), config.MODEL.PRETRAINED)

@moxing_wrapper(pre_process=modelarts_pre_process)
def run_train():
    print('batch size :{}'.format(config.TRAIN.BATCH_SIZE))
    if config.run_distribute:
        init("nccl")
        rank = get_rank()
        device_num = get_device_num()
        context.set_auto_parallel_context(device_num=device_num,
                                          parallel_mode=ParallelMode.DATA_PARALLEL,
                                          gradients_mean=True)
    else:
        rank = 0
        device_num = 1

    # only rank = 0 can write
    rank_save_flag = False
    if rank == 0 or device_num == 1:
        rank_save_flag = True

    # distribution and context
    if config.device_target == "GPU":
        context.set_context(mode=context.GRAPH_MODE,
                            device_target="GPU",
                            save_graphs=False,
                            device_id=get_rank_id())
    else:
        assert config.device_target == "Ascend"
        context.set_context(mode=context.GRAPH_MODE,
                            device_target="Ascend",
                            save_graphs=False,
                            device_id=get_rank_id())
    dataset, _ = keypoint_dataset(config,
                                  rank=rank,
                                  group_size=device_num,
                                  train_mode=True,
                                  num_parallel_workers=config.TRAIN.NUM_WORKERS)

    # network
    net = get_pose_net(config, True, ckpt_path=config.MODEL.PRETRAINED)
    loss = JointsMSELoss(use_target_weight=True)
    net_with_loss = WithLossCell(net, loss)

    # lr schedule and optim
    dataset_size = dataset.get_dataset_size()
    lr = Tensor(get_lr(config.TRAIN.BEGIN_EPOCH,
                       config.TRAIN.END_EPOCH,
                       dataset_size,
                       lr_init=config.TRAIN.LR,
                       factor=config.TRAIN.LR_FACTOR,
                       epoch_number_to_drop=config.TRAIN.LR_STEP))
    opt = Adam(net.trainable_params(), learning_rate=lr)

    # resume params from checkpoint, resume from config.MODEL.PRETRAINED
    if config.MODEL.RESUME:
        param_dict = load_checkpoint(config.MODEL.PRETRAINED)
        param_dict['learning_rate'] = Parameter(lr, requires_grad=False)
        load_param_into_net(net, param_dict)
        load_param_into_net(opt, param_dict)

    # callback
    time_cb = TimeMonitor(data_size=dataset_size)
    loss_cb = LossMonitor(per_print_times=config.TRAIN.PRINT_STEP)
    cb = [time_cb, loss_cb]
    if config.device_target == "Ascend" and config.run_distribute:
        config.ckpt_save_dir = os.path.join(config.ckpt_save_dir, str(get_rank_id()))
    if config.ckpt_save_dir and rank_save_flag:
        config_ck = CheckpointConfig(save_checkpoint_steps=dataset_size * config.TRAIN.SAVE_EPOCH,\
            keep_checkpoint_max=config.TRAIN.KEEP_CKPT_MAX)
        ckpoint_cb = ModelCheckpoint(prefix="simplepose", directory=config.ckpt_save_dir, config=config_ck)
        cb.append(ckpoint_cb)

    # train model
    model = Model(net_with_loss, loss_fn=None, optimizer=opt, amp_level="O2")
    epoch_size = config.TRAIN.END_EPOCH - config.TRAIN.BEGIN_EPOCH
    print('start training, epoch size = %d' % epoch_size)
    model.train(epoch_size, dataset, callbacks=cb, dataset_sink_mode=config.TRAIN.SINK_MODE)

if __name__ == '__main__':
    run_train()

max_stars_repo_name: mindspore-ai/models

max_issues_repo_name: mindspore-ai/models

max_forks_repo_name: mindspore-ai/models


text:
import numpy as np
import scipy as sp
import scipy.sparse.linalg as splinalg

def eig2_nL(g, tol_eigs = 1.0e-6, normalize:bool = True, dim:int=1):
    """
       DESCRIPTION
       -----------

       Computes the eigenvector that corresponds to the second smallest eigenvalue
       of the normalized Laplacian matrix then it uses sweep cut to round the solution.

       PARAMETERS (mandatory)
       ----------------------

       g:         graph object

       PARAMETERS (optional)
       ---------------------

       dim: positive, int
            default == 1
            The number of eigenvectors or dimensions to compute.

       tol_eigs: positive float, double
                 default == 1.0e-6
                 Tolerance for computation of the eigenvector that corresponds to
                 the second smallest eigenvalue of the normalized Laplacian matrix.

       normalize: bool,
                  default == True
                  True if we should return the eigenvectors of the generalized
                  eigenvalue problem associated with the normalized Laplacian.
                  This should be on unless you know what you are doing.

       RETURNS
       ------

       p:    Eigenvector or Eigenvector matrixthat
             corresponds to the second smallest eigenvalue of the
             normalized Laplacian matrix and larger eigenvectors if dim >= 0.
    """

    n = g.adjacency_matrix.shape[0]

    D_sqrt_neg = sp.sparse.spdiags(g.dn_sqrt.transpose(), 0, n, n)

    L = sp.sparse.identity(n) - D_sqrt_neg.dot((g.adjacency_matrix.dot(D_sqrt_neg)))

    emb_eig_val, p = splinalg.eigsh(L, which='SM', k=1+dim, tol = tol_eigs)

    F = np.real(p[:,1:])
    if normalize:
        F *= g.dn_sqrt[:,np.newaxis]
    return F, emb_eig_val

"""
Random walks and local cuts in graphs, Chung, LAA 2007
We just form the sub-matrix of the Laplacian and use the eigenvector there.
"""
def eig2nL_subgraph(g, ref_nodes, tol_eigs = 1.0e-6, normalize: bool = True):
    A_sub = g.adjacency_matrix.tocsr()[ref_nodes, :].tocsc()[:, ref_nodes]
    nref = len(ref_nodes)
    D_sqrt_neg = sp.sparse.spdiags(g.dn_sqrt[ref_nodes].transpose(), 0, nref, nref)
    L_sub = sp.sparse.identity(nref) - D_sqrt_neg.dot((A_sub.dot(D_sqrt_neg)))
    emb_eig_val, emb_eig = splinalg.eigsh(L_sub, which='SM', k=1, tol=tol_eigs)
    emb_eig *= -1 if max(emb_eig) < 0 else 1
    f = emb_eig[:,0]
    if normalize:
        f *= g.dn_sqrt[ref_nodes]
    return ((ref_nodes,f), emb_eig_val)

max_stars_repo_name: vishalbelsare/LocalGraphClustering

max_issues_repo_name: vishalbelsare/LocalGraphClustering

max_forks_repo_name: vishalbelsare/LocalGraphClustering

reason: import numpy,import scipy