text:
#C1949699
#
#
import math
import random
import re
import time
from cv2 import INTER_AREA, INTER_BITS, INTER_CUBIC, INTER_LANCZOS4, INTER_LINEAR, INTER_LINEAR_EXACT, INTER_MAX, imread, imshow, waitKey
import numpy as np
import cv2
import os
from multiprocessing import Process, Manager

data_path = os.getcwd()
dataset_path = data_path+"\\cross_validation\\"
imageDataLocation = dataset_path+"\\images\\"
truthDataLocation = dataset_path+"\\truth\\"
LOG = []
datasetTrain = []
datasetVal = []
PARAMETERS = []
runTimeNumber = int(1)

def calculateSTD(evaluationList, evaluationMean):
    n = 1
    if len(evaluationList)!=1:
        n = len(evaluationList)-1
    sumX = 0
    for score in evaluationList:
        sumX+=(score-evaluationMean)**2
    standardDeviation = sumX / n
    standardDeviation = math.sqrt(standardDeviation)
    return standardDeviation

def runTimeCount():
    i = 1
    fileList = os.listdir(data_path)
    while "log"+str(i)+".txt" in fileList:
            i+=1
    return int(i)

def saveLog():
    i = 1
    fileList = os.listdir(data_path)
    if "log1.txt" not in fileList:
        with open("log"+str(i)+".txt","a") as f:
            for line in LOG:
                f.write(line+"\n")
    else:
        while "log"+str(i)+".txt" in fileList:
            i+=1
        with open("log"+str(i)+".txt","a") as f:
            for line in LOG:
                f.write(line+"\n")
    LOG.clear()

def addToLog(line,varname):
    # print("Line",line)
    if varname == "BinaryMasks":
        LOG.append(varname,line)
    elif isinstance(line, list):
        # log.append(varname)
        LOG.append(str(varname+" "+f'{line}'.split('=')[0]))
    elif isinstance(line, str or int):
        # log.append(varname)
        LOG.append(str(varname)+" "+str(line))
    elif isinstance(line, float):
        LOG.append(str(varname)+" "+str(line))

def calc_IoU(mask1, mask2):   # From the question.
    mask1_area = np.count_nonzero(mask1)
    mask2_area = np.count_nonzero(mask2)
    # print(mask1_area, " : ", mask2_area)
    intersection = np.count_nonzero(np.logical_and( mask1,  mask2))
    # print("intersection",intersection)
    iou = intersection/(mask1_area+mask2_area-intersection)
    return iou

# def runTest():
def atoi(text):
    return int(text) if text.isdigit() else text

def natural_keys(text):
    return [ atoi(c) for c in re.split(r'(\d+)', text) ]

def select_new_dataset():
    datasetTrain.clear()
    datasetVal.clear()
    files = os.listdir(imageDataLocation)
    for i in range(14):
        file = random.choice(files)
        while file in datasetTrain:
            file = random.choice(files)
        datasetTrain.append(file)
    for file in files:
        if file not in datasetTrain:
            datasetVal.append(file)
    # datasetTrain.sort(key=natural_keys)
    # datasetVal.sort(key=natural_keys)

listOfMethods = [cv2.THRESH_OTSU,cv2.THRESH_TRIANGLE]
listOfTypes = [cv2.THRESH_BINARY,cv2.THRESH_TOZERO]
confidenceList = np.divide(np.arange(10,91,15),100)

def selectParameters2():#selecting the parameters that will be swapped for each iteration
    PARAMETERS.clear()
    PARAMETERS.append(random.choice(listOfMethods))
    PARAMETERS.append(random.choice(listOfTypes))
    PARAMETERS.append(random.choice(confidenceList))

configPath = 'ssd_mobilenet_v3_large_coco_2020_01_14.pbtxt'
weightsPath = 'frozen_inference_graph.pb'

net2 = cv2.dnn_DetectionModel(weightsPath,configPath)
net2.setInputSize(320,320)
net2.setInputScale(1.0/127.5)
net2.setInputMean((127.5,127.5,127.5))
net2.setInputSwapRB(True)

def v2(img,numb,PARAMETERS,dictOfBinaryMask):
    thres = PARAMETERS[2]

    classIds, confs, bbox = net2.detect(img,confThreshold = thres)
    boxes = []
    confidenceList = []
    if not isinstance(classIds,tuple):
        for classId,confidence,box in zip(classIds.flatten(),confs.flatten(),bbox):
            x1 = box[0]
            y1 = box[1]
            x2 = box[2]
            y2 = box[3]
            boxes.append([x1,y1,x2,y2])
            confidenceList.append(confidence)

    saliency = cv2.saliency.StaticSaliencyFineGrained_create()
    (success, saliencyMap) = saliency.computeSaliency(img)
    newSaliencyMap=saliencyMap*255
    threshMap = cv2.threshold(newSaliencyMap.astype("uint8"), 0, 255, PARAMETERS[0] | PARAMETERS[1])[1]#globally thresholding the image using otsu's algorithm creating a binary mask
    threshMap = cv2.threshold(threshMap.astype("uint8"), 80, 255, cv2.THRESH_BINARY)[1]
    output_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)#creating a greyscale copy of the original image
    testGrey = img.copy()#creates a colour copy of the original image
    for i in range(len(img)):
        for j in range(len(img[0])):
            testGrey[i][j] = np.array([output_img[i][j],output_img[i][j],output_img[i][j]])

    maxX = 0
    maxY = 0
    minX = len(img)*len(img[0])
    minY = len(img)*len(img[0])
    newThrMap = threshMap.copy()
    newThrMap.fill(0)
    # boxes.remove(boxes[0])
    for b in boxes:
        maxX = max(maxX,b[0],b[2])#finding the overall bounding area of all the boxes
        maxY = max(maxY,b[1],b[3])
        minX = min(minX,b[2],b[0])
        minY = min(minY,b[3],b[1])
    # print(minX,":",minY,"  ",maxX,":",maxY)
    if bool(boxes):#if the list is empty it will return the threshmap
        dictOfBinaryMask[numb] = threshMap
    else:
        for b in boxes: # finding the bounding boxes area
            x1 = b[0]
            y1 = b[1]
            x2 = b[2]+b[0]#the b[2] and b[3] represents a length not a coordinate so adding b[0] to b[2] finds the coord of the second point
            y2 = b[3]+b[1]

            for i in range(y1, y2):
                for j in range(x1, x2):
                    if (threshMap[i][j] == 255):#selecting pixels that are only inside the bounding boxes and where the mask is white, making the ROI in colour
                        newThrMap[i][j] = 255

        dictOfBinaryMask[numb] = newThrMap

def runTestV2():
    listTestEvaluations = []
    for i in range(50):
        if i % 10 == 0:
            print(i*2,"% Done")
        addToLog(i,"Loop: ")
        select_new_dataset()
        addToLog(datasetTrain,f'{datasetTrain=}'.split('=')[0])
        addToLog(datasetVal,f'{datasetVal=}'.split('=')[0])
        # listOfBinaryMask = []
        # for i in range(10):
        listOfEvaluations = []
        evaluationParameters = []
        averageIOU = 0
        optimalParams = []
        with Manager() as manager:
            dictOfBinaryMask = manager.dict()
            # listOfBinaryMask.fromkeys(datasetTrain)
            for i in range(1000):#Only runs for the maximum number of combinations
                selectParameters2()
                if PARAMETERS in evaluationParameters:
                    continue
                evaluationParameters.append(PARAMETERS.copy())#[PARAMETERS[0],PARAMETERS[1],PARAMETERS[2]]
                # print(evaluationParameters)
                # addToLog(PARAMETERS.copy(),"Parameters")
                jobs = []
                for image in datasetTrain:
                    current = imread(imageDataLocation+image)
                    p1 = Process(target=v2,args=[current,image,PARAMETERS,dictOfBinaryMask])
                    p1.start()
                    jobs.append(p1)
                for job in jobs:
                    job.join()
                averageIOU=0
                for image in datasetTrain:
                    imageNumber = int(re.compile(r'\d+(?:\.\d+)?').findall(image)[0])
                    max_IoU = 0
                    for i in range (1,6):
                        mask2 = str(imageNumber)+"_gt"+str(i)+".jpg"
                        mask2 = imread(truthDataLocation+mask2)
                        mask2 = cv2.cvtColor(mask2,cv2.COLOR_BGR2GRAY)
                        max_IoU = max(max_IoU,calc_IoU(dictOfBinaryMask[image],mask2))
                    averageIOU+=max_IoU
                averageIOU /= len(datasetTrain)
                # print(averageIOU)
                listOfEvaluations.append(averageIOU)
            optimalParamsResults = listOfEvaluations[listOfEvaluations.index(max(listOfEvaluations))]
            optimalParams = evaluationParameters[listOfEvaluations.index(max(listOfEvaluations))]
            addToLog(optimalParamsResults,"Evaluation Score Train Set")
            jobs.clear()
            for image in datasetVal:
                current = imread(imageDataLocation+image)
                p1 = Process(target=v2,args=[current,image,optimalParams,dictOfBinaryMask])
                p1.start()
                jobs.append(p1)
            for job in jobs:
                job.join()
            averageIOU=0
            for image in datasetVal:
                imageNumber = int(re.compile(r'\d+(?:\.\d+)?').findall(image)[0])
                max_IoU = 0
                for i in range (1,6):
                    mask2 = str(imageNumber)+"_gt"+str(i)+".jpg"
                    mask2 = imread(truthDataLocation+mask2)
                    mask2 = cv2.cvtColor(mask2,cv2.COLOR_BGR2GRAY)
                    max_IoU = max(max_IoU,calc_IoU(dictOfBinaryMask[image],mask2))
                averageIOU+=max_IoU
            averageIOU /= len(datasetVal)
            addToLog(optimalParams,"Optimal Parameters")
            addToLog(averageIOU,"Evaluation Score Validation Set")
            listTestEvaluations.append(averageIOU)
    averageTestScore = sum(listTestEvaluations)/len(listTestEvaluations)
    addToLog(averageTestScore,"Average Validation Score")
    addToLog(calculateSTD(listTestEvaluations,averageTestScore), "Standard Deviation")
    addToLog("V2","Pipeline: ")

if __name__ == "__main__":
    start_time = time.time()
    runTestV2()
    print("--- %s seconds ---" % (time.time() - start_time))
    runTimeNumber = str(runTimeCount())
    saveLog()
    print("Run time number ",runTimeNumber)

max_stars_repo_name: jwilson-hall/Spot-Colour

max_issues_repo_name: jwilson-hall/Spot-Colour

max_forks_repo_name: jwilson-hall/Spot-Colour


text:
import numpy as np
import scipy.stats as stats
import xarray as xr

def add_in_nan(time, data):
    """
    Procedure add_in_nan
    --------------------
    This procedure adds in NaNs for given time periods in time when there is no
    corresponding data available. This is useful for timeseries that have
    irregular gaps in data.

    Parameters:
    -----------
    time: 1D array of np.datetime64
        List of times in the timeseries
    data: 1 or 2D array
        Array containing the data. The 0 axis corresponds to time.

    Returns:
    --------
    d_time: xarray DataArray
        The xarray DataArray containing the new times at regular intervals.
        The intervals are determined by the mode of the timestep in *time*.
    d_data: xarray DataArray
        The xarray DataArray containing the NaN-filled data.
    """

    diff = time.diff(dim='time', n=1) / np.timedelta64(1, 's')
    mode = stats.mode(diff).mode[0]
    index = np.where(diff.values > 2. * mode)
    d_data = np.asarray(data)
    d_time = np.asarray(time)

    offset = 0
    for i in index[0]:
        n_obs = np.floor(
            (time[i + 1] - time[i]) / mode / np.timedelta64(1, 's'))
        time_arr = [
            d_time[i + offset] + np.timedelta64(int((n + 1) * mode), 's')
            for n in range(int(n_obs))]
        S = d_data.shape
        if len(S) == 2:
            data_arr = np.empty([len(time_arr), S[1]])
        else:
            data_arr = np.empty([len(time_arr)])
        data_arr[:] = np.nan

        d_time = np.insert(d_time, i + 1 + offset, time_arr)
        d_data = np.insert(d_data, i + 1 + offset, data_arr, axis=0)
        offset += len(time_arr)

    d_time = xr.DataArray(d_time)
    d_data = xr.DataArray(d_data)

    return d_time, d_data

ext: py

max_stars_repo_name: scollis/ACT

max_issues_repo_name: scollis/ACT

max_forks_repo_name: scollis/ACT

reason: import numpy,import scipy


text:
# -----------------------------------------------------------
# Copyright (C) 2020 NVIDIA Corporation. All rights reserved.
# Nvidia Source Code License-NC
# Code written by Xueting Li.
# -----------------------------------------------------------
import numpy as np
import os
import ntpath
import time
import termcolor

# convert to colored strings
def red(content): return termcolor.colored(str(content),"red",attrs=["bold"])
def green(content): return termcolor.colored(str(content),"green",attrs=["bold"])
def blue(content): return termcolor.colored(str(content),"blue",attrs=["bold"])
def cyan(content): return termcolor.colored(str(content),"cyan",attrs=["bold"])
def yellow(content): return termcolor.colored(str(content),"yellow",attrs=["bold"])
def magenta(content): return termcolor.colored(str(content),"magenta",attrs=["bold"])

class Visualizer():
    def __init__(self, opt):
        # self.opt = opt
        self.log_name = os.path.join(opt.checkpoint_dir, opt.name, 'loss_log.txt')
        with open(self.log_name, "a") as log_file:
            now = time.strftime("%c")
            log_file.write('================ Training Loss (%s) ================\n' % now)

    # scalars: same format as |scalars| of plot_current_scalars
    def print_current_scalars(self, epoch, i, scalars):
        message = green('(epoch: %d, iters: %d) ' % (epoch, i))
        for k, v in scalars.items():
            if("lr" in k):
                message += '%s: %.6f ' % (k, v)
            else:
                message += '%s: %.3f ' % (k, v)

        print(message)
        with open(self.log_name, "a") as log_file:
            log_file.write('%s\n' % message)

max_stars_repo_name: NVlabs/UMR

max_issues_repo_name: gengshan-y/UMR

max_forks_repo_name: gengshan-y/UMR

reason: import numpy


text:
import detectron2
from detectron2.utils.logger import setup_logger
setup_logger()

# import some common libraries
import numpy as np
import os, json, cv2, random
import pycocotools
import skimage.draw
from PIL import Image, ImageDraw
from progress.bar import Bar
import datetime

# import some common detectron2 utilities
from detectron2 import model_zoo
from detectron2.engine import DefaultPredictor, DefaultTrainer
from detectron2.config import get_cfg
from detectron2.utils.visualizer import Visualizer
from detectron2.data import MetadataCatalog, DatasetCatalog

from detectron2.structures import BoxMode

def convert_to_rle(annotation, height, width):
    """Convert polygons and complex polygons to COCO RLE format.
   Returns: a dictionary with
    """

    # complex polygons have multiple "paths" (polygons)
    if 'complex_polygon' in annotation:
        polygons = annotation['complex_polygon']['path']
    else:
        polygons = [annotation['polygon']['path']]

    mask = np.zeros([height, width, len(polygons)], dtype=np.uint8)

    for ind_pol, pol in enumerate(polygons):

        all_points_y = []; all_points_x = [];

        # not clear why this worked in TF...
        for pt in pol:
            if pt['x'] >= width: pt['x'] = width - 1
            if pt['x'] < 0: pt['x'] = 0
            if pt['y'] >= height: pt['y'] = height - 1
            if pt['y'] < 0: pt['y'] = 0

            all_points_y.append(pt['y'])
            all_points_x.append(pt['x'])

        # Get indexes of pixels inside the polygon and set them to 1
        rr, cc = skimage.draw.polygon(all_points_y, all_points_x)
        mask[rr, cc, ind_pol] = 1

    # once we sum all the polygons any even values are holes (this should allow for "ring" holes, but it is now tested)
    mask = ((np.sum(mask, axis=2)%2) == 1).astype(np.uint8)

    # Return mask, and array of class IDs of each instance
    return pycocotools.mask.encode(np.asarray(mask, order="F")), Image.fromarray(mask).getbbox()

def get_darwin_dataset(img_dir, train_val):

    json_file = os.path.join(img_dir, train_val, train_val + ".json")
    with open(json_file) as f:
        imgs = json.load(f)

    imgs = imgs[0:10]

    dataset_dicts = []

    bar = Bar('Importing Dataset', max=len(imgs))

    for idx, img in enumerate(imgs):

        record = {}

        filename = os.path.join(img_dir,'images',img["image"]["original_filename"])
        height, width = cv2.imread(filename).shape[:2]

        record["file_name"] = filename
        record["image_id"] = idx
        record["height"] = height
        record["width"] = width

        annos = img["annotations"]
        objs = []

        for anno in annos:

            poly, bbox = convert_to_rle(anno, height, width)

            #check bounding boxes are healthy
            # test_mask = pycocotools.mask.decode(poly)
            # mask_img = Image.fromarray(test_mask.astype(np.bool)).convert('RGB')
            # draw = ImageDraw.Draw(mask_img)
            # draw.rectangle(bbox, fill=None, outline='red', width=3)
            # mask_img.save('img.jpg')

            obj = {
                "bbox": bbox,
                "bbox_mode": BoxMode.XYXY_ABS,
                "segmentation": poly,
                "category_id": 0,# change in the future for more than one category
            }

            objs.append(obj)

        record["annotations"] = objs
        dataset_dicts.append(record)

        # # check masks are healthy
        # test_mask = np.zeros([height, width, len(record["annotations"])], dtype=np.uint8)
        # for idx, obj in enumerate(record["annotations"]):
        #     # decode RLE for all objects, create global mask and save image
        #     test_mask[:,:,idx] = pycocotools.mask.decode(obj['segmentation'])
        # Image.fromarray(np.sum(test_mask, axis=2).astype(np.bool)).save('masks/' + img["image"]["original_filename"])

        bar.next()

    bar.finish()

    return dataset_dicts

dataset_directory = "/home/ndserv05/Documents/Data/Corrosion"

# register training and validation datasets with detectron
for d in ["train", "val"]:
    # get_darwin_dataset(dataset_directory, d)
    DatasetCatalog.register("corrosion_" + d, lambda d=d: get_darwin_dataset(dataset_directory, d))
    MetadataCatalog.get("corrosion_" + d).set(thing_classes=["Corrosion"])

corrosion_metadata = MetadataCatalog.get("corrosion_val")

# print(corrosion_metadata)

# check annotations
# dataset_dicts = get_darwin_dataset(dataset_directory, 'val')
# for d in random.sample(dataset_dicts, 3):
#     img = Image.open(d["file_name"])
#     visualizer = Visualizer(img, metadata=corrosion_metadata, scale=1)
#     out = visualizer.draw_dataset_dict(d)
#     Image.fromarray(out.get_image()).save(str(d['image_id']) + '.jpg')

# CONFIGURATION
cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml"))
cfg.OUTPUT_DIR = "./output/" + "Corrosion_" + "{:%Y%m%dT%H%M}".format(datetime.datetime.now())
cfg.INPUT.MASK_FORMAT = "bitmask"
cfg.DATASETS.TRAIN = ("corrosion_train",)
cfg.DATASETS.TEST = ()
cfg.DATALOADER.NUM_WORKERS = 2
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml")  # Let training initialize from model zoo
cfg.SOLVER.IMS_PER_BATCH = 2
cfg.SOLVER.BASE_LR = 0.00025  # pick a good LR
cfg.SOLVER.MAX_ITER = 300    # 300 iterations seems good enough for this toy dataset; you will need to train longer for a practical dataset
cfg.SOLVER.STEPS = []        # do not decay learning rate
cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 512   # faster, and good enough for this toy dataset (default: 512)
cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1  # only has one class (corrosion)

# TRAIN
os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)
trainer = DefaultTrainer(cfg)
trainer.resume_or_load(resume=False)
trainer.train()

# INFERENCE
cfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, "model_final.pth")  # path to the model we just trained
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.7   # set a custom testing threshold
predictor = DefaultPredictor(cfg)

# TEST INFERENCE
dataset_dicts = get_darwin_dataset(dataset_directory, 'val')
for d in random.sample(dataset_dicts, 3):
    im = cv2.imread(d["file_name"])
    # im = Image.open(d["file_name"])
    outputs = predictor(im)  # format is documented at https://detectron2.readthedocs.io/tutorials/models.html#model-output-format
    print(outputs)
    v = Visualizer(im, metadata=corrosion_metadata,  scale=1)
    out = v.draw_instance_predictions(outputs["instances"].to("cpu"))
    # cv2_imshow(out.get_image()[:, :, ::-1])
    Image.fromarray(out.get_image()[:, :, ::-1]).save(str(d['image_id']) + '.jpg')


text:
""" Extracts, organizes, and plots data from an Emu history file.
History file must be in column-wise format with each column being exactly 15 characters
wide. The first row of the history file may contain anything, but must end with the
number of node histories preceded by whitespace. The second row of the history file must
be the column titles.

The history file name (and path) is expected as the first command-line argument. If it is
omitted, the file name is assumed to be "./emu.his".
"""
import numpy as np
import sys
import matplotlib.pyplot as plt
import chart_studio.plotly as py
import plotly.offline as pyoff
import plotly.graph_objs as go

#########################################################################################
#                                      PARAMETERS                                       #
#########################################################################################
# fmt: off
NAME_MAP = {
    "iter":      "Timestep",
    "timex":     "Time",
    "dt":        "Step size",
    "phid":      "Dissipated energy",
    "ekin":      "Kinetic energy",
    "work_bdry": "Boundary work"
}

CATEGORIES = ("x1",  "x2",    "x3",
              "u1",  "u2",    "u3",
              "v1",  "v2",    "v3",
              "dmg", "stmax", "scmax",
              "force")
# fmt: on

#########################################################################################
#                                      FUNCTIONS                                        #
#########################################################################################
def get_filename():
    """ Get history file name from command line argument. Assign default if not found.

    Returns:
        filename (str): name of history file that will be read
    """
    if len(sys.argv) < 2:
        filename = "./emu.his"
        print("No filename provided. Assuming ./emu.his ...")
    else:
        filename = sys.argv[1]
        print("Reading", filename, "...")

    return filename

def get_number_of_node_histories(filename):
    """ Get number of node histories in history file from first line of history file.
    Requires that last item on first line of history file is the number of node
    histories, separated by whitespace.

    Args:
        filename (str): name of history file

    Returns:
        num_histories (int): number of node histories
    """
    with open(filename) as file:
        first_line = file.readline()
        num_histories = int(first_line.split()[-1])
    print("Reading from %i node histories" % (num_histories))
    return num_histories

def read_his_data(filename):
    """ Parse data from history file.
    Requires that the second line of the history file contains the column names and that
    each column is exactly 15 characters wide. There is no requirement for number of
    columns or rows.

    Args:
        filename (str): name of history file

    Returns:
        (structured np array): all extracted data with column names. Number of rows is
                               equal to the number of timesteps or history dumps
    """
    return np.genfromtxt(
        filename, dtype=None, delimiter=",", names=True, skip_header=1
    )

def remap_column_names(data_names, name_map):
    """ Remap data array column names using dictionary map.
    For each column name that matches a key in name_map, the column name is replaced with
    that key's value.

    Args:
        data_names (str, nx1 tuple): list of column names taken from structured np array
        name_map (dict): dictionary with keys matching history file column names to be
                         replaced by the corresponding values

    Returns:
        (str, nx1 tuple): New list of column names
    """
    return tuple(name_map.get(name, name) for name in data_names)
    # get(name, name) means it will keep the current name if not found in dictionary

def populate_datasets(categories, data_names):
    """ Populate dictionary of dataset labels with the corresponding column names.
    Creates a dictionary with keys from the list "categories". For each key, if a column
    name contains that key, the name is added to a tuple of strings stored under that
    key. This is used to access a set of data columns using the names stored under each
    dataset key. Intended to gather node history datasets, which should have columns
    names in the form u1_001, u1_002, u1_003, ... to be stored under the dataset
    label "u1"

    Args:
        categories (str, nx1 tuple): list of dataset labels used to locate corresponding
                                     column names
        data_names (str, mx1 tuple): list of column names taken from structured np array

    Returns:
        datasets (dict): dictionary containing sets of column names
    """
    datasets = {}
    for key in categories:
        datasets.update({key: tuple(name for name in data_names if key in name)})
    return datasets

#########################################################################################
#                                      EXECUTION                                        #
#########################################################################################
filename = get_filename()

num_histories = get_number_of_node_histories(filename)

data = read_his_data(filename)

print("Reading from %i node histories" % (num_histories))
print(data.dtype.names)

data.dtype.names = remap_column_names(data.dtype.names, NAME_MAP)

datasets = populate_datasets(CATEGORIES, data.dtype.names)
# For example, the following line gets u1 for all node histories (u1_001, u1_002, ...)
#     for label in datasets['u1']: data[label]

# Add dataset that contains all columns
datasets.update({"all": data.dtype.names})

# Separate time and timestep for easy access
time = data["Time"]
timestep = data["Timestep"]

""" Plot data """
plotdata = []
for name in datasets["all"]:
    trace = go.Scatter(x=time, y=data[name], mode="lines", name=name)
    plotdata.append(trace)

layout = go.Layout(
    autosize=False,
    width=800,
    height=600,
    margin=go.layout.Margin(l=50, r=50, b=100, t=100, pad=4),
    xaxis=go.layout.XAxis(title="Time (s)", titlefont=dict(size=30), automargin=True),
    paper_bgcolor="#7f7f7f",
    plot_bgcolor="#c7c7c7",
)

fig = go.Figure(data=plotdata, layout=layout)
pyoff.plot(fig)

max_stars_repo_name: CMSM-VCU/tools

max_issues_repo_name: CMSM-VCU/tools

max_forks_repo_name: CMSM-VCU/tools


text:
##############################################################################
# Institute for the Design of Advanced Energy Systems Process Systems
# Engineering Framework (IDAES PSE Framework) Copyright (c) 2018-2019, by the
# software owners: The Regents of the University of California, through
# Lawrence Berkeley National Laboratory,  National Technology & Engineering
# Solutions of Sandia, LLC, Carnegie Mellon University, West Virginia
# University Research Corporation, et al. All rights reserved.
#
# Please see the files COPYRIGHT.txt and LICENSE.txt for full copyright and
# license information, respectively. Both files are also available online
# at the URL "https://github.com/IDAES/idaes-pse".
##############################################################################

from pyomo.environ import value, TerminationCondition, SolverStatus
from pyomo.core.base.var import IndexedVar

def solve_successful(status):
    if (
        status.solver.termination_condition == TerminationCondition.optimal
        or status.solver.status == SolverStatus.ok
    ):
        return True
    return False

def print_ports_summary(ports):
    # find the list of variables to print
    vars_idxs = []
    varname_set = set()
    for prt in ports:
        for vname in prt.vars:
            if vname in varname_set:
                continue

            varname_set.add(vname)
            var = prt.vars[vname]

            for k in var.keys():
                vars_idxs.append((vname, k, "{}[{}]".format(vname, k)))
    vname_len = max([len(vidx[2]) for vidx in vars_idxs])
    vname_len = max(vname_len, len("Variable"))

    # print the header
    print("{0:>{1}s}".format("Variable", vname_len), end="\t")

    for prt in ports:
        print(prt.name, end="\t")
    print()

    print("".join(["-"] * vname_len), end="\t")
    for prt in ports:
        print("".join(["-"] * len(prt.name)), end="\t")
    print()

    for vname, idx, vname_idx in vars_idxs:
        print("{0:>{1}s}".format(vname_idx, vname_len), end="\t")
        for prt in ports:
            print(
                "{0:{1}f}".format(value(prt.vars[vname][idx]), len(prt.name)), end="\t"
            )
        print()

max_stars_repo_name: dt-schwartz/NGFC

max_issues_repo_name: dt-schwartz/NGFC

max_forks_repo_name: dt-schwartz/NGFC

reason: from pyomo


text:
"""
    Example of a simultaneous fit model for REFL1D, as generated by the application.
"""
import numpy
import os
from refl1d.names import *
from math import *
import warnings
import matplotlib.cbook
warnings.filterwarnings("ignore",category=matplotlib.cbook.mplDeprecation)
warnings.simplefilter('ignore', UserWarning)

# run ref_l/160601 #############################################################
# Maximum Q-value ##############################################################
q_min = 0.0
q_max = 1.0

reduced_file = "/SNS/users/m2d/__data725.txt"

Q, R, dR, dQ = numpy.loadtxt(reduced_file).T
i_min = min([i for i in range(len(Q)) if Q[i]>q_min])
i_max = max([i for i in range(len(Q)) if Q[i]<q_max])+1

# SNS data is FWHM
dQ_std = dQ/2.35
probe725 = QProbe(Q[i_min:i_max], dQ_std[i_min:i_max], data=(R[i_min:i_max], dR[i_min:i_max]))

# Materials ####################################################################
Si = SLD(name='Si', rho=2.07, irho=0.0)
Electrolyte = SLD(name='Electrolyte', rho=5.48256, irho=0.0)
SiOx = SLD(name='SiOx', rho=1.67867, irho=0.0)
Cu = SLD(name='Cu', rho=6.42039, irho=0.0)
aSi = SLD(name='aSi', rho=1.40821, irho=0.0)
PAA = SLD(name='PAA', rho=3.0, irho=0.0)

# Film definition ##############################################################
sample725 = (  Electrolyte(0, 14.5834) | PAA(361.52, 98.88) | aSi(512.311, 11.917) | Cu(181.161, 1.0) | SiOx(5.0, 2.23098) | Si )

sample725['SiOx'].thickness.range(5.0, 33.0)
sample725['SiOx'].material.rho.range(1.0, 4.0)
sample725['SiOx'].interface.range(1.0, 5.0)
sample725['Cu'].thickness.range(99.0, 222.0)
sample725['Cu'].material.rho.range(6.0, 7.0)
sample725['Cu'].interface.range(1.0, 10.0)
sample725['aSi'].thickness.range(10.0, 700.0)
sample725['aSi'].material.rho.range(1.0, 2.1)
sample725['aSi'].interface.range(1.0, 33.0)
sample725['PAA'].thickness.range(10.0, 444.0)
sample725['PAA'].material.rho.range(1.0, 3.0)
sample725['PAA'].interface.range(1.0, 111.0)

probe725.intensity=Parameter(value=1.0,name='normalization')
probe725.background.range(0.0, 1e-06)
sample725['Electrolyte'].material.rho.range(0.0, 6.0)
sample725['Electrolyte'].interface.range(1.0, 66.0)

################################################################################

expt725 = Experiment(probe=probe725, sample=sample725)

# run m2d/243027 #############################################################
# Maximum Q-value ##############################################################
q_min = 0.0
q_max = 1.0

reduced_file = "/SNS/users/m2d/__data723.txt"

Q, R, dR, dQ = numpy.loadtxt(reduced_file).T
i_min = min([i for i in range(len(Q)) if Q[i]>q_min])
i_max = max([i for i in range(len(Q)) if Q[i]<q_max])+1

# SNS data is FWHM
dQ_std = dQ/2.35
probe723 = QProbe(Q[i_min:i_max], dQ_std[i_min:i_max], data=(R[i_min:i_max], dR[i_min:i_max]))

# Materials ####################################################################
air = SLD(name='air', rho=0.0, irho=0.0)
Si = SLD(name='Si', rho=2.07, irho=0.0)
aSi = SLD(name='aSi', rho=1.89237, irho=0.0)
Cu = SLD(name='Cu', rho=6.47662, irho=0.0)
SiOx = SLD(name='SiOx', rho=2.63338, irho=0.0)

# Film definition ##############################################################
sample723 = (  Si(0, 5.0) | SiOx(19.4988, 1.00002) | Cu(177.157, 10.7316) | aSi(549.002, 14.5254) | air )

sample723['aSi'].thickness.range(250.0, 650.0)
sample723['aSi'].material.rho.range(1.0, 2.0)
sample723['aSi'].interface.range(1.0, 33.0)
sample723['Cu'].thickness.range(100.0, 200.0)
sample723['Cu'].material.rho.range(1.0, 7.0)
sample723['Cu'].interface.range(1.0, 33.0)
sample723['SiOx'].thickness.range(10.0, 33.0)
sample723['SiOx'].material.rho.range(1.0, 4.0)
sample723['SiOx'].interface.range(1.0, 10.0)

probe723.intensity=Parameter(value=1.0,name='normalization')
probe723.background=Parameter(value=0.0,name='background')
sample723['Si'].interface.range(1.0, 5.0)

################################################################################

expt723 = Experiment(probe=probe723, sample=sample723)

# run ref_l/160617 #############################################################
# Maximum Q-value ##############################################################
q_min = 0.0
q_max = 1.0

reduced_file = "/SNS/users/m2d/__data727.txt"

Q, R, dR, dQ = numpy.loadtxt(reduced_file).T
i_min = min([i for i in range(len(Q)) if Q[i]>q_min])
i_max = max([i for i in range(len(Q)) if Q[i]<q_max])+1

# SNS data is FWHM
dQ_std = dQ/2.35
probe727 = QProbe(Q[i_min:i_max], dQ_std[i_min:i_max], data=(R[i_min:i_max], dR[i_min:i_max]))

# Materials ####################################################################
Si = SLD(name='Si', rho=2.07, irho=0.0)
Electrolyte = SLD(name='Electrolyte', rho=5.48, irho=0.0)
SiOx = SLD(name='SiOx', rho=2.18805, irho=0.0)
Cu = SLD(name='Cu', rho=6.63232, irho=0.0)
aSi = SLD(name='aSi', rho=1.79408, irho=0.0)
SEI = SLD(name='SEI', rho=1.19976, irho=0.0)
PAA = SLD(name='PAA', rho=1.99995, irho=0.0)

# Film definition ##############################################################
sample727 = (  Electrolyte(0, 17.2561) | PAA(137.796, 39.271) | SEI(42.0405, 2.62061) | aSi(695.848, 10.9552) | Cu(178.377, 2.76232) | SiOx(5.16418, 2.5141) | Si )

sample727['SiOx'].thickness.range(5.0, 33.0)
sample727['SiOx'].material.rho.range(1.0, 4.0)
sample727['SiOx'].interface.range(1.0, 5.0)
sample727['Cu'].thickness.range(99.0, 222.0)
sample727['Cu'].material.rho.range(6.0, 7.0)
sample727['Cu'].interface.range(1.0, 10.0)
sample727['aSi'].thickness.range(10.0, 700.0)
sample727['aSi'].material.rho.range(1.0, 2.1)
sample727['aSi'].interface.range(1.0, 33.0)
sample727['SEI'].thickness.range(10.0, 100.0)
sample727['SEI'].material.rho.range(1.0, 4.0)
sample727['SEI'].interface.range(1.0, 10.0)
sample727['PAA'].thickness.range(10.0, 200.0)
sample727['PAA'].material.rho.range(1.0, 2.0)
sample727['PAA'].interface.range(1.0, 40.0)

probe727.intensity=Parameter(value=1.0003,name='normalization')
probe727.background.range(0.0, 8e-06)
sample727['Electrolyte'].interface.range(1.0, 40.0)

################################################################################

expt727 = Experiment(probe=probe727, sample=sample727)

# Constraints ##################################################################
sample723['aSi'].thickness = sample725['aSi'].thickness
sample723['Cu'].thickness = sample725['Cu'].thickness
sample723['SiOx'].thickness = sample725['SiOx'].thickness
sample723['Cu'].material.rho = sample725['Cu'].material.rho
sample723['SiOx'].material.rho = sample725['SiOx'].material.rho
sample723['aSi'].material.rho = sample725['aSi'].material.rho
sample723['Si'].interface = sample725['SiOx'].interface
sample723['SiOx'].interface = sample725['Cu'].interface

problem = FitProblem([expt725,expt723,expt727])


text:
# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import copy

import networkx
from networkx.algorithms.cycles import find_cycle
from networkx.algorithms.dag import topological_sort
from oslo_log import log as logging

from deckhand.engine import document_validation
from deckhand.engine import document_wrapper
from deckhand.engine import secrets_manager
from deckhand.engine import utils as engine_utils
from deckhand import errors
from deckhand import types

LOG = logging.getLogger(__name__)

class DocumentLayering(object):
    """Class responsible for handling document layering.

    Layering is controlled in two places:

    1. The ``LayeringPolicy`` control document, which defines the valid layers
       and their order of precedence.
    2. In the ``metadata.layeringDefinition`` section of normal
       (``metadata.schema=metadata/Document/v1.0``) documents.

    .. note::

        Only documents with the same ``schema`` are allowed to be layered
        together into a fully rendered document.
    """

    SUPPORTED_METHODS = ('merge', 'replace', 'delete')

    def _replace_older_parent_with_younger_parent(self, child, parent,
                                                  all_children):
        # If child has layer N, parent N+1, and current_parent N+2, then swap
        # parent with current_parent. In other words, if parent's layer is
        # closer to child's layer than current_parent's layer, then use parent.
        current_parent = self._parents.get((child.name, child.schema))
        if current_parent:
            if (self._layer_order.index(parent.layer) >
                self._layer_order.index(current_parent.layer)):
                self._parents[(child.name, child.schema)] = parent
                self._children[
                    (current_parent.name, current_parent.schema)].remove(child)
                all_children[child] -= 1
        else:
            self._parents.setdefault((child.name, child.schema), parent)

    def _is_actual_child_document(self, document, potential_child):
        if document == potential_child:
            return False

        document_layer_idx = self._layer_order.index(document.layer)
        child_layer_idx = self._layer_order.index(potential_child.layer)

        parent_selector = potential_child.parent_selector
        labels = document.labels
        # Labels are key-value pairs which are unhashable, so use ``all``
        # instead.
        is_actual_child = all(
            labels.get(x) == y for x, y in parent_selector.items())

        if is_actual_child:
            # Documents with different `schema`s are never layered together,
            # so consider only documents with same schema as candidates.
            if potential_child.schema != document.schema:
                reason = ('Child has parentSelector which references parent, '
                          'but their `schema`s do not match.')
                LOG.error(reason)
                raise errors.InvalidDocumentParent(
                    parent_schema=document.schema, parent_name=document.name,
                    document_schema=potential_child.schema,
                    document_name=potential_child.name, reason=reason)

            # The highest order is 0, so the parent should be lower than the
            # child.
            if document_layer_idx >= child_layer_idx:
                reason = ('Child has parentSelector which references parent, '
                          'but the child layer %s must be lower than the '
                          'parent layer %s for layerOrder %s.' % (
                              potential_child.layer, document.layer,
                              ', '.join(self._layer_order)))
                LOG.error(reason)
                raise errors.InvalidDocumentParent(
                    parent_schema=document.schema, parent_name=document.name,
                    document_schema=potential_child.schema,
                    document_name=potential_child.name, reason=reason)

        return is_actual_child

    def _calc_document_children(self, document):
        potential_children = []
        for label_key, label_val in document.labels.items():
            _potential_children = self._documents_by_labels.get(
                (label_key, label_val), [])
            potential_children.extend(_potential_children)
        # NOTE(fmontei): The intention here is to preserve the order of all
        # the documents that were sorted by `_topologically_sort_documents`
        # in order to substitute documents in the right order. But at the same
        # time, only unique children should be found. So, this trick below
        # maintains the order (unlike set) and guarantees uniqueness.
        unique_potential_children = collections.OrderedDict.fromkeys(
            potential_children)

        for potential_child in unique_potential_children:
            if self._is_actual_child_document(document, potential_child):
                yield potential_child

    def _calc_all_document_children(self):
        """Determine each document's children.

        For each document, attempts to find the document's children. Adds a new
        key called "children" to the document's dictionary.

        .. note::

            A document should only have exactly one parent.

            If a document does not have a parent, then its layer must be
            the topmost layer defined by the ``layerOrder``.

        :returns: Ordered list of documents that need to be layered. Each
            document contains a "children" property in addition to original
            data. List of documents returned is ordered from highest to lowest
            layer.
        :rtype: List[:class:`DocumentDict`]
        :raises IndeterminateDocumentParent: If more than one parent document
            was found for a document.
        """
        # ``all_children`` is a counter utility for verifying that each
        # document has exactly one parent.
        all_children = collections.Counter()
        # Mapping of (doc.name, doc.metadata.name) => children, where children
        # are the documents whose `parentSelector` references the doc.
        self._children = {}
        self._parents = {}
        self._parentless_documents = []

        for layer in self._layer_order:
            documents_in_layer = self._documents_by_layer.get(layer, [])
            for document in documents_in_layer:
                children = list(self._calc_document_children(document))
                self._children[(document.name, document.schema)] = children

                if children:
                    all_children.update(children)

                for child in children:
                    self._replace_older_parent_with_younger_parent(
                        child, document, all_children)

        all_children_elements = list(all_children.elements())
        secondary_documents = []
        for layer, documents in self._documents_by_layer.items():
            if self._layer_order and layer != self._layer_order[0]:
                secondary_documents.extend(documents)

        for doc in secondary_documents:
            # Unless the document is the topmost document in the
            # `layerOrder` of the LayeringPolicy, it should be a child document
            # of another document.
            if doc not in all_children_elements:
                LOG.info('Could not find parent for document with name=%s, '
                         'schema=%s, layer=%s, parentSelector=%s.',
                         doc.name, doc.schema, doc.layer, doc.parent_selector)
                self._parentless_documents.append(doc)
            # If the document is a child document of more than 1 parent, then
            # the document has too many parents, which is a validation error.
            elif all_children[doc] > 1:
                LOG.info('%d parent documents were found for child document '
                         'with name=%s, schema=%s, layer=%s, parentSelector=%s'
                         '. Each document must have exactly 1 parent.',
                         all_children[doc], doc.name, doc.schema, doc.layer,
                         doc.parent_selector)
                raise errors.IndeterminateDocumentParent(document=doc)

    def _get_layering_order(self, layering_policy):
        # Pre-processing stage that removes empty layers from the
        # ``layerOrder`` in the layering policy.
        layer_order = list(layering_policy.layer_order)
        for layer in layer_order[:]:
            documents_by_layer = self._documents_by_layer.get(layer, [])
            if not documents_by_layer:
                LOG.info('%s is an empty layer with no documents. It will be '
                         'discarded from the layerOrder during the layering '
                         'process.', layer)
                layer_order.remove(layer)
        if not layer_order:
            LOG.info('Either the layerOrder in the LayeringPolicy was empty '
                     'to begin with or no document layers were found in the '
                     'layerOrder, causing it to become empty. No layering '
                     'will be performed.')
        return layer_order

    def _topologically_sort_documents(self, documents):
        """Topologically sorts the DAG formed from the documents' substitution
        dependency chain.
        """
        documents_by_name = {}
        result = []

        g = networkx.DiGraph()
        for document in documents:
            document = document_wrapper.DocumentDict(document)
            documents_by_name.setdefault((document.schema, document.name),
                                         document)
            for sub in document.substitutions:
                g.add_edge((document.schema, document.name),
                           (sub['src']['schema'], sub['src']['name']))

        try:
            cycle = find_cycle(g)
        except networkx.exception.NetworkXNoCycle:
            pass
        else:
            LOG.error('Cannot determine substitution order as a dependency '
                      'cycle exists for the following documents: %s.', cycle)
            raise errors.SubstitutionDependencyCycle(cycle=cycle)

        sorted_documents = reversed(list(topological_sort(g)))

        for document in sorted_documents:
            if document in documents_by_name:
                result.append(documents_by_name.pop(document))
        for document in documents_by_name.values():
            result.append(document)

        return result

    def _validate_documents(self, documents):
        LOG.debug('%s performing document pre-validation.',
                  self.__class__.__name__)
        validator = document_validation.DocumentValidation(
            documents, pre_validate=True)
        results = validator.validate_all()
        val_errors = []
        for result in results:
            val_errors.extend(
                [(e['schema'], e['name'], e['message'])
                    for e in result['errors']])
        if val_errors:
            for error in val_errors:
                LOG.error(
                    'Document [%s] %s failed with pre-validation error: %s.',
                    *error)
            raise errors.InvalidDocumentFormat(
                details='The following pre-validation errors occurred '
                        '(schema, name, error): %s.' % val_errors)

    def __init__(self, documents, substitution_sources=None, validate=True):
        """Contructor for ``DocumentLayering``.

        :param layering_policy: The document with schema
            ``deckhand/LayeringPolicy`` needed for layering.
        :param documents: List of all other documents to be layered together
            in accordance with the ``layerOrder`` defined by the
            LayeringPolicy document.
        :type documents: List[dict]
        :param substitution_sources: List of documents that are potential
            sources for substitution. Should only include concrete documents.
        :type substitution_sources: List[dict]
        :param validate: Whether to pre-validate documents using built-in
            schema validation. Default is True.
        :type validate: bool

        :raises LayeringPolicyNotFound: If no LayeringPolicy was found among
            list of ``documents``.
        :raises InvalidDocumentLayer: If document layer not found in layerOrder
            for provided LayeringPolicy.
        :raises InvalidDocumentParent: If child references parent but they
            don't have the same schema or their layers are incompatible.
        :raises IndeterminateDocumentParent: If more than one parent document
            was found for a document.
        """
        self._documents_to_layer = []
        self._documents_by_layer = {}
        self._documents_by_labels = {}
        self._layering_policy = None

        if validate:
            self._validate_documents(documents)

        layering_policies = list(
            filter(lambda x: x.get('schema').startswith(
                   types.LAYERING_POLICY_SCHEMA), documents))
        if layering_policies:
            self._layering_policy = document_wrapper.DocumentDict(
                layering_policies[0])
            if len(layering_policies) > 1:
                LOG.warning('More than one layering policy document was '
                            'passed in. Using the first one found: [%s] %s.',
                            self._layering_policy.schema,
                            self._layering_policy.name)

        if self._layering_policy is None:
            error_msg = (
                'No layering policy found in the system so could not render '
                'documents.')
            LOG.error(error_msg)
            raise errors.LayeringPolicyNotFound()

        sorted_documents = self._topologically_sort_documents(documents)

        for document in sorted_documents:
            document = document_wrapper.DocumentDict(document)
            if document.layering_definition:
                self._documents_to_layer.append(document)
            if document.layer:
                if document.layer not in self._layering_policy.layer_order:
                    LOG.error('Document layer %s for document [%s] %s not '
                              'in layerOrder: %s.', document.layer,
                              document.schema, document.name,
                              self._layering_policy.layer_order)
                    raise errors.InvalidDocumentLayer(
                        document_layer=document.layer,
                        document_schema=document.schema,
                        document_name=document.name,
                        layer_order=', '.join(
                            self._layering_policy.layer_order),
                        layering_policy_name=self._layering_policy.name)
                self._documents_by_layer.setdefault(document.layer, [])
                self._documents_by_layer[document.layer].append(document)
            if document.parent_selector:
                for label_key, label_val in document.parent_selector.items():
                    self._documents_by_labels.setdefault(
                        (label_key, label_val), [])
                    self._documents_by_labels[
                        (label_key, label_val)].append(document)

        self._layer_order = self._get_layering_order(self._layering_policy)
        self._calc_all_document_children()
        self._substitution_sources = substitution_sources or []
        self.secrets_substitution = secrets_manager.SecretsSubstitution(
            self._substitution_sources)

        del self._documents_by_layer
        del self._documents_by_labels

    def _apply_action(self, action, child_data, overall_data):
        """Apply actions to each layer that is rendered.

        Supported actions include:

            * `merge` - a "deep" merge that layers new and modified data onto
              existing data
            * `replace` - overwrite data at the specified path and replace it
              with the data given in this document
            * `delete` - remove the data at the specified path

        :raises UnsupportedActionMethod: If the layering action isn't found
            among ``self.SUPPORTED_METHODS``.
        :raises MissingDocumentKey: If a layering action path isn't found
            in both the parent and child documents being layered together.
        """
        method = action['method']
        if method not in self.SUPPORTED_METHODS:
            raise errors.UnsupportedActionMethod(
                action=action, document=child_data)

        # Use copy to prevent these data from being updated referentially.
        overall_data = copy.deepcopy(overall_data)
        child_data = copy.deepcopy(child_data)
        rendered_data = overall_data

        # Remove empty string paths and ensure that "data" is always present.
        path = action['path'].split('.')
        path = [p for p in path if p != '']
        path.insert(0, 'data')
        last_key = 'data' if not path[-1] else path[-1]

        for attr in path:
            if attr == path[-1]:
                break
            rendered_data = rendered_data.get(attr)
            child_data = child_data.get(attr)

        if method == 'delete':
            # If the entire document is passed (i.e. the dict including
            # metadata, data, schema, etc.) then reset data to an empty dict.
            if last_key == 'data':
                rendered_data['data'] = {}
            elif last_key in rendered_data:
                del rendered_data[last_key]
            elif last_key not in rendered_data:
                # If the key does not exist in `rendered_data`, this is a
                # validation error.
                raise errors.MissingDocumentKey(
                    child=child_data, parent=rendered_data, key=last_key)
        elif method == 'merge':
            if last_key in rendered_data and last_key in child_data:
                # If both entries are dictionaries, do a deep merge. Otherwise
                # do a simple merge.
                if (isinstance(rendered_data[last_key], dict)
                    and isinstance(child_data[last_key], dict)):
                    engine_utils.deep_merge(
                        rendered_data[last_key], child_data[last_key])
                else:
                    rendered_data.setdefault(last_key, child_data[last_key])
            elif last_key in child_data:
                rendered_data.setdefault(last_key, child_data[last_key])
            else:
                # If the key does not exist in the child document, this is a
                # validation error.
                raise errors.MissingDocumentKey(
                    child=child_data, parent=rendered_data, key=last_key)
        elif method == 'replace':
            if last_key in rendered_data and last_key in child_data:
                rendered_data[last_key] = child_data[last_key]
            elif last_key in child_data:
                rendered_data.setdefault(last_key, child_data[last_key])
            elif last_key not in child_data:
                # If the key does not exist in the child document, this is a
                # validation error.
                raise errors.MissingDocumentKey(
                    child=child_data, parent=rendered_data, key=last_key)

        return overall_data

    def _get_children(self, document):
        """Recursively retrieve all children.

        Used in the layering module when calculating children for each
        document.

        :returns: List of nested children.
        :rtype: Generator[:class:`DocumentDict`]
        """
        for child in self._children.get((document.name, document.schema), []):
            yield child
            grandchildren = self._get_children(child)
            for grandchild in grandchildren:
                yield grandchild

    def render(self):
        """Perform layering on the list of documents passed to ``__init__``.

        Each concrete document will undergo layering according to the actions
        defined by its ``metadata.layeringDefinition``. Documents are layered
        with their parents. A parent document's ``schema`` must match that of
        the child, and its ``metadata.labels`` must much the child's
        ``metadata.layeringDefinition.parentSelector``.

        :returns: The list of concrete rendered documents.
        :rtype: List[dict]

        :raises UnsupportedActionMethod: If the layering action isn't found
            among ``self.SUPPORTED_METHODS``.
        :raises MissingDocumentKey: If a layering action path isn't found
            in both the parent and child documents being layered together.
        """
        # ``rendered_data_by_layer`` tracks the set of changes across all
        # actions across each layer for a specific document.
        rendered_data_by_layer = document_wrapper.DocumentDict()

        # NOTE(fmontei): ``global_docs`` represents the topmost documents in
        # the system. It should probably be impossible for more than 1
        # top-level doc to exist, but handle multiple for now.
        global_docs = [
            doc for doc in self._documents_to_layer
            if self._layer_order and doc.layer == self._layer_order[0]
        ]

        for doc in global_docs:
            layer_idx = self._layer_order.index(doc.layer)
            if doc.substitutions:
                substituted_data = list(
                    self.secrets_substitution.substitute_all(doc))
                if substituted_data:
                    rendered_data_by_layer[layer_idx] = substituted_data[0]
            else:
                rendered_data_by_layer[layer_idx] = doc

            # Keep iterating as long as a child exists.
            for child in self._get_children(doc):
                # Retrieve the most up-to-date rendered_data (by referencing
                # the child's parent's data).
                child_layer_idx = self._layer_order.index(child.layer)
                parent = self._parents[child.name, child.schema]
                parent_layer_idx = self._layer_order.index(parent.layer)
                rendered_data = rendered_data_by_layer[parent_layer_idx]

                # Apply each action to the current document.
                for action in child.actions:
                    LOG.debug('Applying action %s to child document with '
                              'name=%s, schema=%s, layer=%s.', action,
                              child.name, child.schema, child.layer)
                    rendered_data = self._apply_action(
                        action, child, rendered_data)

                # Update the actual document data if concrete.
                if not child.is_abstract:
                    child_index = self._documents_to_layer.index(child)
                    child.data = rendered_data.data
                    substituted_data = list(
                        self.secrets_substitution.substitute_all(child))
                    if substituted_data:
                        rendered_data = substituted_data[0]
                    self._documents_to_layer[child_index].data = (
                        rendered_data.data)

                # Update ``rendered_data_by_layer`` for this layer so that
                # children in deeper layers can reference the most up-to-date
                # changes.
                rendered_data_by_layer[child_layer_idx] = rendered_data

        # Handle edge case for parentless documents that require substitution.
        # If a document has no parent, then the for loop above doesn't iterate
        # over the parentless document, so substitution must be done here for
        # parentless documents.
        for doc in self._parentless_documents:
            if not doc.is_abstract and doc.substitutions:
                substituted_data = list(
                    self.secrets_substitution.substitute_all(doc))
                if substituted_data:
                    doc = substituted_data[0]

        # Return only concrete documents.
        return [d for d in self._documents_to_layer if d.is_abstract is False]

max_stars_repo_name: att-comdev/test-submit

max_issues_repo_name: att-comdev/test-submit

max_forks_repo_name: att-comdev/test-submit

reason: import networkx,from networkx


text:
# Copyright 2016 The TensorFlow Authors All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Inference routine for k-cardinality-constrained graphical models.

This implements the "FFT Tree" algorithm from
Tarlow et al., "Fast Exact Inference for Recursive Cardinality Models"
http://www.cs.toronto.edu/~dtarlow/tszaf-fast_cardinality.pdf
"""

from __future__ import absolute_import

import numpy as np
import tensorflow as tf

from shared_util import create_mask
from shared_util import normalize
from shared_util import repeat
from shared_util import sample_categorical

def fft_tree_indep_vars(yp,
                        zp,
                        num_samples,
                        batch_size,
                        max_len,
                        k_constraints=None,
                        lam=0.001):
  """Runs FFT Tree inference on a set of (otherwise) independent variables.

  Args:
    yp: The unary potentials for the binary variables y_i.
        This should have shape [batch_size, num_y].
    zp: The cardinality potential (log-scores for allowable k)
        This should have shape [batch_size, num_y+1].
    num_samples: # of samples to return from inference. Scalar.
    batch_size: batch size.
    max_len: max length of input sequence.
    k_constraints: If the z-potentials have a hard cutoff for some
        k-sparsity, we can be much more numerically stable
        by sparsifying the intermediate messages before normalizing.
    lam: lambda for message-damping, improves numerical stability
        at the cost of some accuracy in the inference/sampling

  Returns:
    marginals: Marginal distribution for each y_i.
               This should have shape [batch_size, num_y].
    samples: A set of samples for each y_i.
             This should have shape [batch_size, num_samples, num_y].
    log_z: Log normalizing constant for distribution.
           This should have shape [batch_size].
  """

  tree_depth = np.ceil(np.log(max_len) / np.log(2.0)).astype(np.int32) + 1
  width = np.power(2, tree_depth).astype(np.int32)

  # have to pad it to always take on zeros on the pad values
  yp_unary_padded = tf.concat(
      1, [yp, tf.ones([batch_size, width / 2 - max_len]) * -100.0])
  yp_stitched = tf.reshape(
      tf.concat(1, [tf.zeros([batch_size * width / 2, 1]),
                    tf.reshape(yp_unary_padded, [-1, 1])]), [batch_size, -1])
  yb_padded = tf.exp(yp_stitched)

  log_z = 0

  yb_padded = tf.concat(0, tf.split(1, width / 2, yb_padded))
  yb_padded, yb_z = normalize(yb_padded)

  yb_padded = tf.concat(1, tf.split(0, width / 2, yb_padded))

  yb_z = tf.concat(1, tf.split(0, width / 2, yb_z))

  with tf.name_scope("initial_log_z"):
    log_z += tf.expand_dims(tf.reduce_sum(tf.log(yb_z), 1), 1)

  upward_msgs = []
  upward_msgs.append(yb_padded)

  prev_msg = yb_padded

  with tf.name_scope("upward_msgs"):
    for d in xrange(1, tree_depth):
      # for lv 0, we want blocks of 4
      block_size = np.power(2, d + 1)
      num_splits = width / block_size
      sub_probs = tf.split(1, num_splits, prev_msg)

      left, right = tf.split(1, 2, tf.concat(0, sub_probs))
      left = tf.pad(left, [[0, 0], [0, block_size / 2]])
      right = tf.pad(right, [[0, 0], [0, block_size / 2]])
      convs = positive_conv(left, right)

      new_msg, new_msg_z = normalize(convs)
      new_msg_z = tf.concat(1, tf.split(0, num_splits, new_msg_z))
      log_z += tf.expand_dims(tf.reduce_sum(tf.log(new_msg_z), 1), 1)

      if k_constraints is not None:
        k_mask = create_mask(
            tf.reshape(
                repeat(num_splits, tf.expand_dims(
                    tf.minimum(2**d + 1, k_constraints + 1), 1)), [-1]),
            block_size)
        new_msg *= k_mask
        k_uni = k_mask / tf.maximum(
            tf.expand_dims(tf.reduce_sum(k_mask, 1), 1), 1.0)
        new_msg = new_msg * (1.0 - lam) + k_uni * lam
      else:
        uni_mask = create_mask(
            tf.reshape(
                repeat(num_splits, tf.tile(
                    tf.expand_dims(tf.reshape(2**d + 1, [-1]), 1),
                    [batch_size, 1])), [-1]), block_size)
        uni = uni_mask / tf.maximum(
            tf.expand_dims(tf.reduce_sum(uni_mask, 1), 1), 1.0)
        new_msg = new_msg * (1.0 - lam) + uni * lam

      new_msg = tf.concat(1, tf.split(0, num_splits, new_msg))
      upward_msgs.append(new_msg)
      prev_msg = new_msg

  with tf.name_scope("k_beliefs"):
    k_pot_msg = tf.pad(tf.exp(zp), [[0, 0], [0, width - max_len - 1]])
    k_pot_msg, k_pot_z = normalize(k_pot_msg)
    log_z += tf.log(k_pot_z)

    all_marginals = []

    k_belief, k_z = normalize(k_pot_msg * prev_msg)
    log_k_z = tf.log(k_z)

  log_z += log_k_z

  all_marginals.append(k_belief)

  samples = []
  sample_indices = []

  with tf.name_scope("k_samples"):
    if num_samples > 0:
      rep_k_b = repeat(num_samples, k_belief)
      k_samples, k_indices = sample_categorical(tf.log(rep_k_b))
      samples.append(k_samples)
      sample_indices.append(k_indices)
      prev_indices = k_indices

  downward_msgs = []
  downward_msgs.append(k_pot_msg)
  prev_msg = k_pot_msg

  with tf.name_scope("bwd_msgs"):
    for d in reversed(xrange(1, tree_depth)):
      up_msgs = upward_msgs[d - 1]
      block_size = np.power(2, d + 1)
      num_splits = width / block_size
      prev_msgs = tf.concat(0, tf.split(1, num_splits, prev_msg))
      left_right_up_msgs = tf.concat(0, tf.split(1, num_splits, up_msgs))
      left_up_msgs, right_up_msgs = tf.split(1, 2, left_right_up_msgs)

      if num_samples > 0:
        reshaped_prev_indices = tf.concat(0, tf.split(1, num_splits,
                                                      prev_indices))
        prev_domain_lens = reshaped_prev_indices + 1
        rep_up_msgs = repeat(num_samples, up_msgs)
        left_up_msgs_repl, right_up_msgs_repl = tf.split(1, 2, tf.concat(
            0, tf.split(1, num_splits, rep_up_msgs)))

        num_rows = num_splits * batch_size * num_samples

        len_mask = create_mask(
            tf.reshape(prev_domain_lens, [-1]), block_size / 2)
        right_up_msgs_repl *= len_mask
        right_up_msgs_repl = tf.pad(right_up_msgs_repl,
                                    [[0, 0], [0, block_size / 2]])
        left_up_msgs_repl *= len_mask
        left_up_msgs_repl = tf.pad(left_up_msgs_repl,
                                   [[0, 0], [0, block_size / 2]])
        left_up_msgs_repl, _ = normalize(left_up_msgs_repl)

        right_up_msgs_repl = tf.reverse_sequence(right_up_msgs_repl, tf.reshape(
            tf.to_int64(prev_domain_lens), [-1]), 1)
        right_up_msgs_repl, _ = normalize(right_up_msgs_repl)

        sliced_factor_marg, _ = normalize(left_up_msgs_repl *
                                          right_up_msgs_repl)
        sliced_factor_marg = tf.slice(sliced_factor_marg, [0, 0],
                                      [num_rows, block_size / 2])

        left_samples, left_indices = sample_categorical(
            tf.log(sliced_factor_marg))
        left_samples = tf.slice(left_samples, [0, 0],
                                [num_rows, block_size / 2])

        right_indices = prev_domain_lens - left_indices - 1
        right_samples = tf.one_hot(
            tf.reshape(right_indices, [-1]), block_size / 2)

        cur_samples = tf.concat(1, tf.split(0, num_splits, tf.concat(
            1, [left_samples, right_samples])))
        cur_indices = tf.concat(1, tf.split(0, num_splits, tf.concat(
            1, [left_indices, right_indices])))
        samples.append(cur_samples)
        sample_indices.append(cur_indices)
        prev_indices = cur_indices

      left_down_msgs = positive_correl(prev_msgs,
                                       tf.pad(right_up_msgs,
                                              [[0, 0], [0, block_size / 2]]))
      left_down_msgs = tf.slice(left_down_msgs, [0, 0],
                                [num_splits * batch_size, block_size / 2])
      left_down_msgs, _ = normalize(left_down_msgs)
      left_margs, _ = normalize(left_up_msgs * left_down_msgs)

      right_down_msgs = positive_correl(prev_msgs,
                                        tf.pad(left_up_msgs,
                                               [[0, 0], [0, block_size / 2]]))
      right_down_msgs = tf.slice(right_down_msgs, [0, 0],
                                 [num_splits * batch_size, block_size / 2])
      right_down_msgs, _ = normalize(right_down_msgs)
      right_margs, _ = normalize(right_up_msgs * right_down_msgs)

      new_msg = tf.concat(1, tf.split(0, num_splits, tf.concat(
          1, [left_down_msgs, right_down_msgs])))
      margs = tf.concat(1, tf.split(0, num_splits,
                                    tf.concat(1, [left_margs, right_margs])))

      all_marginals.append(margs)
      downward_msgs.append(new_msg)
      prev_msg = new_msg

  final_marginals = all_marginals[-1]

  final_unary_marginals = tf.reshape(
      tf.split(1, 2, tf.reshape(final_marginals, [-1, 2]))[1], [batch_size, -1])

  final_samples = samples[-1]
  final_unary_samples = tf.reshape(
      tf.split(1, 2, tf.reshape(final_samples, [-1, 2]))[1],
      [num_samples, batch_size, -1])

  final_unary_samples = tf.transpose(final_unary_samples, [1, 0, 2])

  final_unary_marginals = tf.slice(final_unary_marginals, [0, 0],
                                   [batch_size, max_len])
  final_unary_samples = tf.slice(final_unary_samples, [0, 0, 0],
                                 [batch_size, num_samples, max_len])

  return final_unary_marginals, final_unary_samples, log_z

def positive_conv(a, b):
  """Pairwise convolution on the positive domain of batches of 1-d vectors.

  Args:
    a: discrete function on the positive domain (e.g. real-valued vector
       with a[0] = f(0), etc). Shape of [batch_size, domain_size].
    b: same as a.
  Returns:
    Discrete function on positive domain representing convolution of a and b.

  """
  batch_size = a.get_shape().dims[0].value
  width = a.get_shape().dims[1].value
  a = tf.pad(a, [[0, 0], [width, 0]])
  a = tf.transpose(a)
  b = tf.pad(b, [[0, 0], [width, 0]])
  b = tf.reverse(b, [False, True])
  b = tf.transpose(b)
  reshaped_a = tf.reshape(a, [1, 1, width * 2, batch_size])
  reshaped_b = tf.reshape(b, [1, width * 2, batch_size, 1])
  res = tf.nn.depthwise_conv2d(
      reshaped_a, reshaped_b, strides=[1, 1, 1, 1], padding="SAME")
  res = tf.reshape(tf.transpose(res), [batch_size, width * 2])
  res = tf.slice(res, [0, width], [batch_size, width])
  return res

def positive_correl(a, b):
  """Pairwise cross-correlation of 1-d vectors on the positive domain.

  Args:
    a: discrete function batch on the positive domain (e.g. real-valued vector
       with a[0] = f(0), etc). Shape of [batch_size, domain_size].
    b: same as a.
  Returns:
    Discrete function on positive domain representing correlation of a and b.

  """
  batch_size = a.get_shape().dims[0].value
  width = a.get_shape().dims[1].value
  a = tf.pad(a, [[0, 0], [width - 1, 1]])
  a = tf.transpose(a)
  b = tf.pad(b, [[0, 0], [width - 1, 1]])
  b = tf.transpose(b)
  reshaped_a = tf.reshape(a, [1, 1, width * 2, batch_size])
  reshaped_b = tf.reshape(b, [1, width * 2, batch_size, 1])
  res = tf.nn.depthwise_conv2d(
      reshaped_a, reshaped_b, strides=[1, 1, 1, 1], padding="SAME")
  res = tf.reshape(tf.transpose(res), [batch_size, width * 2])
  res = tf.slice(res, [0, width - 1], [batch_size, width])
  return res


text:
import tensorflow as tf
import numpy as np
import copy

from tf_metric_learning.layers import SoftTripleLoss
from tf_metric_learning.utils.projector import TBProjectorCallback
from tf_metric_learning.utils.recall import AnnoyEvaluatorCallback
from tf_metric_learning.utils.constants import *

def normalize_images(images):
    return images / 255.0

# load data images
BATCH_SIZE = 32
embedding_size, num_class, num_centers = 64, 10, 10
input_shape = (32, 32, 3)
(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.cifar10.load_data()

# define base network for embeddings
inputs = tf.keras.Input(shape=input_shape, name="images")
model = tf.keras.applications.MobileNetV2(input_shape=input_shape, include_top=False, weights="imagenet")(inputs)
pool = tf.keras.layers.GlobalAveragePooling2D()(model)
dropout = tf.keras.layers.Dropout(0.5)(pool)
embeddings = tf.keras.layers.Dense(units=embedding_size)(dropout)
base_network = tf.keras.Model(inputs=inputs, outputs=embeddings)

# define the input and output tensors
input_label = tf.keras.layers.Input(shape=(1,), name="labels")
output_tensor = SoftTripleLoss(num_class, num_centers, embedding_size)(
    {EMBEDDINGS: base_network.outputs[0], LABELS: input_label}
)

# define the model and compile it
model = tf.keras.Model(inputs=[inputs, input_label], outputs=output_tensor)
model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.0001))

train_data = {"images": normalize_images(train_images), "labels": train_labels}

validation_data = {"images": normalize_images(test_images), "labels": test_labels}

# callbacks
tensorboard = tf.keras.callbacks.TensorBoard(log_dir="tb")

projector = TBProjectorCallback(
    base_network,
    "tb/projector",
    copy.deepcopy(test_images),
    np.squeeze(test_labels),
    batch_size=BATCH_SIZE,
    image_size=32,
    normalize_fn=normalize_images,
    normalize_eb=True,
    freq=1,
)

evaluator = AnnoyEvaluatorCallback(
    base_network,
    {"images": validation_data["images"][:5000], "labels": np.squeeze(validation_data["labels"][:5000])},
    {"images": validation_data["images"][5000:], "labels": np.squeeze(validation_data["labels"][5000:])},
    normalize_eb=True,
    eb_size=embedding_size,
    progress=False,
)

model.fit(
    train_data,
    train_labels,
    validation_data=(validation_data, test_labels),
    callbacks=[tensorboard, evaluator, projector],
    shuffle=True,
    epochs=20,
    batch_size=BATCH_SIZE,
)

max_stars_repo_name: Ximilar-com/tf-metric-learning

max_issues_repo_name: Ximilar-com/tf-metric-learning

max_forks_repo_name: Ximilar-com/tf-metric-learning


text:
# Copyright 2018 Tensorforce Team. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

from collections import OrderedDict

import numpy as np
import tensorflow as tf

from tensorforce import TensorforceError, util
from tensorforce.core.models import Model

class RandomModel(Model):
    """
    Utility class to return random actions of a desired shape and with given bounds.
    """

    def __init__(
        self,
        # Model
        name, device, parallel_interactions, buffer_observe, seed, summarizer, config, states,
        actions
    ):
        super().__init__(
            # Model
            name=name, device=None, parallel_interactions=parallel_interactions,
            buffer_observe=buffer_observe, seed=seed, execution=None, saver=None,
            summarizer=summarizer, config=config, states=states, internals=OrderedDict(),
            actions=actions, preprocessing=None, exploration=0.0, variable_noise=0.0,
            l2_regularization=0.0
        )

    def tf_core_act(self, states, internals, auxiliaries):
        if len(internals) > 0:
            raise TensorforceError.unexpected()

        actions = OrderedDict()
        for name, spec in self.actions_spec.items():
            batch_size = tf.shape(input=next(iter(states.values())))[0:1]
            shape = tf.constant(value=spec['shape'], dtype=tf.int32)
            shape = tf.concat(values=(batch_size, shape), axis=0)
            dtype = util.tf_dtype(dtype=spec['type'])

            if spec['type'] == 'bool':
                actions[name] = tf.math.less(
                    x=tf.random.uniform(shape=shape, dtype=util.tf_dtype(dtype='float')),
                    y=tf.constant(value=0.5, dtype=util.tf_dtype(dtype='float'))
                )

            elif spec['type'] == 'int':
                # (Same code as for Exploration)
                int_dtype = util.tf_dtype(dtype='int')
                float_dtype = util.tf_dtype(dtype='float')

                # Action choices
                choices = list(range(spec['num_values']))
                choices_tile = ((1,) + spec['shape'] + (1,))
                choices = np.tile(A=[choices], reps=choices_tile)
                choices_shape = ((1,) + spec['shape'] + (spec['num_values'],))
                choices = tf.constant(value=choices, dtype=int_dtype, shape=choices_shape)
                ones = tf.ones(shape=(len(spec['shape']) + 1,), dtype=int_dtype)
                batch_size = tf.dtypes.cast(x=shape[0:1], dtype=int_dtype)
                multiples = tf.concat(values=(batch_size, ones), axis=0)
                choices = tf.tile(input=choices, multiples=multiples)

                # Random unmasked action
                mask = auxiliaries[name + '_mask']
                num_values = tf.math.count_nonzero(input=mask, axis=-1, dtype=int_dtype)
                action = tf.random.uniform(shape=shape, dtype=float_dtype)
                action = tf.dtypes.cast(
                    x=(action * tf.dtypes.cast(x=num_values, dtype=float_dtype)), dtype=int_dtype
                )

                # Correct for masked actions
                choices = tf.boolean_mask(tensor=choices, mask=mask)
                offset = tf.math.cumsum(x=num_values, axis=-1, exclusive=True)
                actions[name] = tf.gather(params=choices, indices=(action + offset))

            elif spec['type'] == 'float':
                if 'min_value' in spec:
                    min_value = tf.constant(value=spec['min_value'], dtype=dtype)
                    max_value = tf.constant(value=spec['max_value'], dtype=dtype)
                    actions[name] = tf.random.uniform(
                        shape=shape, minval=min_value, maxval=max_value, dtype=dtype
                    )

                else:
                    actions[name] = tf.random.uniform(shape=shape, dtype=dtype)

        return actions, OrderedDict()

    def tf_core_observe(self, states, internals, auxiliaries, actions, terminal, reward):
        return util.no_operation()

max_stars_repo_name: stheid/tensorforce

max_issues_repo_name: stheid/tensorforce

max_forks_repo_name: stheid/tensorforce


text:
import os
import pickle
from unittest import TestCase

import numpy as np
import pandas as pd
import torch

from WordPairGenerator import WordPairGenerator

class WordEmbeddingFake():
    def __init__(self, device='auto'):
        pass

    def get_word_embeddings(self, words):
        words_list = words.split()
        word_embedding = np.random.random([len(words_list), 50])

        return torch.tensor(word_embedding), words_list

class TestWordPairGenerator(TestCase):

    def test_process_df(self):
        we = WordEmbeddingFake()
        words_pairs_dict, emb_pairs_dict = {}, {}
        train = pd.read_csv(os.path.join('G:\\Drive condivisi\\SoftLab\\Dataset\\Entity Matching\\BeerAdvo-RateBeer',
                                         'train_merged.csv'))
        self.model_files_path = os.path.join(
            'G:\\Drive condivisi\\SoftLab\\Projects\\Concept level EM (exclusive-inclluse words)\\dataset_files\\BeerAdvo-RateBeer\\BERT\\')
        self.embeddings = {}
        self.words = {}
        for df_name in ['table_A', 'table_B']:
            tmp_path = os.path.join(self.model_files_path, 'emb_' + df_name + '.csv')
            with open(tmp_path, 'rb') as file:
                self.embeddings[df_name] = pickle.load(file)
            tmp_path = os.path.join(self.model_files_path, 'words_list_' + df_name + '.csv')
            with open(tmp_path, 'rb') as file:
                self.words[df_name] = pickle.load(file)

        word_pair_generator = WordPairGenerator(self.words, self.embeddings, df=train)

        word_pairs, emb_pairs = word_pair_generator.process_df(train.sample(100))

        df = pd.DataFrame(word_pairs)

        self.assertIsInstance(df, pd.DataFrame)

    def test_stable_marriage(self):
        A = np.array([[0, 2, 1, 3, 4],
                      [4, 2, 1, 0, 3],
                      [1, 4, 2, 3, 0]])
        B = np.array([[1, 0, 2],
                      [1, 2, 0],
                      [0, 1, 2],
                      [0, 2, 1],
                      [2, 0, 1]])
        res = WordPairGenerator.stable_marriage(A, B)
        self.assertIsInstance(res, np.ndarray)
        self.assertTrue(res.shape == (3, 2))

    def test_most_similar_pairs(self):
        sim_mat = np.array([[.2, .3, .6],
                            [.5, .2, .7],
                            [.2, .1, .15],
                            [.5, .6, .3]])
        pairs, sim = WordPairGenerator.most_similar_pairs(sim_mat=sim_mat)
        self.assertTrue(pairs.shape == (4, 2))

max_stars_repo_name: softlab-unimore/WYM

max_issues_repo_name: softlab-unimore/WYM

max_forks_repo_name: softlab-unimore/WYM


text:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
plt.style.use('seaborn-darkgrid')
import argparse
import csv
import pandas
import os
import sys
import pickle
import numpy as np
from os.path import join

# matplotlib
titlesize = 33
xsize = 30
ysize = 30
ticksize = 25
legendsize = 25
error_region_alpha = 0.25

def smoothed(x, w):
    """Smooth x by averaging over sliding windows of w, assuming sufficient length.
    """
    if len(x) <= w:
        return x
    smooth = []
    for i in range(1, w):
        smooth.append( np.mean(x[0:i]) )
    for i in range(w, len(x)+1):
        smooth.append( np.mean(x[i-w:i]) )
    assert len(x) == len(smooth), "lengths: {}, {}".format(len(x), len(smooth))
    return np.array(smooth)

def _get_stuff_from_monitor(mon):
    """Get stuff from `monitor` log files.

    Monitor files are named `0.envidx.monitor.csv` and have one line for each
    episode that finished in that CPU 'core', with the reward, length (number
    of steps) and the time (in seconds). The lengths are not cumulative, but
    time is cumulative.
    """
    scores = []
    steps  = []
    times  = []
    with open(mon, 'r') as csv_file:
        csv_reader = csv.reader(csv_file, delimiter=',')
        line_count = 0
        for csv_row in csv_reader:
            # First two lines don't contain interesting stuff.
            if line_count == 0 or line_count == 1:
                line_count += 1
                continue
            scores.append(float(csv_row[0]))
            steps.append(int(csv_row[1]))
            times.append(float(csv_row[2]))
            line_count += 1
    print("finished: {}".format(mon))
    return scores, steps, times

def plot(args):
    """Load monitor curves and the progress csv file. And plot from those.
    """
    nrows, ncols = 1, 2
    fig, ax = plt.subplots(nrows, ncols, squeeze=False, sharey=True, figsize=(11*ncols,7*nrows))
    title = args.title

    # Global statistics across all monitors
    scores_all = []
    steps_all = []
    times_all = []
    total_train_steps = 0
    train_hours = 0

    monitors = sorted(
        [x for x in os.listdir(args.path) if 'monitor.csv' in x and '.swp' not in x]
    )
    progfile = join(args.path,'progress.csv')

    # First row, info from all the monitors, i.e., number of CPUs.
    for env_idx,mon in enumerate(monitors):
        monitor_path = join(args.path, mon)
        scores, steps, times = _get_stuff_from_monitor(monitor_path)

        # Now process to see as a function of episodes and training steps, etc.
        num_episodes = len(scores)
        tr_episodes = np.arange(num_episodes)
        tr_steps = np.cumsum(steps)
        tr_times = np.array(times) / 60.0 # get it in minutes

        # Plot for individual monitors.
        envlabel = 'env {}'.format(env_idx)
        sm_10 = smoothed(scores, w=10)
        ax[0,0].plot(tr_steps, sm_10, label=envlabel+'; avg {:.1f} last {:.1f}'.format(
                np.mean(sm_10), sm_10[-1]))
        sm_100 = smoothed(scores, w=100)
        ax[0,1].plot(tr_times, sm_100, label=envlabel+'; avg {:.1f} last {:.1f}'.format(
                np.mean(sm_100), sm_100[-1]))

        # Handle global stuff.
        total_train_steps += tr_steps[-1]
        train_hours = max(train_hours, tr_times[-1] / 60.0)

    # Bells and whistles
    for row in range(nrows):
        for col in range(ncols):
            ax[row,col].set_ylabel("Scores", fontsize=30)
            ax[row,col].tick_params(axis='x', labelsize=25)
            ax[row,col].tick_params(axis='y', labelsize=25)
            leg = ax[row,col].legend(loc="best", ncol=1, prop={'size':25})
            for legobj in leg.legendHandles:
                legobj.set_linewidth(5.0)
    ax[0,0].set_title(title+', Smoothed (w=10)', fontsize=titlesize)
    ax[0,0].set_xlabel("Train Steps (total {})".format(total_train_steps), fontsize=xsize)
    ax[0,1].set_title(title+', Smoothed (w=100)', fontsize=titlesize)
    ax[0,1].set_xlabel("Train Time (in Hours {:.2f})".format(train_hours), fontsize=xsize)
    plt.tight_layout()
    figname = '{}.png'.format(title)
    plt.savefig(figname)
    print("\nJust saved: {}".format(figname))

if __name__ == "__main__":
    pp = argparse.ArgumentParser()
    pp.add_argument('--path', type=str)
    pp.add_argument('--title', type=str)
    args = pp.parse_args()
    plot(args)

max_stars_repo_name: yunshengtian/PGMORL

max_issues_repo_name: yunshengtian/PGMORL

max_forks_repo_name: mit-gfx/PGMORL


text:
# -*- coding: utf-8 -*-
"""
Created on Sat Aug 15 21:45:14 2020
@author: Thomas DeWitt
Contributor: Andrew Lu
"""

# ArmOfState2

import os
import discord
import csv
import pickle
import pandas as pd
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.preprocessing.sequence import pad_sequences
print(tf.__version__)
print(keras.__version__)

print('All modules successfully loaded.  Initializing...')

# Path
dirs = os.getcwd()

# Discord token
def read_token():
    with open(f"{dirs}/token.txt", "r") as file:
        lines = file.readlines()
        return lines[0].strip()

# Format: <string uid> <float 2 score>\n
def load_scores():
    scores = {}
    try:
        with open(f"{dirs}/scores.csv", "r", newline="") as csvfile:
            lines = csv.reader(csvfile)
            for line in lines:
                scores[str(line[0])] = float(line[1])
        return scores
    except:
        with open(f"{dirs}/scores.csv", "w"):
            return scores

def save_scores():  # have it update every 10 minutes or something eventually
    with open(f"{dirs}/scores.csv", "w", newline="") as csvfile:
        save = csv.writer(csvfile)
        for pairs in toxiscores.items():
            save.writerow([str(pairs[0]), str(pairs[1])])

def load_preprocessing():
    with open(f"{dirs}/tokenizer2.pickle", "rb") as handle:
        tokenizer = pickle.load(handle)  # load network tokenizer
    max_length = 200  # cut/pad all sentences to 200 tokens (words)
    trunc_type = 'post' #leave as-is
    padding_type = 'post' #leave as-is
    penalizing_threshold = 0.27 #min individual
    modifier = lambda x : np.round(pow(1.075, -3/8 * x), 2)
    gain = 1
    loss = 1 # Toxicity score modifier & factor modifier should be multiplied by for gaining/losing toxicity, 1 gain/loss default
    return tokenizer, max_length, trunc_type, padding_type, penalizing_threshold, modifier, gain, loss

# Load model and hyperparameters
model = tf.keras.models.load_model(f'{dirs}/classifier/AoS_GPnet_V2') #leave as-is
tokenizer, max_length, trunc_type, padding_type, penalizing_threshold, modifier, gain, loss = load_preprocessing()

# scoring and score management
def score_text(toxic_preds):
    if max(toxic_preds) <= penalizing_threshold:
        score = 0
    else:
        scale = max(toxic_preds)
        rank = np.where(toxic_preds == scale)
        rank = rank[0] + 1
        score = scale * rank
    return score

def manage_toxiscores(uid, message_score):
    if uid not in toxiscores.keys():
        toxiscores[uid] = 0.00

    user_score = toxiscores[uid]

    if message_score <= 0.00:  # Losing toxicity
        user_score -= modifier(user_score) * loss
        user_score = 0.00 if toxiscores[uid] <= 0.00 else user_score
    else:  # Gaining toxicity
        user_score += modifier(user_score) * message_score * gain

    toxiscores[uid] = np.round(user_score, 2)

# Tokens
token = read_token()

# Master users
master_users = ["641816848865689611"]

# List of commands
commands = ["command (authorization): function",
            "^help (standard): Call command list",
            "^hello (standard): Greet the bot",
            "^watchlist (standard): Get top 20 on toxicity watchlist",
            "!#init (master): Load model and preprocessing methods",
            "!#startup (master): Run model",
            "!#standby (master): Pause analysis while keeping bot online",
            "!#scrape (master): Initialize data scraping (for later use in unsupported features)",
            "!#shutdown (master): Terminate runtime on EC2 instance",
            "!#debug (master): Initialize debug mode",
            "!#nodebug (master): Deactivate debug mode"
           ]

# Scraping info (Eventually write to file instead)
messages = []
authors = []

# Model statuses
scrape_messages = False
prep_to_analyze = False
model_ready = False
debug_mode = False

# Load toxicity scores
toxiscores = load_scores()

client = discord.Client()

print('Connected to client.')

@client.event
async def on_message(message):
    # we need these lmao
    global scrape_messages
    global prep_to_analyze
    global model_ready
    global toxiscores
    global debug_mode

    # Don't talk to bots (including self)
    if message.author.bot:
        return

    if message.content.startswith("!#debug"):
        debug_mode = True

    if message.content.startswith("!#nodebug"):
        debug_mode = False

    if message.content.startswith("!#scrape"):
        if str(message.author.id) in master_users:
            scrape_messages = True
            await message.channel.send("WARNING: all messages following this message will be filed for training purpose.")

    if message.content.startswith("!#init"):
        if str(message.author.id) in master_users:
            model_ready = True
            await message.channel.send("Model initialized.  Standing by.")
            #stringlist = []
            #model.summary(print_fn=lambda x: stringlist.append(x))
            #short_model_summary = "\n".join(stringlist)
            #await message.channel.send(short_model_summary)

    if message.content.startswith("!#startup"):
            if str(message.author.id) in master_users:
                if model_ready == True:
                    prep_to_analyze = True
                    await message.channel.send("Analysis framework active.")
                else:
                    await message.channel.send("Model not loaded. Please load model.")

    if message.content.startswith("!#shutdown"): #shutdown bot and disconnect from all servers
        if str(message.author.id) in master_users:
            await message.channel.send("Saving data...")
            data = pd.DataFrame(data={"user": authors, "message": messages})
            data.to_csv(f"{dirs}/discord_messages.csv", sep=",", index=False)  # Change this eventually
            save_scores()
            await message.channel.send("Tschuss.")
            await client.logout()

    if message.content.startswith("!#standby"): #prevent model from running without terminating bot (for example, if burning too much cloud memory)
        if str(message.author.id) in master_users:
            await message.channel.send("Deactivating analysis framework.")
            prep_to_analyze = False

    if scrape_messages == True:  # Eventually directly write to a buffer then file, also maybe only from one channel (specific channel id)
        messages.append(str(message.content))
        authors.append(str(message.author.id))

    if prep_to_analyze == True:
        words = str(message.content)
        toxic_tokens = tokenizer.texts_to_sequences(words)
        toxic_ready = pad_sequences(toxic_tokens, maxlen=max_length, padding=padding_type, truncating=trunc_type)
        toxic_preds = model.predict(toxic_ready)
        toxic_results = np.amax(toxic_preds, axis=0)
        toxic_score = score_text(toxic_results)
        if debug_mode:
            print("Raw model out: ",str(toxic_preds))
            print(str(toxic_results))
            print(str(toxic_score))
        offender = str(message.author.id)
        manage_toxiscores(offender, toxic_score)

    if message.content == "^hello":
        reply = "Greetings, citizen."
        await message.channel.send(reply)

    if message.content == "^help":
        reply = ""
        for command in commands:
            reply += f"{command} \n"
        await message.channel.send(reply)

    if message.content == "^watchlist":  # Untested
        tmp = dict(sorted(toxiscores.items(), key=lambda x: x[1], reverse=True))
        tmp_uid = [uid for uid, score in tmp.items()]
        tmp_scores = [score for uid, score in tmp.items()]

        reply = "{: <40}".format("```Toxicity Watchlist") + ": Score\n"

        place = 0
        while place < 20 and place < len(tmp):
            reply += "{: <40}".format(f"{str(place+1)}. {str(client.get_user(int(tmp_uid[place])))}") + ": {:2.2f}\n".format(tmp_scores[place])
            place += 1
        await message.channel.send(reply + "```")

client.run(token)

print('Connection closed.')

max_stars_repo_name: dinitrogen-tetroxide/ArmOfState

max_issues_repo_name: dinitrogen-tetroxide/ArmOfState

max_forks_repo_name: dinitrogen-tetroxide/ArmOfState