text:
# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
#
# NVIDIA CORPORATION and its licensors retain all intellectual property
# and proprietary rights in and to this software, related documentation
# and any modifications thereto.  Any use, reproduction, disclosure or
# distribution of this software and related documentation without an express
# license agreement from NVIDIA CORPORATION is strictly prohibited.

import os
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"   # see issue #152
import numpy as np
import torch
import random
import torch.nn as nn
import pickle
torch.manual_seed(0)
import scipy.misc
import json
import torch.nn.functional as F
import os
import imageio

device_ids = [0]
from PIL import Image
from tqdm import tqdm
from torch.utils.data import Dataset, DataLoader
from models.encoder.encoder import FPNEncoder

from utils.data_utils import *
from utils.model_utils import *

import torch.optim as optim
import argparse
import glob
import lpips

device = 'cuda' if torch.cuda.is_available() else 'cpu'

def embed_one_example(args, path, stylegan_encoder, g_all, upsamplers,
                      inter, percept, steps, sv_dir,
                      skip_exist=False):

    if os.path.exists(sv_dir):
        if skip_exist:
            return 0,0,[], []
        else:
            pass
    else:
        os.system('mkdir -p %s' % (sv_dir))
    print('SV folder at: %s' % (sv_dir))
    image_path = path
    label_im_tensor, im_id = load_one_image_for_embedding(image_path, args['im_size'])

    print("****** Run optimization for ", path, " ******")

    label_im_tensor = label_im_tensor.to(device)
    label_im_tensor = label_im_tensor * 2.0 - 1.0
    label_im_tensor = label_im_tensor.unsqueeze(0)
    latent_in = stylegan_encoder(label_im_tensor)
    im_out_wo_encoder, _ = latent_to_image(g_all, upsamplers, latent_in,
                                           process_out=True, use_style_latents=True,
                                           return_only_im=True)

    out = run_embedding_optimization(args, g_all,
                                     upsamplers, inter, percept,
                                     label_im_tensor, latent_in, steps=steps,
                                     stylegan_encoder=stylegan_encoder,
                                     use_noise=args['use_noise'],
                                     noise_loss_weight=args['noise_loss_weight']
                                     )
    if args['use_noise']:
        optimized_latent, optimized_noise, loss_cache = out
        optimized_noise = [torch.from_numpy(noise).cuda() for noise in optimized_noise]
    else:
        optimized_latent, optimized_noise, loss_cache = out
        optimized_noise = None
    print("Curr loss, ", loss_cache[0], loss_cache[-1] )

    optimized_latent_np = optimized_latent.detach().cpu().numpy()[0]
    if args['use_noise']:
        loss_cache_np = [noise.detach().cpu().numpy() for noise in optimized_noise]
    else:
        loss_cache_np = []
    # vis
    img_out, _ = latent_to_image(g_all, upsamplers, optimized_latent,
                                 process_out=True, use_style_latents=True,
                                 return_only_im=True, noise=optimized_noise)

    raw_im_show = (np.transpose(label_im_tensor.cpu().numpy(), (0, 2, 3, 1))) * 255.

    vis_list = [im_out_wo_encoder[0], img_out[0]
                ]

    curr_vis = np.concatenate(
        vis_list, 0)
    imageio.imsave(os.path.join(sv_dir, "reconstruction.jpg"),
                      curr_vis)

    imageio.imsave(os.path.join(sv_dir, "real_im.jpg"),
                      raw_im_show[0])

    return loss_cache[0], loss_cache[-1], optimized_latent_np, loss_cache_np

def test(args, resume, steps,  latent_sv_folder='', skip_exist=False):

    g_all, _, upsamplers, _, avg_latent = prepare_model(args)
    inter = Interpolate(args['im_size'][1], 'bilinear')

    percept = lpips.PerceptualLoss(
        model='net-lin', net='vgg', use_gpu=device.startswith('cuda'), normalize=args['normalize']
    ).to(device)
    stylegan_encoder = FPNEncoder(3, n_latent=args['n_latent'], only_last_layer=args['use_w'])
    stylegan_encoder = stylegan_encoder.to(device)
    stylegan_encoder.load_state_dict(torch.load(resume, map_location=device)['model_state_dict'], strict=True)

    assert latent_sv_folder != ""
    all_images = []
    all_id = []

    curr_images_all = glob.glob(args['testing_data_path'] +  "*/*")
    curr_images_all = [data for data in curr_images_all if ('jpg' in data or 'webp' in data or 'png' in data  or 'jpeg' in data or 'JPG' in data) and not os.path.isdir(data)  and not 'npy' in data ]

    for i, image in enumerate(curr_images_all):
        all_id.append(image.split("/")[-1].split(".")[0])
        all_images.append(image)

    print("All files, " , len(all_images))

    all_loss_before_opti, all_loss_after_opti = [], []
    for i, id in enumerate(tqdm(all_id)):

        print("Curr dir,", id)

        sv_folder = os.path.join(latent_sv_folder,  id, 'crop_latent_' + str(steps))

        loss_before_opti, loss_after_opti , all_final_latent, all_final_noise = embed_one_example(args, all_images[i],
                                                                                stylegan_encoder, g_all,
                                                                                upsamplers, inter, percept, steps,
                                                                                sv_folder, skip_exist=skip_exist)

        all_loss_before_opti.append(loss_before_opti)
        all_loss_after_opti.append(loss_after_opti)

        id_num = id.split("_")[-1]
        latent_name = latent_sv_folder +  '/latents_image_%s.npy' % str(id_num)
        np.save(latent_name, all_final_latent)

        if len(all_final_noise) != 0:

            latent_name = latent_sv_folder + '/latents_image_%s_npose.npy' % str(id_num)

            with open(latent_name, 'wb') as handle:
                pickle.dump(all_final_noise, handle)

    result = {"before:": np.mean(all_loss_before_opti), "after": np.mean(all_loss_after_opti)}

    with open(latent_sv_folder + '/result.json', 'w') as f:
        json.dump(result, f)

def main(args, resume):

    base_path = os.path.join(args['exp_dir'], "checkpoint")
    if not os.path.exists(base_path):
        os.mkdir(base_path)
    g_all, _, upsamplers, _, avg_latent = prepare_model(args)
    percept = lpips.PerceptualLoss(
        model='net-lin', net='vgg',
        use_gpu=device.startswith('cuda'), normalize=args['normalize']
    ).to(device)
    stylegan_encoder = FPNEncoder(3, n_latent=args['n_latent'], only_last_layer=args['use_w'], same_view_code=args['same_view_code'])
    stylegan_encoder = stylegan_encoder.to(device)

    if resume != "":
        stylegan_encoder.load_state_dict(torch.load(resume, map_location=device)['model_state_dict'])

    optimizer = optim.Adam(stylegan_encoder.parameters(), lr=args['lr'])
    inter = Interpolate(args['im_size'][1], 'bilinear')
    images_all = glob.glob(args['training_data_path'] + "/*")

    images_all = [data for data in images_all if 'jpg' in data or 'webp' in data or 'png' in data]
    if args['debug']:
        images_all = images_all[:1]
    print( "Training data length, ", len(images_all))

    if 'car' in args['category']:
        fill_blank = True
    else:
        fill_blank = False

    train_data = trainData(images_all, img_size=args['im_size'], fill_blank=fill_blank)
    shuffle = True
    if args['debug']:
        args['bs'] = 1
        shuffle = False

    train_data_loader = DataLoader(train_data, batch_size=args['bs'], shuffle=shuffle, num_workers=8)

    if args['debug']:
        np.random.seed(41)
        sampling_latent = np.random.randn(1, 512)

    best_loss = 999999999
    for epoch in range(100):

        for i, da, in enumerate(train_data_loader):
            if i % 10 == 0:
                gc.collect()
            img_tensor = da[0].to(device)
            img_tensor = img_tensor  *  2.0 - 1.0
            stylegan_encoder.train()
            latent_in = stylegan_encoder(img_tensor)
            if args['truncation']:
                latent_in = g_all.module.truncation(latent_in)

            img_out , _ = latent_to_image(g_all, upsamplers, latent_in, use_style_latents=True,
                                          return_only_im=True, process_out=False)

            img_out = inter(img_out)
            img_tensor = (img_tensor + 1.0) / 2.0
            img_out = (img_out + 1.0) / 2.0

            p_loss = percept(img_out, img_tensor).mean()
            mse_loss = F.mse_loss(img_out, img_tensor)

            loss = p_loss * args['loss_dict']['p_loss'] + \
                   mse_loss * args['loss_dict']['mse_loss']

            if epoch > args['train_real_start_epochs']:
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()
            if args['sampling_training']:

                sampling_loss = 0
                sample_img = []
                sample_latnet = []
                for round in range(args['sample_bs']):
                    if args['debug']:
                        latent = sampling_latent

                    else:
                        latent = np.random.randn(1, 512)
                    latent = torch.from_numpy(latent).type(torch.FloatTensor).to(device)

                    with torch.no_grad():
                        curr_sample_img, curr_sample_latnet = latent_to_image(g_all, upsamplers, latent,
                                                             return_only_im=True, process_out=False)

                    sample_img.append(curr_sample_img)
                    sample_latnet.append(curr_sample_latnet)

                sample_img = torch.cat(sample_img, 0)

                sample_img = inter(sample_img)

                sample_latnet = torch.cat(sample_latnet, 0)

                encode_latent = stylegan_encoder(sample_img)

                encoder_loss = F.mse_loss(encode_latent, sample_latnet)

                sampling_loss += encoder_loss * args['loss_dict']['encoder_loss']

                if args['truncation']:
                    encode_latent = g_all.module.truncation(encode_latent)

                img_out_sampling, _ = latent_to_image(g_all, upsamplers, encode_latent, use_style_latents=True, return_only_im=True, process_out=False)
                img_out_sampling = inter(img_out_sampling)

                img_out_sampling = (img_out_sampling + 1.0) / 2.0
                sample_img = (sample_img + 1.0) / 2.0
                p_loss = percept(img_out_sampling, sample_img).mean()
                mse_loss = F.mse_loss(img_out_sampling, sample_img)
                sampling_loss += p_loss * args['loss_dict']['p_loss'] + \
                       mse_loss * args['loss_dict']['mse_loss']
                optimizer.zero_grad()
                sampling_loss.backward()
                optimizer.step()

            if args['debug']:
                image_name = os.path.join(args['exp_dir'], 'real.png')

                img_out = 255 * np.transpose( (img_out.detach().cpu().numpy()[0]), (1,2,0))
                img_gt = 255 * np.transpose( (img_tensor.cpu().numpy()[0]), (1,2,0))

                img_save = (np.concatenate( (img_out, img_gt), 1)).astype(np.uint8)
                img_out = Image.fromarray(img_save)
                img_out.save(image_name)

                if args['sampling_training']:
                    image_name = os.path.join(args['exp_dir'], 'sampling.png')

                    img_out = 255 * np.transpose((img_out_sampling.detach().cpu().numpy()[0]), (1, 2, 0))
                    img_gt = 255 * np.transpose((sample_img.detach().cpu().numpy()[0]), (1, 2, 0))

                    img_save = (np.concatenate((img_out, img_gt), 1)).astype(np.uint8)
                    img_out = Image.fromarray(img_save)
                    img_out.save(image_name)
                exit()
            if i % 10 == 0 :
                if args['sampling_training']:
                    print(epoch, 'epoch', 'iteration', i, 'loss: ', loss.item(), " sampling loss: ", sampling_loss.item())
                else:
                    print(epoch, 'epoch', 'iteration', i, 'loss', loss)

            if i % 2000 == 0 and i!=0 and not args['debug']:
                image_name = os.path.join(args['exp_dir'], 'Epoch_' + str(epoch)  + "_iter_"+ str(i) + '.png')
                img_out = np.transpose(img_out.detach().cpu().numpy()[0], (1,2,0))
                img_gt = np.transpose(img_tensor.cpu().numpy()[0], (1,2,0))
                img_save=  (255 * np.concatenate( (img_out, img_gt), 0)).astype(np.uint8)
                img_out = Image.fromarray(img_save)
                img_out.save(image_name)
                image_name = os.path.join(args['exp_dir'], 'Epoch_' + str(epoch) + "_iter_" + str(i) + '_sampling.png')
                img_out = 255 * np.transpose((img_out_sampling.detach().cpu().numpy()[0]), (1, 2, 0))
                img_gt = 255 * np.transpose((sample_img.detach().cpu().numpy()[0]), (1, 2, 0))
                img_save = (np.concatenate((img_out, img_gt), 1)).astype(np.uint8)
                img_out = Image.fromarray(img_save)
                img_out.save(image_name)
                loss = loss.item()
                model_path = os.path.join(base_path, 'epoch_' + str(epoch) + '_iter_' + str(i) + '_loss_' + str(loss) + '.pth')
                print('Save to:', model_path)
                torch.save({'model_state_dict': stylegan_encoder.state_dict()},
                           model_path)
                if epoch > 2:
                    if loss < best_loss:
                        best_loss = loss
                        model_path = os.path.join(base_path, 'BEST_' + 'loss' + str(loss) + '.pth')
                        print('Save to:', model_path)
                        torch.save({'model_state_dict': stylegan_encoder.state_dict()},
                                   model_path)

            del loss
            if args['sampling_training']:
                del sampling_loss

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('--exp', type=str)
    parser.add_argument('--resume', type=str, default="")
    parser.add_argument('--test', type=bool, default=False)

    parser.add_argument('--testing_path', type=str, default='')
    parser.add_argument('--latent_sv_folder', type=str, default='')
    parser.add_argument('--skip_exist', type=bool, default=False)
    parser.add_argument('--steps', type=int, default=500)

    parser.add_argument('--use_noise', type=bool, default=False)
    parser.add_argument('--noise_loss_weight', type=float, default=100)

    args = parser.parse_args()
    opts = json.load(open(args.exp, 'r'))

    path =opts['exp_dir']
    if os.path.exists(path):
        pass
    else:
        os.system('mkdir -p %s' % (path))
        print('Experiment folder created at: %s' % (path))

    os.system('cp %s %s' % (args.exp, opts['exp_dir']))
    opts['use_noise'] = args.use_noise
    opts['noise_loss_weight'] = args.noise_loss_weight

    print("Opt", opts)

    if args.testing_path != "":
        opts['testing_data_path'] = args.testing_path

    if args.test:
        test(opts, args.resume, args.steps,
             latent_sv_folder=args.latent_sv_folder, skip_exist=args.skip_exist)
    else:
        main(opts, args.resume)

max_stars_repo_name: dubtor/EditGAN-Robert

max_issues_repo_name: dubtor/EditGAN-Robert

max_forks_repo_name: dubtor/EditGAN-Robert


text:
from typing import Union, Optional, List
from torch import Tensor
from torch_geometric.data import Dataset

import torch
import numpy as np

from torch_geometric.io.planetoid import index_to_mask

def get_idx_split_arxiv(dataset: Dataset) -> Dataset:
    """create dataset split for the ogbn-arxiv dataset compatible with our pipeline

    Args:
        dataset (Dataset): dataset object for which the split should be created

    Returns:
        Dataset: modified dataset
    """

    split_idx = dataset.get_idx_split()
    train_idx, valid_idx, test_idx = split_idx["train"], split_idx["valid"], split_idx["test"]

    num_nodes = dataset.data.y.size(0)
    dataset.data.train_mask = index_to_mask(train_idx, num_nodes)
    dataset.data.val_mask = index_to_mask(valid_idx, num_nodes)
    dataset.data.test_mask = index_to_mask(test_idx, num_nodes)

    return dataset

def get_idx_split(
        dataset: Dataset,
        split: str = 'random',
        train_samples_per_class: Union[int, float] = None,
        val_samples_per_class: Union[int, float] = None,
        test_samples_per_class: Union[int, float] = None,
        train_size: Optional[int] = None,
        val_size: Optional[int] = None,
        test_size: Optional[int] = None) -> Dataset:
    """utility function for creating train/test/val split for a dataset.

    The split is either created by specifying the number or fraction of samples per class or the overall size
    the training/validation/test set. If the fraction of samples per class is chosen, the fraction is relative
    to the number of labeled data points for each class separately.

    code taken partially from (https://github.com/shchur/gnn-benchmark)

    Args:
        dataset (Dataset): dataset object
        split (str, optional): selected split ('random' or 'public'). Defaults to 'random'.
        train_samples_per_class (Union[int, float], optional): number of fraction of samples per class in the training set. Defaults to None.
        val_samples_per_class (Union[int, float], optional): number or fraction of samples per class in the validation set. Defaults to None.
        test_samples_per_class (Union[int, float], optional): number or fraction of samples per cleass in the test set. Defaults to None.
        train_size (Optional[int], optional): size of the training set. Defaults to None.
        val_size (Optional[int], optional): size of the validation set. Defaults to None.
        test_size (Optional[int], optional): size of the test set. Defaults to None.

    Returns:
        Dataset: modified dataset object containing the dataset split
    """

    data = dataset.data

    assert (train_size is None) ^ (train_samples_per_class is None)
    assert (val_size is None) ^ (val_samples_per_class is None)
    assert (test_size is None) ^ (test_samples_per_class is None)

    if split == 'public':
        assert hasattr(data, 'train_mask') and hasattr(data, 'test_mask') \
            and hasattr(data, 'test_mask')
        return dataset

    labels = data.y
    num_nodes = labels.size(0)
    num_classes = max(labels) + 1
    classes = range(num_classes)
    remaining_indices = list(range(num_nodes))

    # drop classes which do not have enough samples
    _train_samples_per_class = 0
    _val_samples_per_class = 0
    _test_samples_per_class = 0
    if isinstance(train_samples_per_class, int):
        _train_samples_per_class = train_samples_per_class
    if isinstance(val_samples_per_class, int):
        _val_samples_per_class = val_samples_per_class
    if isinstance(test_samples_per_class, int):
        _test_samples_per_class = test_samples_per_class

    forbidden_indices = None
    min_samples_per_class = _train_samples_per_class + _val_samples_per_class + _test_samples_per_class

    if min_samples_per_class > 0:
        dropped_classes = []
        forbidden_indices = []
        for c in classes:
            class_indices = (labels == c).nonzero(as_tuple=False).squeeze()
            if (labels == c).sum() < min_samples_per_class:
                dropped_classes.append(c)
                forbidden_indices.append(class_indices.numpy())

        if len(forbidden_indices) > 0:
            forbidden_indices = np.concatenate(forbidden_indices)

    else:
        dropped_classes = []
        forbidden_indices = []

    classes = [c for c in classes if c not in dropped_classes]

    # train indices
    if train_samples_per_class is not None:
        train_indices = sample_per_class(labels, num_nodes, classes,
                                         train_samples_per_class, forbidden_indices=forbidden_indices)
    else:
        remaining_indices = np.setdiff1d(remaining_indices, forbidden_indices)
        train_indices = np.random.choice(remaining_indices, train_size, replace=False)

    # validation indices (exclude train indices)
    forbidden_indices = np.concatenate((forbidden_indices, train_indices))
    if val_samples_per_class is not None:
        val_indices = sample_per_class(labels, num_nodes, classes,
                                       val_samples_per_class, forbidden_indices=train_indices)
    else:
        remaining_indices = np.setdiff1d(remaining_indices, forbidden_indices)
        val_indices = np.random.choice(remaining_indices, val_size, replace=False)

    # test indices (exclude test indices)
    forbidden_indices = np.concatenate((forbidden_indices, val_indices))
    if test_samples_per_class is not None:
        test_indices = sample_per_class(labels, num_nodes, classes,
                                        test_samples_per_class,
                                        forbidden_indices=forbidden_indices)
    elif test_size is not None:
        remaining_indices = np.setdiff1d(remaining_indices, forbidden_indices)
        test_indices = np.random.choice(remaining_indices, test_size, replace=False)
    else:
        test_indices = np.setdiff1d(remaining_indices, forbidden_indices)

    # assert that there are no duplicates in sets
    assert len(set(train_indices)) == len(train_indices)
    assert len(set(val_indices)) == len(val_indices)
    assert len(set(test_indices)) == len(test_indices)
    # assert sets are mutually exclusive
    assert len(set(train_indices) - set(val_indices)) == len(set(train_indices))
    assert len(set(train_indices) - set(test_indices)) == len(set(train_indices))
    assert len(set(val_indices) - set(test_indices)) == len(set(val_indices))

    data.train_mask = index_to_mask(train_indices, num_nodes)
    data.test_mask = index_to_mask(test_indices, num_nodes)
    data.val_mask = index_to_mask(val_indices, num_nodes)

    if len(dropped_classes) > 0:
        data.dropped_classes = torch.LongTensor(dropped_classes)

    return dataset

def sample_per_class(labels: Tensor, num_nodes: int, classes: List[int],
                     samples_per_class: Union[int, float],
                     forbidden_indices: np.array = None) -> np.array:
    """samples a subset of indices based on specified number of samples per class

    Args:
        labels (Tensor): tensor of ground-truth labels
        num_nodes (int): number nof nodes
        classes (List[int]): classes (labels) for which the subset is sampled
        samples_per_class (Union[int, float]): number or fraction of samples per class
        forbidden_indices (np.array, optional): indices to ignore for sampling. Defaults to None.

    Returns:
        np.array: sampled indices
    """

    sample_indices_per_class = {index: [] for index in classes}
    num_samples_per_class = {index: None for index in classes}

    # get indices sorted by class
    for class_index in classes:
        for sample_index in range(num_nodes):
            if labels[sample_index] == class_index:
                if forbidden_indices is None or sample_index not in forbidden_indices:
                    sample_indices_per_class[class_index].append(sample_index)

    for class_index in classes:
        if isinstance(samples_per_class, float):
            class_labels = sample_indices_per_class[class_index]
            num_samples_per_class[class_index] = int(samples_per_class * len(class_labels))
        else:
            num_samples_per_class[class_index] = samples_per_class

    # get specified number of indices for each class
    return np.concatenate(
        [np.random.choice(sample_indices_per_class[class_index], num_samples_per_class[class_index], replace=False)
         for class_index in classes
        ])

ext: py

max_stars_repo_name: WodkaRHR/Graph-Posterior-Network

max_issues_repo_name: WodkaRHR/Graph-Posterior-Network

max_forks_repo_name: WodkaRHR/Graph-Posterior-Network

reason: import numpy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2020-07-19 13:25:53
# @Author  : helingjie-maskmind (helingjie440@gmail.com)
# @Link    : ${link}
# @Version : $Id$

import os
import numpy as np

c=np.loadtxt('data.csv',delimiter=',',usecols=(6,),unpack=True)
# diff比输入数组少一个元素
l=np.diff(c)
# 输出原数组与diff之后的数组
print(c)
print(l)
print(c[:-1])
l_return=np.diff(c)/c[:-1]
# std函数计算标准差
print('标准差',np.std(l_return))
# 对数收益率
log_return=np.diff(np.log(c))

afterProcessLog=np.where(l_return>0)
print('>0的索引',afterProcessLog)

annual_vol=np.std(log_return)/np.mean(log_return)
annual_vol=annual_vol/np.sqrt(1./252.)

print('年波动率',annual_vol)
print('月波动率',annual_vol*np.sqrt(1./12.))

max_stars_repo_name: Mrhelingjie/learnNumPyTour

max_issues_repo_name: Mrhelingjie/learnNumPyTour

max_forks_repo_name: Mrhelingjie/learnNumPyTour


text:
from collections import Counter

import networkx

from textdb.makeNetworkView import DataBasePlotter
from utils.cytoscape_grapher import CytoscapeGrapher
from utils.tmutils import normalize_gene_names

cbn2edges = {
"CV-IPN-Endothelial_cell-monocyte_interaction_1": {('CXCL16', 'CXCL16'), ('ITGAX', 'ITGAX'), ('CX3CR1', 'CX3CR1'), ('CCL2', 'PIK3CA'), ('ITGAX', 'ITGB2'), ('CXCR6', 'CXCR6'), ('SIRPA', 'ITGB2'), ('ITGB2', 'ITGB2'), ('ITGA4', 'VCAM1'), ('CCL2', 'CCL2'), ('SELE', 'SELE'), ('IL1A', 'CCR2'), ('CCL2', 'RHOA'), ('CX3CL1', 'STAT5A'), ('LTB4R', 'ITGAM'), ('IL8', 'ITGAM'), ('ITGA4', 'ITGA4'), ('CX3CR1', 'ICAM1'), ('ICAM1', 'ETS1'), ('SELP', 'SELPLG'), ('ITGA4', 'ITGB1'), ('ITGAM', 'ITGAM'), ('THY1', 'THY1'), ('CX3CL1', 'CX3CL1'), ('ITGAL', 'ITGB2'), ('ITGAM', 'ICAM1'), ('CX3CL1', 'CX3CR1'), ('ICAM1', 'ITGB2'), ('IFNG', 'CXCL16'), ('ETS1', 'CCL2'), ('ICAM1', 'ICAM1'), ('IL8', 'ITGB2'), ('CXCR6', 'CXCL16'), ('ITGB2', 'ITGAL'), ('STAT5A', 'ICAM1'), ('SELP', 'SELP'), ('ETS1', 'ETS1'), ('SELPLG', 'SELPLG'), ('ITGAL', 'ITGAL'), ('IL8', 'IL8'), ('STAT5A', 'STAT5A'), ('CCR2', 'CCR2'), ('ITGB2', 'ITGAM'), ('VCAM1', 'VCAM1'), ('CCL2', 'CCR2'), ('STAB1', 'STAB1')},
"CV-IPN-Endothelial_cell_activation_1": {('CSF3', 'CSF3'), ('SOCS2', 'STAT3'), ('CRP', 'OLR1'), ('VEGFA', 'EGR1'), ('OLR1', 'MMP14'), ('HMOX1', 'CCL2'), ('CAV1', 'NOS3'), ('IL1B', 'VCAM1'), ('MMP14', 'RAC1'), ('ATF4', 'VEGFA'), ('RHOA', 'RHOA'), ('RHOA', 'NOS3'), ('TRAF2', 'TRAF2'), ('VEGFA', 'VEGFA'), ('MMP1', 'MMP1'), ('VEGFA', 'PTK2'), ('PPARG', 'ICAM1'), ('CXCL12', 'VEGFA'), ('OLR1', 'CD40'), ('VEGFA', 'PLVAP'), ('RAC1', 'RAC1'), ('GSK3B', 'GATA6'), ('PRKCZ', 'PRKCZ'), ('PTEN', 'PTEN'), ('PPARA', 'PPARA'), ('CD40', 'CD40'), ('ETS1', 'ETS1'), ('PLCG1', 'PLCG1'), ('OLR1', 'OLR1'), ('S1PR1', 'S1PR1'), ('RELA', 'CAV1'), ('PRKACA', 'FOXO1'), ('CD40', 'MAPK8'), ('MAPK3', 'MAPK3'), ('VCAM1', 'VCAM1'), ('OLR1', 'ICAM1'), ('JUN', 'JUN'), ('CXCL12', 'EGR1'), ('TNFRSF1B', 'TNFRSF1B'), ('STAT3', 'IL8'), ('IL1B', 'SELE'), ('AKT1', 'CCL2'), ('JAK2', 'STAT3'), ('ATF4', 'IL6'), ('CSF2', 'CSF2'), ('AGTR1', 'AGTR1'), ('AGTR1', 'CCL2'), ('FOXO1', 'NOS2'), ('FOXO1', 'VCAM1'), ('CCL2', 'CCL2'), ('CD40LG', 'EGR1'), ('CDKN1A', 'EP300'), ('NR3C2', 'NOX4'), ('PRKCB', 'MMP1'), ('IL6', 'JAK2'), ('HMOX1', 'VCAM1'), ('REL', 'F3'), ('VEGFA', 'VWF'), ('TNF', 'SELE'), ('CD40', 'MAPK3'), ('CXCL2', 'CXCL2'), ('TNF', 'VWF'), ('OLR1', 'IL8'), ('STAT3', 'ICAM1'), ('CSF2', 'SELP'), ('PTK2', 'PTK2'), ('ROCK2', 'ROCK2'), ('TNF', 'MAPK3'), ('STAT1', 'ICAM1'), ('IFNG', 'CD40'), ('CXCL3', 'CXCL3'), ('IL10', 'PECAM1'), ('SCARB1', 'NOS3'), ('NOS3', 'NOS3'), ('TNFRSF1B', 'TRAF2'), ('MMP14', 'MMP14'), ('TNFRSF1A', 'TRAF2'), ('SP1', 'SP1'), ('ANGPT1', 'PRKCZ'), ('EGR1', 'EGR1'), ('RHOA', 'PTK2'), ('PPARG', 'VCAM1'), ('RAC1', 'PTK2'), ('CSF1', 'CSF1'), ('PRKCB', 'MMP3'), ('CD40', 'PLCG1'), ('REL', 'REL'), ('GATA6', 'VCAM1'), ('TRAF2', 'CHUK'), ('IL1A', 'CD40'), ('VEGFA', 'RELA'), ('GSK3B', 'GSK3B'), ('OLR1', 'CSF3'), ('RELA', 'RELA'), ('IL1B', 'VWF'), ('IL1B', 'CD40'), ('PPARG', 'PPARG'), ('ITGAV', 'CAV1'), ('EGR1', 'PECAM1'), ('ITGAV', 'ITGAV'), ('TNF', 'CCL2'), ('EIF2AK3', 'EIF2AK3'), ('TNF', 'GATA6'), ('TNF', 'ICAM1'), ('CREB1', 'CREB1'), ('FOXO1', 'FOXO1'), ('ITGAV', 'ITGB3'), ('SELE', 'SELE'), ('CD40LG', 'REL'), ('TNF', 'VCAM1'), ('S1PR1', 'MAPK3'), ('VWF', 'VWF'), ('S1PR1', 'MAPK1'), ('EP300', 'EP300'), ('ATF4', 'CCL2'), ('STAT1', 'STAT1'), ('TNF', 'CD40'), ('TNF', 'CDKN1A'), ('IFNB1', 'CD40'), ('GATA6', 'GATA6'), ('MMP14', 'RHOA'), ('HMGB1', 'AGER'), ('NOS2', 'NOS2'), ('PRKCB', 'PRKCB'), ('SPHK1', 'SPHK1'), ('F3', 'F3'), ('AGER', 'AGER'), ('ICAM1', 'ICAM1'), ('OLR1', 'CXCL3'), ('SCARB1', 'SCARB1'), ('NR3C2', 'NR3C2'), ('IL6', 'CAV1'), ('EP300', 'ETS1'), ('RELA', 'ICAM1'), ('SELP', 'SELP'), ('MMP3', 'MMP3'), ('CD40LG', 'CD40'), ('MAPK8', 'MAPK8'), ('EIF2S1', 'EIF2S1'), ('AKT1', 'AKT1'), ('IL6', 'IL6'), ('EIF2AK3', 'EIF2S1'), ('JAK2', 'STAT1'), ('GATA3', 'GATA3'), ('ATF6', 'ATF6'), ('PLVAP', 'PLVAP'), ('PECAM1', 'PECAM1'), ('TNF', 'TNF'), ('ROCK2', 'CXCL2'), ('PPARA', 'VCAM1'), ('OLR1', 'VCAM1'), ('F2', 'F2'), ('CAV1', 'CAV1'), ('ROCK2', 'IL8'), ('SRC', 'SRC'), ('CDKN1A', 'CDKN1A'), ('TRAF2', 'IKBKB'), ('TNF', 'MAPK1'), ('NOX4', 'NOX4'), ('STAT3', 'STAT3'), ('ESR1', 'ESR1'), ('OLR1', 'ROCK2'), ('JAK2', 'JAK2'), ('F2', 'JUN'), ('CREB1', 'HMOX1'), ('EGR1', 'F3'), ('CD40', 'MAPK1'), ('IL1B', 'ICAM1'), ('AGTR1', 'OLR1'), ('F2', 'PRKCZ'), ('SRC', 'JAK2'), ('CHUK', 'CHUK'), ('ERN1', 'ERN1'), ('TNFRSF1A', 'TNFRSF1A'), ('IFNG', 'JAK2'), ('ESR1', 'NOS3'), ('ATF4', 'IL8'), ('ATF4', 'ATF4'), ('TNF', 'TNFRSF1B'), ('TNF', 'TNFRSF1A'), ('ITGB3', 'ITGB3'), ('IKBKB', 'IKBKB'), ('EGR1', 'ICAM1'), ('SP1', 'NOS3'), ('ETS1', 'CCL2'), ('TNF', 'CAV1'), ('OLR1', 'CXCL2'), ('TNF', 'CHUK'), ('MAPK1', 'MAPK1'), ('CAV1', 'VCAM1'), ('STAT3', 'CCL2'), ('HMOX1', 'HMOX1'), ('IL8', 'IL8'), ('GATA3', 'VCAM1'), ('PPARG', 'SELE'), ('OLR1', 'PRKCB'), ('TNF', 'GATA3')},
"CV-IPN-Plaque_destabilization_1": {('MSR1', 'STAT1'), ('ATP2A2', 'ATP2A2'), ('IL17A', 'MMP1'), ('PPARA', 'TNF'), ('CD36', 'TLR4'), ('CD36', 'IL6'), ('CIITA', 'COL1A2'), ('TGFB1', 'COL3A1'), ('TLR2', 'IL1B'), ('TLR4', 'TLR4'), ('CD36', 'CD36'), ('IL18', 'MMP9'), ('CD36', 'TLR2'), ('MFGE8', 'IFNG'), ('TLR4', 'CXCL2'), ('IL1B', 'FGF2'), ('TIMP1', 'MMP9'), ('PPARA', 'IL6'), ('VEGFA', 'PLAU'), ('CD40', 'CEBPA'), ('TLR4', 'STAT1'), ('IL1B', 'CD40LG'), ('IL6', 'MMP9'), ('CD36', 'CXCL2'), ('PTGS2', 'MMP9'), ('MSR1', 'PLAU'), ('TLR4', 'TNF'), ('TNF', 'COL1A2'), ('IRF5', 'TNF'), ('MMP1', 'MMP1'), ('MMP9', 'MMP9'), ('HIF1A', 'VEGFA'), ('IRAK4', 'CXCL10'), ('F2', 'CXCL2'), ('CEBPA', 'MMP8'), ('IRAK4', 'NFKBIA'), ('ELK1', 'ELK1'), ('PPARA', 'F3'), ('PPARA', 'PPARA'), ('CD40', 'CD40'), ('ETS1', 'FASLG'), ('TIMP1', 'TIMP1'), ('PON2', 'PON2'), ('TLR2', 'CCL2'), ('CD36', 'CCL5'), ('CD40LG', 'SLC9A1'), ('PPARA', 'ABCA1'), ('CD40', 'VCAM1'), ('BAX', 'BAX'), ('VCAM1', 'VCAM1'), ('IRF5', 'IL6'), ('COL1A2', 'COL1A2'), ('CEBPA', 'CEBPA'), ('MMP13', 'MMP13'), ('ABCA1', 'ABCA1'), ('MSR1', 'MSR1'), ('CXCL10', 'CXCL10'), ('CD40', 'ICAM1'), ('PLAU', 'PLAU'), ('CIITA', 'COL1A1'), ('CCL2', 'CCL2'), ('IRAK4', 'TNF'), ('IL10', 'F3'), ('CCL2', 'PECAM1'), ('PON3', 'CCL2'), ('DDIT3', 'BAX'), ('TNF', 'MMP9'), ('CD40', 'SELE'), ('IFNG', 'ABCA1'), ('PON2', 'IL6'), ('TLR2', 'MYD88'), ('SLC9A1', 'IL12B'), ('MYD88', 'PTGS2'), ('TNF', 'MAP3K1'), ('TLR2', 'TNF'), ('F2', 'CCL2'), ('VCAM1', 'MMP2'), ('EGR1', 'EGR1'), ('STAT1', 'CIITA'), ('IL1B', 'MMP2'), ('CD40LG', 'MMP1'), ('TLR4', 'VCAM1'), ('IL17A', 'VCAM1'), ('PPARG', 'VCAM1'), ('CIITA', 'CIITA'), ('PECAM1', 'VCAM1'), ('IRAK4', 'IL1B'), ('TLR4', 'CCL2'), ('CD40LG', 'SELE'), ('IL1B', 'IL1B'), ('F2', 'PLAU'), ('CD36', 'MAPK9'), ('TNF', 'DDIT3'), ('CCL2', 'MMP2'), ('IFNG', 'CIITA'), ('IRAK4', 'IRAK4'), ('APOE', 'APOE'), ('PPARG', 'PPARG'), ('MAPK9', 'MAPK9'), ('IRAK4', 'CCL4'), ('CD40LG', 'PTGS2'), ('PPARA', 'CD36'), ('TLR4', 'MYD88'), ('IFNG', 'HLA-DRA'), ('TGFB1', 'COL1A2'), ('MMP8', 'MMP8'), ('F3', 'F2'), ('SELE', 'SELE'), ('FGF2', 'NOS2'), ('PPARG', 'CCL2'), ('TNF', 'VCAM1'), ('IFNG', 'COL1A2'), ('STAT1', 'STAT1'), ('CD36', 'TNF'), ('TIMP2', 'MMP2'), ('IL17A', 'MMP3'), ('TP53', 'BAX'), ('IFNG', 'COL1A1'), ('COL3A1', 'COL3A1'), ('NOS2', 'NOS2'), ('IFNG', 'IFNG'), ('F3', 'F3'), ('FGF2', 'MMP9'), ('CIITA', 'HLA-DRA'), ('TNF', 'MMP1'), ('CD36', 'CCL2'), ('ICAM1', 'ICAM1'), ('SOD1', 'IL6'), ('MAPK9', 'MSR1'), ('IKBKB', 'NFKBIA'), ('IL6', 'IL6'), ('MMP3', 'MMP3'), ('MAP3K1', 'MAP2K6'), ('CD40LG', 'CD40'), ('VEGFA', 'TIMP2'), ('DDIT3', 'DDIT3'), ('PPARA', 'ICAM1'), ('PPARA', 'CCL2'), ('MMP1', 'COL1A1'), ('PPARA', 'PTGS2'), ('FASLG', 'FASLG'), ('PPARA', 'ABCG1'), ('JAK2', 'STAT1'), ('MMP2', 'MMP2'), ('PECAM1', 'PECAM1'), ('SLC9A1', 'SLC9A1'), ('TGFB1', 'COL1A1'), ('PPARA', 'IL1B'), ('IL1B', 'MMP8'), ('TNF', 'TNF'), ('PPARA', 'VCAM1'), ('MMP1', 'COL1A2'), ('F2', 'F2'), ('TIMP3', 'TIMP3'), ('ABCG1', 'ABCG1'), ('JAK2', 'JAK2'), ('F2', 'ICAM1'), ('HLA-DRA', 'HLA-DRA'), ('IL10', 'ELK1'), ('MERTK', 'MERTK'), ('CD40', 'F3'), ('NOX4', 'NOX4'), ('CCL5', 'CCL5'), ('EGR1', 'F3'), ('SOD1', 'IL1B'), ('TGM2', 'ABCA1'), ('TLR4', 'IL1B'), ('IRAK4', 'IL6'), ('CASP1', 'CASP1'), ('CTGF', 'CTGF'), ('ERN1', 'ERN1'), ('MMP1', 'COL3A1'), ('IRAK4', 'CXCL1'), ('IRAK4', 'CCL2'), ('TIMP2', 'TIMP2'), ('IFNG', 'JAK2'), ('MYD88', 'IRF5'), ('HIF1A', 'HIF1A'), ('SOD1', 'SOD1'), ('PTGS2', 'PTGS2'), ('IL10', 'EGR1'), ('PON2', 'TNF'), ('ETS1', 'MMP1'), ('FCER1A', 'IL6'), ('TLR2', 'TLR2'), ('VEGFA', 'KDR'), ('CD40LG', 'F3'), ('SLC9A1', 'IL6'), ('IL1B', 'NFKBIA'), ('IRAK4', 'CXCL2'), ('IL1B', 'MMP9')},
"CV-IPN-Platelet_activation_1": {('F2', 'F2'), ('CXCL12', 'CXCL12'), ('P2RY1', 'P2RY1'), ('TBXA2R', 'TBXA2R'), ('TLR4', 'MYD88'), ('TLR6', 'TLR6'), ('MYD88', 'MYD88'), ('THBS1', 'CD36'), ('MAP2K4', 'MAPK9'), ('PTK2B', 'PIK3CB'), ('TLR1', 'TLR1'), ('GSN', 'GSN'), ('RAP1B', 'ITGA2B'), ('VWF', 'VWF'), ('CCL5', 'CCL5'), ('PPBP', 'PPBP'), ('P2RY12', 'P2RY12'), ('CD36', 'SYK'), ('TLR1', 'MYD88'), ('VWF', 'PRKG1'), ('ITGA2', 'PTK2B'), ('F2R', 'MAPK3'), ('CD36', 'CD36'), ('TLR4', 'TLR4'), ('PIK3CB', 'AKT1'), ('F2RL3', 'PRKCD'), ('F2RL3', 'AKT1'), ('LYN', 'LYN'), ('MAP2K4', 'MAP2K4'), ('F2R', 'F2R'), ('PRKCD', 'PRKCD'), ('PRKG1', 'PRKG1'), ('RAP1B', 'RAP1B'), ('PTGER3', 'PTGER3'), ('ITGA2B', 'ITGA2B'), ('TLR2', 'MYD88'), ('F2', 'F2RL3'), ('PRKG1', 'VASP'), ('SYK', 'SYK'), ('PRKG1', 'MAPK1'), ('PTEN', 'AKT1'), ('VEGFA', 'VEGFA'), ('F2R', 'MAPK1'), ('TLR2', 'TLR2'), ('CD40LG', 'CD40LG'), ('F2', 'F2R'), ('MAPK9', 'MAPK9'), ('PRKG1', 'RAP1B'), ('VWF', 'LYN'), ('CD36', 'VASP'), ('MAPK1', 'MAPK1'), ('ENTPD1', 'ENTPD1'), ('AKT1', 'AKT1'), ('SELP', 'SELP'), ('PF4', 'PF4'), ('F2RL3', 'F2RL3'), ('PTK2B', 'PTK2B'), ('VASP', 'VASP'), ('PIK3CB', 'PIK3CB'), ('INSR', 'AKT1'), ('IL1B', 'IL1B'), ('ITGA2B', 'ITGB3'), ('CD36', 'MAPK9'), ('FGA', 'FGA'), ('MAPK3', 'MAPK3'), ('AKT1', 'ITGA2B'), ('FGB', 'FGB')},
"CV-IPN-Smooth_muscle_cell_activation_1": {('CCND1', 'CCND1'), ('PDGFB', 'PDGFB'), ('LRP6', 'PDGFRB'), ('LRP1', 'PDGFRB'), ('PLAU', 'PTPN11'), ('CREB1', 'NR4A3'), ('TLR4', 'TLR4'), ('CREB1', 'CCND1'), ('RHOA', 'RHOA'), ('CREB1', 'PTGS2'), ('RB1', 'RB1'), ('RB1', 'E2F1'), ('CEBPD', 'CEBPD'), ('PPARA', 'CDKN2A'), ('IGF1', 'NOX4'), ('CD40LG', 'CASP1'), ('CDK4', 'E2F1'), ('PPARA', 'PPARA'), ('RAC1', 'RAC1'), ('LRP6', 'LRP6'), ('CD40', 'CD40'), ('NR4A3', 'NR4A3'), ('PDGFRA', 'PDGFRA'), ('PDGFB', 'PDGFRA'), ('LRP1', 'IRAK1'), ('MAPK3', 'MAPK3'), ('CD40', 'RELA'), ('HIF1A', 'IL8'), ('CDK4', 'CDK4'), ('HIF1A', 'NOS2'), ('AGTR1', 'AGTR1'), ('LRP1', 'RHOA'), ('PDGFRA', 'MAPK1'), ('CEBPD', 'IL6'), ('HIF1A', 'PTGS2'), ('DCBLD2', 'PDGFRB'), ('CTNNB1', 'CTNNB1'), ('IL1B', 'IRAK1'), ('HIF1A', 'IL1B'), ('LTBR', 'RELA'), ('PDGFRB', 'MAPK3'), ('MAPK3', 'CREB1'), ('E2F1', 'CCND1'), ('CCNE1', 'CCNE1'), ('MAPK1', 'CREB1'), ('IL1B', 'IL1B'), ('CTNNB1', 'CCND1'), ('AKR1B1', 'RB1'), ('GSK3B', 'GSK3B'), ('IGF1', 'RAC1'), ('PTPN11', 'PTPN11'), ('TLR4', 'MYD88'), ('CREB1', 'CREB1'), ('IRAK1', 'IRAK1'), ('MYD88', 'MYD88'), ('PDGFB', 'RAC1'), ('AKT1', 'GSK3B'), ('E2F1', 'MYC'), ('PDGFRB', 'PDGFRB'), ('PDGFA', 'PDGFRA'), ('PRKCD', 'HIF1A'), ('NOS2', 'NOS2'), ('F3', 'F3'), ('AGTR1', 'PTPN11'), ('PRKCD', 'CCND1'), ('AKR1B1', 'CDK4'), ('AKT1', 'AKT1'), ('IL6', 'IL6'), ('CD40LG', 'CD40'), ('PDGFRB', 'MAPK1'), ('TNF', 'TNF'), ('MMP2', 'MMP2'), ('LRP1', 'LRP1'), ('E2F1', 'PCNA'), ('TNF', 'AKR1B1'), ('PLAUR', 'PLAUR'), ('PCNA', 'PCNA'), ('E2F1', 'E2F1'), ('APOE', 'LRP1'), ('E2F1', 'CCNE1'), ('NOX4', 'MMP2'), ('CCR3', 'MMP2'), ('CASP1', 'CASP1'), ('CCL11', 'CCR3'), ('THBS1', 'MMP2'), ('HIF1A', 'HIF1A'), ('PTGS2', 'PTGS2'), ('GSK3B', 'CTNNB1'), ('PDGFB', 'PDGFRB'), ('PLAU', 'PLAUR'), ('IL1B', 'CEBPD'), ('RHOA', 'F3'), ('MYC', 'MYC'), ('MAPK1', 'MAPK1'), ('AKR1B1', 'AKR1B1'), ('PDGFRA', 'MAPK3'), ('IL8', 'IL8'), ('NR4A3', 'CCND1')},
"CV-IPN-Foam_cell_formation_1": {('PTK2B', 'MAPK3'), ('MSR1', 'STAT1'), ('STAT1', 'ACAT1'), ('BIRC5', 'BIRC5'), ('TNFSF15', 'CD36'), ('EDN1', 'EDNRB'), ('ADAM17', 'OLR1'), ('PRKCZ', 'VEGFA'), ('APOA1', 'APOA1'), ('SYK', 'MAPK1'), ('CD36', 'TLR4'), ('CD36', 'PTK2B'), ('TLR4', 'SYK'), ('TLR4', 'TLR4'), ('CD36', 'CD36'), ('SYK', 'MAPK3'), ('CD36', 'TLR2'), ('MAPK1', 'SREBF2'), ('FABP4', 'ABCA1'), ('PPARD', 'PPARD'), ('PTK2B', 'MAPK1'), ('VEGFA', 'VEGFA'), ('TRAF2', 'TRAF2'), ('AGTR1', 'ABCA1'), ('TLR9', 'RELA'), ('PPARG', 'ABCA1'), ('PPARA', 'CPT1B'), ('HMGB1', 'TLR4'), ('PPARA', 'PPARA'), ('RAC1', 'RAC1'), ('SREBF1', 'SREBF1'), ('CPT1A', 'CPT1A'), ('OLR1', 'OLR1'), ('PPARA', 'ABCA1'), ('SCAP', 'SREBF2'), ('MAPK3', 'MAPK3'), ('TNFSF15', 'ABCG1'), ('TLR9', 'TLR9'), ('TNFRSF1A', 'TNFRSF1A'), ('TLR9', 'MYD88'), ('CD14', 'TLR4'), ('CD14', 'CD14'), ('NCOR1', 'PPARG'), ('AGTR1', 'AGTR1'), ('FABP4', 'PPARG'), ('ABCA1', 'ABCA1'), ('MSR1', 'MSR1'), ('MAPK3', 'RAC1'), ('MAPK1', 'CDC42'), ('XDH', 'XDH'), ('IFNG', 'ABCA1'), ('EDNRB', 'EDNRB'), ('IL6', 'STAT3'), ('NCOR1', 'PPARD'), ('SPI1', 'MSR1'), ('SREBF1', 'ACAT1'), ('OLR1', 'MSR1'), ('STAT3', 'ABCA1'), ('SREBF2', 'LDLR'), ('PPARD', 'MSR1'), ('CD36', 'MAPK9'), ('MAPK9', 'MAPK9'), ('SIRT1', 'SIRT1'), ('MYD88', 'MYD88'), ('TNFSF15', 'MSR1'), ('PPARA', 'CD36'), ('SREBF2', 'ACAT1'), ('TLR4', 'IL10'), ('NCOR1', 'NCOR1'), ('IFNA1', 'MSR1'), ('TNFSF14', 'TNFSF14'), ('STAT1', 'STAT1'), ('NR1H3', 'NR1H3'), ('HMGB1', 'TLR2'), ('MYD88', 'MAPK1'), ('TNFSF15', 'SCARB1'), ('MYD88', 'MAPK3'), ('EDNRA', 'EDNRA'), ('SREBF2', 'SCARB1'), ('SREBF2', 'SREBF2'), ('NOX1', 'NOX1'), ('PPARD', 'ABCA1'), ('MAPK3', 'EGR1'), ('SCARB1', 'SCARB1'), ('NR1H3', 'ABCA1'), ('PPARA', 'CPT1A'), ('MAP3K7', 'CHUK'), ('APOA1', 'LCAT'), ('IKBKB', 'NFKBIA'), ('SIRT1', 'NR1H3'), ('CYP27A1', 'CYP27A1'), ('CDC42', 'CDC42'), ('MAP3K7', 'MAP3K7'), ('TNFRSF1A', 'TRADD'), ('MAPK1', 'EGR1'), ('PTK2B', 'PTK2B'), ('AKT1', 'AKT1'), ('CCR7', 'CCR7'), ('FABP4', 'FABP4'), ('TNFSF15', 'ABCA1'), ('PTAFR', 'PTAFR'), ('TLR4', 'IRF3'), ('PPARA', 'ABCG1'), ('LCAT', 'LCAT'), ('TRAF2', 'MAP3K7'), ('SCAP', 'SCAP'), ('ACAT1', 'ACAT1'), ('LDLR', 'LDLR'), ('MAPK3', 'CDC42'), ('TGFB1', 'NR1H3'), ('NR1H2', 'NR1H2'), ('ABCG1', 'ABCG1'), ('CPT1B', 'CPT1B'), ('SPI1', 'SPI1'), ('CHUK', 'NFKBIA'), ('STAT3', 'STAT3'), ('PPARG', 'APOA1'), ('NR1H2', 'CCR7'), ('NR3C1', 'SCAP'), ('SREBF1', 'LDLR'), ('IRF3', 'NR1H3'), ('CHUK', 'CHUK'), ('AKT1', 'MSR1'), ('TRADD', 'TRADD'), ('TNF', 'TNFRSF1A'), ('STAT3', 'XDH'), ('SOD1', 'SOD1'), ('IFNG', 'CYP27A1'), ('EDN1', 'EDNRA'), ('IKBKB', 'IKBKB'), ('TLR2', 'TLR2'), ('IFNG', 'SPI1'), ('TRADD', 'TRAF2'), ('TLR2', 'FABP4'), ('NR1H2', 'ABCA1'), ('TLR9', 'CHUK'), ('PPARG', 'XDH'), ('NR3C1', 'NR3C1'), ('MAPK1', 'MAPK1'), ('NR1H3', 'CCR7'), ('PPARG', 'CD36'), ('INSR', 'AKT1'), ('MAP3K7', 'IKBKB')}
}
genes2name = {'CXCL12': {'CXCL12'}, 'CXCL13': {'CXCL13'}, 'CXCL14': {'CXCL14'}, 'PPBP': {'PPBP'}, 'XCL1': {'XCL1'},
              'CCL2': {'CCL2'}, 'PF4': {'PF4'}, 'CXCL10': {'CXCL10'}, 'CXCL5': {'CXCL5'},
              'XCL2': {'XCL1', 'XCL2'}, 'CXCL1': {'CXCL1'}, 'CCL13': {'CCL13'}, 'CXCL11': {'CXCL11'},
              'CXCL8': {'CXCL8'}, 'CCL14': {'CCL14'}, 'CCL3': {'CCL3', 'CCL3L3'}, 'CCL5': {'CCL5'},
              'CXCL2': {'CXCL2'}, 'CCL15': {'CCL15'}, 'CXCL3': {'CXCL3'},
              'CCL21': {'CCL21C', 'CCL21A', 'CCL21', 'GM10591', 'GM13304', 'GM21541', 'CCL21B'}, 'CCL17': {'CCL17'},
              'CXCL6': {'CXCL6'}, 'CCL11': {'CCL11'}, 'CCL7': {'CCL7'}, 'CCL4': {'CCL4'}, 'CCL1': {'CCL1'},
              'CXCL16': {'CXCL16'}, 'CCL18': {'CCL18'}, 'CCL19': {'GM2564', 'CCL19'}, 'CXCL9': {'CXCL9'},
              'CCL8': {'CCL8', 'CCL12'}, 'CCL20': {'CCL20'}, 'C5': {'C5', 'HC'}, 'CCL22': {'CCL22'}, 'CCL24': {'CCL24'},
              'CX3CL1': {'CX3CL1'}, 'CCL25': {'CCL25'}, 'CCL28': {'CCL28'}, 'CCL23': {'CCL23'},
              'CCL26': {'CCL26'}, 'CCL27': {'CCL27A', 'CCL27', 'CCL27B', 'GM13306'}, 'CCL6': {'CCL6'}, 'CCL9': {'CCL9'},
              }

normGeneSymbols = normalize_gene_names(path="/mnt/c/ownCloud/data/miRExplore/obodir/" + "/hgnc_no_withdrawn.syn")

cbn2restrict = {

    "CV-IPN-Endothelial_cell_activation_1": {
            'sentences': "false",
            "cells": [{
              "group": "cells",
              "name": "endothelial cell",
              "termid": "META:52"
            }],
        "go": [{ "group": "go", "name": "low-density lipoprotein particle", "termid": "GO:0034362" },
               { "group": "go", "name": "collagen binding", "termid": "GO:0005518" },
               { "group": "go", "name": "collagen trimer", "termid": "GO:0005581" }
        ]
        },

}

def acceptEvidence(ev):

    if ev['data_source'] == 'miRTarBase':

        if "Weak" in ev['functional_type']:
            return False

    return True

cbn2graph = {}

totalNWGenes = set()

for cbnNW in cbn2edges:

    allNWGenes = set()
    newEdges = set()

    for edge in cbn2edges[cbnNW]:

        src = edge[0].upper()
        tgt = edge[1].upper()

        if src in normGeneSymbols:
            src = normGeneSymbols[src]

        if tgt in normGeneSymbols:
            tgt = normGeneSymbols[tgt]

        allNWGenes.add(src)
        allNWGenes.add(tgt)

        totalNWGenes.add(src)
        totalNWGenes.add(tgt)

        newEdges.add((src, tgt))

    cbn2edges[cbnNW] = newEdges

    #print(cbnNW, allNWGenes)

    requestData = cbn2restrict[cbnNW]
    requestData['gene'] = list(allNWGenes)

    graph, nodeCounts, evcount = DataBasePlotter.fetchGenes(requestData, gene2name=genes2name, minPMIDEvCount=0, minTgtCount=0, acceptEv=acceptEvidence)

    newNodes = []
    mirnaNodes = []

    for (node, nodeAttr) in graph.nodes(data=True):

        if node in nodeCounts:
            nodeAttr['size'] = 20 + nodeCounts[node]

        if nodeAttr['color'] == 'blue':
            mirnaNodes.append(node)

        newNodes.append((node, nodeAttr))

    removeNodes = [node for node, degree in graph.degree().items() if degree >= 2 and node in mirnaNodes]
    graph.remove_nodes_from(removeNodes)

    removeEdges = set()

    singleEvidenceCounter = Counter()
    edgeEvidenceCounter = Counter()

    for (edgeSrc, edgeTgt, edgeAttr) in graph.edges(data=True):

        edge = (edgeSrc, edgeTgt)
        edgeR = (edgeTgt, edgeSrc)

        evC = None

        if edge in evcount:
            evC = evcount[edge]
        elif edgeR in evcount:
            evC = evcount[edgeR]

        if evC != None:
            if len(evC) == 1 and 'DIANA' in evC:
                edgeAttr['color'] = '#00ff00'
                removeEdges.add(edge)
            elif len(evC) == 1 and not 'pmid' in evC:

                for x in evC:
                    singleEvidenceCounter[x] += 1

                edgeAttr['color'] = '#00ff00'

            elabel = ""
            for x in sorted([x for x in evC]):
                elabel += x[0]

            edgeAttr["label"] = elabel

            edgeEvidenceCounter[elabel] += 1

        else:
            print(edge, "not in evcount")

    print(singleEvidenceCounter)
    print(edgeEvidenceCounter)

    for edge in removeEdges:
        graph.remove_edge(edge[0], edge[1])

    graph.remove_nodes_from(networkx.isolates(graph))

    for edge in cbn2edges[cbnNW]:
        if edge[0] == edge[1]:
            continue

        graph.add_edge(edge[0], edge[1], {'color': '#aa0000'})

    cbn2graph[cbnNW] = graph

for cbnNW in cbn2graph:

    graph = cbn2graph[cbnNW]

    for (node, nodeAttr) in graph.nodes(data=True):

        if 'color' in nodeAttr and nodeAttr['color'] == 'blue':

            otherGraphHasNode = False

            for oNW in cbn2graph:
                if oNW == cbnNW:
                    continue

                ograph = cbn2graph[oNW]

                if ograph.has_node(node):
                    otherGraphHasNode = True
                    break

            if not otherGraphHasNode:
                nodeAttr['color'] = '#d942f4'

    mygraph = CytoscapeGrapher.showGraph(graph, location='/mnt/c/Users/mjopp/Desktop/yanc_network/', name=cbnNW)

max_stars_repo_name: mjoppich/miRExplore

max_issues_repo_name: mjoppich/miRExplore

max_forks_repo_name: mjoppich/miRExplore

reason: import networkx


text:
import argparse
import sys
import numpy as np
from io import StringIO

def parse_plist(plist):
    """Parse dbert files grep'd by EEP with M= formatting cut out"""
    # genfromtext will not allow "." in column name
    ptcriname = plist[0].strip().split()[-1].replace('.', '').replace('#', '')
    col_keys = ['age', 'logl', 'logte', 'idx', ptcriname, 'ptcriname']
    dtype = [(c, '<f8') for c in col_keys]
    dtype[-1] = (col_keys[-1], '|S15')
    pd = np.genfromtxt(StringIO(''.join(plist)), dtype=dtype)
    return pd[ptcriname], ptcriname

def format_ptcriHB(filename, nptcri=5):
    """
    Reformat ptcri*.dat.HB (from dbert/) file to a ptcri*.dat file (from data/)
    """
    # ../F7/CAF09_MC_S15/OV0.6_Z0.0005_Y0.249/Z0.0005Y0.249OUTA1.74_F7_M20.0.PMS
    pref = filename.replace('ptcri_', '').split('.dat')[0]
    pref = "../F7/" + pref
    outfile = filename.replace('.HB', '').replace('ptcri_', 'ptcri_hb_')
    with open(filename, 'r') as inp:
        lines = inp.readlines()

    # grep EEPs
    lines = [l for l in lines if 'He' in l]

    # split by EEP
    plists = [lines[i::nptcri] for i in range(nptcri)]

    ptcrinames = []
    for plist in plists:
        # first EEP has different formatting:
        # e.g., after EEP: M= 0.49500000 npt=771  kind=1.0000000
        if 'M' in plist[0]:
            metad = [l.split('M')[1] for l in plist]
            # position 1 because "=" would be poisiton 0
            mass = np.array([m.strip().split()[1] for m in metad], dtype=float)
            kind = np.array([float(m.strip().split('=')[-1])
                             for m in metad], dtype=int)
            data = np.column_stack((np.arange(len(mass)) + 1, mass, kind))
            plist = [l.split('M')[0]+'\n' for l in plist]

        pdata, ptcriname = parse_plist(plist)
        data = np.column_stack((data, pdata))
        ptcrinames.append(ptcriname)

    header = 'i mass kind ' + ' '.join(ptcrinames)
    fmt = '%i %f %i ' + ' '.join('%i' * len(ptcrinames))

    np.savetxt(outfile, data, header=header, fmt=fmt)
    print(('wrote {}'.format(outfile)))

def main(argv):
    """main function for format_ptcriHB"""
    parser = argparse.ArgumentParser(
        description="convert a dbert HB file to a data HB file")

    parser.add_argument('filename', type=str, nargs='*',
                        help='ptcri*.dat.HB files to work on')

    args = parser.parse_args(argv)

    [format_ptcriHB(f) for f in args.filename]

if __name__ == "__main__":
    main(sys.argv[1:])


text:
import math
import numpy as np
import torch

'''
Point cloud transform:
    pc:
        torch: [b, 3, p]
        np: [(b, )3, p]
'''

# def R_to_hom_np(R):
#     assert R.ndim == 2
#     R = np.concatenate((R,[[0,0,0]]),axis=0)
#     return np.concatenate((R,np.array([[0,0,0,1]]).T),axis=1)

# homogeneous transform with transformation matrix
# T: [(b, ){3/4}, 4] x [(b, ){3/4}, p]
# return: [(b, ){3/4}, p]
def transform(x, T, bnc=False):
    if bnc:
        x = x.permute(0,2,1).contiguous()

    assert x.shape[-2] == 4, 'homography required'
    if T.ndim == 2:
        x = torch.matmul(T, x)
    elif T.ndim == 3:
        x = torch.bmm(T, x)

    if bnc:
        x = x.permute(0,2,1).contiguous()
    return x

def transform_np(x, T, bnc=False, batch=False):
    if bnc:
        x = x.transpose(0, 2, 1)

    assert x.shape[-2] == 4, 'homography required'
    if batch:
        if T.ndim == 2:
            x =  np.einsum('pk,bkl->bpl', T, x)
        elif T.ndim == 3:
            x = np.einsum('bpk,bkl->bpl', T, x)
    else:
        x = np.matmul(T, x)

    if bnc:
        x = x.transpose(0, 2, 1)
    return x

# rotate
# R [(b, )3, 3] / [(b, ){3/4}, 4] (hom)
# return: [(b, )3, p]
def rotate(x, R):
    if R.shape[-1] == 4:
        R = R[..., :3, :3].contiguous()
    if R.ndim == 2:
        x = torch.matmul(R, x)
    elif R.ndim == 3:
        x = torch.bmm(R, x)
    return x

def rotate_np(x, R, batch=False):
    if R.shape[-1] == 4:
        R = R[..., :3, :3]
    if batch:
        if R.ndim == 2:
            return np.einsum('pk,bkl->bpl', R, x)
        elif R.ndim == 3:
            return np.einsum('bpk,bkl->bpl', R, x)
    else:
        return np.matmul(R, x)

'''
euler angle [x,y,z] to rotation matrix [3,3]
'''

def R_from_euler_np(angles):
    '''
    angles: [(b, )3]
    '''
    # R_x = np.array([[1,         0,                  0                   ],
    #                 [0,         math.cos(angles[0]), -math.sin(angles[0]) ],
    #                 [0,         math.sin(angles[0]), math.cos(angles[0])  ]
    #                 ])
    # R_y = np.array([[math.cos(angles[1]),    0,      math.sin(angles[1])  ],
    #                 [0,                     1,      0                   ],
    #                 [-math.sin(angles[1]),   0,      math.cos(angles[1])  ]
    #                 ])

    # R_z = np.array([[math.cos(angles[2]),    -math.sin(angles[2]),    0],
    #                 [math.sin(angles[2]),    math.cos(angles[2]),     0],
    #                 [0,                     0,                      1]
    #                 ])
    # return np.dot(R_z, np.dot( R_y, R_x ))
    raise NotImplementedError()

# def rotate_from_euler(pc, angles, to_hom=False):
#     R = torch.from_numpy(R_from_euler_np(angles).astype(np.float32)).cuda()
#     pc = rotate(pc.transpose(1,2), R)
#     return pc.transpose(1,2)


text:
def resample_to_1km( x ):
	'''
	template_raster_mask should be a mask in in the res/extent/origin/crs of the
	existing TEM IEM products.
	'''
	fn, output_filename, template_raster_mask_fn = x
	import rasterio, os
	from rasterio.warp import RESAMPLING, reproject
	import numpy as np

	rst = rasterio.open( fn )
	rst_arr = rst.read( 1 )

	fn = os.path.basename( fn )
	fn_split = fn.split( '.' )[0].split( '_' )

	template_raster_mask = rasterio.open( template_raster_mask_fn )
	template_arr = template_raster_mask.read( 1 )
	template_meta = template_raster_mask.meta
	template_meta.update( compress='lzw', nodata=rst.nodata, dtype='float32' )

	if 'transform' in template_meta.keys():
		template_meta.pop( 'transform' )

	output_arr = np.empty_like( template_arr.astype( np.float32 ) )
	output_arr[ template_arr == 0 ] = rst.nodata

	epsg3338 = {'init':'epsg:3338'}
	reproject( rst_arr, output_arr, src_transform=rst.affine, src_crs=epsg3338, src_nodata=rst.nodata, \
			dst_transform=template_raster_mask.affine, dst_crs=epsg3338,\
			dst_nodata=rst.nodata, resampling=RESAMPLING.cubic_spline ) # , num_threads=2

	with rasterio.open( output_filename, 'w', **template_meta ) as out:
		output_arr[ template_arr == 0 ] = rst.nodata
		out.write( output_arr, 1 )
	return output_filename

def path_slicer_ar5( input_path ):
	'''
	slicer works for the /Data AR5 data outputs produced by leonawicz
	and is hardwired to that data structure.
	'''
	scenario, model, variable = input_path.split( os.path.sep )[-3:]
	return scenario, model, variable

def generate_output_fn( input_fn, output_path, name ):
	'''
	add id as an element in the new filename
	'''
	base = os.path.basename( input_fn )
	basesplit = base.split( '_' )
	basesplit.insert( 3, name ) #name = 'iem' or 'akcan'
	return os.path.join( output_path, '_'.join( basesplit ) )

if __name__ == '__main__':
	import os, glob, rasterio
	import numpy as np1
	import pandas as pd
	from functools import partial
	import itertools
	from pathos import multiprocessing as mp
	from pathos.mp_map import mp_map
	import argparse

	parser = argparse.ArgumentParser( description='crop mask and resample the alf inputs to different extents and resolutions' )
	parser.add_argument( '-p', '--input_path', action='store', dest='input_path', type=str, help='path to input files' )
	parser.add_argument( '-o', '--output_base_path', action='store', dest='output_base_path', type=str, help='path to output base directory' )
	parser.add_argument( '-m', '--template_raster_mask_fn', action='store', dest='template_raster_mask_fn', type=str, help='path to mask raster' )
	parser.add_argument( '-g', '--group', action='store', dest='group', type=str, help='name to identify raster type in filename. i.e. "iem", "alf" ')

	args = parser.parse_args()
	input_path = args.input_path
	output_base_path = args.output_base_path
	template_raster_mask_fn = args.template_raster_mask_fn
	group = args.group
	ncores = 32

	# more prep
	scenario, model, variable = path_slicer_ar5( input_path )
	template_raster_mask = rasterio.open( template_raster_mask_fn )
	output_path = os.path.join( output_base_path, scenario, model, variable )

	try:
		if not os.path.exists( output_path ):
			os.makedirs( output_path )
	except:
		pass

	file_list = glob.glob( os.path.join( input_path, '*.tif' ) )
	output_filenames = [ generate_output_fn( input_fn, output_path, group ) for input_fn in file_list ]

	args = zip( file_list, output_filenames, itertools.repeat( template_raster_mask_fn, len(file_list) ) )
	_ = mp_map( resample_to_1km, args, nproc=ncores )

# # # RUN THE ABOVE # # # #
# import os, glob
# import numpy as np

# # list the data we want
# input_path = '/Data/Base_Data/Climate/AK_CAN_2km/projected/AR5_CMIP5_models'
# out = [ root for root, subs, files in os.walk( input_path ) \
# 		if len( glob.glob( os.path.join( root, '*.tif' ) ) ) > 0 and not 'derived' in root ]
# input_paths = np.unique( out ).tolist()

# model = 'CCSM4' # [ 'CCSM4', 'CNRM-CM5', 'GFDL-CM3', 'GISS-E2-R', 'IPSL-CM5A-LR', 'MPI-ESM-LR', 'MRI-CGCM3', '5modelAvg' ]
# scenarios = [ 'rcp26', 'rcp45', 'rcp60', 'rcp85' ]

# # # IEM 1km
# output_base_path = '/atlas_scratch/malindgren/CMIP5/IEM_1km'
# template_raster_mask_fn = '/workspace/Shared/Tech_Projects/ALFRESCO_Inputs/project_data/TEM_Data/extents/IEM_Mask_1km.tif'
# group = 'iem'

# for input_path in input_paths:
# 	if model in input_path:
# 		# print input_path
# 		os.chdir( '/workspace/Shared/Tech_Projects/ALFRESCO_Inputs/project_data/CODE/tem_ar5_inputs/downscale_cmip5/bin' )
# 		os.system( 'python alfresco_cmip5_crop_mask_resample_non_tem.py -p ' + input_path + ' -o ' + output_base_path + ' -m ' + template_raster_mask_fn + ' -g ' + group )

# # # AKCAN 1km
# output_base_path = '/atlas_scratch/malindgren/CMIP5/AKCAN_1km'
# template_raster_mask_fn = '/workspace/UA/malindgren/projects/ALFRESCO_DATA_CLIP/mask_tmp/AKCanada_PRISM_Mask_1km_ALFRESCO_commonExtent.tif'
# group = 'alf'

# for input_path in input_paths:
# 	if model in input_path:
# 		# print input_path
# 		os.chdir( '/workspace/Shared/Tech_Projects/ALFRESCO_Inputs/project_data/CODE/tem_ar5_inputs/downscale_cmip5/bin' )
# 		os.system( 'python alfresco_cmip5_crop_mask_resample_non_tem.py -p ' + input_path + ' -o ' + output_base_path + ' -m ' + template_raster_mask_fn + ' -g ' + group )

# # # NOTES:
# some setup:
# cru_path = '/Data/Base_Data/Climate/AK_CAN_2km/historical/CRU/CRU_TS32'
# ar5_path = '/Data/Base_Data/Climate/AK_CAN_2km/projected/AR5_CMIP5_models'
# input_path = '/Data/Base_Data/Climate/AK_CAN_2km/projected/AR5_CMIP5_models'
# output_base_path = '/atlas_scratch/malindgren/CMIP5'

max_stars_repo_name: ua-snap/downscale

max_issues_repo_name: ua-snap/downscale

max_forks_repo_name: ua-snap/downscale


text:
import numpy as np
import cv2
import os
import math
import scipy.io as sio
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit

class AverageMeter(object):
	def __init__(self):
		self.reset()

	def reset(self):
		self.val = 0
		self.avg = 0
		self.sum = 0
		self.count = 0

	def update(self, val, n=1):
		self.val = val
		self.sum += val * n
		self.count += n
		self.avg = self.sum / self.count

def ReadImg(filename):
	img = cv2.imread(filename)
	img = img[:,:,::-1] / 255.0
	img = np.array(img).astype('float32')

	return img

####################################################
#################### noise model ###################
####################################################

def func(x, a):
    return np.power(x, a)

def CRF_curve_fit(I, B):
    popt, pcov = curve_fit(func, I, B)
    return popt

def CRF_function_transfer(x, y):
    para = []
    for crf in range(201):
        temp_x = np.array(x[crf, :])
        temp_y = np.array(y[crf, :])
        para.append(CRF_curve_fit(temp_x, temp_y))
    return para

def mosaic_bayer(rgb, pattern, noiselevel):

    w, h, c = rgb.shape
    if pattern == 1:
        num = [1, 2, 0, 1]
    elif pattern == 2:
        num = [1, 0, 2, 1]
    elif pattern == 3:
        num = [2, 1, 1, 0]
    elif pattern == 4:
        num = [0, 1, 1, 2]
    elif pattern == 5:
        return rgb

    mosaic = np.zeros((w, h, 3))
    mask = np.zeros((w, h, 3))
    B = np.zeros((w, h))

    B[0:w:2, 0:h:2] = rgb[0:w:2, 0:h:2, num[0]]
    B[0:w:2, 1:h:2] = rgb[0:w:2, 1:h:2, num[1]]
    B[1:w:2, 0:h:2] = rgb[1:w:2, 0:h:2, num[2]]
    B[1:w:2, 1:h:2] = rgb[1:w:2, 1:h:2, num[3]]

    gauss = np.random.normal(0, noiselevel/255.,(w, h))
    gauss = gauss.reshape(w, h)
    B = B + gauss

    return (B, mask, mosaic)

def ICRF_Map(Img, I, B):
    w, h, c = Img.shape
    output_Img = Img.copy()
    prebin = I.shape[0]
    tiny_bin = 9.7656e-04
    min_tiny_bin = 0.0039
    for i in range(w):
       for j in range(h):
           for k in range(c):
               temp = output_Img[i, j, k]
               start_bin = 0
               if temp > min_tiny_bin:
                   start_bin = math.floor(temp/tiny_bin - 1) - 1
               for b in range(start_bin, prebin):
                   tempB = B[b]
                   if tempB >= temp:
                       index = b
                       if index > 0:
                           comp1 = tempB - temp
                           comp2 = temp - B[index-1]
                           if comp2 < comp1:
                               index = index - 1
                       output_Img[i, j, k] = I[index]
                       break

    return output_Img

def CRF_Map(Img, I, B):
    w, h, c = Img.shape
    output_Img = Img.copy()
    prebin = I.shape[0]
    tiny_bin = 9.7656e-04
    min_tiny_bin = 0.0039
    for i in range(w):
        for j in range(h):
            for k in range(c):
                temp = output_Img[i, j, k]

                if temp < 0:
                    temp = 0
                    Img[i, j, k] = 0
                elif temp > 1:
                    temp = 1
                    Img[i, j, k] = 1
                start_bin = 0
                if temp > min_tiny_bin:
                    start_bin = math.floor(temp/tiny_bin - 1) - 1

                for b in range(start_bin, prebin):
                    tempB = I[b]
                    if tempB >= temp:
                        index = b
                        if index > 0:
                            comp1 = tempB - temp
                            comp2 = temp - B[index-1]
                            if comp2 < comp1:
                                index = index - 1
                        output_Img[i, j, k] = B[index]
                        break
    return output_Img

def CRF_Map_opt(Img, popt):
    w, h, c = Img.shape
    output_Img = Img.copy()

    output_Img = func(output_Img, *popt)
    return output_Img

def Demosaic(B_b, pattern):

    B_b = B_b * 255
    B_b = B_b.astype(np.uint16)

    if pattern == 1:
        lin_rgb = cv2.demosaicing(B_b, cv2.COLOR_BayerGB2BGR)
    elif pattern == 2:
        lin_rgb = cv2.demosaicing(B_b, cv2.COLOR_BayerGR2BGR)
    elif pattern == 3:
        lin_rgb = cv2.demosaicing(B_b, cv2.COLOR_BayerBG2BGR)
    elif pattern == 4:
        lin_rgb = cv2.demosaicing(B_b, cv2.COLOR_BayerRG2BGR)
    elif pattern == 5:
        lin_rgb = B_b

    lin_rgb = lin_rgb[:,:,::-1] / 255.
    return lin_rgb

def AddNoiseMosai(x, CRF_para, iCRF_para, I, B, Iinv, Binv, sigma_s, sigma_c, crf_index, pattern, opt = 1):
    w, h, c = x.shape
    temp_x = CRF_Map_opt(x, iCRF_para[crf_index] )

    sigma_s = np.reshape(sigma_s, (1, 1, c))
    noise_s_map = np.multiply(sigma_s, temp_x)
    noise_s = np.random.randn(w, h, c) * noise_s_map
    temp_x_n = temp_x + noise_s

    noise_c = np.zeros((w, h, c))
    for chn in range(3):
        noise_c [:, :, chn] = np.random.normal(0, sigma_c[chn], (w, h))

    temp_x_n = temp_x_n + noise_c
    temp_x_n = np.clip(temp_x_n, 0.0, 1.0)
    temp_x_n = CRF_Map_opt(temp_x_n, CRF_para[crf_index])

    if opt == 1:
        temp_x = CRF_Map_opt(temp_x, CRF_para[crf_index])

    B_b_n = mosaic_bayer(temp_x_n[:,:,::-1], pattern, 0)[0]
    lin_rgb_n = Demosaic(B_b_n, pattern)
    result = lin_rgb_n
    if opt == 1:
        B_b = mosaic_bayer(temp_x[:,:,::-1], pattern, 0)[0]
        lin_rgb = Demosaic(B_b, pattern)
        diff = lin_rgb_n - lin_rgb
        result = x + diff

    return result

def AddRealNoise(image, CRF_para, iCRF_para, I_gl, B_gl, I_inv_gl, B_inv_gl):
    sigma_s = np.random.uniform(0.0, 0.16, (3,))
    sigma_c = np.random.uniform(0.0, 0.06, (3,))
    CRF_index = np.random.choice(201)
    pattern = np.random.choice(4) + 1
    noise_img = AddNoiseMosai(image, CRF_para, iCRF_para, I_gl, B_gl, I_inv_gl, B_inv_gl, sigma_s, sigma_c, CRF_index, pattern, 0)
    noise_level = sigma_s * np.power(image, 0.5) + sigma_c

    return noise_img, noise_level

max_stars_repo_name: IDKiro/CBDNet-tensorflow

max_issues_repo_name: IDKiro/CBDNet-tensorflow

max_forks_repo_name: IDKiro/CBDNet-tensorflow

reason: import numpy,import scipy,from scipy


text:
"""
This Modules performs the basic preprocessing on the raw data:

1. R-Peaks are detected to seperate the signal into individual heartbeats
2. Heartbeats with less than 5% unique values are clipped
3. Every heartbeat is interpolated to dimension 100
4. Individual interpolated heartbeats are saved

Outputs several processed data files named based off their "index" which is the alphanurmeric
sorted position of the raw data file in the Data_H5_Files directory

Mod_Four_Lead_[idx].npy : 4 x n Numpy array of the four lead signals with lead disconnections clipped out
HB_Peaks_[idx].npy      : (num heartbeats - 1) Numpy vector of the indicies of the R-Peaks in Mod_Four_Lead
Fixed_Dim_Hbs_[idx].npy : 4 x 100 x (num heartbeats) Numpy array of the four lead signals seperated into
						  individual heartbeats and interpolated to dimension 100
Cleaned_HR_[idx].npy    : n Numpy vector of the heartrate from monitor, with lead disconnections clipped out
HB_Lens_[idx].npy       : (num heartbeats) Numpy vector of individual heartbeat lengths in samples; before interpolation

"""

import numpy as np
import os

import matplotlib.pyplot as plt

from src.utils.file_indexer import get_filenames, get_patient_ids
from src.preprocessing.noise_filtering import remove_noise

from src.utils import plotting_utils, dsp_utils, h5_interface

def detect_gaps(signal_data, peaks):
	"""
    Checks a list of detected heartbeat slices from ECG signal, and detects flatlines
    :param signal_data: [np array[float64]] raw ECG signal
    :param peaks: [np array[int]] indicies of the detected peaks
    :return: [list[slice[int:int]]] slices of flatline signal
    """
	#Look for gaps (based of unique values)
	bad_hbs = []
	# plt.plot(signal_data)
	# plt.show()
	vals = np.unique(signal_data[0:peaks[0]])
	#leading heartbeat
	if len(vals) < .05 * (peaks[0]):
		bad_hbs = [slice(0, peaks[0])]
	#scan heartbeats
	for i in range(1, len(peaks)):
		vals = np.unique(signal_data[peaks[i-1]:peaks[i]])
		if len(vals) < .05 * (peaks[i] - peaks[i-1]):
			"""
			#Visual Inspection of bad heartbeat removal
			plt.plot(signal_data[peaks[i-1]: peaks[i]])
			plt.show()
			"""
			bad_hbs.append(slice(peaks[i-1], peaks[i]))
	vals = np.unique(signal_data[peaks[-1]:])
	#Trailing heartbeat
	if len(vals) < .05 * (len(signal_data) - peaks[-1]):
		bad_hbs.append( slice(peaks[-1], len(signal_data)) )

	return bad_hbs
def find_lengths(peaks):
	"""
    Returns the heartbeat lengths
    :param peaks: [np array[int]] indicies of the detected peaks
    :return: [np array[int]] heartbeat lengths
    """
	return np.diff(np.concatenate( (np.array([0]), peaks) ) )

def find_long_heartbeats(peaks, total_len, threshold):
	"""
    Find slices that are longer than a threshold
    :param peaks: [np array[int]] indicies of the detected peaks
    :param total_len: [int] length of signal
    :param threshold: [float] threshold length
    :return: [list[slice[int:int]]] slices that are above threshold
    """
	long_slices = []
	rm_idx = []

	removed_lengths = []
	for i in range(1, len(peaks)):
		hb_len = peaks[i] - peaks[i-1]
		if hb_len > threshold:
			long_slices.append(slice(peaks[i-1], peaks[i]))
			removed_lengths.append(hb_len)
			rm_idx.append(i-1)
	if total_len - peaks[-1] > threshold:
		long_slices.append(slice(peaks[-1], total_len))
		peaks = peaks[:-1]
	for i in range(len(rm_idx)):
		peaks[rm_idx[i]+1:] = peaks[rm_idx[i]+1:] - removed_lengths[i]
	return long_slices, np.delete(peaks, rm_idx)

def delete_slices(slices, length, removals):
	"""
    Deletes slices from ECG
    :param slices: [list[slice[int:int]]] slices to be removed
    :param length: [int] Length of signal
    :param removals: [list[np array[float64]]] list of signals to be modified
    :return: [list[np array[float64]]] modified signals
    """
	ind = np.indices((length,))[0]
	rm = np.hstack(list(ind[i] for i in slices))
	good_indices = sorted(set(ind)-set(rm))

	for i in range(len(removals)):
		 removals[i] = np.take(removals[i], good_indices)

	return removals

def build_hb_matrix_centered(four_lead, peaks, dimension, plotting = False):
	"""
    Build the data matrix
    :param four lead: [np array[np array[float64]]] four lead ECG signal
    :param peaks: [np array[int]] indicies of the detected peaks
    :param dimension: [int] Dimension for each heartbeat to be interpolated to
    :param plotting: [bool] periodically show the heartbeat
    :return: [np array[np array[np array[float64]]]] 3-D Data Matrix
    """
	if len(peaks) % 2 == 0:
		four_lead = four_lead[:,:peaks[-1]]
		peaks = peaks[:-1]
	lead_length = len(four_lead[1,:]) - 1
	#Save an array of dimension Num heartbeats x 200 (heartbeat length) x Leads (4)
	fixed_dimension_hbs = np.zeros(((len(peaks)+3) // 2, dimension * 2, 4))
	for lead_num in range(4):
		#First heartbeat in data
		first_half = dsp_utils.change_dim(four_lead[lead_num, peaks[0] // 10:peaks[0]], dimension)
		second_half = dsp_utils.change_dim(four_lead[lead_num, peaks[0]:peaks[1] - (peaks[1] - peaks[0]) // 10], dimension)
		fixed_dimension_hbs[0,:,lead_num] = np.concatenate((first_half, second_half))
		#Last heartbeat in data
		first_half = dsp_utils.change_dim(four_lead[lead_num, peaks[-2] + (peaks[-1] - peaks[-2]) // 10:peaks[-1]], dimension)
		second_half = dsp_utils.change_dim(four_lead[lead_num, peaks[-1]:lead_length - (lead_length - peaks[-1]) // 10], dimension)
		fixed_dimension_hbs[(len(peaks) + 1) // 2,:,lead_num] = np.concatenate((first_half, second_half))

		#iterate through the rest of heartbeats
		for hb_num in range(2, len(peaks) - 1, 2):
			first_half = dsp_utils.change_dim(four_lead[lead_num, peaks[hb_num] + (peaks[hb_num + 1] - peaks[hb_num]) // 10:peaks[hb_num + 1]], dimension)
			second_half = dsp_utils.change_dim(four_lead[lead_num, peaks[hb_num + 1]:peaks[hb_num + 2] - (peaks[hb_num + 2] - peaks[hb_num + 1]) // 10], dimension)
			fixed_dimension_hbs[hb_num // 2,:,lead_num] = np.concatenate((first_half, second_half))

			#Periodic Visual inspection of dimension fixed heartbeat
			if plotting and hb_num % 15000 == 0:
				plt.plot(fixed_dimension_hbs[hb_num // 2,:,lead_num])
				plt.title(label = f"Lead {lead_num} hb_num {hb_num // 2}")
				plt.show()
	#plt.plot(fixed_dimension_hbs[-1,:,1])
	#plt.show()
	return fixed_dimension_hbs

def build_hb_matrix(four_lead, peaks, dimension, time, beats_per_vector = 1, plotting = False):
	"""
	Build the data matrix
	:param four lead: [np array[np array[float64]]] four lead ECG signal
	:param peaks: [np array[int]] indicies of the detected peaks
	:param dimension: [int] Dimension for each heartbeat to be interpolated to
	:param time: [np array[int]] time vector
	:param beats_per_vector: [int] number of heartbeats in each feature vector
	:param plotting: [bool] periodically show the heartbeat
	:return: [np array[np array[np array[float64]]]] 3-D Data Matrix
	"""
	crimp = 0
	#number of samples to cut off each end of a set of beats
	if beats_per_vector > 1:
		crimp = 10

	peaks = np.concatenate((np.array([0]), peaks, np.array([four_lead.shape[1]])))
	num_beats = len(peaks) - 1
	if not beats_per_vector == 1:
		peaks = peaks[num_beats % beats_per_vector:]
	peaks = np.take(peaks, list(range(0, len(peaks),beats_per_vector)))
	#Save an array of dimension Num heartbeats x 100 (heartbeat length) x Leads (4)
	fixed_dimension_hbs = np.zeros((num_beats // beats_per_vector, beats_per_vector * dimension, 4))
	for lead_num in range(4):
		#First heartbeat in data
		#fixed_dimension_hbs[0,:,lead_num] = dsp_utils.change_dim(four_lead[lead_num, 0:peaks[0]], dimension)
		#Last heartbeat in data
		#fixed_dimension_hbs[len(peaks),:,lead_num] = dsp_utils.change_dim(four_lead[lead_num, peaks[-1]:], dimension)
		#iterate through the rest of heartbeats
		for hb_num, start_peak in enumerate(peaks[:-1]):
			individual_hb = four_lead[lead_num,start_peak + crimp:peaks[hb_num+1] - crimp]
			try:
				fixed_dimension_hbs[hb_num,:,lead_num] = dsp_utils.change_dim(individual_hb, beats_per_vector * dimension)
			except:
				print(f"WARNING : failed to interpolate heartbeat num: {hb_num} of length: {peaks[hb_num + 1] - start_peak}")
			#Periodic Visual inspection of dimension fixed heartbeat
			if plotting and hb_num % 15000 == 0:
				plt.plot(fixed_dimension_hbs[hb_num,:,lead_num])
				plt.show()
	#plt.plot(fixed_dimension_hbs[-1,:,1])
	#plt.show()
	return fixed_dimension_hbs

def load_np(filename, control=False):
	"""
    Load the raw data from h5 files
    :param filename: [string] filename to be read in
    :return: 4 x [np array[float64]] leads, [np array[int]] time vector,
             [np array[int]] heartrate vector from machine, [np array[float64]] abs pos sum ECG signal
    """
	print("Opening file : " + filename)

	h5f = h5_interface.readh5(filename, control)

	lead1, lead2, lead3, lead4, time, heartrate = h5_interface.ecg_np(h5f, split = True)

	four_lead = np.vstack((lead1, lead2, lead3, lead4))

	pos_sum = dsp_utils.combine_four_lead(four_lead)
	'''
	plt.plot(pos_sum)
	plt.title("Example Heartbeat")
	plt.show()
	'''
	return lead1, lead2, lead3, lead4, time, heartrate, pos_sum

def writeout(curr_index, orig_num_hbs, four_lead, fixed_dimension_hbs, heartrate, peaks, hb_lengths, time, percent, prefix = "",directory = "Working_Data"):
	"""
    Save the intermidiate data after preprocessing. Also write stats to log file
    :param curr_index: [int] index of the current patient
    :param orig_num_hbs: [int] original number of heartbeats before filtering
    :param four lead: [np array[np array[float64]]] four lead ECG signal
    :param fixed_dimension_hbs: [np array[np array[np array[float64]]]] 3D data matrix
    :param heartrate: [np array[float]] Heartrate vector from ECG machine
    :param peaks: [np array[int]] indicies of the detected peaks
    :param hb_lengths: [np array[int]] heartbeat lengths
    :param time: [np array[int]] Time vector
    :param percent: [float] percent of original signal kept after filtering
    :return: None
    """
	#logging setup
	log_filepath = os.path.join(directory, "Heartbeat_Stats_Idx" + curr_index + ".txt")
	os.makedirs(os.path.dirname(log_filepath), exist_ok=True)
	log = open(log_filepath, 'w')

	log.write("Average valid heartbeat length : " + str(np.average(hb_lengths)) + "\n")
	log.write("Total valid heartbeats : " + str(len(peaks))+ "\n")
	log.write("Total invalid/removed heartbeats : " + str(orig_num_hbs - len(peaks))+ "\n")
	log.write("Percent unavaliable : " + str(percent))
	#Save the four lead signals with gaps cut out
	mod_four_lead_savename = os.path.join(directory, prefix + "Mod_Four_Lead_Idx" + curr_index + ".npy")
	#Save the processed heartbeat arrays
	data_savename = os.path.join(directory, prefix + "Fixed_Dim_HBs_Idx" + curr_index + ".npy")
	#Save the clipped heartrate vector from the ECG machine
	hr_savename = os.path.join(directory, prefix + "Cleaned_HR_Idx" + curr_index + ".npy")
	#Save the peak indicies
	peaks_savename = os.path.join(directory, prefix + "HB_Peaks_Idx" + curr_index + ".npy")
	#Save the heartbeat lengths
	HB_lens_savename = os.path.join(directory, prefix + "HB_Lens_Idx" + curr_index + ".npy")
	#Save the heartbeat timestamps
	HB_timestamps_savename = os.path.join(directory, prefix + "HB_Timestamps_Idx" + curr_index + ".npy")

	np.save(mod_four_lead_savename, four_lead)
	np.save(data_savename, fixed_dimension_hbs)
	np.save(hr_savename, heartrate)
	np.save(peaks_savename, peaks)
	np.save(HB_lens_savename, hb_lengths)
	np.save(HB_timestamps_savename, time[peaks[:-1]])
	log.close()

def denoise(time, lead1, lead2, lead3, lead4):
	"""
    Build the data matrix
    :param time: [np array[int]] time vector
    :param lead: [np array[float64]] ECG lead signal
    :return: [np array[float64]] denoised leads
    """
	# Removing baseline wander and high frequency noise
	lead1 = remove_noise(time, lead1)
	lead2 = remove_noise(time, lead2)
	lead3 = remove_noise(time, lead3)
	lead4 = remove_noise(time, lead4)
	return lead1, lead2, lead3, lead4, np.vstack((lead1, lead2, lead3, lead4))
'''
Inputs: Indicies of the patient files to process
Outputs: Saves multiple files of processed data, in addition to a text file log for each
'''
def preprocess_sum(filename, curr_index, beats_per_datapoint = 1, file_prefix = "", control=False):
	"""
    Main function to run the preprocessing using the absolute value sum method
    :param filename: [string] filename to be read in
    :param curr_index: [int] index of the current patient
    :param four lead: [np array[np array[float64]]] four lead ECG signal
    :param beats_per_datapoint: [int] number of heartbeats in each feature vector
    :param file_prefix: [string] optional filename prefix for savefiles
    :return: None
    """
	lead1, lead2, lead3, lead4, time, heartrate, pos_sum = load_np(filename, control)

	if heartrate is None:
		heartrate = np.full((len(lead1),), 120)

	peaks = dsp_utils.get_peaks_dynamic(pos_sum, heartrate) # indices on the signal where we found a peak
	peaks = peaks.astype(int)

	orig_len = len(lead1)
	orig_num_hbs = len(peaks)

	#get the bad "heartbeats"
	gap_beats = detect_gaps(pos_sum, peaks)
	'''
	for gap in gap_beats:
		plt.plot(lead1[gap])
		plt.title("Flatline \"Lead-off\" heartbeat")
		plt.xlabel("Time")
		plt.show()
	'''
	#Delete the bad heartbeats
	if gap_beats:
		if heartrate is not None:
			pos_sum, time, heartrate, lead1, lead2, lead3, lead4 = delete_slices(gap_beats, len(pos_sum), [pos_sum, time, heartrate, lead1, lead2, lead3, lead4])
		else:
			pos_sum, time, lead1, lead2, lead3, lead4 = delete_slices(gap_beats, len(pos_sum), [pos_sum, time, lead1, lead2, lead3, lead4])
		#refind peaks
		peaks = dsp_utils.get_peaks_dynamic(pos_sum, heartrate) # indices on the signal where we found a peak
		peaks = peaks.astype(int)

	lead1, lead2, lead3, lead4, four_lead = denoise(time, lead1, lead2, lead3, lead4)

	#hb_lengths = find_lengths(peaks, len(pos_sum), pos_sum = pos_sum)
	#print(min(hb_lengths), sum(hb_lengths) / len(hb_lengths))
	#try to refind peaks on long heartbeats, mostly to split missed edge beats
	#edge beats are peaks on the edge of a dynamic window in dynamic peakfind, they get missed
	new_peaks = []
	new_inds = []
	for i in range(1, len(peaks)):
		if peaks[i] - peaks[i - 1] > 200:
			extra_peaks = dsp_utils.get_peaks_dynamic(pos_sum[peaks[i - 1]:peaks[i]], heartrate)
			if len(extra_peaks) == 1:
				if extra_peaks[0] > 40:
					new_peaks.append(peaks[i-1] + extra_peaks[0])
					new_inds.append(i)
	peaks = np.insert(peaks, new_inds, new_peaks)

	#hb_lengths = find_lengths(peaks, len(pos_sum))
	#print(min(hb_lengths), sum(hb_lengths) / len(hb_lengths))
	#print(len(peaks))
	too_longs, peaks = find_long_heartbeats(peaks, len(pos_sum), 200)
	if too_longs:
		if heartrate is not None:
				pos_sum, time, heartrate, lead1, lead2, lead3, lead4 = delete_slices(too_longs, len(pos_sum), [pos_sum, time, heartrate, lead1, lead2, lead3, lead4])
		else:
			pos_sum, time, lead1, lead2, lead3, lead4 = delete_slices(too_longs, len(pos_sum), [pos_sum, time, lead1, lead2, lead3, lead4])
	#print(len(peaks), len(pos_sum),peaks[-1])
	four_lead = np.vstack((lead1, lead2, lead3, lead4))
	percent_unavaliable = len(lead1) / orig_len
	#print(four_lead.shape)

	"""
	import matplotlib
	matplotlib.rcParams['figure.dpi'] = 200
	#Visual Test for R-Peak identification
	plt.plot(lead1)
	plt.title("Peak Detection Example")
	#plt.vlines(x = peaks, ymin = 0, ymax = 8, colors = "red", linewidth = 2)
	peaks = peaks - 1 #lol the pos sum must be off by one from lead1
	plt.plot(peaks, lead1[peaks], 'ro')
	plt.axis([10000,11125,-1.5,2])
	plt.show()
	"""
	if not beats_per_datapoint == 2:
		fixed_dimension_hbs = build_hb_matrix(four_lead, peaks, 100, time, beats_per_vector = beats_per_datapoint, plotting = False)
	else:
		fixed_dimension_hbs = build_hb_matrix_centered(four_lead, peaks, 100, plotting = False)

	#Find the lengths of the heartbeats
	hb_lengths = find_lengths(peaks)

	writeout(str(curr_index), orig_num_hbs, four_lead, fixed_dimension_hbs, heartrate, peaks, hb_lengths, time, percent_unavaliable, prefix = file_prefix)

def preprocess_all(num_beats):
	"""
	Preprocess all patients
	:return:
	"""
	flags = [True, False]

	for control_patients in flags:
		indicies = get_patient_ids(control=control_patients)
		for idx, filename in zip(indicies, get_filenames(original=(not control_patients), control=control_patients)):
			idx = str(idx)
			try:
				preprocess_sum(filename, idx, beats_per_datapoint=num_beats, control=control_patients)
			except:
				print(idx + 'bad')
	# preprocess_seperate(filename, idx)

def plot_heartbeat(patient, lead, beat_index):
	"""
	Plots a single heartbeat
	:param patient: patient index
	:param lead: lead number (0-3)
	:param beat_index: index of the heartbeat into the array of heartbeats
	:return:
	"""
	plt.figure(dpi=500)
	data = np.load(f"Working_Data/Fixed_Dim_HBs_Idx{patient}.npy")
	heartbeat = data[beat_index, :, lead]
	plt.plot(heartbeat[50:150])
	plt.show()
	return

if __name__ == "__main__":
	# control_patients = True
	#
	# plotting_utils.set_font_size()
	# indicies = get_patient_ids(control=control_patients)
	# for idx, filename in zip(indicies, get_filenames(original=(not control_patients), control=control_patients)):
	# 	idx = str(idx)
	# 	try:
	# 		preprocess_sum(filename, idx, beats_per_datapoint=10, control=control_patients)
	# 	except:
	# 		print(idx+'bad')
		# preprocess_seperate(filename, idx)
	# preprocess_all(10)
	plot_heartbeat(1, 0, 100)
	plot_heartbeat(1, 0, 1000)
	plot_heartbeat(1, 0, 1200)

	pass


text:
# -*- coding:utf-8 -*-

import numpy as np
import matplotlib.pyplot as plt

class Windy_Grid_World(object):
    def __init__(self, epsilon=0.1, step_size=0.5, discount=1):
        self.world_size = (7, 10)    # first number is row, second number is column
        self.num_action = 4     # the number of actions
        self.actions = np.arange(self.num_action)    # 0:up, 1:down, 2:left, 3:right
        self.epsilon = epsilon    # epsilon-greedy
        self.step_size = step_size    # step size for Sarsa
        self.discount = discount

        self.start = np.array([3, 0])
        self.end = np.array([3, 7])
        self.reward = -1
        self.wind = np.array([0, 0, 0, 1, 1, 1, 2, 2, 1, 0])    # the strength of wind for every column

    def epsilon_greedy(self, q_value, state):
        rand_num = np.random.rand()
        if rand_num < self.epsilon:
            return np.random.choice(self.actions)
        else:
            state_value = q_value[:, state[0], state[1]]
            return np.random.choice([act for act, val in enumerate(state_value) if val == np.max(state_value)])

    def find_next_state(self, current_state, current_action):
        row, col = current_state
        # take the up action
        if current_action == self.actions[0]:
            temp = max(row - 1 - self.wind[col], 0)
            return np.array([temp, col])
        # take the down action
        elif current_action == self.actions[1]:
            temp = min(row + 1 - self.wind[col], self.world_size[0]-1)
            temp = max(temp, 0)
            return np.array([temp, col])
        # take the left action
        elif current_action == self.actions[2]:
            temp = max(row - self.wind[col], 0)
            return np.array([temp, max(col-1, 0)])
        # take the right action
        else:
            temp = min(col + 1, self.world_size[1] - 1)
            return np.array([max(row - self.wind[col], 0), temp])

    def sarsa_algorithm(self, q_value):
        state = self.start
        time_step = 0   # to track the time steps during the episode

        # to use epsilon-greedy algorithm to select action
        action = self.epsilon_greedy(q_value, state)
        while any(state != self.end):
            # according to current state and action to find next state
            next_state = self.find_next_state(state, action)
            next_action = self.epsilon_greedy(q_value, next_state)

            # to update the action value
            q_value[action, state[0], state[1]] += \
                self.step_size*(self.reward + self.discount*q_value[next_action, next_state[0], next_state[1]] -
                                q_value[action, state[0], state[1]])
            state = next_state
            action = next_action
            time_step += 1
        return time_step

if __name__ == "__main__":
    windy_grid_word = Windy_Grid_World()
    q_value_fun = np.zeros((windy_grid_word.num_action, windy_grid_word.world_size[0],
                            windy_grid_word.world_size[1]))
    '''
    stat = np.array([3, 0])
    time = 200
    actions = np.zeros(time)
    for idx in range(time):
        actions[idx] += windy_grid_word.epsilon_greedy(q_value_fun, stat)
    print(actions)

    '''
    episode = 170
    time_steps = np.zeros(episode)
    for idx in range(episode):
        time_steps[idx] += windy_grid_word.sarsa_algorithm(q_value_fun)
    time_steps = np.add.accumulate(time_steps)

    average_episode = time_steps[-1]/episode
    print("the average episode length at about %.f steps" % average_episode)

    plt.figure(1)
    plt.plot(time_steps, np.arange(episode))
    plt.xlabel("Time Steps")
    plt.ylabel("Episodes")
    plt.savefig("./images/Example6-5.png")
    plt.show()
    plt.close()
    print("Completed!!! You can check it in the 'images' directory")

max_stars_repo_name: liloganle/Reinforcement-Learning

max_issues_repo_name: liloganle/Reinforcement-Learning

max_forks_repo_name: liloganle/Reinforcement-Learning


text:
caffe_root = '../'  # this file is expected to be in {caffe_root}/examples/fcn
import sys
sys.path.insert(0, caffe_root + 'python')
import caffe
import numpy as np
from PIL import Image
import os
import scipy

path1 =  '../data/camvid/701_StillsRaw_full/'
path2 = '../data/camvid/argmax/'
# load net
net = caffe.Net('Googlenet_8s/deploy.prototxt', 'Googlenet_8s/snapshot/train_iter_80000.caffemodel', caffe.TEST)

listing = os.listdir(path1)
for file in listing:
    im = np.array(Image.open(path1 + file))
    in_ = np.array(im, dtype=np.float32)
    in_ = in_[:,:,::-1]
    in_ -= np.array((104.00698793,116.66876762,122.67891434))
    in_ = in_.transpose((2,0,1))
    # shape for input (data blob is N x C x H x W), set data
    net.blobs['data'].reshape(1, *in_.shape)
    net.blobs['data'].data[...] = in_
    # run net and take argmax for prediction
    net.forward()
    out = net.blobs['score'].data[0].argmax(axis=0)
    # save image
    thisFile = file
    base = os.path.splitext(thisFile)[0]
    scipy.misc.imsave(path2+'argmax'+base+'.pgm' , out)

print 'camvid done'

#path1 =  '../../Desktop/Dusk_test/'
#path2 = '../../Desktop/argmax_seq05/'
#
#listing = os.listdir(path1)
#for file in listing:
#    im = np.array(Image.open(path1 + file))
#    in_ = np.array(im, dtype=np.float32)
#    in_ = in_[:,:,::-1]
#    in_ -= np.array((104.00698793,116.66876762,122.67891434))
#    in_ = in_.transpose((2,0,1))
#    # load net
#    net = caffe.Net('Alexnet_8s/deploy.prototxt', 'Alexnet_8s/alexnet8s.caffemodel', caffe.TEST)
#    # shape for input (data blob is N x C x H x W), set data
#    net.blobs['data'].reshape(1, *in_.shape)
#    net.blobs['data'].data[...] = in_
#    # run net and take argmax for prediction
#    net.forward()
#    out = net.blobs['score'].data[0].argmax(axis=0)
#    # save image
#    thisFile = file
#    base = os.path.splitext(thisFile)[0]
#    scipy.misc.imsave(path2+'argmax'+base+'.pgm' , out)
#
#print 'dusk test sequence done'

max_stars_repo_name: TrevinoCarlos/caffe

max_issues_repo_name: TrevinoCarlos/caffe

max_forks_repo_name: TrevinoCarlos/caffe

reason: import numpy,import scipy