text:
#!/usr/bin/env python
# coding: utf-8

# - In this notebook -
#     - generate random edits of type (s',r',o') and (s',r',o')
#     - i.e. generate totally random edits, not restricted to the neighboourhood of target triple
#     - this random baseline is needed for comparison with decoy composition where edits are not restricted to neighbourhood of target

# In[1]:

import pickle
from typing import Dict, Tuple, List
import os
import numpy as np
import json
import torch
import logging
import argparse
import math
from pprint import pprint
import errno

import torch
from torch.utils.data import DataLoader
import torch.backends.cudnn as cudnn

from dataset import TrainDataset, BidirectionalOneShotIterator
from evaluation import evaluation
from model import Distmult, Complex, Conve, Transe

# In[2]:

def add_arguments():
    parser = argparse.ArgumentParser(description='Link prediction for knowledge graphs')

    parser.add_argument('--data', type=str, default='FB15k-237', help='Dataset to use: {FB15k-237, YAGO3-10, WN18RR, umls, nations, kinship}, default: FB15k-237')
    parser.add_argument('--model', type=str, default='conve', help='Choose from: {conve, distmult, complex}')
    parser.add_argument('--add-reciprocals', action='store_true', help='Option to add reciprocal relations')

    parser.add_argument('--transe-margin', type=float, default=12.0, help='Margin value for TransE scoring function. Default:12.0')
    parser.add_argument('--transe-norm', type=int, default=2, help='P-norm value for TransE scoring function. Default:2')

    parser.add_argument('--epochs', type=int, default=400, help='Number of epochs to train (default: 400)')
    parser.add_argument('--lr', type=float, default=0.001, help='Learning rate (default: 0.001)')#maybe 0.1
    parser.add_argument('--lr-decay', type=float, default=0.0, help='Weight decay value to use in the optimizer. Default: 0.0')

    parser.add_argument('--num-batches', type=int, default=400, help='Number of batches for training (default: 400)') #maybe 200?
    parser.add_argument('--test-batch-size', type=int, default=128, help='Batch size for test split (default: 128)')
    parser.add_argument('--valid-batch-size', type=int, default=128, help='Batch size for valid split (default: 128)')
    parser.add_argument('--num-workers', type=int, default=4, help='Number of workers to use for the batch loaders on GPU. Default: 4')

    parser.add_argument('--embedding-dim', type=int, default=200, help='The embedding dimension (1D). Default: 200')

    parser.add_argument('--stack_width', type=int, default=20, help='The first dimension of the reshaped/stacked 2D embedding. Second dimension is inferred. Default: 20')
    #parser.add_argument('--stack_height', type=int, default=10, help='The second dimension of the reshaped/stacked 2D embedding. Default: 10')
    parser.add_argument('--hidden-drop', type=float, default=0.3, help='Dropout for the hidden layer. Default: 0.3.')
    parser.add_argument('--input-drop', type=float, default=0.2, help='Dropout for the input embeddings. Default: 0.2.')
    parser.add_argument('--feat-drop', type=float, default=0.3, help='Dropout for the convolutional features. Default: 0.2.')
    parser.add_argument('-num-filters', default=32,   type=int, help='Number of filters for convolution')
    parser.add_argument('-kernel-size', default=3, type=int, help='Kernel Size for convolution')

    parser.add_argument('--use-bias', action='store_true', help='Use a bias in the convolutional layer. Default: True')
    parser.add_argument('--label-smoothing', type=float, default=0.1, help='Label smoothing value to use. Default: 0.1')

    parser.add_argument('--reg-weight', type=float, default=5e-12, help='Weight for regularization. Default: 5e-12')#maybe 5e-2?
    parser.add_argument('--reg-norm', type=int, default=2, help='Norm for regularization. Default: 3')

    parser.add_argument('--resume', action='store_true', help='Restore a saved model.')
    parser.add_argument('--resume-split', type=str, default='test', help='Split to evaluate a restored model')
    parser.add_argument('--seed', type=int, default=17, metavar='S', help='Random seed (default: 17)')

    return parser

def generate_dicts(data_path):
    with open (os.path.join(data_path, 'entities_dict.json'), 'r') as f:
        ent_to_id = json.load(f)
    with open (os.path.join(data_path, 'relations_dict.json'), 'r') as f:
        rel_to_id = json.load(f)
    n_ent = len(list(ent_to_id.keys()))
    n_rel = len(list(rel_to_id.keys()))

    return n_ent, n_rel, ent_to_id, rel_to_id

import pandas as pd
def load_data(data_path):
    data = {}
    for split in ['train', 'valid', 'test']:
        df = pd.read_csv(os.path.join(data_path, split+'.txt'), sep='\t', header=None, names=None, dtype=int)
        df = df.drop_duplicates()
        data[split] = df.values

    return data

def add_model(args, n_ent, n_rel):
    if args.add_reciprocals:
        if args.model is None:
            model = Conve(args, n_ent, 2*n_rel)
        elif args.model == 'conve':
            model = Conve(args, n_ent, 2*n_rel)
        elif args.model == 'distmult':
            model = Distmult(args, n_ent, 2*n_rel)
        elif args.model == 'complex':
            model = Complex(args, n_ent, 2*n_rel)
        elif args.model == 'transe':
            model = Transe(args, n_ent, 2*n_rel)
        else:
            logger.info('Unknown model: {0}', args.model)
            raise Exception("Unknown model!")
    else:
        if args.model is None:
            model = Conve(args, n_ent, n_rel)
        elif args.model == 'conve':
            model = Conve(args, n_ent, n_rel)
        elif args.model == 'distmult':
            model = Distmult(args, n_ent, n_rel)
        elif args.model == 'complex':
            model = Complex(args, n_ent, n_rel)
        elif args.model == 'transe':
            model = Transe(args, n_ent, n_rel)
        else:
            logger.info('Unknown model: {0}', args.model)
            raise Exception("Unknown model!")

    #model.to(self.device)
    return model

if __name__ == '__main__':
    parser = add_arguments()
    parser.add_argument('--target-split', type=int, default=1, help='Ranks to use for target set. Values are 1 for ranks <=10; 2 for ranks>10 and ranks<=100. Default: 1')
    parser.add_argument('--budget', type=int, default=1, help='Budget for each target triple for each corruption side')
    parser.add_argument('--rand-run', type=int, default=1, help='A number assigned to the random run of experiment')

    # In[5]:

    args = parser.parse_args()
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    # In[6]:

    #args.target_split = 1 # which target split to use
    #Values are 1 for ranks <=10; 2 for ranks>10 and ranks<=100.
    #args.budget = 1 #indicates the num of adversarial edits for each target triple for each corruption side
    #args.rand_run = 1 #  a number assigned to the random run of the experiment

    args.seed = args.seed + (args.rand_run - 1) # default seed is 17

    #args.model = 'distmult'
    #args.data = 'FB15k-237'
    # Below is based on hyperparams for original model
    if args.data == 'WN18RR':
        if args.model == 'distmult':
            args.lr = 0.01
            args.num_batches = 50
        elif args.model == 'complex':
            args.lr = 0.01
        elif args.model == 'conve':
            args.lr =  0.001
        elif args.model == 'transe':
            args.lr = 0.005
            args.input_drop = 0.0
            args.transe_margin = 9.0
            args.num_batches = 1000
            args.epochs = 100
            args.reg_weight = 1e-12
        else:
            print("New model:{0},{1}. Set hyperparams".format(args.data, args.model))
    elif args.data == 'FB15k-237':
        if args.model == 'distmult':
            args.lr = 0.005
            args.input_drop = 0.5
        elif args.model == 'complex':
            args.lr = 0.005
            args.input_drop = 0.5
        elif args.model == 'conve':
            args.lr = 0.001
            args.hidden_drop = 0.5
        elif args.model == 'transe':
            args.lr = 0.001
            args.input_drop = 0.0
            args.transe_margin = 9.0
            args.num_batches = 800
            args.epochs = 100
            args.reg_weight = 1e-10
        else:
            print("New model:{0},{1}. Set hyperparams".format(args.data, args.model))
    else:
        print("New dataset:{0}. Set hyperparams".format(args.data))

    # In[7]:

    # Fixing random seeds for reproducibility -https://pytorch.org/docs/stable/notes/randomness.html
    torch.manual_seed(args.seed)
    cudnn.deterministic = True
    cudnn.benchmark = False
    np.random.seed(args.seed)
    rng = np.random.default_rng(seed=args.seed)

    args.epochs = -1 #no training here
    model_name = '{0}_{1}_{2}_{3}_{4}'.format(args.model, args.embedding_dim, args.input_drop, args.hidden_drop, args.feat_drop)
    model_path = 'saved_models/{0}_{1}.model'.format(args.data, model_name)
    #log_path = 'logs/inv_add_1_{0}_{1}_{2}_{3}.log'.format(args.data, model_name, args.num_batches, args.epochs)

    logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                            datefmt = '%m/%d/%Y %H:%M:%S',
                            level = logging.INFO
                           )
    logger = logging.getLogger(__name__)

    data_path = 'data/target_{0}_{1}_{2}'.format(args.model, args.data, args.target_split)

    n_ent, n_rel, ent_to_id, rel_to_id = generate_dicts(data_path)

    ##### load data####
    data  = load_data(data_path)
    train_data, valid_data, test_data = data['train'], data['valid'], data['test']

    inp_f = open(os.path.join(data_path, 'to_skip_eval.pickle'), 'rb')
    to_skip_eval: Dict[str, Dict[Tuple[int, int], List[int]]] = pickle.load(inp_f)
    inp_f.close()
    to_skip_eval['lhs'] = {(int(k[0]), int(k[1])): v for k,v in to_skip_eval['lhs'].items()}
    to_skip_eval['rhs'] = {(int(k[0]), int(k[1])): v for k,v in to_skip_eval['rhs'].items()}

    # In[ ]:

    # #### Pseudocode
    #  - Sample an entity from list of entities and relation from list of relations (the list should exclude target entity and relation)
    #  - If the random triple already exists in the data, sample again

    # In[7]:

    #np.where(np.asarray(list(rel_to_id.values()))!=2), n_rel

    # In[8]:

    ents = np.asarray(list(ent_to_id.values()))
    rels = np.asarray(list(rel_to_id.values()))

    # In[9]:

    train_data.shape

    # In[11]:

    # Finding corruptions of type 1; checking existence in train set and adding them
    per_tr = np.empty_like(train_data)
    per_tr[:] = train_data

    summary_dict = {}
    logger.info('------------ Generating random corruptions -----------------')
    trip_to_add = [] # of type s`r`o`
    test_trip_idx = 0
    while test_trip_idx < len(test_data):
        if test_trip_idx%500 == 0:
            logger.info('Processing triple ' + str(test_trip_idx))

        test_trip = test_data[test_trip_idx]
        s = test_trip[0]
        r = test_trip[1]
        o = test_trip[2]
        #print(s,r,o)

        summary_list = []
        summary_list.append(list(map(int, [s,r,o])))

        budget_idx = 0
        # depending on how many triples already exist in data and how many triples are added, this might turn into an infinite loop
        # 2*budget because there are two sides to corrupt
        while budget_idx < 2*args.budget:

            rel_choices = rels[np.where(rels!=r)]
            ent_choices = ents
            #ent_choices = ents[np.where((ents!=o) & (ents!=s))]

            rand_r1 = rng.choice(a=rel_choices, size = 1, replace=True)[0]
            ch = rng.choice(a=ent_choices, size = 2, replace=False)
            rand_s, rand_o = ch[0], ch[1]

            adv = [rand_s, rand_r1, rand_o]
            # mask for  triple in training set
            m1 = (np.isin(per_tr[:,0], [adv[0]])
                  & np.isin(per_tr[:,1], [adv[1]])
                  & np.isin(per_tr[:,2], [adv[2]]))
            if np.any(m1):
                logger.info('Random triple already exists...generating another random triple')
            else:
                trip_to_add.append(adv)
                per_tr = np.append(per_tr, np.asarray(adv).reshape(-1,3), axis=0)
                #summary_dict[(s,r,o)] = tuple(adv)
                summary_list.append(list(map(int, adv)))
                budget_idx += 1

        summary_dict[test_trip_idx] = summary_list
        test_trip_idx += 1

    del per_tr

    # In[11]:

    #summary_dict = {'rhs':summary_dict_o, 'lhs':summary_dict_s}

    # In[12]:

    logger.info(len(trip_to_add))
    logger.info(test_data.shape[0])

    # In[13]:

    trips_to_add = np.asarray(trip_to_add)

    # In[14]:

    new_train = np.concatenate((trips_to_add, train_data))

    # In[15]:

    logger.info ('Length of original training set: ' + str(train_data.shape[0]))
    logger.info ('Length of new poisoned training set: ' + str(new_train.shape[0]))

    # In[16]:

    num_en_or = np.unique(np.concatenate((train_data[:,0], train_data[:,2]))).shape[0]
    num_en_pos = np.unique(np.concatenate((new_train[:,0], new_train[:,2]))).shape[0]

    # In[17]:

    #decoy_trips = np.concatenate((np.asarray(decoy_trip_o), np.asarray(decoy_trip_s)))

    # In[17]:

    logger.info ('Length of original test set: ' + str(test_data.shape[0]))
    logger.info ('Number of edits generated : ' + str(trips_to_add.shape[0]))
    #print ('Number of triples in decoy test set: ' + str(decoy_trips.shape[0]))

    # In[19]:

    save_path = 'data/rand_add_g_{0}_{1}_{2}_{3}_{4}'.format( args.model, args.data, args.target_split, args.budget, args.rand_run)

    # In[20]:

    try :
        os.makedirs(save_path)
    except OSError as e:
        if e.errno == errno.EEXIST:
            logger.info(e)
            logger.info('Using the existing folder {0} for processed data'.format(save_path))
        else:
            raise

    # In[21]:

    with open(os.path.join(save_path, 'train.txt'), 'w') as out:
        for item in new_train:
            out.write("%s\n" % "\t".join(map(str, item)))

    out = open(os.path.join(save_path, 'train.pickle'), 'wb')
    pickle.dump(new_train.astype('uint64'), out)
    out.close()

    # In[22]:

    with open(os.path.join(save_path, 'entities_dict.json'), 'w') as f:
        f.write(json.dumps(ent_to_id)  + '\n')

    with open(os.path.join(save_path, 'relations_dict.json'), 'w') as f:
        f.write(json.dumps(rel_to_id)  + '\n')

    # In[23]:

    with open(os.path.join(save_path, 'valid.txt'), 'w') as out:
        for item in valid_data:
            out.write("%s\n" % "\t".join(map(str, item)))

    out = open(os.path.join(save_path, 'valid.pickle'), 'wb')
    pickle.dump(valid_data.astype('uint64'), out)
    out.close()

    # In[24]:

    with open(os.path.join(save_path, 'test.txt'), 'w') as out:
        for item in test_data:
            out.write("%s\n" % "\t".join(map(str, item)))

    out = open(os.path.join(save_path, 'test.pickle'), 'wb')
    pickle.dump(test_data.astype('uint64'), out)
    out.close()

    # In[25]:

    # In[26]:

    with open(os.path.join(save_path, 'summary_edits.json'), 'w') as out:
        out.write(json.dumps(summary_dict)  + '\n')

    # In[27]:

    # In[28]:

    with open(os.path.join(save_path, 'stats.txt'), 'w') as f:
        f.write('Length of original training set: {0} \n'. format(train_data.shape[0]))
        f.write('Length of new poisoned training set: {0} \n'. format(new_train.shape[0]))
        f.write('Number of entities in original training set: {0} \n'. format(num_en_or))
        f.write('Number of entities in poisoned training set: {0} \n'. format(num_en_pos))
        f.write('Length of original test set: {0} \n'. format(test_data.shape[0]))
        f.write('Number of triples added from corrupting both sides: {0}\n'. format(trips_to_add.shape[0]))
        f.write('This attack version is generated from global random edits \n')
        f.write('---------------------------------------------------------------------- \n')

    # In[ ]:

    # In[ ]:

    # In[ ]:

    # In[ ]:

    # In[ ]:

    # In[ ]:

max_stars_repo_name: PeruBhardwaj/InferenceAttack

max_issues_repo_name: PeruBhardwaj/InferenceAttack

max_forks_repo_name: PeruBhardwaj/InferenceAttack


text:
# The MIT License (MIT)
# Copyright (c) 2016 satojkovic

# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:

# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function
import tensorflow as tf
import numpy as np
from six.moves import cPickle as pickle
from six.moves import range
import sys
import os

FLAGS = tf.app.flags.FLAGS

tf.app.flags.DEFINE_string(
    "train_dir", "flickr_logos_27_dataset",
    "Directory where to write event logs and checkpoint.")
tf.app.flags.DEFINE_integer("max_steps", 10001, "Number of batches to run.")
tf.app.flags.DEFINE_integer("image_width", 64, "A width of an input image.")
tf.app.flags.DEFINE_integer("image_height", 32, "A height of an input image.")
tf.app.flags.DEFINE_integer("num_classes", 27, "Number of logo classes.")
tf.app.flags.DEFINE_integer("learning_rate", 0.0001, "Learning rate")
tf.app.flags.DEFINE_integer("batch_size", 64, "A batch size")
tf.app.flags.DEFINE_integer("num_channels", 3,
                            "A number of channels of an input image.")
tf.app.flags.DEFINE_integer("patch_size", 5,
                            "A patch size of convolution filter")

PICKLE_FILENAME = 'deep_logo.pickle'

def accuracy(predictions, labels):
    return (100 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1)) /
            predictions.shape[0])

def reformat(dataset, labels):
    dataset = dataset.reshape((-1, FLAGS.image_width, FLAGS.image_height,
                               FLAGS.num_channels)).astype(np.float32)
    labels = (
        np.arange(FLAGS.num_classes) == labels[:, None]).astype(np.float32)
    return dataset, labels

def model(data, w_conv1, b_conv1, w_conv2, b_conv2, w_conv3, b_conv3, w_fc1,
          b_fc1, w_fc2, b_fc2):
    # First layer
    h_conv1 = tf.nn.relu(
        tf.nn.conv2d(data, w_conv1, [1, 1, 1, 1], padding='SAME') + b_conv1)
    h_pool1 = tf.nn.max_pool(
        h_conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')

    # Second layer
    h_conv2 = tf.nn.relu(
        tf.nn.conv2d(h_pool1, w_conv2, [1, 1, 1, 1], padding='SAME') + b_conv2)
    h_pool2 = tf.nn.max_pool(
        h_conv2, ksize=[1, 1, 2, 1], strides=[1, 1, 2, 1], padding='SAME')

    # Third layer
    h_conv3 = tf.nn.relu(
        tf.nn.conv2d(h_pool2, w_conv3, [1, 1, 1, 1], padding='SAME') + b_conv3)
    h_pool3 = tf.nn.max_pool(
        h_conv3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')

    # Fully connected layer
    conv_layer_flat = tf.reshape(h_pool3, [-1, 16 * 4 * 128])
    h_fc1 = tf.nn.relu(tf.matmul(conv_layer_flat, w_fc1) + b_fc1)

    # Output layer
    out = tf.matmul(h_fc1, w_fc2) + b_fc2

    return out

def read_data():
    with open(PICKLE_FILENAME, 'rb') as f:
        save = pickle.load(f)
        train_dataset = save['train_dataset']
        train_labels = save['train_labels']
        valid_dataset = save['valid_dataset']
        valid_labels = save['valid_labels']
        test_dataset = save['test_dataset']
        test_labels = save['test_labels']
        del save
        print('Training set', train_dataset.shape, train_labels.shape)
        print('Valid set', valid_dataset.shape, valid_labels.shape)
        print('Test set', test_dataset.shape, test_labels.shape)

    return [train_dataset, valid_dataset,
            test_dataset], [train_labels, valid_labels, test_labels]

def main():
    if len(sys.argv) > 1:
        f = np.load(sys.argv[1])

        # f.files has unordered keys ['arr_8', 'arr_9', 'arr_6'...]
        # Sorting keys by value of numbers
        initial_weights = [
            f[n] for n in sorted(f.files, key=lambda s: int(s[4:]))
        ]
    else:
        initial_weights = None

    # read input data
    dataset, labels = read_data()

    train_dataset, train_labels = reformat(dataset[0], labels[0])
    valid_dataset, valid_labels = reformat(dataset[1], labels[1])
    test_dataset, test_labels = reformat(dataset[2], labels[2])
    print('Training set', train_dataset.shape, train_labels.shape)
    print('Valid set', valid_dataset.shape, valid_labels.shape)
    print('Test set', test_dataset.shape, test_labels.shape)

    # Training model
    graph = tf.Graph()
    with graph.as_default():
        # Variables
        w_conv1 = tf.Variable(
            tf.truncated_normal(
                [FLAGS.patch_size, FLAGS.patch_size, FLAGS.num_channels, 48],
                stddev=0.1))
        b_conv1 = tf.Variable(tf.constant(0.1, shape=[48]))

        w_conv2 = tf.Variable(
            tf.truncated_normal(
                [FLAGS.patch_size, FLAGS.patch_size, 48, 64], stddev=0.1))
        b_conv2 = tf.Variable(tf.constant(0.1, shape=[64]))

        w_conv3 = tf.Variable(
            tf.truncated_normal(
                [FLAGS.patch_size, FLAGS.patch_size, 64, 128], stddev=0.1))
        b_conv3 = tf.Variable(tf.constant(0.1, shape=[128]))

        w_fc1 = tf.Variable(
            tf.truncated_normal([16 * 4 * 128, 2048], stddev=0.1))
        b_fc1 = tf.Variable(tf.constant(0.1, shape=[2048]))

        w_fc2 = tf.Variable(tf.truncated_normal([2048, FLAGS.num_classes]))
        b_fc2 = tf.Variable(tf.constant(0.1, shape=[FLAGS.num_classes]))

        # Params
        params = [
            w_conv1, b_conv1, w_conv2, b_conv2, w_conv3, b_conv3, w_fc1, b_fc1,
            w_fc2, b_fc2
        ]

        # Initial weights
        if initial_weights is not None:
            assert len(params) == len(initial_weights)
            assign_ops = [w.assign(v) for w, v in zip(params, initial_weights)]

        # Input data
        tf_train_dataset = tf.placeholder(
            tf.float32,
            shape=(FLAGS.batch_size, FLAGS.image_width, FLAGS.image_height,
                   FLAGS.num_channels))
        tf_train_labels = tf.placeholder(
            tf.float32, shape=(FLAGS.batch_size, FLAGS.num_classes))
        tf_valid_dataset = tf.constant(valid_dataset)
        tf_test_dataset = tf.constant(test_dataset)

        # Training computation
        logits = model(tf_train_dataset, w_conv1, b_conv1, w_conv2, b_conv2,
                       w_conv3, b_conv3, w_fc1, b_fc1, w_fc2, b_fc2)
        with tf.name_scope('loss'):
            loss = tf.reduce_sum(
                tf.nn.softmax_cross_entropy_with_logits(
                    logits=logits, labels=tf_train_labels))
            tf.summary.scalar('loss', loss)
        optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss)

        # Predictions for the training, validation, and test data
        train_prediction = tf.nn.softmax(logits)
        valid_prediction = tf.nn.softmax(
            model(tf_valid_dataset, w_conv1, b_conv1, w_conv2, b_conv2,
                  w_conv3, b_conv3, w_fc1, b_fc1, w_fc2, b_fc2))
        test_prediction = tf.nn.softmax(
            model(tf_test_dataset, w_conv1, b_conv1, w_conv2, b_conv2, w_conv3,
                  b_conv3, w_fc1, b_fc1, w_fc2, b_fc2))
        # Merge all summaries
        merged = tf.summary.merge_all()
        train_writer = tf.summary.FileWriter(FLAGS.train_dir + '/train')

        # Add ops to save and restore all the variables
        saver = tf.train.Saver()

    # Do training
    with tf.Session(graph=graph) as session:
        tf.global_variables_initializer().run()
        if initial_weights is not None:
            session.run(assign_ops)
            print('initialized by pre-learned values')
        else:
            print('initialized')
        for step in range(FLAGS.max_steps):
            offset = (step * FLAGS.batch_size) % (
                train_labels.shape[0] - FLAGS.batch_size)
            batch_data = train_dataset[offset:(offset + FLAGS.batch_size
                                               ), :, :, :]
            batch_labels = train_labels[offset:(offset + FLAGS.batch_size), :]
            feed_dict = {
                tf_train_dataset: batch_data,
                tf_train_labels: batch_labels
            }
            try:
                _, l, predictions = session.run(
                    [optimizer, loss, train_prediction], feed_dict=feed_dict)
                if step % 50 == 0:
                    summary, _ = session.run(
                        [merged, optimizer], feed_dict=feed_dict)
                    train_writer.add_summary(summary, step)
                    print('Minibatch loss at step %d: %f' % (step, l))
                    print('Minibatch accuracy: %.1f%%' %
                          accuracy(predictions, batch_labels))
                    print('Validation accuracy: %.1f%%' %
                          accuracy(valid_prediction.eval(), valid_labels))
            except KeyboardInterrupt:
                last_weights = [p.eval() for p in params]
                np.savez("weights.npz", *last_weights)
                return last_weights

        print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(),
                                                 test_labels))

        # Save the variables to disk.
        save_dir = "models"
        if not os.path.exists(save_dir):
            os.makedirs(save_dir)
        save_path = os.path.join(save_dir, "deep_logo_model")
        saved = saver.save(session, save_path)
        print("Model saved in file: %s" % saved)

if __name__ == '__main__':
    main()

ext: py

max_stars_repo_name: sarwojowo/DeepLogo

max_issues_repo_name: sarwojowo/DeepLogo

max_forks_repo_name: sarwojowo/DeepLogo

reason: import numpy


text:
import dlib
import cv2
import numpy as np
from PIL import Image

import face_blend_common as fbc

def load_image(image_bytes):
    print('Reading image')
    img = cv2.imdecode(np.frombuffer(image_bytes, np.uint8), -1)
    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
    return img

def load_models(predictor_path):
    print('Initializing the face detector instance')
    face_detector = dlib.get_frontal_face_detector()

    print('Loading the shape predictor model')
    landmark_detector = dlib.shape_predictor(predictor_path)

    return face_detector, landmark_detector

def normalize_image(img, points):
    print('Aligning Image')
    img_norm, points = fbc.normalizeImagesAndLandmarks((600, 600), img, points)
    img_norm = np.uint8(img_norm * 255)
    return img_norm

def align_image(predictor_path, image_bytes):
    # Load models and image
    face_detector, landmark_detector = load_models(predictor_path)
    img = load_image(image_bytes)

    # Detect landmarks
    status, result = fbc.getLandmarks(face_detector, landmark_detector, img)
    if status == 'fail':
        return status, result

    points = np.array(result)

    # Convert image to floating point in the range 0 to 1
    img = np.float32(img) / 255.0

    # Align image
    output = normalize_image(img, points)

    return status, Image.fromarray(output)

max_stars_repo_path: 03 - Face Recognition Part 1/deployment/face_alignment/face_alignment.py


text:
from keras.layers import Input, Dense, Flatten, Concatenate
from keras.models import Model
import numpy as np
import pandas as pd
import keras
import pickle

df = pd.read_csv('sequence_data.tsv',sep='\t')
with open('all','rb') as f:
    test = pickle.load(f)

status = df.iloc[test].status
status = np.array(status)
metrics = [keras.metrics.AUC(name='auc'),
      keras.metrics.TruePositives(name='tp'),
      keras.metrics.FalsePositives(name='fp'),
      keras.metrics.TrueNegatives(name='tn'),
      keras.metrics.FalseNegatives(name='fn'),
      keras.metrics.BinaryAccuracy(name='accuracy'),
      keras.metrics.Precision(name='precision'),
      keras.metrics.Recall(name='recall')]
pos = 6931
neg = 101232
total = pos + neg

weight_0 = (1 / neg)*(total) / 2.0
weight_1 = (1 / pos)*(total) / 2.0

weights = {0: weight_0, 1: weight_1}
#weights = {0: 1., 1: 101232 / 6931}
gen = np.load('genomic.npy')
gen = np.reshape(gen[test], (gen[test].shape[0], gen[test].shape[1], gen[test].shape[2]))
signal = np.load('signal.npy')
signal = np.reshape(signal[test], (signal[test].shape[0], signal[test].shape[1], signal[test].shape[2]))
clip = np.load('softclipped.npy')
clip = np.reshape(clip[test], (clip[test].shape[0], clip[test].shape[1], clip[test].shape[2]))
seq_shape = np.load('shape.npy')
seq_shape = np.reshape(seq_shape[test], (seq_shape[test].shape[0], seq_shape.shape[1], seq_shape.shape[2]))

seq_input = Input(shape = (21,5))
flat_seq = Flatten()(seq_input)
clip_input = Input(shape = (clip.shape[1], clip.shape[2]))
flat_clip = Flatten()(clip_input)
sig_input = Input(shape = (signal.shape[1], signal.shape[2]))
flat_sig = Flatten()(sig_input)
shape_input = Input(shape = (seq_shape.shape[1], seq_shape.shape[2]))
flat_shape = Flatten()(shape_input)

inputs = Concatenate()([flat_seq, flat_clip, flat_sig, flat_shape])
out = Dense(1, activation = 'sigmoid')(inputs)

model = Model(inputs = [seq_input,clip_input,sig_input,shape_input], outputs = out)
opt = keras.optimizers.adam(lr=0.0003)
model.compile(optimizer = opt, loss = 'binary_crossentropy',
                                  metrics = metrics)
# To do the training without balancing the loss function, then save weights:
model.fit([gen, clip, signal, seq_shape], status, epochs = 25, batch_size = 32, validation_split = 0.1)
model.save_weights('logit_unbalance.h5')
# Use weights above to balance loss function, then save weights:
model.fit([gen, clip, signal, seq_shape], status, epochs = 25, batch_size = 32, validation_split = 0.1,
                                                                       class_weight = weights)
model.save_weights('logit_balance.h5')


text:
import cv2
from PIL import Image
import pytesseract
import numpy as np
import glob
images = glob.glob('../split/*.jpg')
print(str(len(images)) + ' images to process total!')
c = 0
for path in images:
    name = path[9:-4]
    im = Image.open(path)
    im.save('temp.png')
    img = cv2.imread('temp.png')

    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

    ret, thresh1 = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU | cv2.THRESH_BINARY_INV)
    rect_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (6, 6))
    dilation = cv2.dilate(thresh1, rect_kernel, iterations = 1)
    contours, hierarchy = cv2.findContours(dilation, cv2.RETR_EXTERNAL,
                                                    cv2.CHAIN_APPROX_NONE)
    im2 = img.copy()

    counter = 0
    for cnt in contours:
        counter += 1
        x, y, w, h = cv2.boundingRect(cnt)
        cropped = im2[y:y + h, x:x + w]

        crop_panel = im2[y:y+h,x:x+w]
        cv2.imwrite('../indiv_panels/' + name + '_panel_' + str(counter) + '.png', crop_panel)
    c+=1
    print(str(round(100*c/len(images),2)) + '%, processed ' + name)


text:
import logging
from libdvid import DVIDNodeService, DVIDServerService
import numpy as np
import json_tricks.np as json
from pluginsystem.plugin_manager import TrackingPluginManager

def dataToBlock(data, dtype=np.uint8, block_size=32):
    if len(data.shape) == 2:
        data = np.expand_dims(data, axis=2)
    elif len(data.shape) != 3:
        raise ValueError("Cannot prepare data of shape that is not a 2D or 3D frame")

    old_shape = data.shape
    new_shape = []
    for d in old_shape:
        if d % block_size == 0:
            new_shape.append(d)
        else:
            new_shape.append(((d // block_size) + 1) * block_size)

    logging.debug("transformed data from {} to {}".format(old_shape, new_shape))
    new_data = np.zeros(new_shape, dtype=dtype)
    new_data[0 : old_shape[0], 0 : old_shape[1], 0 : old_shape[2]] = data
    return new_data

if __name__ == "__main__":
    """
    Upload raw data and segmentation of a dataset to dvid.

    Example: python dvid/upload_dataset.py --dvid-address 104.196.46.138:80 --label-image /Users/chaubold/hci/data/animal-tracking/FlyBowlTracking/FlyBowlTracking.ilp --raw /Users/chaubold/hci/data/animal-tracking/FlyBowlTracking/FlyBowlMovie.h5 --raw-path data --dataset-name flybowl-test-2016-04-07 --time-range 0 10
    """
    import argparse

    parser = argparse.ArgumentParser(
        description="Upload raw data and segmentation to dvid",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser.add_argument(
        "--dataset-name",
        required=True,
        type=str,
        dest="datasetName",
        help="Datset name that will be seen in the DVID web interface",
    )
    parser.add_argument(
        "--dvid-address",
        required=True,
        type=str,
        dest="dvidAddress",
        help="<IP>:<Port> of the dvid server",
    )
    parser.add_argument(
        "--label-image",
        required=True,
        type=str,
        dest="ilpFilename",
        help="Filename of the HDF5 file containing the label images",
    )
    parser.add_argument(
        "--raw",
        required=True,
        type=str,
        dest="rawFilename",
        help="Filename of the hdf5 file containing the raw data",
    )
    parser.add_argument(
        "--raw-path",
        required=True,
        type=str,
        dest="rawPath",
        help="Path inside HDF5 file to raw volume",
    )
    parser.add_argument(
        "--label-image-path",
        type=str,
        dest="labelImagePath",
        help="Path inside ilastik project file to the label image",
        default="/TrackingFeatureExtraction/LabelImage/0000/[[%d, 0, 0, 0, 0], [%d, %d, %d, %d, 1]]",
    )
    parser.add_argument(
        "--object-count-classifier-file",
        type=str,
        dest="objectCountClassifierFile",
        help="HDF5 file containing the object count classifier",
        default=None,
    )
    parser.add_argument(
        "--object-count-classifier-path",
        type=str,
        dest="objectCountClassifierPath",
        help="Path inside HDF5 file to the object count classifier",
        default="/CountClassification",
    )
    parser.add_argument(
        "--rf-zero-padding",
        type=int,
        dest="rfZeroPadding",
        default=4,
        help="Number of digits per forest index inside the ClassifierForests HDF5 group",
    )
    parser.add_argument(
        "--time-range",
        type=int,
        nargs=2,
        dest="timeRange",
        help="Set time range to upload (inclusive!)",
    )
    parser.add_argument(
        "--verbose", type=bool, dest="verbose", default=False, help="verbose logs"
    )

    args = parser.parse_args()
    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    # initialize plugin manager
    plugin_manager = TrackingPluginManager(verbose=False)
    image_provider = plugin_manager.getImageProvider()

    # create dataset on server and get uuid
    server_address = args.dvidAddress
    server_service = DVIDServerService(server_address)
    uuid = server_service.create_new_repo(args.datasetName, "description")
    logging.info("UUID:\n{}".format(uuid))

    # get node service
    node_service = DVIDNodeService(server_address, uuid)

    # get dataset size and store in dvid
    shape = image_provider.getImageShape(args.ilpFilename, args.labelImagePath)
    time_range = image_provider.getTimeRange(args.ilpFilename, args.labelImagePath)
    if args.timeRange is not None:
        time_range = (
            max(time_range[0], args.timeRange[0]),
            min(time_range[1], args.timeRange[1]),
        )
    logging.info("Uploading time range {} to {}".format(time_range, server_address))
    keyvalue_store = "config"
    node_service.create_keyvalue(keyvalue_store)
    settings = {"shape": shape, "time_range": time_range}
    node_service.put(keyvalue_store, "imageInfo", json.dumps(settings))

    # upload all frames
    for frame in range(time_range[0], time_range[1]):
        logging.info("Uploading frame {}".format(frame))
        label_image = image_provider.getLabelImageForFrame(
            args.ilpFilename, args.labelImagePath, frame
        )
        raw_image = image_provider.getImageDataAtTimeFrame(
            args.rawFilename, args.rawPath, frame
        )

        raw_name = "raw-{}".format(frame)
        seg_name = "seg-{}".format(frame)
        node_service.create_grayscale8(raw_name)
        node_service.put_gray3D(
            raw_name, dataToBlock(raw_image, dtype=np.uint8), (0, 0, 0)
        )
        node_service.create_labelblk(seg_name)
        node_service.put_labels3D(
            seg_name, dataToBlock(label_image, dtype=np.uint64), (0, 0, 0)
        )

    # TODO: upload classifier

max_stars_repo_name: m-novikov/hytra

max_issues_repo_name: m-novikov/hytra

max_forks_repo_name: m-novikov/hytra


text:
# exercise 1.5.3
import numpy as np

from scipy.io import loadmat
# You can load the matlab data (matlab's m-file) to Python environment with
# 'loadmat()' function imported from 'scipy.io' module.
# The matlab workspace is loaded as a dictionary, with keys corresponding to
# matlab variable names, and values to arrays representing matlab matrices.

# Load Matlab data file to python dict structure
iris_mat = loadmat('../Data/iris.mat', squeeze_me=True)
# The argument squeeze_me ensures that there the variables we get from the
# MATLAB filed are not stored within "unneeded" array dimensions.

# You can check which variables are in the loaded dict by calling
# the function keys() for the dict:
#mat_data.keys()
# this will tell you that X, y, M, N and C are stored in the dictionary,
# as well as some extra information about e.g. the used MATLAB version.

# We'll extract the needed variables by using these keys:
X = iris_mat['X']
y = iris_mat['y']
M = iris_mat['M']
N = iris_mat['N']
C = iris_mat['C']
attributeNames = iris_mat['attributeNames']
classNames = iris_mat['classNames']

# Loading the Iris data from the .mat-file was quite easy, because all the work
# of putting it into the correct format was already done. This is of course
# likely not the case for your own data, where you'll need to do something
# similar to the two previous exercises. We will, however, sometimes in the
# course use .mat-files in the exercises.

reason: import numpy,from scipy


text:
import numpy as np
from pathlib import Path
import sys, os

if __name__ == "__main__":
    """
    Jobs:
    1) M-EPS-VAE for data=[dsprites, celeba, chairs] with hypars derived from beta-TCVAE training loss
    """

    # absolute path
    my_path = Path(__file__).parent.resolve().expanduser()
    main_path = my_path.parent.parent.parent

    # hypars
    epochs_list = [120, 800, 1200]
    seed = 1234
    nlat = 64
    batchs = 64
    lr = 1e-5
    n_stddevs = 3

    datasets = ["dsprites", "celeba", "chairs"]
    eps_A = 270
    eps_G = 5
    eps_Bs_dsprites_chairs = [-255, -260, -265, -270]
    eps_Bs_celeba = [-230, -235, -240, -245]

    # cherry-pick data samples as done in repo
    cherry_picked = ["92595 339150 656090",
                     "88413 176606 179144 32260 191281 143307 101535 70059 87889 131612",
                     "40919 5172 22330", ]

    # .sh filename
    fname = my_path / f'run_jobs_2.sh'

    # clear .sh file
    os.system(f'rm {fname}')

    # multi-epsilon-VAE from chaos
    for data, epochs, cherries in zip(datasets, epochs_list, cherry_picked):
        if data=="celeba":
            eps_Bs = eps_Bs_celeba
        else:
            eps_Bs = eps_Bs_dsprites_chairs
        for eps_B in eps_Bs:

            cmd = (
                f"python main.py qualitative/mepsvae_{data}_z{nlat}_epsA{eps_A}_epsB{eps_B}_epsG{eps_G} -s {seed} "
                f"--checkpoint-every 50 -d {data} -e {epochs} -b {batchs} "
                f"-z {nlat} -l mepsvae --lr {lr} --mepsvae-epsilon-alpha {eps_A} --mepsvae-epsilon-beta {eps_B} --mepsvae-epsilon-gamma {eps_G} "
                f'--no-progress-bar -F {str(my_path / f"mepsvae_{data}_z{nlat}_epsA{eps_A}_epsB{eps_B}_epsG{eps_G}.out")} '
                f"--record-loss-every=50 --pin-dataset-gpu \n"
                f"python main_viz.py qualitative/mepsvae_{data}_z{nlat}_epsA{eps_A}_epsB{eps_B}_epsG{eps_G} all -i {cherries} "
                f"-s {seed} -c 10 -r 10 -t 2 --is-show-loss --is-posterior \n"
            )

            with open(fname, 'a') as f:
                f.write(cmd)

    # multi-epsilon-VAE from VAE
    for data, epochs, cherries in zip(datasets, epochs_list, cherry_picked):
        if data=="celeba":
            eps_Bs = eps_Bs_celeba
        else:
            eps_Bs = eps_Bs_dsprites_chairs
        for eps_B in eps_Bs:

            cmd = (
                f"python main.py qualitative/mepsvae_fromVAE_{data}_z{nlat}_epsA{eps_A}_epsB{eps_B}_epsG{eps_G} -s {seed} "
                f"--checkpoint-every 50 -d {data} -e {epochs} -b {batchs} --mepsvae-warmup 0 --continue-train-from VAE_{data}_z64 "
                f"-z {nlat} -l mepsvae --lr {lr} --mepsvae-epsilon-alpha {eps_A} --mepsvae-epsilon-beta {eps_B} --mepsvae-epsilon-gamma {eps_G} "
                f'--no-progress-bar -F {str(my_path / f"mepsvae_fromVAE_{data}_z{nlat}_epsA{eps_A}_epsB{eps_B}_epsG{eps_G}.out")} '
                f"--record-loss-every=50 --pin-dataset-gpu \n"
                f"python main_viz.py qualitative/mepsvae_fromVAE_{data}_z{nlat}_epsA{eps_A}_epsB{eps_B}_epsG{eps_G} all -i {cherries} "
                f"-s {seed} -c 10 -r 10 -t 2 --is-show-loss --is-posterior \n"
            )

            with open(fname, 'a') as f:
                f.write(cmd)

max_stars_repo_name: kuangdai/disentangling-vae

max_issues_repo_name: kuangdai/disentangling-vae

max_forks_repo_name: kuangdai/disentangling-vae


text:
# -*- coding: utf-8 -*-
"""
Class for reading/writing analog signals in a text file.
Each column represents an AnalogSignal. All AnalogSignals have the same sampling rate.
Covers many case when part of a file can be viewed as a CSV format.

Supported : Read/Write

Author: sgarcia

"""

import csv
import os

import numpy as np
import quantities as pq

from neo.io.baseio import BaseIO
from neo.core import AnalogSignal, Segment

class AsciiSignalIO(BaseIO):
    """

    Class for reading signal in generic ascii format.
    Columns respresents signals. They all share the same sampling rate.
    The sampling rate is externally known or the first columns could hold the time
    vector.

    Usage:
        >>> from neo import io
        >>> r = io.AsciiSignalIO(filename='File_asciisignal_2.txt')
        >>> seg = r.read_segment(lazy=False, cascade=True)
        >>> print seg.analogsignals
        [<AnalogSignal(array([ 39.0625    ,   0.        ,   0.        , ..., -26.85546875 ...

    """

    is_readable        = True
    is_writable        = True

    supported_objects  = [ Segment , AnalogSignal]
    readable_objects   = [ Segment]
    writeable_objects  = [Segment]

    has_header         = False
    is_streameable     = False

    read_params        = {
                            Segment : [
                                        ('delimiter' , {'value' :  '\t', 'possible' : ['\t' , ' ' , ',' , ';'] }) ,
                                        ('usecols' , { 'value' : None , 'type' : int } ),
                                        ('skiprows' , { 'value' :0 } ),
                                        ('timecolumn' , { 'value' : None, 'type' : int } ) ,
                                        ('unit' , { 'value' : 'V', } ),
                                        ('sampling_rate' , { 'value' : 1000., } ),
                                        ('t_start' , { 'value' : 0., } ),
                                        ('method' , { 'value' : 'homemade', 'possible' : ['genfromtxt' , 'csv' , 'homemade' ] }) ,
                                        ]
                            }
    write_params       = {
                            Segment : [
                                        ('delimiter' , {'value' :  '\t', 'possible' : ['\t' , ' ' , ',' , ';'] }) ,
                                        ('writetimecolumn' , { 'value' : True,  } ) ,
                                        ]
                            }

    name               = None
    extensions          = [ 'txt' , 'asc', ]

    mode = 'file'

    def __init__(self , filename = None) :
        """
        This class read/write AnalogSignal in a text file.
        Each signal is a column.
        One of the column can be the time vector

        Arguments:
            filename : the filename to read/write
        """
        BaseIO.__init__(self)
        self.filename = filename

    def read_segment(self,
                                        lazy = False,
                                        cascade = True,
                                        delimiter = '\t',
                                        usecols = None,
                                        skiprows =0,

                                        timecolumn = None,
                                        sampling_rate = 1.*pq.Hz,
                                        t_start = 0.*pq.s,

                                        unit = pq.V,

                                        method = 'genfromtxt',

                                        ):
        """
        Arguments:
            delimiter  :  columns delimiter in file  '\t' or one space or two space or ',' or ';'
            usecols : if None take all columns otherwise a list for selected columns
            skiprows : skip n first lines in case they contains header informations
            timecolumn :  None or a valid int that point the time vector
            samplerate : the samplerate of signals if timecolumn is not None this is not take in account
            t_start : time of the first sample
            unit : unit of AnalogSignal can be a str or directly a Quantities

            method :  'genfromtxt' or 'csv' or 'homemade'
                        in case of bugs you can try one of this methods

                        'genfromtxt' use numpy.genfromtxt
                        'csv' use cvs module
                        'homemade' use a intuitive more robust but slow method

        """
        seg = Segment(file_origin = os.path.basename(self.filename))
        if not cascade:
            return seg

        if type(sampling_rate) == float or type(sampling_rate)==int:
            # if not quantitities Hz by default
            sampling_rate = sampling_rate*pq.Hz

        if type(t_start) == float or type(t_start)==int:
            # if not quantitities s by default
            t_start = t_start*pq.s

        unit = pq.Quantity(1, unit)

        #loadtxt
        if method == 'genfromtxt' :
            sig = np.genfromtxt(self.filename,
                                        delimiter = delimiter,
                                        usecols = usecols ,
                                        skip_header = skiprows,
                                        dtype = 'f')
            if len(sig.shape) ==1:
                sig = sig[:, np.newaxis]
        elif method == 'csv' :
            tab = [l for l in  csv.reader( file(self.filename,'rU') , delimiter = delimiter ) ]
            tab = tab[skiprows:]
            sig = np.array( tab , dtype = 'f')
        elif method == 'homemade' :
            fid = open(self.filename,'rU')
            for l in range(skiprows):
                fid.readline()
            tab = [ ]
            for line in fid.readlines():
                line = line.replace('\r','')
                line = line.replace('\n','')
                l = line.split(delimiter)
                while '' in l :
                    l.remove('')
                tab.append(l)
            sig = np.array( tab , dtype = 'f')

        if timecolumn is not None:
            sampling_rate = 1./np.mean(np.diff(sig[:,timecolumn])) * pq.Hz
            t_start = sig[0,timecolumn] * pq.s

        for i in range(sig.shape[1]) :
            if timecolumn == i : continue
            if usecols is not None and i not in usecols: continue

            if lazy:
                signal = [ ]*unit
            else:
                signal = sig[:,i]*unit

            anaSig = AnalogSignal(signal, sampling_rate=sampling_rate,
                                  t_start=t_start, channel_index=i,
                                  name='Column %d'%i)
            if lazy:
                anaSig.lazy_shape = sig.shape
            seg.analogsignals.append( anaSig )

        seg.create_many_to_one_relationship()
        return seg

    def write_segment(self, segment,
                                delimiter = '\t',

                                skiprows =0,
                                writetimecolumn = True,

                                ):
        """
        Write a segment and AnalogSignal in a text file.

         **Arguments**
            delimiter  :  columns delimiter in file  '\t' or one space or two space or ',' or ';'
            writetimecolumn :  True or Flase write time vector as first column
        """
        if skiprows:
            raise NotImplementedError('skiprows values other than 0 are not ' +
                                      'supported')
        l = [ ]
        if writetimecolumn is not None:
            l.append(segment.analogsignals[0].times[:, np.newaxis])
        for anaSig in segment.analogsignals:
            l.append(anaSig.magnitude[:, np.newaxis])
        sigs = np.concatenate(l, axis=1)
        #print sigs.shape
        np.savetxt(self.filename , sigs , delimiter = delimiter)

max_stars_repo_name: CINPLA/python-neo

max_issues_repo_name: CINPLA/python-neo

max_forks_repo_name: CINPLA/python-neo


text:
#!/usr/bin/env python3

import os
import os.path as path
import sys

import numpy as np
from scipy.sparse import csr_matrix
from scipy.sparse.csgraph import reverse_cuthill_mckee
from tensorflow.keras.datasets import cifar100
from tensorflow.keras.models import load_model

from utils import cifar100_names

if len(sys.argv) != 2:
    print('Usage: export_confusion.py <model_file.h5>', file=sys.stderr)
    sys.exit(1)

# Load data and the model
_, (x, y_true) = cifar100.load_data(label_mode='fine')
y_true = np.squeeze(y_true, axis=-1)
model_filename = sys.argv[1]
model = load_model(model_filename)

# Determine model predictions
y_pred = model.predict(x)
n_classes = y_pred.shape[1]
y_pred = np.argmax(y_pred, axis=-1)

# Populate the confusion matrix (i: true class, j: predicted class)
confusion = np.zeros((n_classes, n_classes))
for i in range(n_classes):
    true_i = (y_true == i)
    for j in y_pred[true_i]:
        confusion[i, j] += 1
    confusion[i] /= np.count_nonzero(true_i)

# Compute and apply the Cuthill McKee ordering
order_cm = reverse_cuthill_mckee(csr_matrix(confusion))
confusion_cm = confusion[order_cm][:, order_cm]

model_name = path.splitext(path.basename(model_filename))[0]
os.makedirs('confusion', exist_ok=True)
names_f = cifar100_names(label_mode='fine')

def write_confusion_csv(matrix, names, filename_append):
    csv_filename = path.join(
        'confusion', '{}{}.csv'.format(model_name, filename_append))
    with open(csv_filename, 'w') as csv_file:
        print('Writing CSV file "{}"...'.format(csv_filename))
        csv_file.write(',{}\n'.format(','.join(names)))
        for k in range(n_classes):
            # One row for each ground truth class
            csv_file.write(
                '{},{}\n'.format(names[k], ','.join(map(str, matrix[k]))))
        print('Done.')

# Write .csv files with the confusion matrices
write_confusion_csv(confusion, names_f, '')
write_confusion_csv(confusion_cm, [names_f[i] for i in order_cm], '_cm')

# Write a .npz file (more compact but less portable)
npz_filename = path.join('confusion', '{}.npz'.format(model_name))
print('Writing NPZ file "{}"...'.format(npz_filename))
np.savez_compressed(
    npz_filename, **{
        'confusion':    confusion,
        'order_cm':     order_cm,
        'confusion_cm': confusion_cm,
    })
print('Done.')

max_stars_repo_name: mattdutson/ml-vis

max_issues_repo_name: mattdutson/ml-vis

max_forks_repo_name: mattdutson/ml-vis


text:
#!/usr/bin/env python
import numpy
import torch

class DomainConfoundedDataset(torch.utils.data.Dataset):
    def __init__(self, datasetclass1, datasetclass2):
        self.ds1 = datasetclass1
        self.ds2 = datasetclass2
        self.len1 = len(self.ds1)
        self.len2 = len(self.ds2)
        self.labels = self.ds1.labels
    def init_worker(self, worker_id):
        self.ds1.init_worker(worker_id)
        self.ds2.init_worker(worker_id)
    def __getitem__(self, idx):
        if idx < self.len1:
            item = self.ds1[idx]
        else: # if idx >= self.len1
            item = self.ds2[idx-self.len1]
        return item[0], item[1], item[2], item[3]
    def __len__(self):
        return self.len1+self.len2
    def get_all_labels(self):
        labels1 = self.ds1.get_all_labels()
        labels2 = self.ds2.get_all_labels()
        if labels1.shape[0] == 1:
            return numpy.concatenate((self.ds1.get_all_labels(),
                                      self.ds2.get_all_labels()), axis=1)
        else:
            return numpy.concatenate((self.ds1.get_all_labels(),
                                      self.ds2.get_all_labels()), axis=0)


text:
import networkx as nx
from openfermion import QubitOperator
from .problem import Problem

class MaxCut(Problem):
    def _build_hamiltonian(self, graph: nx.Graph) -> QubitOperator:
        """Converts a MAXCUT instance, as described by a weighted graph, to an Ising
        Hamiltonian. It allows for different convention in the choice of the Hamiltonian.
        The returned Hamiltonian is consistent with the definitions from
        "A Quantum Approximate Optimization Algorithm" by E. Farhi, eq. 12
        (https://arxiv.org/pdf/1411.4028.pdf)
        and
        "Performance of the Quantum Approximate Optimization Algorithm on the Maximum Cut Problem" eq. 1
        (https://arxiv.org/pdf/1811.08419.pdf).

        Note: In the convention we assumed, values of the cuts are negative
        to frame the problem as a minimization problem.
        So for a linear graph 0--1--2 with weights all equal 1, and the solution [0,1,0],
        the returned value will be equal to -2.

        Args:
            graph: undirected weighted graph defining the problem
            scale_factor: constant by which all the coefficients in the Hamiltonian will be multiplied
            offset: coefficient of the constant term added to the Hamiltonian to shift its energy levels

        Returns:
            operator describing the Hamiltonian
        """
        hamiltonian = QubitOperator()
        shift = 0.0

        for i, j in graph.edges:
            try:
                weight = graph.adj[i][j]["weight"]
            except KeyError:
                weight = 1

            hamiltonian += weight * QubitOperator(f"Z{i} Z{j}")
            shift -= weight

        return 0.5 * (hamiltonian + shift)

max_stars_repo_name: zapatacomputing/z-quantum-qaoa

max_issues_repo_name: zapatacomputing/z-quantum-qaoa

max_forks_repo_name: zapatacomputing/z-quantum-qaoa

reason: import networkx


text:
# AMFinder - amfinder_superresolution.py
#
# MIT License
# Copyright (c) 2021 Edouard Evangelisti, Carl Turner
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

import os
import sys
import cv2
import numpy as np
import datetime
import matplotlib.pyplot as plt

from tensorflow.keras.optimizers import Adam
from keras.layers import Input, Dense, BatchNormalization, Add
from keras.layers.advanced_activations import PReLU, LeakyReLU
from keras.layers.convolutional import UpSampling2D, Conv2D
from keras.models import Model
# From keras 2.5, the following will be replaced by:
# from tf_agents.networks import network
# Thanks to matevzl533 for reporting this.
# Reference: https://www.tensorflow.org/agents/tutorials/8_networks_tutorial
from keras.engine.network import Network
from keras import backend as K

import amfinder_model as AmfModel
import amfinder_config as AmfConfig
import amfinder_segmentation as AmfSegm

BIN_SIZE = 3
CHANNELS = 3
HR_EDGE = 126
LR_EDGE = HR_EDGE // BIN_SIZE
LR_SHAPE = (LR_EDGE, LR_EDGE, CHANNELS)
HR_SHAPE = (HR_EDGE, HR_EDGE, CHANNELS)
OPTIMIZER = Adam(0.0002, 0.5)

def preprocess(tile_list):
    """
    For super resolution, images must be normalized to [-1; 1] for use with
    tanh activation function.
    """
    return (np.array(tile_list, np.float32) / 127.5) - 1.0

def restore(tile_list):

    return np.uint8((tile_list + 1) * 127.5)

# Reference: https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio
def PSNR(y_true, y_pred):
    """
    PSNR stands for Peek Signal to Noise Ratio.
    The equation is:
    PSNR = 20 * log10(MAX_I) - 10 * log10(MSE)

    Since input is scaled from -1 to 1, MAX_I = 1, and
    thus 20 * log10(1) = 0. Only the last part of
    the equation is therefore necessary.
    """
    return -10.0 * K.log(K.mean(K.square(y_pred - y_true))) / K.log(10.0)

# Reference: https://stackoverflow.com/a/56135413
def fast_downsample(tile):
    """
    Fast image downsampling method.
    """

    tmp = tile.reshape((15, BIN_SIZE, 15, BIN_SIZE, 3)).max(3).max(1)
    return cv2.resize(tmp, 42, cv2.INTER_NEAREST)

def get_tiles(paths):

    hr_tiles = []
    lr_tiles = []

    # Note: using tiles smaller than 126x126 pixels as high-resolution
    # tiles for SRGAN training does not make sense.
    edge = max(AmfConfig.get('tile_edge'), 126)
    AmfConfig.set('tile_edge', edge)

    for path in paths:

        image = AmfSegm.load(path)

        hr_tile_set = AmfSegm.mosaic(image, edge=edge)
        hr_tiles.extend(hr_tile_set)

        lr_tile_set = [fast_downsample(tile) for tile in hr_tile_set]
        lr_tiles.extend(lr_tile_set)

    # (N, HR_EDGE, HR_EDGE, CHANNELS)
    hr_tiles = preprocess(hr_tiles)
    # (N, LR_EDGE, LR_EDGE, CHANNELS)
    lr_tiles = preprocess(lr_tiles)

    return list(zip(hr_tiles, lr_tiles))

def build_feature_extractor():

    # CNN2 model gives good results.
    model = AmfModel.load(name='CNN2v1.h5')
    # Layer C22 has an output shape of (56, 56, 64).
    model.outputs = [model.get_layer('C22').output]
    img = Input(shape=HR_SHAPE)
    img_features = model(img)
    return Model(img, img_features)

def d_block(layer_input, filters, strides=1, batch_norm=True):
    """Discriminator layer"""
    d = Conv2D(filters,
               kernel_size=3,
               strides=strides,
               padding='same')(layer_input)
    d = LeakyReLU(alpha=0.2)(d)
    if batch_norm:
        d = BatchNormalization(momentum=0.8)(d)
    return d

def build_discriminator():

    df = 64

    d0 = Input(shape=HR_SHAPE)
    d1 = d_block(d0, df, batch_norm=False)
    d2 = d_block(d1, df, strides=2)
    d3 = d_block(d2, df * 2)
    d4 = d_block(d3, df * 2, strides=2)
    d5 = d_block(d4, df * 4)
    d6 = d_block(d5, df * 4, strides=2)
    d7 = d_block(d6, df * 8)
    d8 = d_block(d7, df * 8, strides=2)
    d9 = Dense(df * 16)(d8)
    d10 = LeakyReLU(alpha=0.2)(d9)
    validity = Dense(1, activation='sigmoid')(d10)
    return Model(d0, validity, name='discriminator')

def residual_block(layer_input):
    """Residual block described in paper"""
    d = Conv2D(64, kernel_size=3, strides=1, padding='same')(layer_input)
    d = BatchNormalization(momentum=0.8)(d)
    d = PReLU(shared_axes = [1,2])(d)
    d = Conv2D(64, kernel_size=3, strides=1, padding='same')(d)
    d = BatchNormalization(momentum=0.8)(d)
    d = Add()([d, layer_input])
    return d

def upscale_block(layer_input):
    """Layers used during upsampling"""
    u = Conv2D(256, kernel_size=3, strides=1, padding='same')(layer_input)
    u = UpSampling2D(size=3)(u)
    u = PReLU(shared_axes=[1,2])(u)
    return u

def build_generator(residual_blocks=5):

    # Low resolution image input
    img_lr = Input(shape=LR_SHAPE)

    # Pre-residual block
    c1 = Conv2D(64, kernel_size=9, strides=1, padding='same')(img_lr)
    c1 = PReLU(shared_axes=[1,2])(c1)

    # Propogate through residual blocks
    r = residual_block(c1)
    for _ in range(residual_blocks - 1):
        r = residual_block(r)

    # Post-residual block
    c2 = Conv2D(64, kernel_size=3, strides=1, padding='same')(r)
    c2 = PReLU(shared_axes=[1,2])(c2)
    #c2 = BatchNormalization(momentum=0.8)(c2)
    c2 = Add()([c2, c1])

    # Upsampling (original code had two blocks to achieve x4)
    # Instead we have a single block which achieves x3.
    u2 = upscale_block(c2)

    # Generate high resolution output. For tanh activation function, see:
    # https://towardsdatascience.com/gan-ways-to-improve-gan-performance-acf37f9f59b
    gen_hr = Conv2D(CHANNELS,
                    kernel_size=9,
                    strides=1,
                    padding='same',
                    activation='tanh')(u2)

    return Model(img_lr, gen_hr, name='generator')

def get_random_tile_pairs(tiles, batch_size=1, training=True):
    """
    Return randomly selected tile pairs.
    """

    indices = np.random.choice(len(tiles), size=batch_size)

    may_flip = lambda x: x

    if training and np.random.random() < 0.5:

        may_flip = lambda x: np.fliplr(x)

    selection = [tiles[i] for i in indices]

    hr_tiles = np.array([may_flip(x[0]) for x in selection])
    lr_tiles = np.array([may_flip(x[1]) for x in selection])

    return hr_tiles, lr_tiles

def train(paths, batch_size=2, sample_interval=20):
    """
    Trains a SRGAN.
    """

    print('* (super-resolution) Building feature extractor')

    feature_extractor = build_feature_extractor()
    feature_extractor.trainable = False
    feature_extractor.compile(loss='mse',
                              optimizer=OPTIMIZER,
                              metrics=['accuracy'])

    print('* (super-resolution) Building discriminator')

    discriminator = build_discriminator()

    if AmfConfig.get('discriminator') is not None:

        discriminator.load_weights(AmfConfig.get('discriminator'))

    discriminator.compile(loss='mse',
                          optimizer=OPTIMIZER,
                          metrics=['accuracy'])

    print('* (super-resolution) Building generator')

    generator = build_generator()

    if AmfConfig.get('generator') is not None:

        generator.load_weights(AmfConfig.get('generator'))

    generator.compile(loss='mse',
                      optimizer=OPTIMIZER,
                      metrics=['mse', PSNR])

    # High res. and low res. images
    img_hr = Input(shape=HR_SHAPE)
    img_lr = Input(shape=LR_SHAPE)

    # Generate high res. version from low res.
    fake_hr = generator(img_lr)

    # Extract image features of the generated img
    fake_features = feature_extractor(fake_hr)

    # For the combined model we will only train the generator
    # Using Network removes warnings.
    # https://github.com/keras-team/keras/issues/8585#issuecomment-412728017
    frozen_discriminator = Network(discriminator.inputs,
                                   discriminator.outputs,
                                   name='frozen_discriminator')

    frozen_discriminator.trainable = False

    # Discriminator determines validity of generated high res. images
    validity = frozen_discriminator(fake_hr)

    gan_model = Model(inputs=[img_lr, img_hr],
                      outputs=[validity, fake_features],
                      name='SRGAN')

    # TODO: display in verbose mode?
    #generator.summary()
    #discriminator.summary()
    #gan_model.summary()
    #sys.exit(0)

    gan_model.compile(loss=['binary_crossentropy', 'mse'],
                      loss_weights=[1e-3, 1],
                      optimizer=OPTIMIZER)

    tiles = get_tiles(paths)
    epochs = len(tiles) * 200

    # Annotations for original high-resolution images (valid), and
    # newly generated images (fake). Annotations are given so the
    # discriminator learns to identify fake images. Both tables
    # have shape (batch_size, 8, 8, 1) to tell which parts
    # of the generated images were identified as fake.
    valid = np.ones((batch_size,) + (8, 8, 1))
    fake = np.zeros((batch_size,) + (8, 8, 1))
    start_time = datetime.datetime.now()

    base_dir = os.path.join(AmfConfig.get_appdir(), 'trained_networks')

    for epoch in range(epochs):

        print('Epoch {}/{}... '.format(epoch + 1, epochs), end='')

        hr_tile, lr_tile = get_random_tile_pairs(tiles,
                                                 batch_size=batch_size)

        # From low res. image generate high res. version
        # (N, LR_EDGE * 4, LR_EDGE * 4, CHANNELS)
        fake_hr = generator.predict(lr_tile)

        # Train the discriminators
        # (original images = real / generated = Fake)
        d_loss_real = discriminator.train_on_batch(hr_tile, valid)
        d_loss_fake = discriminator.train_on_batch(fake_hr, fake)
        d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)

        hr_tile, lr_tile = get_random_tile_pairs(tiles,
                                                 batch_size=batch_size)

        # The generators want the discriminators
        # to label the generated images as real
        #valid = np.ones((batch_size, 1))

        # Extract ground truth image features using pre-trained CNN1 model
        image_features = feature_extractor.predict(hr_tile)

        # Train the generators
        g_loss, _, _ = gan_model.train_on_batch([lr_tile, hr_tile],
                                                [valid, image_features])

        elapsed_time = datetime.datetime.now() - start_time
        # Plot the progress
        print('time:', elapsed_time, 'g_loss:', g_loss, 'd_loss:', d_loss)

        # Save image samples to see progress.
        if (epoch + 1) % sample_interval == 0:
            save_sample_images(epoch, generator, tiles)
            discriminator.save_weights(os.path.join(base_dir, 'disc/srgan_disc{}.h5'.format(epoch+1)))
            generator.save_weights(os.path.join(base_dir, 'gen/srgan_gen{}.h5'.format(epoch+1)))

    # TODO: improve this.
    discriminator.save_weights(os.path.join(base_dir, 'disc/srgan_disc_final.h5'))
    generator.save_weights(os.path.join(base_dir, 'gen/srgan_gen_final.h5'))

def initialize(nrows, ncols):
    """
    Creates an empty image.
    """

    canvas = None

    if AmfConfig.get('super_resolution'):

        canvas = np.zeros([nrows * 126, ncols * 126, 3], dtype=np.uint8)
        canvas.fill(255)

    return canvas

def generate(sr_image, row, r):

    # Super-resolution is not active, then return the row unaltered.
    if sr_image is None:

        return row

    else:

        sr_row = improve(row)

        x_ini = 0
        y_ini = r * 126
        # Update the image.

        for i, tile in enumerate(sr_row):

            sr_image[y_ini:y_ini + 126, i * 126:(i + 1) * 126] = tile

        return sr_row

GENERATOR = None
def improve(raw_tiles):
    """
    Generate high-resolution versions of the given tiles.
    """

    global GENERATOR

    if GENERATOR is None:

        GENERATOR = build_generator()

        GENERATOR.load_weights(AmfConfig.get('generator'))

        GENERATOR.compile(loss='mse',
                          optimizer=OPTIMIZER,
                          metrics=['mse', PSNR])

    return restore(GENERATOR.predict(preprocess(raw_tiles)))

def save_sample_images(epoch, generator, tiles, batch_size=2):
    r, c = batch_size, 4

    imgs_hr, imgs_lr = get_random_tile_pairs(tiles,
                                             batch_size=batch_size,
                                             training=False)

    fake_hr = generator.predict(imgs_lr)

    # Restore pixel values.
    imgs_lr = restore(imgs_lr)
    imgs_in = [cv2.resize(x,
                          dsize=(HR_EDGE, HR_EDGE),
                          interpolation=cv2.INTER_CUBIC) for x in imgs_lr]
    fake_hr = restore(fake_hr)
    imgs_hr = restore(imgs_hr)

    # Save generated images together with originals.
    titles = ['Low resolution', 'Interpolation',
              'Generated', 'High resolution']
    fig, axs = plt.subplots(r, c)
    fig.suptitle('AMFinder SRGAN epoch {}'.format(epoch + 1), fontsize=20)

    for row in range(r):
        for col, image in enumerate([imgs_lr, imgs_in, fake_hr, imgs_hr]):
            axs[row, col].imshow(image[row])
            axs[row, col].set_title(titles[col])
            axs[row, col].axis('off')

    fig.savefig('tmp/sr-output/SR_output_{}.png'.format(epoch + 1))
    plt.close()

max_stars_repo_name: EEvangelisti/amfinder

max_issues_repo_name: EEvangelisti/amfinder

max_forks_repo_name: EEvangelisti/amfinder