text:
import caffe
import numpy as np
import os
import sys

import track_model_train as track_model
import train_config

def compute_accuracy(scores, labels):
    is_pos = (labels != 0)
    is_neg = np.logical_not(is_pos)
    num_all = labels.shape[0]
    num_pos = np.sum(is_pos)
    num_neg = num_all - num_pos

    is_correct = np.logical_xor(scores < 0, is_pos)
    accuracy_all = np.sum(is_correct) / float(num_all)
    accuracy_pos = np.sum(is_correct[is_pos]) / float(num_pos)
    accuracy_neg = np.sum(is_correct[is_neg]) / float(num_neg)
    return accuracy_all, accuracy_pos, accuracy_neg

def train(config):
    with open('./track_model/proto_train.prototxt', 'w') as f:
        f.write(str(track_model.generate_model('train', config)))

    caffe.set_device(config.gpu_id)
    caffe.set_mode_gpu()

    solver = caffe.get_solver('./track_model/solver_sgd.prototxt')
    solver.net.copy_from(config.weights)
    #solver.net.save('./snapshots/track_model/_iter_0.caffemodel')
    #solver.restore('./snapshots/track_model/_iter_50000.solverstate')

    loss_avg = 0.0
    avg_accuracy_all, avg_accuracy_pos, avg_accuracy_neg = 0, 0, 0
    decay = 0.99

    for it in range(config.max_iter):
        solver.step(1)

        loss_val = solver.net.blobs['loss'].data
        #scores_val = solver.net.blobs['fcn_scaled_scores'].data.copy()
        #label_val = solver.net.blobs['label'].data.copy()

        #import ipdb; ipdb.set_trace()

        loss_avg = decay*loss_avg + (1-decay)*loss_val
        if it % config.iter_display == 0:
            print('\titer = %d, cls_loss (cur) = %f, cls_loss (avg) = %f'
                % (it, loss_val, loss_avg))

        # Accuracy
        #accuracy_all, accuracy_pos, accuracy_neg = compute_accuracy(scores_val, label_val)
        #avg_accuracy_all = decay*avg_accuracy_all + (1-decay)*accuracy_all
        #avg_accuracy_pos = decay*avg_accuracy_pos + (1-decay)*accuracy_pos
        #avg_accuracy_neg = decay*avg_accuracy_neg + (1-decay)*accuracy_neg
        #print('\titer = %d, accuracy (cur) = %f (all), %f (pos), %f (neg)'
        #      % (it, accuracy_all, accuracy_pos, accuracy_neg))
        #print('\titer = %d, accuracy (avg) = %f (all), %f (pos), %f (neg)'
        #      % (it, avg_accuracy_all, avg_accuracy_pos, avg_accuracy_neg))

if __name__ == '__main__':
    config = train_config.Config()
    train(config)

max_stars_repo_name: QUVA-Lab/lang-tracker

max_issues_repo_name: zhenyangli/lang-tracker

max_forks_repo_name: QUVA-Lab/lang-tracker


text:
import argparse
import csv
import ctypes
from functools import partial
import os
import multiprocessing
from subprocess import call
import tempfile

from numpy import array, savetxt
import numpy as np
import SimpleITK as sitk

####################################################
# Process arguments
####################################################
def parse_args():
    parser = argparse.ArgumentParser(description=""
                "Make a tiff dataset from DDSM raws of normal cases.")
    parser.add_argument('read_from', type=str,
                        help="the path to the DDSM normals directory")
    parser.add_argument('write_to', type=str,
                        help="the destination directory for the converted "
                             "(tiff) files")
    parser.add_argument('--clip_noise', type=bool, default=True,
                        help="clip out low and high values of image after "
                             "conversion from raw for 'noise reduction'")
    parser.add_argument('--resize', type=int, default=None,
                        help="resize images to a square of this size")
    parser.add_argument('--force', action='store_true',
                        help="force overwrite of existing files")
    args = parser.parse_args()
    return args

####################################################
# CSV fields to record
####################################################
fields = ['patient_id',
          'breast_density',
          'side',
          'view',
          'scanner_type',
          'scan_institution',
          'width',
          'height',
          'bpp',
          'resolution',
          'od_img_path']

####################################################
# Extract value from split list of data
####################################################
def get_value(lst, row_name, idx):
    """
    :param lst: data list, each entry is another list with whitespace separated data
    :param row_name: name of the row to find data
    :param idx: numeric index of desired value
    :return: value
    """
    val = None
    for l in lst:
        if not l:
            continue

        if l[0] == row_name:
            try:
                val = l[idx]
            except Exception:
                print(row_name, idx)
                print(lst)
                val = None
            break
    return val

####################################################
# ICS Information extraction
####################################################
scanner_map = {
    ('A', 'DBA'): 'MGH',
    ('A', 'HOWTEK'): 'MGH',
    ('B', 'LUMISYS'): 'WFU',
    ('C', 'LUMISYS'): 'WFU',
    ('D', 'HOWTEK'): 'ISMD'
}

def get_ics_info(ics_file_path):
    """
    :param ics_file_path: path to ics file
    :return: dictionary containing all relevant data in ics file
    """
    # get letter for scanner type
    ics_file_name = os.path.basename(ics_file_path)
    letter = ics_file_name[0]

    # get data from ics file
    with open(ics_file_path, 'r') as f:
        lines = list(map(lambda s: s.strip().split(), f.readlines()))

    # map ics data to values
    ics_dict = {
        'patient_id': get_value(lines, 'filename', 1),
        'age': get_value(lines, 'PATIENT_AGE', 1),
        'scanner_type': get_value(lines, 'DIGITIZER', 1),
        'scan_institution': scanner_map[(letter, get_value(lines,
                                                           'DIGITIZER', 1))],
        'density': get_value(lines, 'DENSITY', 1)
    }

    for sequence in ['LEFT_CC', 'RIGHT_CC', 'LEFT_MLO', 'RIGHT_MLO']:
        if get_value(lines, sequence, 0) is None:
            continue

        sequence_dict = {
            'height': int(get_value(lines, sequence, 2)),
            'width': int(get_value(lines, sequence, 4)),
            'bpp': int(get_value(lines, sequence, 6)),
            'resolution': float(get_value(lines, sequence, 8))
        }

        ics_dict[sequence] = sequence_dict

    return ics_dict

####################################################
# Process an image from a normal DDSM case
####################################################
class ddsm_normal_case_image(object):
    def __init__(self, path, ics_dict):

        fname = os.path.basename(path)
        case_id, sequence, ext = fname.split('.')
        self.case_id = case_id
        self.sequence = sequence
        self.ext = ext

        # file path data
        self.path = path

        # image information
        self.height = ics_dict[sequence]['height']
        self.width = ics_dict[sequence]['width']
        self.bpp = ics_dict[sequence]['bpp']
        self.resolution = ics_dict[sequence]['resolution']
        self.scanner_type = ics_dict['scanner_type']
        self.scan_institution = ics_dict['scan_institution']

        # patient information
        self.patient_id = ics_dict['patient_id']
        self.side, self.view = sequence.split('_')
        self.breast_density = ics_dict['density']

        # image information
        self._raw_image = None

    def _decompress_ljpeg(self, log_file_path='ljpeg_decompression_log.txt'):
        """
        :param im_path: base path for ljpeg
        :param log_file_path: path to log for writing these
        :return: None
        """
        with open(log_file_path, 'a') as log_file:
            # Assume jpeg binary is in PATH
            # TODO: install jpeg to a binary directory with setup.py
            call_lst = ['jpeg', '-d', '-s', self.path]
            call(call_lst, stdout=log_file)

    def _read_raw_image(self, force=False):
        """
        Read in a raw image into a numpy array
        :param force: boolean flag if we should force a read if we already have this image
        :return: None
        """

        # only read if we haven't already or
        # we aren't trying to force a read
        if (self._raw_image is not None) and not force:
            return

        try:
            # make sure decompressed image exists
            self._decompress_ljpeg()

            # read it in and make it correct
            raw_im_path = "{}.1".format(self.path)
            im = np.fromfile(raw_im_path, dtype=np.uint16)
            im.shape = (self.height, self.width)
            self._raw_image = im.byteswap()  # switch endian
        finally:
            # clean up : delete decompressed ljpeg
            os.remove("{}.1".format(self.path))

    def _od_correct(self, im, clip_noise=True):
        """
        Map gray levels to optical density level
        :param im: image
        :return: optical density image
        """
        im_od = np.zeros_like(im, dtype=np.float64)
        MAXVAL = 2**16-1
        if (self.scan_institution == 'MGH') and (self.scanner_type == 'DBA'):
            im_clip = np.clip(im,0,MAXVAL-1)+1 # add 1 to keep from log(0)
            im_od = (np.log10(im_clip) - 4.80662) / -1.07553
        elif (self.scan_institution == 'MGH') and (self.scanner_type == 'HOWTEK'):
            im_od = (-0.00094568 * im) + 3.789
        elif (self.scan_institution == 'WFU') and (self.scanner_type == 'LUMISYS'):
            im_od = (im - 4096.99) / -1009.01
        elif (self.scan_institution == 'ISMD') and (self.scanner_type == 'HOWTEK'):
            im_od = (-0.00099055807612 * im) + 3.96604095240593

        # Clip into [0, 4]
        im_od = np.clip(im_od, 0, 4)

        # perform heath noise correction
        if clip_noise:
            im_od = np.clip(im_od, 0.05, 3)
            im_od = (im_od-0.05)*4/(3-0.05) # rescale back into [0, 4]
        return im_od

    def save_image(self,
                   out_dir,
                   od_correct=False,
                   clip_noise=True,
                   resize=None,
                   force=False):
        """
        save the image data as a tiff file (without correction)
        :param out_dir: directory to put this image
        :param od_correct: boolean to decide to perform od_correction
        :param clip_noise: boolean for whether to clip out low and high values
        :param force: force if this image already exists
        :return: path of the image
        """
        if not os.path.exists(out_dir):
            os.makedirs(out_dir)
        # construct image path
        out_name = os.path.split(self.path)[1].replace(".LJPEG", ".tif")
        im_path = os.path.join(out_dir, out_name)

        # don't write if image exists and we aren't forcing it
        if os.path.exists(im_path) and not force:
            return im_path

        # make sure we have an image to save
        if self._raw_image is None:
            self._read_raw_image()

        im_array = np.copy(self._raw_image)

        # convert to optical density
        if od_correct:
            im_array = self._od_correct(im_array, clip_noise=clip_noise)
            im_array = np.interp(im_array, (0.0, 4.0), (65535, 0))
            im_array = im_array.astype(np.uint16)

        # create image object
        im = sitk.GetImageFromArray(im_array)

        # resize if necessary
        if resize:
            im = sitk.Resample(im,
                               (resize, resize),
                               sitk.Transform(),
                               sitk.sitkLinear,
                               im.GetOrigin(),
                               im.GetSpacing(),
                               im.GetDirection(),
                               0,
                               im.GetPixelID())

        # save image (make this atomic in case of interruption)
        tmp = tempfile.NamedTemporaryFile(delete=False,
                                          dir=out_dir,
                                          suffix='.tif')
        tmp_path = os.path.join(out_dir, tmp.name)
        sitk.WriteImage(im, tmp_path, True)  # True for useCompression
        os.rename(tmp_path, im_path)         # Atomic move

        # return location of image
        return im_path

####################################################
# Create the dataset (read, convert, save)
####################################################
def make_data_set(read_from, write_to,
                  clip_noise=True, resize=None, force=False):
    if not os.path.exists(write_to):
        os.makedirs(write_to)

    # Walk through image directory tree; load ics files and note down
    # all image paths for later multiprocessing.
    image_list = []     # tuples of (image_path, ics_dict)
    for curdir, dirs, files in os.walk(read_from):
        raw_image_filenames = []
        ics_filename = None
        for f in files:
            if f.endswith('.LJPEG'):
                raw_image_filenames.append(f)
            elif f.endswith('.ics'):
                ics_filename = f
        if not ics_filename:
            continue

        # Read ics info.
        ics_path = os.path.join(curdir, ics_filename)
        ics_dict = get_ics_info(ics_path)

        # Record paths.
        image_list.extend([(os.path.join(curdir, fn), ics_dict)
                           for fn in raw_image_filenames])

    # Use multiprocessing to convert images.
    args = (read_from, write_to, resize, force)
    with multiprocessing.Pool() as pool:
        log = pool.map(partial(convert_image,
                               read_from=read_from,
                               write_to=write_to,
                               clip_noise=clip_noise,
                               resize=resize,
                               force=force),
                       image_list)

    # Parse results. Record output in csv file.
    outfile = open(os.path.join(write_to, 'ddsm_normal_cases.csv'), 'w')
    outfile_writer = csv.writer(outfile, delimiter=',')
    outfile_writer.writerow(fields)
    count_success = 0
    count_failure = 0
    for result in log:
        if result[0]==0:
            count_success += 1
            outfile_writer.writerow(result[1])
        else:
            count_failure += 1
    outfile.close()

    # Report on the number of successes and failures.
    print("SUCCESS: {}, FAILURE: {}".format(count_success, count_failure))

####################################################
# Convert a raw image to a tif (and save)
####################################################
def convert_image(image_tuple, read_from, write_to, clip_noise, resize, force):
    path, ics_dict = image_tuple
    rel_path = os.path.relpath(path, read_from)
    case = ddsm_normal_case_image(path, ics_dict)
    dir_write_to = os.path.join(write_to, os.path.dirname(rel_path))
    print("Converting {} ... ".format(rel_path), end="")
    try:
        # uint8 optical density
        save_path = case.save_image(out_dir=dir_write_to,
                                    od_correct=True,
                                    clip_noise=clip_noise,
                                    resize=resize,
                                    force=force)
        case.od_img_path = save_path    # to save in csv
        print("done")
        return (0, [getattr(case, f) for f in fields])
    except Exception as e:
        print("error : {}".format(e))
    return (1, None)

if __name__=='__main__':
    args = parse_args()
    make_data_set(read_from=args.read_from,
                  write_to=args.write_to,
                  clip_noise=args.clip_noise,
                  resize=args.resize,
                  force=args.force)

ext: py

reason: import numpy,from numpy


text:
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import numpy as np
from bigdl.nano.pytorch.optim import SparseAdam
import torch
from torch import nn

class Example18(nn.Module):
    def __init__(self, vocab_size, embedding_dim, sparse=False):
        super(Example18, self).__init__()
        self.model = nn.Sequential()
        self.embedding = nn.Embedding(vocab_size, embedding_dim, sparse=sparse)

    def forward(self, inputs):
        embeds = self.embedding(inputs)
        return self.model(embeds)

def test_optim_sparseadam():
    # # Test different results of SparseAdam
    # # when applied in sparse=False and sparse=True
    # model1: sparse=False
    # model2: sparse=True
    model1 = Example18(30, 16, sparse=False)
    model2 = Example18(30, 16, sparse=True)

    loss_function = nn.MSELoss()

    optimizer1 = SparseAdam(model1.parameters())
    optimizer2 = SparseAdam(model2.parameters())

    nums_epochs = 1
    nums_batches = 2
    input = torch.LongTensor([[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]])
    labels_np = np.random.randint(30, size=model1(input).shape)
    labels = torch.FloatTensor(labels_np)

    sparse_dic = {}

    for epoch in range(nums_epochs):
        for i in range(nums_batches):
            x = input[i]
            y = labels[i]

            model1.train()
            total_loss = torch.Tensor([0])
            model1.zero_grad()
            result = model1(x)
            loss = loss_function(result, y)
            loss.backward()
            optimizer1.step()
            total_loss += loss.data

            sparse_dic["result{0}".format(i)] = model1.state_dict()['embedding.weight'][0:5, :].clone()

    for epoch in range(nums_epochs):
        for i in range(2):
            x = input[i]
            y = labels[i]

            model2.train()
            total_loss = torch.Tensor([0])
            model2.zero_grad()
            result = model2(x)
            loss = loss_function(result, y)
            loss.backward()
            optimizer2.step()
            total_loss += loss.data

            sparse_dic["result{0}".format(i + 2)] = model2.state_dict()['embedding.weight'][0:5, :].clone()

    # check if the 0-5 cows of model1 weights have changed after training
    # check if the 0-5 cows of model2 weights remain unchanged after training
    assert ((sparse_dic['result0'] != sparse_dic['result1']).all())
    assert ((sparse_dic['result2'] == sparse_dic['result3']).all())

max_stars_repo_name: DirkFi/BigDL

max_issues_repo_name: DirkFi/BigDL

max_forks_repo_name: DirkFi/BigDL

reason: import numpy


text:
#!/usr/bin/env python
from __future__ import print_function
from __future__ import absolute_import
__author__ = 'Kevin Lee - kevin_lee@claris.com'

import argparse
import numpy as np
import os
import pdb
import sys
import torch
import torchvision
from torch.autograd import Variable
from torchvision import transforms
from PIL import Image

from .classes.dataset.ImageDataLoader import transform
from .classes.model.pix2code import Pix2Code

from .classes.Utils import Utils
from .classes.Vocabulary import *

def build_model_and_vocab(vocab_path, weights_path, device='cpu'):

    # Model Hyperparams
    embed_size = 1024
    hidden_size = 1024
    num_layers = 2

    # Load vocabulary
    vocab = Utils.build_vocab(vocab_path)

    vocab_size = len(vocab)

    print(vocab.word2idx)

    #Load data from checkpoint
    checkpoint = torch.load(weights_path, map_location=device)
    if 'hyp' in checkpoint:
        embed_size = checkpoint['hyp']['embed_size']
        hidden_size = checkpoint['hyp']['hidden_size']
        num_layers = checkpoint['hyp']['num_layers']
        assert vocab_size == checkpoint['hyp']['vocab_size'], 'incompatible vocab_sizes {} and {}'.format(vocab_size, checkpoint['hyp']['vocab_size'])

    # Create models
    model = Pix2Code(embed_size, hidden_size, vocab_size, num_layers)

    # Convert device
    model = model.to(device)

    # Load trained models
    model.load_state_dict(checkpoint['model'], strict=True)

    # Set model to eval mode
    model.eval()

    return model, vocab

def sample(image, model, vocab, beam_size, device='cpu'):

    image = image.convert('RGB')
    image = transform(image)

    image_tensor = Variable(image.unsqueeze(0).to(device))

    #Sample
    with torch.no_grad():
        sampled_ids = model.sample(image_tensor, vocab, beam_size=beam_size)

    #Convert tensor to numpy array
    sampled_ids = sampled_ids.cpu().data.numpy()

    predicted = Utils.transform_idx_to_words(vocab, sampled_ids)

    predicted = ''.join(predicted)
    predicted = predicted.replace('{', "{\n").replace('}', "\n}\n").replace('}\n\n}', "}\n}")

    #Strip START and END tokens
    predicted = predicted.replace(START_TOKEN, "").replace(END_TOKEN, "")

    return predicted

def main():

    # Determine device
    device = Utils.get_device(opt.gpu_id)

    # Load model and vocab
    model, vocab = build_model_and_vocab(opt.vocab, opt.weights, device)

    # Get image from filesystem
    assert opt.input.endswith('.png'), 'unsupported input format'

    image = Image.open(opt.input)

    # Sample
    predicted = sample(image, model, vocab, opt.beam_size, device)

    if(opt.output):
        output_path = opt.output

        if not output_path.endswith('.gui'):
            output_path += '.gui'

        output_dir = os.path.dirname(output_path)
    else:
        output_dir = os.path.dirname(opt.input)
        output_path = opt.input.replace('.png', '.gui')

    #Create output directory if it doesn't exist
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    #Open file and write
    with open(output_path, 'w') as out_f:
        out_f.write(predicted)

    print('done!')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--input', '-i', type=str, required=True, help='input image path')
    parser.add_argument('--output', '-o', type=str, required=False, help='output path (<path>/*.gui)')
    parser.add_argument('--vocab', '-v', type=str, required=True, default='../bootstrap.vocab', help='*-config.json path')
    parser.add_argument('--weights', '-w', type=str, required=True, default='', help='weights to preload into model')
    parser.add_argument('--gpu-id', type=int, required=False, default=0, help='GPU ID to use')
    parser.add_argument('--beam-size', type=int, required=False, default=3, help='Beam size for beam search')
    opt = parser.parse_args()
    main()


text:
#!/usr/bin/env python
import numpy as np
import sys
import random
import torch
if len(sys.argv) >= 4:
    seed_num = int(sys.argv[2]) + 3
    np.random.seed(seed_num)
    random.seed(seed_num)
    torch.manual_seed(seed_num)
    print("\033[93m" + 'random seed is set to '+ str(seed_num) + "\033[0m")

import time
import config
import math

from mat_factor import tester
from sec_hop import selector
from experiment import Experiment
import matplotlib.pyplot as plt
import net_init
from simple_model.simple_model import NetworkSim
from sec_hop.experiment import Experiment as SecHopExperiment

# assuming uniform, num_pub => num region, if this info is known
# otherwise use a large number
def calculate_T(miss_prob, num_node, num_pub, num_topo, top_n_peer):
    T = math.ceil(math.log(miss_prob) / math.log(1.0 - 1.0/num_pub)) / top_n_peer
    return int(T * num_topo*top_n_peer)

num_out = 4
num_in =  8
num_2hop = 0
num_rand = 1
num_msg = 40 # 20 *2

def run_simple_model():
    subcommand = sys.argv[1]
    seed = int(sys.argv[2])
    topo = sys.argv[3]
    plt_name = sys.argv[4]
    num_star = int(sys.argv[5])
    num_epoch = int(sys.argv[6])
    churn_rate = float(sys.argv[7])
    print_log = sys.argv[8]=='y'

    num_topo = 3
    update_interval = 2

    top_n_peer = 2
    # extra info, extra choose appropriate T as hyperparameter
    num_pub, num_node, pubs = net_init.get_num_pub_node(topo)
    # T = calculate_T(0.001, num_node, num_pub, num_topo, top_n_peer)
    T = num_msg*num_topo
    print('pubs', pubs)

    # pools = [i for i in range(num_node) if i not in pubs] # node using adaptive algo
    pools = [i for i in range(num_node)]
    stars = list(np.random.choice(pools, num_star, replace=False))
    print('stars', stars)

    mc_epochs = 2000
    mc_lr = 1e-2
    mc_exit_loss_diff = 1e-3

    m = NetworkSim(topo, num_out, num_in, num_epoch, T, num_topo, stars, mc_epochs, mc_lr, mc_exit_loss_diff, num_rand, top_n_peer, plt_name, print_log, churn_rate, update_interval)
    start_t = time.time()
    m.run()
    print('finish', num_epoch, 'epochs in', round(time.time()-start_t,2), 'sec')

def run_mc():
    subcommand = sys.argv[1]
    seed = int(sys.argv[2])
    data_path = sys.argv[3]
    out_lim = int(sys.argv[4])
    num_new = int(sys.argv[5])
    input_json = sys.argv[6]
    use_logger = sys.argv[7] == 'y'
    loc, link_delay, role, proc_delay = net_init.load_network(input_json)
    record_epochs = [int(i) for i in sys.argv[8:]]
    max_epoch = max(record_epochs) +1
    num_node = len(loc)
    num_msg =  out_lim*3 *2 # assume numIn = numOut, num include both
    window = 2*num_msg
    in_lim = num_node

    print("\033[93m" + 'num msg per topo '+ str(num_msg) +  "\033[0m")
    print("\033[93m" + 'mat complete window size'+ str(window) +  "\033[0m")

    start = time.time()
    adv_nodes = [i for i in range(config.num_adv)]
    perigee = Experiment(
        link_delay,
        role,
        proc_delay,
        num_node,
        in_lim,
        out_lim,
        data_path,
        adv_nodes,
        window,
        'mc',
        loc,
        num_new,
        use_logger,
        seed
        )
    perigee.init_graph_mc()
    perigee.start(max_epoch, record_epochs, num_msg)

def run_mf():
    subcommand = sys.argv[1]
    data_path = sys.argv[3]
    out_lim = int(sys.argv[4])
    num_region = int(sys.argv[5])
    use_node_hash = sys.argv[6]=='y'
    input_json = sys.argv[7]

    assert(config.use_abs_time)
    loc, link_delay = net_init.load_network(input_json)

    record_epochs = [int(i) for i in sys.argv[8:]]
    max_epoch = max(record_epochs) +1
    window = int(config.num_msg*config.window_constant * math.ceil(num_region * math.log(config.num_node))) # T > L log N

    print(window, num_region, config.num_node)
    node_delay = net_init.GenerateInitialDelay(config.num_node)
    node_hash = None
    # [LinkDelay,NodeHash,NodeDelay] = readfiles.Read(NodeDelay, NetworkType, num_node)

    # [ node_delay,
      # node_hash, _,
      # _, _,
      # _, _,
      # _] = initnetwork.GenerateInitialNetwork(
            # config.network_type,
            # config.num_node,
            # subcommand,
            # out_lim)
    # tester.print_mat(link_delay, False)

    if config.use_reduce_link:
        print("\033[91m" + 'Use reduced link latency' + "\033[0m")
        net_ini.reduce_link_latency(config.num_node, int(0.2*config.num_node), link_delay)
    else:
        print("\033[93m" + 'Not use reduced link latency'+ "\033[0m")

    if not use_node_hash:
        print("\033[93m" + 'Not Use asymmetric node hash'+ "\033[0m")
        node_hash = None
    else:
        print("\033[91m" + 'Use asymmetric node hash'+ "\033[0m")

    if config.use_matrix_completion:
        print("\033[93m" + 'Use matrix completion'+ "\033[0m")
        if config.use_abs_time:
            print("\033[93m" + '\tuse absolute time'+ "\033[0m")
        else:
            print("\033[93m" + '\tuse relative time'+ "\033[0m")
    else:
        print("\033[93m" + 'Use 2hop selections'+ "\033[0m")
    print("\033[93m" + 'num region '+ str(num_region) +  "\033[0m")
    print("\033[93m" + 'num msg '+ str(config.num_msg) +  "\033[0m")
    print("\033[93m" + 'window '+ str(window) +  "\033[0m")

    start = time.time()
    adv_nodes = [i for i in range(config.num_adv)]
    perigee = Experiment(
        node_hash,
        link_delay,
        node_delay,
        config.num_node,
        config.in_lim,
        out_lim,
        num_region,
        data_path,
        adv_nodes,
        window,
        'mf-bandit' ,
        loc
        )
    perigee.init_graph()
    perigee.start(max_epoch, record_epochs, config.num_msg)

def run_rel_comp():
    subcommand = sys.argv[1]
    data_path = sys.argv[3]
    out_lim = int(sys.argv[4])
    num_region = int(sys.argv[5])
    use_node_hash = sys.argv[6]=='y'
    input_json = sys.argv[7]

    loc, link_delay = net_init.load_network(input_json)

    # record_epochs = [int(i) for i in sys.argv[8:]]
    # max_epoch = max(record_epochs) +1
    # window = int(config.num_msg*config.window_constant * math.ceil(num_region * math.log(config.num_node))) # T > L log N

    # print(window, num_region, config.num_node)
    # node_delay = initnetwork.GenerateInitialDelay(config.num_node)
    # node_hash = readfiles.ReadHashFile(config.num_node)
    # adv_nodes = []
    # assert(config.use_abs_time == False)
    # perigee = Experiment(
        # node_hash,
        # link_delay,
        # node_delay,
        # config.num_node,
        # config.in_lim,
        # out_lim,
        # num_region,
        # data_path,
        # adv_nodes,
        # window,
        # 'rel_comp' ,
        # loc
        # )
    # perigee.init_graph()
    # perigee.start_rel_comp(max_epoch, record_epochs, config.num_msg)

def run_2hop():
    seed = int(sys.argv[2])

    topo =  sys.argv[3]
    num_epoch = int(sys.argv[4])
    num_adapt = int(sys.argv[5])
    churn_rate = float(sys.argv[6])
    data_path = sys.argv[7]

    num_pub, num_node, pubs = net_init.get_num_pub_node(topo)

    num_keep = num_out - num_rand - num_2hop

    print('pubs', pubs)
    # pools = [i for i in range(num_node) if i not in pubs] # node using adaptive algo
    pools = [i for i in range(num_node)] # node using adaptive algo

    stars = list(np.random.choice(pools, num_adapt, replace=False))
    print('stars', stars)

    sec_hop_model = SecHopExperiment(
        topo,
        num_in,
        num_out,
        data_path,
        num_keep,
        num_2hop,
        num_rand,
        num_epoch,
        stars,
        num_msg,
        churn_rate,
        )
    sec_hop_model.run()

def complete_graph():
    print('./testbed complete_graph 1 n 0 1 2 3 4 5')
    subcommand = sys.argv[1]
    data_path = sys.argv[3]
    out_lim = int(sys.argv[4])
    use_node_hash = sys.argv[5]=='y'

    record_epochs = [int(i) for i in sys.argv[6:]]
    max_epoch = max(record_epochs) +1
    num_region = out_lim

    # [ node_delay,
      # node_hash, link_delay,
      # neighbor_set, IncomingLimit,
      # outs_neighbors, in_lims,
      # bandwidth] = initnetwork.GenerateInitialNetwork(
            # config.network_type,
            # config.num_node,
            # subcommand,
            # out_lim)

    # if out_lim != config.num_node-1:
        # print('Error. A complete graph need correct out lim')
        # sys.exit(1)

    # outs_neighbors = {}
    # for i in range(config.num_node):
        # connections = []
        # for j in range(config.num_node):
            # if i != j:
                # connections.append(j)
        # outs_neighbors[i] = connections

    # print('setup experiment')
    # perigee = Experiment(
            # node_hash,
            # link_delay,
            # node_delay,
            # config.num_node,
            # config.in_lim,
            # out_lim,
            # data_path,
            # [],
            # 0,
            # subcommand
            # )
    # perigee.init_graph(outs_neighbors)
    # perigee.start_complete_graph(max_epoch, record_epochs)

def test_mf():
    if len(sys.argv) < 4:
        print('test-mf N L std num_exp')
        sys.exit(1)
    # N = int(sys.argv[2])
    # L = int(sys.argv[3])
    # std = float(sys.argv[4])
    # num = int(sys.argv[5])
    # T = int(math.ceil(L * math.log(N)))
    # mf_tester = tester.MF_tester(T, N, L, num)
    # mf_tester.test_mf()

def parse_static_input():
    if len(sys.argv) < 13:
        print('Error. Invalud arguments')
        print('test-mf seed[int] out[string] N[int] L[int] max_iter[int] new_msgs[int] noise[float] H_method[unif,log-unif,1D-linear] add_method[rolling,append] num_mask[int] init_method[algo,ref] mask_method[random,bandit] windowM[float]')
        print('example: ./testbed.py mf-static 1 node=20 region=5 new_msgs=1 H_method=1D-linear add_method=append mask_method=bandit init_method=ref name=rand-init num_mask=0 noise=0 max_iter=100 windowM=4 ')
        sys.exit(1)

    for arg in sys.argv[3:]:
        if 'noise=' in arg:
            std = float(arg[6:])
        elif 'node=' in arg:
            N = int(arg[5:])
        elif 'region=' in arg:
            L = int(arg[7:])
        elif 'new_msgs=' in arg:
            num_msg = int(arg[9:])
        elif 'H_method=' in arg:
            H_method = arg[9:]
        elif 'add_method=' in arg:
            add_method = arg[11:]
        elif 'max_iter=' in arg:
            max_iter = int(arg[9:])
        elif 'num_mask=' in arg:
            num_mask_per_row = int(arg[9:])
        elif 'mask_method=' in arg:
            mask_method = arg[12:]
        elif 'name=' in arg:
            name = arg[5:]
        elif 'init_method=' in arg:
            init_method = arg[12:]
        elif 'windowM=' in arg:
            windowM = float(arg[8:])
        else:
            print('Error. Unknown arg', arg)
            sys.exit(1)

    return (std, N, L, num_msg, H_method, add_method, max_iter, num_mask_per_row, mask_method, name, init_method, windowM)

def run_1hop_static():
    (std, N, L, num_msg, H_method,
     add_method, max_iter, num_mask_per_row,
     mask_method, name, init_method, windowM) =  parse_static_input()

    exp_name = ('static1hop_node'+str(N)+'-'+'region'+str(L)+"-"+'noise'+str(int(std))+'-'+
            H_method+'-'+add_method+str(num_msg)+'msg'+'-'+mask_method+'-'+str(num_mask_per_row)+
            'mask'+'-'+init_method+'-'+'windowM'+ str(int(windowM))+'-'+name)

    T = int(windowM*math.ceil(L * math.log(N))) # 3
    static1hop_exp = tester.MF_tester(T, N, L, max_iter, exp_name, add_method, num_mask_per_row,
            H_method, init_method, mask_method)
    static1hop_exp.run_1hop(max_iter, num_msg, std)

def run_mf_static():
    (std, N, L, num_msg, H_method,
     add_method, max_iter, num_mask_per_row,
     mask_method, name, init_method, windowM) =  parse_static_input()

    exp_name = ('node'+str(N)+'-'+'region'+str(L)+"-"+'noise'+str(int(std))+'-'+
            H_method+'-'+add_method+str(num_msg)+'msg'+'-'+mask_method+'-'+str(num_mask_per_row)+
            'mask'+'-'+init_method+'-'+'windowM'+ str(int(windowM))+'-'+name)
    print('exp_name', exp_name)

    T = int(windowM*math.ceil(L * math.log(N))) # 3
    mf_online_exp = tester.MF_tester(T, N, L, max_iter, exp_name, add_method, num_mask_per_row,
            H_method, init_method, mask_method)
    W_scores, H_scores = mf_online_exp.start_mf_online(max_iter, num_msg, std)
    with open(mf_online_exp.result_filepath, 'w') as w:
        # w.write(str(int(std)) + '\t' + str(num_mask_per_row) + '\n')
        w.write('\t'.join([str(a) for a in W_scores]) + '\n')
        w.write('\t'.join([str(a) for a in H_scores]) + '\n')

    iters = [i for i in range(len(W_scores))]
    fig, axs = plt.subplots(2)
    axs[0].scatter(iters, W_scores, c='r', s=10)
    axs[0].set_title(exp_name + '\n'+'W best perm score')
    axs[1].scatter(iters, H_scores, s=10)
    axs[1].set_title('H best perm score')
    plt.tight_layout()
    plt.show()
    fig.savefig(mf_online_exp.fig_filepath)

def get_window_size(num_mask_per_row, N, L):
    pass

def print_help():
    print('subcommand')
    print('\trun-2hop       seed[int] output_dir[str] num_out[int] num_msg[int] use_node_hash[y/n] rounds[intList]')
    print('\trun-mf         seed[int] output_dir[str] num_out[int] num_region[int] use_node_hash[y/n] rounds[intList]')
    print('\trun-mc         seed[int] output_dir[str] num_out[int] num_new[int] input_json[str] rounds[intList]')

    print('\trel-comp         seed[int] output_dir[str] num_out[int] num_region[int] use_node_hash[y/n] rounds[intList]')

    print('\tcomplete-graph seed[int] output_dir[str] num_out[int] num_region[int] use_node_hash[y/n] 1')
    print('\tmf-static      seed[int] help')
    print('\t1hop-static    seed[int] help')
    print('\ttest_mf        N[int] L[int] std[float] num_exp[int]')
    print('Note. Output is stored at analysis/output_dir')

if __name__ == '__main__':
    subcommand = sys.argv[1]
    if subcommand == 'help':
        print_help()
    elif subcommand == 'test-mf':
        test_mf()
    elif subcommand == 'mf-static':
        run_mf_static()
    elif subcommand == 'complete_graph':
        complete_graph()
    elif subcommand == 'run-2hop':
        run_2hop()
    elif subcommand == 'run-mf':
        run_mf()
    elif subcommand == 'rel-comp':
        run_rel_comp()
    elif subcommand == '1hop-static':
        run_1hop_static()
    elif subcommand == 'run-mc':
        run_mc()
    elif subcommand == 'run-simple-model':
        run_simple_model()
    else:
        print('Error. Unknown subcommand', subcommand)
        print_help()
        sys.exit(1)


text:
import base64
import json
import os
import os.path as osp

import numpy as np
import PIL.Image
from labelme import utils

'''
我使用的labelme版本是3.16.7，建议使用该版本的labelme，有些版本的labelme会发生错误
此处生成的标签图是8位彩色图，每个像素点的值就是这个像素点所属的种类
'''
if __name__ == '__main__':
    jpgs_path   = "datasets/JPEGImages"
    pngs_path   = "datasets/SegmentationClass"
    classes     = ["_background_","aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"]
    # classes     = ["_background_","cat","dog"]

    count = os.listdir("./datasets/before/")
    for i in range(0, len(count)):
        path = os.path.join("./datasets/before", count[i])

        if os.path.isfile(path) and path.endswith('json'):
            data = json.load(open(path))

            if data['imageData']:
                imageData = data['imageData']
            else:
                imagePath = os.path.join(os.path.dirname(path), data['imagePath'])
                with open(imagePath, 'rb') as f:
                    imageData = f.read()
                    imageData = base64.b64encode(imageData).decode('utf-8')

            img = utils.img_b64_to_arr(imageData)
            label_name_to_value = {'_background_': 0}
            for shape in data['shapes']:
                label_name = shape['label']
                if label_name in label_name_to_value:
                    label_value = label_name_to_value[label_name]
                else:
                    label_value = len(label_name_to_value)
                    label_name_to_value[label_name] = label_value

            # label_values must be dense
            label_values, label_names = [], []
            for ln, lv in sorted(label_name_to_value.items(), key=lambda x: x[1]):
                label_values.append(lv)
                label_names.append(ln)
            assert label_values == list(range(len(label_values)))

            lbl = utils.shapes_to_label(img.shape, data['shapes'], label_name_to_value)

            PIL.Image.fromarray(img).save(osp.join(jpgs_path, count[i].split(".")[0]+'.jpg'))

            new = np.zeros([np.shape(img)[0],np.shape(img)[1]])
            for name in label_names:
                index_json = label_names.index(name)
                index_all = classes.index(name)
                new = new + index_all*(np.array(lbl) == index_json)

            utils.lblsave(osp.join(pngs_path, count[i].split(".")[0]+'.png'), new)
            print('Saved ' + count[i].split(".")[0] + '.jpg and ' + count[i].split(".")[0] + '.png')

max_stars_repo_name: juranccc/CVerrr

max_issues_repo_name: juranccc/CVerrr

max_forks_repo_name: juranccc/CVerrr


text:
import gensim
import _init_paths
import utils.zl_utils as zl
from numpy.random import randn
from sklearn.decomposition import sparse_encode
import numpy as np
from sklearn.decomposition import SparseCoder
import pycocotools.coco as coco
import random
import zl_config as C
n = m = 100 # dimensions of our input
input_x = randn(n, m)

def get_ilsvrc_word_list():
    ret = []
    lines = [line.strip() for line in open('ilsvrc_det_wordlist.txt')]
    for l in lines:
        ret.append(l.split(' ')[2])
    return ret

def save_ilsvrc_vectors_to_pickle():
    ilsvrc_words = get_ilsvrc_word_list()
    model = gensim.models.Word2Vec.load_word2vec_format('./GoogleNews-vectors-negative300.bin', binary=True)
    word2vec = {}
    for w in ilsvrc_words:
        w = w.replace('_',' ')
        if w in model:
            word2vec[w] = model[w]
        else:
            print '%s not in model'%w
    zl.save('ilsvrc_word2vec',word2vec)

def save_test_words_to_pickle():
    ilsvrc_words =  ['antelope','cup','ball','cooking']
    model = gensim.models.Word2Vec.load_word2vec_format('./GoogleNews-vectors-negative300.bin', binary=True)
    word2vec = {}
    for w in ilsvrc_words:
        w = w.replace('_',' ')
        word2vec[w] = model[w]
    zl.save('test_word2vec',word2vec)

def test_sparse_coder():
    db = coco.COCO('/media/zawlin/ssd/coco/annotations/captions_train2014.json')
    ilsvrc_word2vec = zl.load('ilsvrc_word2vec')
    test_word2vec = zl.load('test_word2vec')
    D = []
    idx2word = []
    for i in xrange(len(ilsvrc_word2vec.keys())):
        idx2word.append(ilsvrc_word2vec.keys()[i])
        D.append(ilsvrc_word2vec[ilsvrc_word2vec.keys()[i]])
    idx2word = np.array(idx2word)
    D = np.array(D)
    print 'loading word2vec'
    model = gensim.models.Word2Vec.load_word2vec_format('./GoogleNews-vectors-negative300.bin', binary=True)
    coder = SparseCoder(dictionary=D, transform_n_nonzero_coefs=5,
                            transform_alpha=None, transform_algorithm='lasso_cd')
    #random.shuffle(db.anns)
    for k in db.anns:
        cap = db.anns[k]['caption']
        splited = cap.split(' ')
        for s in splited:
            if s.lower() in model:
                y = model[s.lower()]
                x  = coder.transform(y.reshape(1,-1))[0]
                print '%s = %s'%(s.lower(),idx2word[np.argsort(x)[::-1]])
                print x[np.argsort(x)[::-1]]
        c = raw_input('press q to quit')
        if c== 'q':
            exit(0)

def test_coco():
    db = coco.COCO('/media/zawlin/ssd/coco/annotations/captions_train2014.json')
    for k in db.imgs:
        print db.imgs[k]
        exit(0)
    pass

def test_word2vec_wiki():
    ilsvrc_words = get_ilsvrc_word_list()
    model = gensim.models.Word2Vec.load(C.word2vec_wiki_path)
    print dir(model.vocab.keys())
    print len(model.vocab.keys())
    word2vec = {}
    for w in ilsvrc_words:
        w = w.replace('_',' ')
        if w in model:
            word2vec[w] = model[w]
        else:
            print '%s not in model'%w

#test_coco()
#test_sparse_coder()
#test_word2vec_wiki()
#save_test_words_to_pickle()
#save_ilsvrc_vectors_to_pickle()
# loaded = zl.load('ilsvrc_word2vec')
# print loaded.keys()