text:
import platform
import os, sys

if platform.system() == 'Linux':
    # To find available GPU on a multi-gpu machine cluster
    import utils.selectgpu as setgpu
    os.environ["CUDA_VISIBLE_DEVICES"] = str(setgpu.pick_gpu_lowest_memory())

import argparse
import importlib

import numpy as np
import math
import time

import tensorflow as tf
from datetime import datetime

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.join(BASE_DIR, 'loss'))
sys.path.append(os.path.join(BASE_DIR, 'utils'))
sys.path.append(os.path.join(BASE_DIR, 'experiments'))

import model
import fourier
import pcf
import plotutils as plot
import ioutils as io
from mathutils import *
from telemetryutils import *
import setup
import sampler
import ioutils
import projection
import evaluation

#============================================================================

def train(env, experiment):

    experimentID = datetime.utcnow().strftime("%Y%m%d%H%M%S")
    logPath = io.joinPath(experiment.LOGS_DIR, experimentID)

    tf.reset_default_graph()
    sess = tf.Session()

    # create input placeholder
    inputNode = tf.placeholder(
        tf.float32,
        shape=(
            env.trainingSetup.batchSize,
            env.trainingSetup.pointCount,
            env.trainingSetup.dimCount),
        name="inputNode")

    # create network
    outputNode = model.createNetwork(env.trainingSetup, inputNode)

    # create loss(es)
    lossNode, spectrumNode, histogramNode = experiment.lossSetup(env, outputNode)

    #-------------------------------------------------

    # create optimizer
    global_step = tf.Variable(0, trainable=False)
    learning_rate = tf.train.exponential_decay(env.trainingSetup.learningRate, global_step, 200, 0.99, staircase=True)
    optimizer = tf.train.AdamOptimizer(learning_rate)
    trainStep = optimizer.minimize(lossNode, global_step=global_step)

    # initialize variables
    sess.run(tf.global_variables_initializer())

    # create telemetry
    writer = tf.summary.FileWriter(logdir=logPath)
    writer.add_graph(sess.graph)
    writer.flush()

    #--------------------------
    def trainTimed(feedDict):
        t0 = time.time()
        _, loss = sess.run((trainStep, lossNode), feedDict)
        t = round(time.time() - t0, 3)
        return loss, t
    #--------------------------

    # training loop
    if env.trainingSetup.trainIterations > 0:
        print("==== Start training...")
        for i in range(env.trainingSetup.trainIterations):

            trainPoints = sampler.griddedRandom(
                env.trainingSetup.batchSize,
                env.trainingSetup.pointCount,
                env.trainingSetup.dimCount,
                env.trainingSetup.griddingDims)

            loss, t = trainTimed({inputNode:trainPoints})

            # monitor
            outputStats(writer, i, loss)
            if i % 10 == 0:
                writer.flush()

                sys.stdout.write('iter ' + str(i) + ' | loss ' + str(round(loss, 3)) + ' | time ' + str(t) + '\r')
                sys.stdout.flush()

            if i != 0 and env.trainingSetup.storeNetwork and i % env.trainingSetup.backupInterval == 0:
                evaluation.simpleCheckpoint(env, sess, logPath, i)

    print("")

    writer.flush()
    writer.close()

    #-------------------------------------------------

    # store trained network
    if env.trainingSetup.storeNetwork:
        evaluation.simpleCheckpoint(env, sess, logPath)

    #-------------------------------------------------

    # evaluation
    print("==== Evaluating...")

    testPoints = sampler.griddedRandom(
        env.trainingSetup.batchSize,
        env.trainingSetup.pointCount,
        env.trainingSetup.dimCount,
        env.trainingSetup.griddingDims)

    #-------------------------------------------------

    # output points visualization
    outPoints = sess.run(outputNode, {inputNode:testPoints})

    # scatter plots of points
    if env.trainingSetup.griddingDims == 0:
        grid = 1/math.sqrt(env.trainingSetup.pointCount) if env.trainingSetup.displayGrid else None
        plot.multiPointPlot(
            np.stack((testPoints[0], outPoints[0]), 0),
            ("input", "output"),
            grid=grid,
            filename = ioutils.joinPath(logPath, "points" + experiment.FILE_EXT))

    # dither masks (when using gridding)
    else:
        if env.trainingSetup.dimCount - env.trainingSetup.griddingDims <= 3:
            textures = plot.convertGriddedToArray(outPoints, env.trainingSetup.griddingDims)
            # 2D textures
            if env.trainingSetup.griddingDims == 2:
                for b in range(env.trainingSetup.batchSize):
                    filename = ioutils.joinPath(logPath, "dithermask_" + str(b) + ".exr")
                    ioutils.saveExr(textures[b], filename)
            # 3D textures (as 2D slices)
            elif env.trainingSetup.griddingDims == 3:
                for b in range(env.trainingSetup.batchSize):
                    for s in range(textures.shape[1]):
                        filename = ioutils.joinPath(logPath, "dithermask_b" + str(b) + "_s" + str(s) + ".exr")
                        ioutils.saveExr(textures[b, s, ...], filename)
            else:
                print("Could not save dither masks: gridding dimension > 3")
        else:
            print("Could not save dither masks: value dimensions > 3")

    #-------------------------------------------------

    # spectrum figures
    if spectrumNode is not None:

        #--------------------------
        def spectrumOutput(spectrumNode, spectrumTarget, path):
            expectedSpectrum = evaluation.produceExpectedOutput(
                env,
                sess,
                spectrumTarget.shape,
                inputNode,
                spectrumNode)
            if len(expectedSpectrum.shape) == 1:
                plot.multiLinePlot((spectrumTarget, expectedSpectrum),
                    title="1d spectra", legend=("target", "result"), filename=path)
            else:
                io.saveExr(expectedSpectrum, filename=path)
        #--------------------------

        spectrumNode = [spectrumNode] if not isinstance(spectrumNode, list) else spectrumNode
        for i, s in enumerate(spectrumNode):
            spectrumPath = io.joinPath(logPath, "spectra_" + str(i) + experiment.FILE_EXT)
            spectrumOutput(s, env.fourierSetupList[i].target, spectrumPath)

    #-------------------------------------------------

    # histogram figures
    if histogramNode is not None:

        #--------------------------
        def histogramOutput(histogramNode, histogramTarget, path):
            expectedHistogram = evaluation.produceExpectedOutput(
                env,
                sess,
                histogramTarget.shape,
                inputNode,
                histogramNode)
            plot.multiLinePlot((histogramTarget, expectedHistogram),
                title="histograms", legend=("target", "result"), filename=path)
        #--------------------------

        histogramNode = [histogramNode] if not isinstance(histogramNode, list) else histogramNode
        for i, h in enumerate(histogramNode):
            histogramPath = io.joinPath(logPath, "histogram" + str(i) + experiment.FILE_EXT)
            histogramOutput(h, env.histogramSetupList[i].target, histogramPath)

    #-------------------------------------------------

    # visualize trained variables
    if env.trainingSetup.storeNetwork:
        print("==== Extracting trained variables...")
        kernelWeights = evaluation.extractModelWeights(sess, env.trainingSetup)

        # plot kernels for each projection in different figure
        for i in range(env.trainingSetup.projectionCount):

            # line plots
            plot.multiLinePlot(
                kernelWeights[i:len(kernelWeights)+1:env.trainingSetup.projectionCount],
                title="kernelWeights" + env.trainingSetup.projectionsStrings[i],
                legend=None,
                filename=ioutils.joinPath(logPath, "kernelVars_" + str(i) + experiment.FILE_EXT))

            # surface plots
            if env.trainingSetup.kernelCount > 1:
                x = np.arange(env.trainingSetup.kernelSampleCount)
                y = np.arange(env.trainingSetup.kernelCount)
                x, y = np.meshgrid(x, y)
                z = np.stack(kernelWeights[i:len(kernelWeights)+1:env.trainingSetup.projectionCount])
                plot.surfacePlot(
                    [x, y, z],
                    title="kernelWeights" + env.trainingSetup.projectionsStrings[i],
                    filename=ioutils.joinPath(logPath, "kernelVars3D_" + str(i) + experiment.FILE_EXT))

    #-------------------------------------------------

    # save realizations
    if env.trainingSetup.saveEvalRealizations:

        realizationPath = ioutils.joinPath(logPath, "realizations/")
        io.makeDir(realizationPath)
        evaluation.saveRealizations(
            env,
            sess,
            inputNode,
            outputNode,
            env.trainingSetup.evalRealizations,
            realizationPath)

    sess.close()

##============================================================================

def main():

    # import experiment
    parser = argparse.ArgumentParser()
    parser.add_argument('-e','--experiment', help='experiment to run from /experiments', required=True)
    args = parser.parse_args()
    print("==== Import", args.experiment, "...")
    experiment = importlib.import_module(args.experiment)

    # setup
    env = experiment.buildEnvironment()

    # train
    train(env, experiment)

#============================================================================

if __name__ == '__main__':
    main()
    print("========= TERMINATED =========")

max_stars_repo_name: sinbag/deepsampling

max_issues_repo_name: sinbag/deepsampling

max_forks_repo_name: sinbag/deepsampling


text:
"""
Train Async-VFL on ModelNet-10 dataset
"""

import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader
from torch.autograd import Variable

import torchvision.transforms as transforms

import argparse
import numpy as np
import time
import os
import copy
import random
import pickle
import math
import subprocess as sp
from tqdm import tqdm

from scipy.optimize import minimize
from scipy.optimize import Bounds
from scipy.optimize import NonlinearConstraint
from scipy.optimize import BFGS

from models.resnet import *
from models.mvcnn import *
from models.mvcnn_top import *
from models.mvcnn_bottom import *
import util
#from logger import Logger
from custom_dataset import MultiViewDataSet
import sys

MVCNN = 'mvcnn'
RESNET = 'resnet'
MODELS = [RESNET,MVCNN]

def get_gpu_memory():
    """
    Utility function for checking GPU memory
    """
    _output_to_list = lambda x: x.decode('ascii').split('\n')[:-1]

    ACCEPTABLE_AVAILABLE_MEMORY = 1024
    COMMAND = "nvidia-smi --query-gpu=memory.free --format=csv"
    memory_free_info = _output_to_list(sp.check_output(COMMAND.split()))[1:]
    memory_free_values = [int(x.split()[0]) for i, x in enumerate(memory_free_info)]
    print(memory_free_values)
    return memory_free_values

# Set up input arguments
num_clients = int(sys.argv[3])

parser = argparse.ArgumentParser(description='MVCNN-PyTorch')
parser.add_argument('data', metavar='DIR', help='path to dataset')
parser.add_argument('--num_clients', type=int, help='Number of clients to split data between vertically',
                        default=2)
parser.add_argument('--depth', choices=[18, 34, 50, 101, 152], type=int, metavar='N', default=18, help='resnet depth (default: resnet18)')
parser.add_argument('--model', '-m', metavar='MODEL', default=RESNET, choices=MODELS,
                    help='pretrained model: ' + ' | '.join(MODELS) + ' (default: {})'.format(RESNET))
parser.add_argument('--epochs', default=100, type=int, metavar='N', help='number of total epochs to run (default: 100)')
parser.add_argument('-b', '--batch-size', default=4, type=int,
                    metavar='N', help='mini-batch size (default: 4)')
parser.add_argument('--lr', '--learning-rate', default=0.0001, type=float,
                    metavar='LR', help='initial learning rate (default: 0.0001)')
parser.add_argument('--momentum', default=0.9, type=float, metavar='M',
                    help='momentum (default: 0.9)')
parser.add_argument('--lr-decay-freq', default=30, type=float,
                    metavar='W', help='learning rate decay (default: 30)')
parser.add_argument('--lr-decay', default=0.1, type=float,
                    metavar='W', help='learning rate decay (default: 0.1)')
parser.add_argument('--print-freq', '-p', default=10, type=int,
                    metavar='N', help='print frequency (default: 10)')
parser.add_argument('-r', '--resume', default='', type=str, metavar='PATH',
                    help='path to latest checkpoint (default: none)')
parser.add_argument('--pretrained', dest='pretrained', action='store_true', help='use pre-trained model')
parser.add_argument('--prob', type=float, nargs=num_clients+1, default=1,
                        help='Indicates probability distribution to use for workers.')
parser.add_argument('--lr_type', type=str, nargs='?', default="hetero",
                        help='Indicates if learning rates should be heterogeneous or homogeneous.')
parser.add_argument('--priori', type=str, nargs='?', default="a",
                        help='Adjust learning rate adaptively or not')
parser.add_argument('--server_time', type=int, nargs='?', default=1,
                        help='Number of local iteration time it takes for server to communicate.')

# Parse input arguments
args = parser.parse_args()

print('Loading data')

transform = transforms.Compose([
    transforms.CenterCrop(500),
    transforms.Resize(224),
    transforms.ToTensor(),
])

# Load dataset
dset_train = MultiViewDataSet(args.data, 'train', transform=transform)
indices = torch.randperm(len(dset_train))
dset_train_sub = torch.utils.data.Subset(dset_train, indices[:int(len(dset_train)/4)])
train_loader = DataLoader(dset_train_sub, batch_size=args.batch_size, shuffle=True, num_workers=1)

dset_val = MultiViewDataSet(args.data, 'test', transform=transform)
test_loader = DataLoader(dset_val, batch_size=args.batch_size, shuffle=True, num_workers=1)

classes = dset_train.classes
print(len(classes), classes)

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
models = []
optimizers = []
# Make models for each client
for i in range(num_clients+1):
    if i == num_clients:
        model = mvcnn_top(pretrained=args.pretrained,num_classes=len(classes))
    else:
        model = mvcnn_bottom(pretrained=args.pretrained,num_classes=len(classes))

    cudnn.benchmark = True

    optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
    models.append(model)
    optimizers.append(optimizer)

# Client views of the server model
server_models = []
for i in range(num_clients):
    server_models.append(copy.deepcopy(models[-1]))

# Loss and Optimizer
n_epochs = args.epochs
criterion = nn.CrossEntropyLoss()
coords_per = int(12/num_clients)

best_acc = 0.0
best_loss = 0.0
start_epoch = 0

# Validation and Testing
def eval(models, data_loader):
    """
    Calculate loss and accuracy for a given data_loader
    """
    total = 0.0
    correct = 0.0

    total_loss = 0.0
    n = 0

    for i, (inputs, targets) in enumerate(data_loader):
        with torch.no_grad():
            # Convert from list of 3D to 4D
            inputs = np.stack(inputs, axis=1)

            inputs = torch.from_numpy(inputs)

            #inputs, targets = inputs.cuda(device), targets.cuda(device)
            inputs, targets = Variable(inputs), Variable(targets)

            # Get current embeddings
            H_new = [None] * num_clients
            for i in range(num_clients):
                x_local = inputs[:,coords_per*i:coords_per*(i+1),:,:,:]
                H_new[i] = models[i](x_local)
            # compute output
            outputs = models[-1](torch.cat(H_new,axis=1))
            loss = criterion(outputs, targets)

            total_loss += loss
            n += 1

            _, predicted = torch.max(outputs.data, 1)
            total += targets.size(0)
            correct += (predicted.cpu() == targets.cpu()).sum()

    avg_test_acc = 100 * correct / total
    avg_loss = total_loss / n

    return avg_test_acc, avg_loss

losses = []
accs_train = []
accs_test = []
orig_prob = copy.deepcopy(args.prob)
prob = args.prob

# Get embeddings for all batches from each client
server_H = np.empty((len(train_loader), num_clients), dtype=object)
client_H = []
for i in range(num_clients):
    client_H.append(np.empty((len(train_loader), num_clients), dtype=object))
for step, (inputs, targets) in tqdm(enumerate(train_loader)):
    # Convert from list of 3D to 4D
    inputs = np.stack(inputs, axis=1)
    inputs = torch.from_numpy(inputs)
    #inputs, targets = inputs.cuda(device), targets.cuda(device)
    inputs, targets = Variable(inputs), Variable(targets)
    # Train for each client
    for i in range(num_clients):
        x_local = inputs[:,coords_per*i:coords_per*(i+1),:,:,:]
        server_H[step,i] = models[i](x_local)
    for k in range(num_clients):
        client_H[k][step] = server_H[step].copy()

train_size = len(train_loader)
server_model = models[-1]
server_optimizer = optimizers[-1]
#server_model.to(device)
#for k in range(num_clients):
#    models[k].to(device)
client_busy = [0] * num_clients
server_busy = 0
queue = {}

# Get initial loss/accuracy
avg_train_acc, avg_loss = eval(models, train_loader)
#avg_test_acc, _ = eval(models, test_loader)

losses.append(avg_loss)
accs_train.append(avg_train_acc)
#accs_test.append(avg_test_acc)

pickle.dump(losses, open(f'loss_mvcnnAsync_NC{args.num_clients}_lr{args.lr_type}_prob{orig_prob}_priori{args.priori}_server{args.server_time}.pkl', 'wb'))
pickle.dump(accs_train, open(f'accs_train_mvcnnAsync_NC{args.num_clients}_lr{args.lr_type}_prob{orig_prob}_priori{args.priori}_server{args.server_time}.pkl', 'wb'))
#pickle.dump(accs_test, open(f'accs_test_mvcnnAsync_NC{args.num_clients}_lr{args.lr_type}_prob{orig_prob}_priori{args.priori}_server{args.server_time}.pkl', 'wb'))

#print('Train Acc: %.2f - Test Acc: %.2f - Loss: %.4f' % (avg_train_acc.item(), avg_test_acc.item(), avg_loss.item()))
print('Train Acc: %.2f - Loss: %.4f'
        % (avg_train_acc.item(), avg_loss.item()))
# Training / Eval loop
for epoch in range(start_epoch, n_epochs):
    # Set offset for saving loss/accuracy every 6 batches
    print('\n-----------------------------------')
    print('Epoch: [%d/%d]' % (epoch+1, n_epochs))
    start = time.time()

    for step, (inputs, targets) in enumerate(train_loader):
        # Convert from list of 3D to 4D
        inputs = np.stack(inputs, axis=1)

        inputs = torch.from_numpy(inputs)

        #inputs, targets = inputs.cuda(device), targets.cuda(device)
        inputs, targets = Variable(inputs), Variable(targets)

        # Train clients
        for i in range(num_clients):
            print(client_busy)
            # Is client waiting on the server?
            if i in queue.keys():
                continue
            client_busy[i] -= 1
            if client_busy[i] > 0:
                continue

            # Train for each client with probability
            if random.random() < 1-prob[i]:
                continue

            x_local = inputs[:,coords_per*i:coords_per*(i+1),:,:,:]
            H = client_H[i][step].copy()
            model = models[i]
            optimizer = optimizers[i]

            # Train
            #model.to(device)
            #server_model.to(device)
            outputs = model(x_local)
            H[i] = outputs.cpu()
            Hlist = torch.cat(H.tolist(),axis=1)
            outputs = server_model(Hlist)
            loss = criterion(outputs, targets)

            # compute gradient and do SGD step
            optimizer.zero_grad()
            server_optimizer.zero_grad()
            loss.backward(retain_graph=True)
            optimizer.step()
            #model.to(torch.device("cpu"))
            #server_model.to(torch.device("cpu"))

            # Check if client is ready to comm with server
            H_new = model(x_local).cpu()
            queue[i] = [step, H_new, copy.deepcopy(targets)]

        # If server is not busy and there are clients in the queue
        if len(queue) > 0:
            # Train server on latest embeddings
            for k in queue:
                k_step = queue[k][0]
                server_H[k_step,k] = queue[k][1]
                y_local = queue[k][2]
                client_busy[k] = args.server_time+len(queue)

                # Train server
                #server_model.to(device)
                # compute output
                Hlist = torch.cat(server_H[k_step].tolist(),axis=1)
                outputs = server_model(Hlist)
                loss = criterion(outputs, y_local)

                # compute gradient and do SGD step
                server_optimizer.zero_grad()
                loss.backward(retain_graph=True)
                server_optimizer.step()
                #server_model.to(torch.device("cpu"))

            # Update client embeddings and server model
            for k in queue:
                client_H[k] = server_H.copy()
                server_models[k] = copy.deepcopy(models[-1])
            queue.clear()

        # Save current loss/accuracy every 15 batches
        if (epoch*63 + step + 1) % 24 == 0:
            avg_train_acc, avg_loss = eval(models, train_loader)
            #avg_test_acc, _ = eval(models, test_loader)

            losses.append(avg_loss)
            accs_train.append(avg_train_acc)
            #accs_test.append(avg_test_acc)

            pickle.dump(losses, open(f'loss_mvcnnAsync_NC{args.num_clients}_lr{args.lr_type}_prob{orig_prob}_priori{args.priori}_server{args.server_time}.pkl', 'wb'))
            pickle.dump(accs_train, open(f'accs_train_mvcnnAsync_NC{args.num_clients}_lr{args.lr_type}_prob{orig_prob}_priori{args.priori}_server{args.server_time}.pkl', 'wb'))
            #pickle.dump(accs_test, open(f'accs_test_mvcnnAsync_NC{args.num_clients}_lr{args.lr_type}_prob{orig_prob}_priori{args.priori}_server{args.server_time}.pkl', 'wb'))

            #print('Iter [%d/%d]: Train Acc: %.2f - Test Acc: %.2f - Loss: %.4f'
            #        % (step + 1, train_size, avg_train_acc.item(), avg_test_acc.item(), avg_loss.item()))
            print('Iter [%d/%d]: Train Acc: %.2f - Loss: %.4f'
                    % (step + 1, train_size, avg_train_acc.item(), avg_loss.item()))
            import pdb; pdb.set_trace()

    print('Time taken: %.2f sec.' % (time.time() - start))

ext: py

reason: import numpy,from scipy


text:
# -*- coding: utf-8 -*-
# flake8: noqa
import logging
from six.moves import cPickle as pickle
from wbia_cnn.models import abstract_models
from wbia_cnn.models.abstract_models import *

(print, rrr, profile) = ut.inject2(__name__)
logger = logging.getLogger()

(print, rrr, profile) = ut.inject2(__name__)
logger = logging.getLogger()

def check_external_training_paths():
    """
    Notes:
        http://cs231n.github.io/neural-networks-3/#distr
    """
    checkpoints_dir = '.'
    checkpoints_dir = '/home/joncrall/.config/wbia_cnn/training_junction/train_patchmetric((315)iofvvdflcllgjkyu)/checkpoints'
    # checkpoints_dir = '/media/raid/work/PZ_MTEST/_ibsdb/_ibeis_cache/nets/train_patchmetric((315)iofvvdflcllgjkyu)/checkpoints/hist_eras2_epochs23_aayzxkezpzjgwupd'
    checkpoints_dir = '/home/joncrall/.config/wbia_cnn/training_junction/train_patchmetric((1576)fxzkszaajypyzqne)/checkpoints'

    checkpoints_dir = (
        '/home/joncrall/.config/wbia_cnn/training_junction/liberty/checkpoints'
    )
    checkpoints_dir = '/home/joncrall/.config/wbia_cnn/training_junction/NNP_Master3_patchmatch-_24285_xatwrytpdbfttoax-/checkpoints'

    from six.moves import cPickle as pickle
    from wbia_cnn.models import abstract_models

    model_fpaths = ut.glob(checkpoints_dir, '*.pkl', recursive=True)
    tmp_model_list = []
    for fpath in model_fpaths:
        tmp_model = abstract_models.BaseModel()
        tmp_model.load_model_state(fpath=fpath)
        tmp_model_list.append(tmp_model)
        # hashid = tmp_model.get_history_hashid()
        # dpath = dirname(fpath)
        # new_dpath = join(dirname(dpath), hashid)
        # ut.move(dpath, new_dpath)

    for model in tmp_model_list:
        model.rrr(verbose=False)

    vallist_ = [tmp_model.get_total_epochs() for tmp_model in tmp_model_list]
    tmp_model_list = ut.sortedby(tmp_model_list, vallist_)

    for tmp_model in tmp_model_list:
        # tmp_model.checkpoint_save_model_info()
        logger.info(tmp_model.best_results['train_loss'])
        logger.info(tmp_model.best_results['valid_loss'])
        logger.info('----')

    for tmp_model in ut.InteractiveIter(tmp_model_list):
        logger.info(fpath)
        logger.info(sum([len(era['epoch_list']) for era in tmp_model.era_history]))
        tmp_model.show_era_loss(fnum=1)

def load_tmp_model():
    from wbia_cnn.models import abstract_models

    fpath = '/home/joncrall/.config/wbia_cnn/training_junction/liberty/model_state.pkl'
    fpath = (
        '/home/joncrall/.config/wbia_cnn/training_junction/liberty/model_state_dozer.pkl'
    )
    fpath = '/home/joncrall/.config/wbia_cnn/training_junction/train_patchmetric((15152)nunivgoaibmjsdbs)/model_state.pkl'

    tmp_model = abstract_models.BaseModel()
    tmp_model.load_model_state(fpath=fpath)
    tmp_model.rrr()
    tmp_model.show_era_loss(fnum=1)
    pt.iup()

def grab_model_from_dozer():
    remote = 'dozer'
    user = 'joncrall'
    # remote_path = '/home/joncrall/.config/wbia_cnn/training/liberty/model_state.pkl'
    # local_path = '/home/joncrall/.config/wbia_cnn/training/liberty/model_state_dozer.pkl'

    remote_path = '/home/joncrall/.config/wbia_cnn/training/liberty/checkpoints/hist_eras3_epochs30_zqwhqylxyihnknxc/model_state_arch_tiloohclkatusmmp'
    local_path = '/home/joncrall/.config/wbia_cnn/training/liberty/checkpoints/hist_eras3_epochs30_zqwhqylxyihnknxc/model_state_arch_tiloohclkatusmmp.pkl'

    remote_path = '/home/joncrall/.config/wbia_cnn/training/liberty/checkpoints'
    local_path = '/home/joncrall/.config/wbia_cnn/training/liberty'

    from os.path import dirname

    ut.ensuredir(dirname(local_path))

    ut.scp_pull(remote_path, local_path, remote, user)

    ut.vd(dirname(local_path))

def convert_old_weights():
    ## DO CONVERSION
    # if False:
    #    old_weights_fpath = ut.truepath('~/Dropbox/wbia_cnn_weights_liberty.pickle')
    #    if ut.checkpath(old_weights_fpath, verbose=True):
    #        self = model
    #        self.load_old_weights_kw(old_weights_fpath)
    #    self.save_model_state()
    #    #self.save_state()
    pass

def theano_gradient_funtimes():
    import theano
    import numpy as np
    import theano.tensor as T
    import lasagne
    import wbia_cnn.theano_ext as theano_ext

    TEST = True

    x_data = np.linspace(-10, 10, 100).astype(np.float32)[:, None, None, None]
    y_data = (x_data ** 2).flatten()[:, None]

    X = T.tensor4('x')
    y = T.matrix('y')

    # x_data_batch =
    # y_data_batch =
    inputs_to_value = {X: x_data[0:16], y: y_data[0:16]}

    l_in = lasagne.layers.InputLayer((16, 1, 1, 1))
    l_out = lasagne.layers.DenseLayer(
        l_in,
        num_units=1,
        nonlinearity=lasagne.nonlinearities.rectify,
        W=lasagne.init.Orthogonal(),
    )

    network_output = lasagne.layers.get_output(l_out, X)

    # TEST NETWORK OUTPUT

    if TEST:
        result = theano_ext.eval_symbol(network_output, inputs_to_value)
        logger.info('network_output = %r' % (result,))

    loss_function = lasagne.objectives.squared_error
    # def loss_function(network_output, labels):
    #    return (network_output - labels) ** 2

    losses = loss_function(network_output, y)
    if TEST:
        result = theano_ext.eval_symbol(losses, inputs_to_value)
        logger.info('losses = %r' % (result,))

    loss = lasagne.objectives.aggregate(losses, mode='mean')

    if TEST:
        result = theano_ext.eval_symbol(loss, inputs_to_value)
        logger.info('loss = %r' % (result,))

    L2 = lasagne.regularization.regularize_network_params(
        l_out, lasagne.regularization.l2
    )
    weight_decay = 0.0001
    loss_regularized = loss + weight_decay * L2
    loss_regularized.name = 'loss_regularized'

    parameters = lasagne.layers.get_all_params(l_out)

    gradients_regularized = theano.grad(loss_regularized, parameters, add_names=True)

    if TEST:
        if False:
            s = T.sum(1 / (1 + T.exp(-X)))
            s.name = 's'
            gs = T.grad(s, X, add_names=True)
            theano.pp(gs)
            inputs_to_value = {X: x_data[0:16], y: y_data[0:16]}
            result = theano_ext.eval_symbol(gs, inputs_to_value)
            logger.info(
                '%s = %r'
                % (
                    gs.name,
                    result,
                )
            )
            inputs_to_value = {X: x_data[16:32], y: y_data[16:32]}
            result = theano_ext.eval_symbol(gs, inputs_to_value)
            logger.info(
                '%s = %r'
                % (
                    gs.name,
                    result,
                )
            )

        for grad in gradients_regularized:
            result = theano_ext.eval_symbol(grad, inputs_to_value)
            logger.info(
                '%s = %r'
                % (
                    grad.name,
                    result,
                )
            )

        grad_on_losses = theano.grad(losses, parameters, add_names=True)

    learning_rate_theano = 0.0001
    momentum = 0.9
    updates = lasagne.updates.nesterov_momentum(
        gradients_regularized, parameters, learning_rate_theano, momentum
    )

    X_batch = T.tensor4('x_batch')
    y_batch = T.fvector('y_batch')

    func = theano.function(
        inputs=[theano.Param(X_batch), theano.Param(y_batch)],
        outputs=[network_output, losses],
        # updates=updates,
        givens={
            X: X_batch,
            y: y_batch,
        },
    )

    y_predict_batch, loss_batch = func(inputs_to_value[X], inputs_to_value[y])

    if ut.inIPython():
        import IPython

        IPython.get_ipython().magic('pylab qt4')

    from wbia import plottool as pt

    pt.plot(x_data, y_predict)
    pt.iup()
    pass

reason: import numpy,import theano


text:
import grpc
import tensorflow as tf

#import run_multilabels_classifier as classifiers

from run_multilabels_classifier import MultiLabelTextProcessor
from run_multilabels_classifier import convert_single_example
from run_multilabels_classifier import create_int_feature

import tokenization
import collections

from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2_grpc
from tensorflow.core.framework import tensor_pb2
from tensorflow.core.framework import tensor_shape_pb2
from tensorflow.core.framework import types_pb2

from flask import Flask
from flask import request

import logging

import random

import numpy as np

app = Flask(__name__)

@app.route("/", methods = ['GET'])
def hello():
  return "Hello BERT predicting Toxic Comments! Try posting a string to this url"

@app.route("/", methods = ['POST'])
def predict():
  # MODEL PARAMS
  max_seq_length = 128

  channel = grpc.insecure_channel("bert-toxic:8500")
  stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)

  # Parse Description
  tokenizer = tokenization.FullTokenizer(
    vocab_file="asset/vocab.txt", do_lower_case=True)
  processor = MultiLabelTextProcessor()
  label_list = [0,0,0,0,0,0]

  # logging.basicConfig(level=logging.DEBUG,format='%(asctime)s - %(levelname)s - %(name)s - %(message)s')
  # logger = logging.getLogger(__name__)
  # logger.info('getting json')
  content = request.get_json()
  #logger.info('JSON: {}'.format(content))
  request_id = str(random.randint(1, 9223372036854775807))

  inputExample = processor.serving_create_example([request_id, content['description']], 'test')
  feature = convert_single_example(0, inputExample, label_list, max_seq_length, tokenizer)

  features = collections.OrderedDict()
  features["input_ids"] = create_int_feature(feature.input_ids)
  features["input_mask"] = create_int_feature(feature.input_mask)
  features["segment_ids"] = create_int_feature(feature.segment_ids)
  features["is_real_example"] = create_int_feature([int(feature.is_real_example)])
  if isinstance(feature.label_id, list):
    label_ids = feature.label_id
  else:
    label_ids = [feature.label_id]
  features["label_ids"] = create_int_feature(label_ids)

  tf_example = tf.train.Example(features=tf.train.Features(feature=features))

  model_input = tf_example.SerializeToString()

  # Send request
  # See prediction_service.proto for gRPC request/response details.
  model_request = predict_pb2.PredictRequest()
  model_request.model_spec.name = 'bert'
  model_request.model_spec.signature_name = 'serving_default'
  dims = [tensor_shape_pb2.TensorShapeProto.Dim(size=1)]
  tensor_shape_proto = tensor_shape_pb2.TensorShapeProto(dim=dims)
  tensor_proto = tensor_pb2.TensorProto(
    dtype=types_pb2.DT_STRING,
    tensor_shape=tensor_shape_proto,
    string_val=[model_input])

  model_request.inputs['examples'].CopyFrom(tensor_proto)
  result = stub.Predict(model_request, 10.0)  # 10 secs timeout
  predict_response_dict = predict_response_to_dict(result)
  keys = [k for k in predict_response_dict]

  label_dict={'toxic':predict_response_dict['probabilities'][0][0],
            'severe_toxic':predict_response_dict['probabilities'][0][1] ,
            'obscene':predict_response_dict['probabilities'][0][2],
            'threat':predict_response_dict['probabilities'][0][3],
            'insult':predict_response_dict['probabilities'][0][4],
            'identity_hate':predict_response_dict['probabilities'][0][5]}

  pretty_result = "Prediction{}: "
  pretty_result = pretty_result.format(label_dict)
  #app.logger.info("Predicted Label: %s", label_list[result[0]])
  return pretty_result

dtype_to_number = {
    'DT_INVALID': 0,
    'DT_FLOAT': 1,
    'DT_DOUBLE': 2,
    'DT_INT32': 3,
    'DT_UINT8': 4,
    'DT_INT16': 5,
    'DT_INT8': 6,
    'DT_STRING': 7,
    'DT_COMPLEX64': 8,
    'DT_INT64': 9,
    'DT_BOOL': 10,
    'DT_QINT8': 11,
    'DT_QUINT8': 12,
    'DT_QINT32': 13,
    'DT_BFLOAT16': 14,
    'DT_QINT16': 15,
    'DT_QUINT16': 16,
    'DT_UINT16': 17,
    'DT_COMPLEX128': 18,
    'DT_HALF': 19,
    'DT_RESOURCE': 20
}

number_to_dtype_value = {
    1: 'float_val',
    2: 'double_val',
    3: 'int_val',
    4: 'int_val',
    5: 'int_val',
    6: 'int_val',
    7: 'string_val',
    8: 'scomplex_val',
    9: 'int64_val',
    10: 'bool_val',
    18: 'dcomplex_val',
    19: 'half_val',
    20: 'resource_handle_val'
}

def predict_response_to_dict(predict_response):
    predict_response_dict = dict()

    for k in predict_response.outputs:
        shape = [x.size for x in predict_response.outputs[k].tensor_shape.dim]

        #logger.debug('Key: ' + k + ', shape: ' + str(shape))

        dtype_constant = predict_response.outputs[k].dtype

        if dtype_constant not in number_to_dtype_value:
            #logger.error('Tensor output data type not supported. Returning empty dict.')
            predict_response_dict[k] = 'value not found'

        if shape == [1]:
            predict_response_dict[k] = eval('predict_response.outputs[k].' + number_to_dtype_value[dtype_constant])[0]
        else:
            predict_response_dict[k] = np.array(
                eval('predict_response.outputs[k].' + number_to_dtype_value[dtype_constant])).reshape(shape)

    return predict_response_dict

if __name__ == '__main__':
  app.run(debug=True, host='0.0.0.0')

reason: import numpy


text:
import numpy as np
import pdb
import sys
from gensim.models.keyedvectors import KeyedVectors
from utils import Progbar
import evaluate as E
import cPickle as cp

def find_closest_word(vector, word2vec):
    closest_val = None
    closest_word = None
    for w in word2vec:
        if closest_val is None or closest_val < E.cosine(word2vec[w], vector):
            closest_val = E.cosine(word2vec[w], vector)
            closest_word = w
    return closest_word

def compute_accuracy(method_metric, analogy_set):
    """
    Computes the word analogy accuracies for each method in the method_metric
        :param method_metric: A dictionary of form name : word2vec
        :param analogy_set: List of 4 tuples: word1, word2, word3, word4
        :return accuracies: A dictionary of form name: accuracy
    """
    accuracies = {name: 0. for name in method_metric}
    total = len(analogy_set)
    bar = Progbar(total)
    ix = 0
    for w1, w2, w3, w4 in analogy_set:
        ix += 1
        for method in accuracies:
            if any([w not in method_metric[method] for w in [w1, w2, w3, w4]]):
                continue
            w_vec = method_metric[method][w1] - method_metric[method][w2] + method_metric[method][w3]
            w = E.get_nearest_k_with_matrix(w_vec, method_metric[method], 5, False)
            if w4 in w:
                accuracies[method] += 1.
        bar.update(ix)
    accuracies = {name: accuracies[name] / total * 100 for name in accuracies}
    return accuracies

def load_word_vector(filename, wordset):
    word2vec = {}
    with open(filename) as f:
        for line in f:
            line = line.strip().split(' ')
            if line[0] in wordset:
                word2vec[line[0]] = np.array([float(x) for x in line[1:]])
    return word2vec

def load_w2v(fname, wordset):
    word_vectors = KeyedVectors.load_word2vec_format(fname, binary=True)
    word2vec = {}
    for w in wordset:
        if w in word_vectors.vocab:
            word2vec[w] = word_vectors[w]
    return word2vec

def load_sw2v(fname, wordset, bpe, subword2ix):
    word2vec = {}
    W_embed = np.load(fname)
    for word in wordset:
        w_seg = bpe.segment(word)
        w_embed = None
        for seg in w_seg:
            if seg in subword2ix:
                if w_embed is None:
                    w_embed = 0.
                w_embed += W_embed[subword2ix[seg]]
        if w_embed is not None:
            word2vec[word] = w_embed
    return word2vec

if __name__ == "__main__":
    analogy_file = "../Data/WordAnalogies/word_analogies.txt"
    analogies = []
    wordset = set()
    with open(analogy_file) as f:
        for line in f:
            if line.startswith(":"):
                continue
            line = [x.lower() for x in line.strip().split(' ')]
            for w in line:
                wordset.add(w)
            analogies.append(line)

    # =========== Loading Google Word2vec ====================================#
    word2vec = load_w2v("../Models/Word2Vec/vectors_skipgram_300.bin", wordset)
    # =========== Loading auxillary info for subword-model ===================#
    sys.path.append('../')
    from Code.constants import *
    SUB_WORD_FILE = DATA_DIR + "BPE/vocab_subwords.txt"
    SUB_WORD_SEPERATOR = "@@"
    CODECS_FILE = DATA_DIR + "BPE/bpe_codecs.txt"
    from Code.bpe import BPE
    bpe = BPE(open(CODECS_FILE), separator=SUB_WORD_SEPERATOR)
    subword2ix = cp.load(open(SUBWORD_VOCAB_FILE))
    # =========================================================================#
    methods = {"Word2vec": word2vec,
               "Word2vec Pytorch": load_word_vector("../Models/Embeddings_file.txt", wordset),
               "Word2vec with syn cons": load_word_vector("../Models/embeddings_with_syn_info_epoch_5_no_lr_decay.txt", wordset),
               "Fast Text": load_word_vector("../Models/fastext.vec", wordset),
               "Counter Fitted Vectors": load_word_vector("../Baselines/Counter-fitted-vectors/counter-fitted-vectors.txt", wordset),
               "Glove Embedding Vectors": load_word_vector("../Models/glove.txt", wordset),
               "Subword Embedding Vectors": load_sw2v("../Models/subword_vocab_matrix_with_syn_ant.npy", wordset, bpe, subword2ix)
               }
    accuracies = compute_accuracy(methods, analogies)
    print '\nPrinting the Analogy scores (@5)'
    for method in accuracies:
        print '\t%26s:\t%.4f' % (method, accuracies[method])

max_stars_repo_name: codedecde/WordEmbeddings

max_issues_repo_name: codedecde/WordEmbeddings

max_forks_repo_name: codedecde/WordEmbeddings


text:
import numpy as np
from nose.tools import assert_equals, assert_almost_equals
from ..metrics import root_mean_square_error, mean_absolute_error,\
                        normalized_mean_absolute_error, precision_recall_fscore, \
                        precision_score, recall_score, f1_score, evaluation_report,\
                        evaluation_error
from numpy.testing import assert_array_almost_equal

def test_root_mean_square_error():
    """Check that the metric Root Mean Squared Error (RMSE) """
    y_real = np.array([0.0, 1.0, 0.0, 2.0, 3.0])
    y_pred = np.array([0.0, 1.0, 0.0, 2.0, 3.0])
    assert_equals(0.0, root_mean_square_error(y_real, y_pred))

    y_real = np.array([3.0, 1.0, 2.0, 1.0, 1.0])
    y_pred = np.array([0.0, 1.0, 0.0, 2.0, 3.0])
    assert_almost_equals(1.8973665961, root_mean_square_error(y_real, y_pred))

def test_root_mean_absolute_error():
    """Check that the metric Mean Absolute Error (MAE) """
    y_real = np.array([0.0, 1.0, 0.0, 2.0, 3.0])
    y_pred = np.array([0.0, 1.0, 0.0, 2.0, 3.0])
    assert_equals(0.0, mean_absolute_error(y_real, y_pred))

    y_real = np.array([3.0, 1.0, 2.0, 1.0, 1.0])
    y_pred = np.array([0.0, 1.0, 0.0, 2.0, 3.0])
    assert_almost_equals(1.6, mean_absolute_error(y_real, y_pred))

def test_root_normalized_mean_absolute_error():
    """Check that the metric Normalized Mean Absolute Error (NMAE) """
    max_rating = 5.0
    min_rating = 1.0
    y_real = np.array([0.0, 1.0, 0.0, 2.0, 3.0])
    y_pred = np.array([0.0, 1.0, 0.0, 2.0, 3.0])
    assert_equals(0.0, normalized_mean_absolute_error(y_real, y_pred, max_rating, min_rating))

    y_real = np.array([3.0, 1.0, 2.0, 1.0, 1.0])
    y_pred = np.array([0.0, 1.0, 0.0, 2.0, 3.0])
    assert_almost_equals(0.4, normalized_mean_absolute_error(y_real, y_pred,
                max_rating, min_rating))

def test_evaluation_error():
    """ Check the error evaluation """
    max_rating = 5.0
    min_rating = 1.0
    y_real = np.array([0.0, 1.0, 0.0, 2.0, 3.0])
    y_pred = np.array([0.0, 1.0, 0.0, 2.0, 3.0])

    mae, nmae, rmse = evaluation_error(y_real, y_pred, max_rating, min_rating)
    assert_equals(mae, 0.0)
    assert_equals(nmae, 0.0)
    assert_equals(rmse, 0.0)

def test_precision_recall_f1_score():
    """Test Precision Recall and F1 Score """
    y_real = np.array([['a', 'b', 'c'], ['a', 'b', 'e', 'f', 'g'], ['a', 'b']])
    y_pred = np.array([['a', 'b', 'c'], ['a', 'b', 'c', 'd'], ['e', 'f']])

    p, r, f = precision_recall_fscore(y_real, y_pred)
    assert_array_almost_equal(p, [1, 0.4, 0], 2)
    assert_array_almost_equal(r, [1., 0.5, 0], 2)
    assert_array_almost_equal(f, [1., 0.44, 0], 2)

    ps = precision_score(y_real, y_pred)
    assert_array_almost_equal(ps, 0.4666, 2)

    rs = recall_score(y_real, y_pred)
    assert_array_almost_equal(rs, 0.5, 2)

    fs = f1_score(y_real, y_pred)
    assert_array_almost_equal(fs, 0.48, 2)

def test_zero_precision_recall():
    """Check that pathological cases do not bring NaNs"""

    try:
        old_error_settings = np.seterr(all='raise')

        y_real = np.array([['a', 'b', 'c']])
        y_pred = np.array([[]])

        assert_array_almost_equal(precision_score(y_real, y_pred), 0.0, 2)
        assert_array_almost_equal(recall_score(y_real, y_pred), 0.0, 2)
        assert_array_almost_equal(f1_score(y_real, y_pred), 0.0, 2)

    finally:
        np.seterr(**old_error_settings)

def test_evaluation_report():
    """Test evaluation report"""
    y_real = np.array([['a', 'b', 'c'], ['a', 'b', 'e', 'f', 'g'], ['a', 'b']])
    y_pred = np.array([['a', 'b', 'c'], ['a', 'b', 'c', 'd'], ['e', 'f']])
    labels = np.array(['user_id1', 'user_id2', 'user_id3'])
    # print evaluation report with class names
    expected_report = """\
             precision    recall  f1-score

   user_id1       1.00      1.00      1.00
   user_id2       0.40      0.50      0.44
   user_id3       0.00      0.00      0.00

avg / total       0.47      0.50      0.48
"""
    report = evaluation_report(
        y_real, y_pred,
        target_names=labels)
    assert_equals(report, expected_report)
    # print classification report with label detection
    expected_report = """\
             precision    recall  f1-score

          0       1.00      1.00      1.00
          1       0.40      0.50      0.44
          2       0.00      0.00      0.00

avg / total       0.47      0.50      0.48
"""
    report = evaluation_report(y_real, y_pred)
    assert_equals(report, expected_report)

max_stars_repo_name: CrazyCaspian/crab

max_issues_repo_name: qqwjq/crab

max_forks_repo_name: qqwjq/crab

reason: import numpy,from numpy


text:
from pymesh.TestCase import TestCase
from pymesh import distance_to_mesh, BVH
from pymesh.meshutils import generate_box_mesh

import numpy as np

class DistanceToMeshTest(TestCase):
    def test_boundary_pts_cgal(self):
        mesh = generate_box_mesh(
                np.array([0, 0, 0]), np.array([1, 1, 1]))
        pts = np.array([
            [0.0, 0.0, 0.0],
            [1.0, 1.0, 1.0] ])

        sq_dist, face_idx, closest_pts = distance_to_mesh(mesh, pts, "cgal")
        self.assert_array_equal(sq_dist, np.zeros(2))

    def test_boundary_pts_geogram(self):
        mesh = generate_box_mesh(
                np.array([0, 0, 0]), np.array([1, 1, 1]))
        pts = np.array([
            [0.0, 0.0, 0.0],
            [1.0, 1.0, 1.0] ])

        if "geogram" in BVH.available_engines:
            sq_dist, face_idx, closest_pts = distance_to_mesh(mesh, pts, "geogram")
            self.assert_array_equal(sq_dist, np.zeros(2))


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2019/12/17 15:38
# @Author  : lillian
# @File    : xmp.py
import cv2 as cv
from aip import AipOcr
import numpy as np
import os
import paramiko
import time
import warnings
from openpyxl import Workbook

""" 初 始 化 部 分 """
#APP_ID = '18044023'
#API_KEY = 'xdMedP6gkH2X6hZbUAfnqvr0'
#SECRET_KEY = 'igeVigIi65Np9Vv0K6yL0U2mTpkyhdsd'
warnings.filterwarnings("ignore")
print(os.getcwd())

with open(r'/home/vjulylun/share/WInAutomation/ipconfig.txt') as f:  # 这边可以加一个异常处理，避免找不到这个档案
#with open('./ipconfig.txt') as f:
    text = f.read()
    name = text.split('\n')
    workingdir = name[2].split('=')
    app_id = name[3].split('=')
    app_key = name[4].split('=')
    secret_key = name[5].split('=')
    bmcip = name[1].split('=')

APP_ID = app_id[1]
API_KEY = app_key[1]
SECRET_KEY = secret_key[1]
#连接百度API端口
client = AipOcr(APP_ID, API_KEY, SECRET_KEY)
#BMC_IP = bmcip[1] #no BMC
# 实例化一个transport对象
#transport = paramiko.Transport((BMC_IP, 22))
# 建立连接
#transport.connect(username='sysadmin', password='superuser')
# 将sshclient的对象的transport指定为以上的transport
#ssh = paramiko.SSHClient()
#ssh._transport = transport
ID = 0
cap = cv.VideoCapture(ID)
'''
def usbCamDetect():
    ID = 0
    global cap
    cap = cv.VideoCapture(ID)
    while(ID < 4):
        if cap.isOpened():
            print(ID)
            cap.release()
            return ID
        else:
            print('id %d has no cam detected...' % ID)
            ID += 1
    cap.release()

usbCamDetect()
'''

"""截取指定位置的图片"""
def video_select_size(a,b,c,d):
    #capture = cv.VideoCapture(ID + cv.CAP_DSHOW)  # 读取当前USB摄像头
    capture = cv.VideoCapture(ID)
    capture.set(3, 1920)
    capture.set(4, 1080)
    alltext=[]
    i = 0
    while (True):
        ret, frame = capture.read()  # 读取当前视频画面的信息，frame是每一帧的图像
        capture.release()
        # print(frame)
        try:
            if frame == None:
                print('No connect video')
                break
        except ValueError:
            sp = frame.shape
            newpic = frame[a:b,c:d]
            cv.imwrite('./result.bmp',newpic)
            image = get_file_content('./result.bmp')
            results = client.basicGeneral(image)["words_result"]
            for result in results:
                alltext.append(result["words"])
        break
    return alltext

"""截取所选栏位"""
def select_place(image):
    #image = 'result3.bmp'
    files = os.listdir('./mark')
    for file in files:
        path = './mark'+'/'+file
        os.remove(path)
    savefile = './mark'

    picture = []
    # 设定颜色HSV范围，假定为红色
    #redLower = np.array([0, 60, 46])
    #redUpper = np.array([5, 255, 255])

    # 读取图像
    #img = cv.imread(image)
    img = image

    # 将图像转化为HSV格式
    #hsv = cv.cvtColor(img, cv.COLOR_BGR2HSV)

    # 去除颜色范围外的其余颜色
    #mask = cv.inRange(hsv, redLower, redUpper)
    #二值化操作，灰度
    gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
    ret, binary = cv.threshold(gray, 15, 255, cv.THRESH_BINARY)
    # 二值化操作
    #ret, binary = cv.threshold(mask, 0, 255, cv.THRESH_BINARY)

    # 膨胀操作，因为是对线条进行提取定位，所以腐蚀可能会造成更大间隔的断点，将线条切断，因此仅做膨胀操作
    kernel = np.ones((5, 5), np.uint8)
    dilation = cv.dilate(binary, kernel, iterations=1)
    cv.imwrite('./select_place_dilation.png', dilation)
    #cv.imwrite('3.jpg', img)
    # 获取图像轮廓坐标，其中contours为坐标值，此处只检测外形轮廓
    #a, contours, hierarchy = cv.findContours(dilation, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE)
    contours, hierarchy = cv.findContours(dilation, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE)  # 新版opencv只需要2个参数
    if len(contours) > 0:
        # cv2.boundingRect()返回轮廓矩阵的坐标值，四个值为x, y, w, h， 其中x, y为左上角坐标，w,h为矩阵的宽和
        boxes = [cv.boundingRect(c) for c in contours]
        for box in boxes:
            x, y, w, h = box
            save_image = os.path.join(savefile, str(x)+'_'+str(y)+'_'+str(w)+'_'+str(h) + '.jpg')
            # 绘制矩形框对轮廓进行定位
            # cv2.rectangle(img, (x, y), (x+w, y+h), (153, 153, 0), 2)
            # 将绘制的图像保存并展示
            if (w > 700) and (10<y<600) and (10<h<50) and (x<10):

                new = img[y:y + h, x:x + w]
                #new = img[y:y + h, x:x + w]
                cv.imwrite(save_image, new)
    return

def select_place2(image):
    #image = 'result3.bmp'
    files = os.listdir('./mark')
    for file in files:
        path = './mark'+'/'+file
        os.remove(path)
    savefile = './mark'

    picture = []
    # 设定颜色HSV范围，假定为红色
    #redLower = np.array([0, 60, 46])
    #redUpper = np.array([5, 255, 255])

    # 读取图像
    #img = cv.imread(image)
    img = image

    # 将图像转化为HSV格式
    #hsv = cv.cvtColor(img, cv.COLOR_BGR2HSV)

    # 去除颜色范围外的其余颜色
    #mask = cv.inRange(hsv, redLower, redUpper)
    #二值化操作，灰度
    gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
    ret, binary = cv.threshold(gray, 15, 255, cv.THRESH_BINARY)
    cv.imwrite("./select_image2_binary.png",binary)
    # 二值化操作
    #ret, binary = cv.threshold(mask, 0, 255, cv.THRESH_BINARY)

    # 膨胀操作，因为是对线条进行提取定位，所以腐蚀可能会造成更大间隔的断点，将线条切断，因此仅做膨胀操作
    kernel = np.ones((5, 5), np.uint8)
    dilation2 = cv.dilate(binary, kernel, iterations=1)
    cv.imwrite("select_image2_dilation.png",dilation2)
    #cv.imwrite('3.jpg', img)
    # 获取图像轮廓坐标，其中contours为坐标值，此处只检测外形轮廓
    #a, contours, hierarchy = cv.findContours(dilation, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE)
    contours, hierarchy = cv.findContours(dilation2, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE)

    if len(contours) > 0:
        # cv2.boundingRect()返回轮廓矩阵的坐标值，四个值为x, y, w, h， 其中x, y为左上角坐标，w,h为矩阵的宽和
        boxes = [cv.boundingRect(c) for c in contours]
        for box in boxes:
            x, y, w, h = box
            save_image = os.path.join(savefile, str(x)+'_'+str(y)+'_'+str(w)+'_'+str(h) + '.jpg')
            # 绘制矩形框对轮廓进行定位
            # cv2.rectangle(img, (x, y), (x+w, y+h), (153, 153, 0), 2)
            # 将绘制的图像保存并展示
            if (w > 400) and (10<y<600) and (10<h<40) and (x<10):
                new = img[y:y + h, x:x + w]
                cv.imwrite(save_image, new)
    return

""" 读取图片 """
def get_file_content(filePath):
    with open(filePath, 'rb') as fp:
        return fp.read()

def video_bios_select(): #视频显示
    #capture1 = cv.VideoCapture(ID+cv.CAP_DSHOW)# 读取当前USB摄像头
    #capture1 = cv.VideoCapture(ID)
    #print(cv.CAP_DSHOW)
    cap.release()
    cap.open(ID)
    cap.set(3,1280)
    cap.set(4,720)
    global videoFlag
    videoFlag = cap.isOpened()
    while (videoFlag):
        ret,frame = cap.read()#读取当前视频画面的信息，frame是每一帧的图像
        try:
            if ret:
                print('saving pic.....')
                newpic = frame[0:720, 0:500]
                select_place2(newpic)
                if os.listdir(r'./mark'):
                    print('select mode,please wait')
                    filenames = os.listdir(r'./mark')
                    """ 调用通用文字识别, 图片参数为本地图片 """
                    if filenames:
                        for n in filenames:
                            path = './mark' + '/' + n
                            image = get_file_content(path)
                            print(client.basicGeneral(image))
                            results = client.basicGeneral(image)["words_result"]  # 还可以使用身份证驾驶证模板，直接得到字典对应所需字段
                            sp = []
                        for result in results:
                            sp.append(result["words"])
                else:
                    print('no select mode,please wait')
                    name = './result.png'
                    cv.imwrite(name, frame)
                    image = get_file_content(name)
                    print(client.basicGeneral(image))
                    results = client.basicGeneral(image)["words_result"]
                    sptext = []
                    for result in results:
                        sptext.append(result["words"])
                break
            else:
                print('No Image....')
                break
        except IOError:
            print("Pic can not be saved")
            break
        cap.release()
    return sptext

def video_demo(): #视频显示
    if ID != None:
        print('Screen Capturing>>>')
        #global cap
        #cap = cv2.VideoCapture(usbid)
        #codec = 0x47504A4D  # MJPG
        #cap.set(cv.CAP_PROP_FPS, 30.0)
        #cap.set(cv.CAP_PROP_FOURCC, codec)
        cap.open(ID)
        cap.set(3,1280)#1024
        cap.set(4,720)#768
        print('Image reading>>>')
        #ret, frame = cap.read()
        global videoFlag
        print('Camera detecting...')
        videoFlag = cap.isOpened()
        while (videoFlag):
            #cv2.waitKey(1)
            time.sleep(1)
            ret, frame = cap.read()
            try:
                if ret:
                    print('saving pic.....')
                    select_place(frame)
                    if os.listdir(r'./mark'):
                        print('select mode,please wait')
                        filenames = os.listdir(r'./mark')
                        """ 调用通用文字识别, 图片参数为本地图片 """
                        if filenames:
                            for n in filenames:
                                path = './mark' + '/' + n
                                image = get_file_content(path)
                                results = client.basicGeneral(image)["words_result"]  # 还可以使用身份证驾驶证模板，直接得到字典对应所需字段
                                sptext = []
                            for result in results:
                                sptext.append(result["words"])
                    else:
                        cap.release()
                        sptext = video_bios_select()
                    break
                else:
                    print('No Image....')
                    break
            except IOError:
                print("Pic can not be saved")
                break
    else:
        print('No Camera detected!')
    print('Camera closing>>>')
    cap.release()
    return sptext

def video_all(): #视频显示
    capture = cv.VideoCapture(ID+cv.CAP_DSHOW)# 读取当前USB摄像头
    #capture = cv.VideoCapture(ID)
    capture.set(3,1280)#1024
    capture.set(4,720)#768
    i=0
    while(True):
        ret,frame = capture.read()#读取当前视频画面的信息，frame是每一帧的图像
        capture.release()
        #print(frame)
        try:
            if frame == None:
                print('No connect video')
                break
        except ValueError:
            #frame = cv.flip(frame,1) #图像镜像变化
            #cv.imshow("video",frame)
            #name = './result'+str(i)+'.bmp'
            #cv.imwrite(name, frame)
            #fname = './result.bmp'
            #print(frame.shape)
            #image = get_file_content(name)
            name = './result.bmp'
            cv.imwrite(name, frame)
            image = get_file_content(name)
            results = client.basicGeneral(image)["words_result"]
            alltext=[]
            for result in results:
                alltext.append(result["words"])
            #img = cv.imread(fname)
            #name2 = './'+str(i)+'.txt'
            #f1 = open(name2,'w')
            #for result in results:
            #    text = result["words"]
             #   f1.write(text)
            #i+=1
            #if c == 27: # 按ESC退出
            return alltext
        break

#高精度识别，一天500次
def video_high_ocr():
    capture = cv.VideoCapture(ID + cv.CAP_DSHOW)  # 读取当前USB摄像头
    #capture = cv.VideoCapture(ID)
    capture.set(3, 1280)
    capture.set(4, 720)
    i = 0
    while (True):
        ret, frame = capture.read()  # 读取当前视频画面的信息，frame是每一帧的图像
        capture.release()
        # print(frame)
        try:
            if frame == None:
                print('No connect video')
                break
        except ValueError:
            name = './result.bmp'
            cv.imwrite(name, frame)
            image = get_file_content(name)
            results = client.basicAccurate(image)["words_result"]
            alltext = []
            for result in results:
                alltext.append(result["words"])
            return alltext
        break

def high_ocr(picture):
    image = get_file_content(picture)
    results = client.basicAccurate(image)["words_result"]
    alltext = []
    for result in results:
        alltext.append(result["words"])
    return alltext

def check_post(text):
    sta='post'
    stdin, stdout, stderr = ssh.exec_command('usb_enable')
    print(text)
    for i in text:
        if 'installed' in i:
            print('New CPU installed,Press F1 to Run SETUP')
            sta = 'newcpu'
        elif 'fail' in i:
            print('DOCP or XMP fail')
            sta = 'docpfail'
        elif 'recover' in i:
            print('flash bios,press F1 to run setup')
            sta = 'newbios'
        elif 'over' in i:
            print(' hardware monitor error ')
            sta = 'tempover'

    return sta

##x_start,x_end,y_start,y_end
#print(text)
#pic_to_file(r'F:\0testexe\opencv\TUF Z370-PLUS GAMING_2401\EZMODE\fast',100,550,0,1000)
#b=video_select_size(100,550,0,1000)
a=video_demo()
print(a)
#stdin, stdout, stderr = ssh.exec_command('usb_enable')
#enter_bios(stdin, stdout, stderr)
#select_onoff('off')

max_stars_repo_name: xfzlun/WinUIAutomation

max_issues_repo_name: xfzlun/WinUIAutomation

max_forks_repo_name: xfzlun/WinUIAutomation


text:
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
"""This module contains code to test SageMaker ``ModelPackageGroup``"""
from __future__ import absolute_import

import unittest.mock
import pytest
from sagemaker.lineage import context

@pytest.fixture
def sagemaker_session():
    return unittest.mock.Mock()

def test_pipeline_execution_arn(sagemaker_session):
    obj = context.ModelPackageGroup(
        sagemaker_session,
        context_name="foo",
        description="test-description",
        properties={"PipelineExecutionArn": "abcd", "k2": "v2"},
        properties_to_remove=["E"],
    )
    actual_result = obj.pipeline_execution_arn()
    expected_result = "abcd"
    assert expected_result == actual_result

max_stars_repo_name: HappyAmazonian/sagemaker-python-sdk

max_issues_repo_name: HappyAmazonian/sagemaker-python-sdk

max_forks_repo_name: HappyAmazonian/sagemaker-python-sdk

reason: from sage


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on June 2019
@author: Thomas Bonald <bonald@enst.fr>
@author: Bertrand Charpentier <bertrand.charpentier@live.fr>
@author: Quentin Lutz <qlutz@enst.fr>
"""

import copy
from collections import defaultdict
from typing import Optional, Union, Tuple

import numpy as np

from sknetwork.utils.check import check_n_clusters, check_dendrogram

def reorder_dendrogram(dendrogram: np.ndarray):
    """Reorder the dendrogram in non-decreasing order of height."""
    n = dendrogram.shape[0] + 1
    order = np.zeros((2, n - 1), float)
    order[0] = np.max(dendrogram[:, :2], axis=1)
    order[1] = dendrogram[:, 2]
    index = np.lexsort(order)
    dendrogram_new = dendrogram[index]
    index_new = np.arange(2 * n - 1)
    index_new[n + index] = np.arange(n, 2 * n - 1)
    dendrogram_new[:, 0] = index_new[dendrogram_new[:, 0].astype(int)]
    dendrogram_new[:, 1] = index_new[dendrogram_new[:, 1].astype(int)]
    return dendrogram_new

def get_labels(dendrogram: np.ndarray, cluster: dict, sort_clusters: bool, return_dendrogram: bool):
    """Returns the labels from clusters."""
    n = len(dendrogram) + 1
    n_clusters = len(cluster)
    clusters = list(cluster.values())
    index = None
    if sort_clusters:
        sizes = np.array([len(nodes) for nodes in clusters])
        index = np.argsort(-sizes)
        clusters = [clusters[i] for i in index]

    labels = np.zeros(n, dtype=int)
    for label, nodes in enumerate(clusters):
        labels[nodes] = label

    if return_dendrogram:
        cluster_index = {i: label for i, label in enumerate(labels)}
        cluster_size = {i: len(cluster) for i, cluster in enumerate(clusters)}
        dendrogram_new = []
        current_cluster = len(labels)
        current_cluster_new = len(clusters)
        for i, j, height, _ in dendrogram:
            i_new = cluster_index.pop(int(i))
            j_new = cluster_index.pop(int(j))
            if i_new != j_new:
                size = cluster_size.pop(i_new) + cluster_size.pop(j_new)
                cluster_size[current_cluster_new] = size
                cluster_index[current_cluster] = current_cluster_new
                dendrogram_new.append([i_new, j_new, height, size])
                current_cluster_new += 1
            else:
                cluster_index[current_cluster] = i_new
            current_cluster += 1
        dendrogram_new = np.array(dendrogram_new)
        return labels, dendrogram_new
    else:
        return labels

def cut_straight(dendrogram: np.ndarray, n_clusters: Optional[int] = None, threshold: Optional[float] = None,
                 sort_clusters: bool = True, return_dendrogram: bool = False) \
                -> Union[np.ndarray, Tuple[np.ndarray, np.ndarray]]:
    """Cut a dendrogram and return the corresponding clustering.

    Parameters
    ----------
    dendrogram:
        Dendrogram.
    n_clusters :
        Number of clusters (optional).
        The number of clusters can be larger than n_clusters in case of equal heights in the dendrogram.
    threshold :
        Threshold on height (optional).
        If both n_clusters and threshold are ``None``, n_clusters is set to 2.
    sort_clusters :
        If ``True``,  sorts clusters in decreasing order of size.
    return_dendrogram :
        If ``True``, returns the dendrogram formed by the clusters up to the root.
    Returns
    -------
    labels : np.ndarray
        Cluster of each node.
    dendrogram_aggregate : np.ndarray
        Dendrogram starting from clusters (leaves = clusters).

    Example
    -------
    >>> from sknetwork.hierarchy import cut_straight
    >>> dendrogram = np.array([[0, 1, 0, 2], [2, 3, 1, 3]])
    >>> cut_straight(dendrogram)
    array([0, 0, 1])
    """
    check_dendrogram(dendrogram)
    n = dendrogram.shape[0] + 1

    if return_dendrogram and not np.all(np.diff(dendrogram[:, 2]) >= 0):
        raise ValueError("The third column of the dendrogram must be non-decreasing.")

    cluster = {i: [i] for i in range(n)}
    if n_clusters is None:
        if threshold is None:
            n_clusters = 2
        else:
            n_clusters = n
    else:
        check_n_clusters(n_clusters, n, n_min=1)
    cut = np.sort(dendrogram[:, 2])[n - n_clusters]
    if threshold is not None:
        cut = max(cut, threshold)
    for t in range(n - 1):
        i = int(dendrogram[t][0])
        j = int(dendrogram[t][1])
        if dendrogram[t][2] < cut and i in cluster and j in cluster:
            cluster[n + t] = cluster.pop(i) + cluster.pop(j)

    return get_labels(dendrogram, cluster, sort_clusters, return_dendrogram)

def cut_balanced(dendrogram: np.ndarray, max_cluster_size: int = 20, sort_clusters: bool = True,
                 return_dendrogram: bool = False) -> Union[np.ndarray, Tuple[np.ndarray, np.ndarray]]:
    """Cuts a dendrogram with a constraint on the cluster size and returns the corresponding clustering.

    Parameters
    ----------
    dendrogram:
        Dendrogram
    max_cluster_size :
        Maximum size of each cluster.
    sort_clusters :
        If ``True``, sort labels in decreasing order of cluster size.
    return_dendrogram :
        If ``True``, returns the dendrogram formed by the clusters up to the root.
    Returns
    -------
    labels : np.ndarray
        Label of each node.
    dendrogram_aggregate : np.ndarray
        Dendrogram starting from clusters (leaves = clusters).

    Example
    -------
    >>> from sknetwork.hierarchy import cut_balanced
    >>> dendrogram = np.array([[0, 1, 0, 2], [2, 3, 1, 3]])
    >>> cut_balanced(dendrogram, 2)
    array([0, 0, 1])
    """
    check_dendrogram(dendrogram)
    n = dendrogram.shape[0] + 1
    if max_cluster_size < 2 or max_cluster_size > n:
        raise ValueError("The maximum cluster size must be between 2 and the number of nodes.")

    cluster = {i: [i] for i in range(n)}
    for t in range(n - 1):
        i = int(dendrogram[t][0])
        j = int(dendrogram[t][1])
        if i in cluster and j in cluster and len(cluster[i]) + len(cluster[j]) <= max_cluster_size:
            cluster[n + t] = cluster.pop(i) + cluster.pop(j)

    return get_labels(dendrogram, cluster, sort_clusters, return_dendrogram)

def aggregate_dendrogram(dendrogram: np.ndarray, n_clusters: int = 2, return_counts: bool = False) \
                        -> Union[np.ndarray, Tuple[np.ndarray, np.ndarray]]:
    """Aggregate a dendrogram in order to get a certain number of leaves.
    The leaves in the output dendrogram correspond to subtrees in the input one.

    Parameters
    ----------
    dendrogram:
        The input to aggregate.
    n_clusters:
        Number of clusters (or leaves) to keep.
    return_counts
        If ``True``, returns an array of counts corresponding to the sizes of the merged subtrees.
        The sum of the counts is equal to the number of samples in the input dendrogram.

    Returns
    -------
    new_dendrogram:
        Aggregated dendrogram. The nodes are reindexed from 0.
    counts:
        Size of the subtrees corresponding to each leaf in new_dendrogram.
    """
    n_nodes: int = dendrogram.shape[0] + 1
    check_n_clusters(n_clusters, n_nodes, n_min=1)

    new_dendrogram = dendrogram[n_nodes - n_clusters:].copy()
    node_indices = np.array(sorted(set(new_dendrogram[:, 0]).union(set(new_dendrogram[:, 1]))))
    new_index = {ix: i for i, ix in enumerate(node_indices)}

    for j in range(2):
        for i in range(new_dendrogram.shape[0]):
            new_dendrogram[i, j] = new_index[new_dendrogram[i, j]]

    if return_counts:
        leaves = node_indices[:n_clusters].astype(int)
        leaves_indices = leaves - n_nodes
        counts = dendrogram[leaves_indices, 3]

        return new_dendrogram, counts.astype(int)
    else:
        return new_dendrogram

def get_index(tree):
    """Reindex a dendrogram from the leaves

    Parameters
    ----------
    tree:
        The tree to be indexed

    Returns
    -------
    index:
        The index of the root of the given tree
    """
    if type(tree) != list:
        return tree
    else:
        return np.max([get_index(t) for t in tree])

def get_dendrogram(tree, dendrogram=None, index=None, depth=0, size=None, copy_tree=False):
    """Get dendrogram from tree.

    Parameters
    ----------
    tree :
        The initial tree
    dendrogram :
        Intermediary dendrogram for recursive use
    index :
        Intermediary index for recursive use
    depth :
        Current depth for recursive use
    size :
        Current leaf count for recursive use
    copy_tree :
        If ``True``, ensure the passed tree remains unchanged.

    Returns
    -------
    dendrogram`:
        The reordered dendrogram
    index :
        The indexing array
    """
    if copy_tree:
        return get_dendrogram(copy.deepcopy(tree))
    else:
        if dendrogram is None:
            dendrogram = []
        if index is None:
            index = get_index(tree)
        if size is None:
            size = defaultdict(lambda: 1)
        if len(tree) > 1:
            lengths = np.array([len(t) for t in tree])
            if np.max(lengths) == 1:
                # merge all
                i = tree.pop()[0]
                j = tree.pop()[0]
                s = size[i] + size[j]
                dendrogram.append([i, j, float(-depth), s])
                index += 1
                while len(tree):
                    s += 1
                    dendrogram.append([index, tree.pop()[0], float(-depth), s])
                    index += 1
                size[index] = s
                tree.append(index)
                return dendrogram, index
            else:
                i = np.argwhere(lengths > 1).ravel()[0]
                dendrogram_, index_ = get_dendrogram(tree[i], None, index, depth + 1, size)
                dendrogram += dendrogram_
                return get_dendrogram(tree, dendrogram, index_, depth, size)
        else:
            return dendrogram, index

def split_dendrogram(dendrogram: np.ndarray, shape: tuple):
    """Split the dendrogram of a bipartite graph into 2 dendrograms, one for each part.

    Parameters
    ----------
    dendrogram :
        Dendrogram of the bipartite graph.
    shape :
        Shape of the biadjacency matrix.
    Returns
    -------
    dendrogram_row :
        Dendrogram for the rows.
    dendrogram_col :
        Dendrogram for the columns.
    """
    n1, n2 = shape
    dendrogram_row = []
    dendrogram_col = []
    id_row_new = n1
    id_col_new = n2
    size_row = {i: 1 for i in range(n1)}
    size_col = {i + n1: 1 for i in range(n2)}
    id_row = {i: i for i in range(n1)}
    id_col = {i + n1: i for i in range(n2)}

    for t in range(n1 + n2 - 1):
        i = dendrogram[t, 0]
        j = dendrogram[t, 1]

        if i in id_row and j in id_row:
            size_row[n1 + n2 + t] = size_row.pop(i) + size_row.pop(j)
            id_row[n1 + n2 + t] = id_row_new
            dendrogram_row.append([id_row.pop(i), id_row.pop(j), dendrogram[t, 2], size_row[n1 + n2 + t]])
            id_row_new += 1
        elif i in id_row:
            size_row[n1 + n2 + t] = size_row.pop(i)
            id_row[n1 + n2 + t] = id_row.pop(i)
        elif j in id_row:
            size_row[n1 + n2 + t] = size_row.pop(j)
            id_row[n1 + n2 + t] = id_row.pop(j)

        if i in id_col and j in id_col:
            size_col[n1 + n2 + t] = size_col.pop(i) + size_col.pop(j)
            id_col[n1 + n2 + t] = id_col_new
            dendrogram_col.append([id_col.pop(i), id_col.pop(j), dendrogram[t, 2], size_col[n1 + n2 + t]])
            id_col_new += 1
        elif i in id_col:
            size_col[n1 + n2 + t] = size_col.pop(i)
            id_col[n1 + n2 + t] = id_col.pop(i)
        elif j in id_col:
            size_col[n1 + n2 + t] = size_col.pop(j)
            id_col[n1 + n2 + t] = id_col.pop(j)

    return np.array(dendrogram_row), np.array(dendrogram_col)

max_stars_repo_name: HerrZYZ/scikit-network

max_issues_repo_name: HerrZYZ/scikit-network

max_forks_repo_name: HerrZYZ/scikit-network


text:
import numpy as np
import pandas as pd
import torch as torch

class Preprocess:

    def __init__(self, max_sequence_length, truncation_side):
        self.__max_sequence_length = max_sequence_length
        self.__truncation_side = truncation_side

    def truncate(self, sequence: np.array) -> np.array:
        truncated = {
            'head': sequence[(sequence.shape[0] - self.__max_sequence_length):, :],
            'tail': sequence[:self.__max_sequence_length, :]
        }
        return truncated[self.__truncation_side]

    def pad_sequence(self, sequence: np.array) -> np.array:
        padding = np.zeros((self.__max_sequence_length - len(sequence), sequence.shape[1]))
        return np.append(padding, sequence, axis=0)

    def transform(self, sequence: pd.DataFrame) -> np.array:
        sequence = sequence.values
        sequence = self.truncate(sequence)

        if len(sequence) < self.__max_sequence_length:
            sequence = self.pad_sequence(sequence)

        return torch.from_numpy(sequence)

max_stars_repo_name: hengwei-chan/attentionRNN


text:
import random

import cv2
import pylab
import matplotlib.pyplot as plt
import torch
import torch.nn.functional as F
from torch.utils.data import Dataset
from torchvision.utils import make_grid
from torchvision import transforms
import numpy as np

from src.models.transporter import Transporter
from src.models.inception3 import perception_inception_net
from src.models.alexnet import perception_alex_net
from src.utils.visualization import gen_eval_imgs
from src.utils.grad_flow import plot_grad_flow
from src.utils.kpt_utils import kpts_2_img_coordinates
from src.losses import perception_loss
from .abstract_agent import AbstractAgent

Tran = transforms.RandomApply([
    # transforms.RandomHorizontalFlip(p=1.0),
    # transforms.RandomVerticalFlip(p=0.3),
    transforms.RandomRotation(degrees=30)
], p=0.5)

class TransporterAgent(AbstractAgent):

    def __init__(self,
                 dataset: Dataset,
                 config: dict):

        super(TransporterAgent, self).__init__(
            name='Transporter Agent',
            dataset=dataset,
            config=config
        )

        if config['training']['loss_function'] in ['inception', 'Inception', 'INCEPTION']:
            self.perception_net = perception_inception_net(config['log_dir']).to(self.device)

        if config['training']['loss_function'] in ['alexnet', 'AlexNet', 'ALEXNET']:
            self.perception_net = perception_alex_net(config['log_dir']).to(self.device)

        self.model = Transporter(config).to(self.device)

    def log_model(self, config: dict):
        pass
        """
        self.writer.add_text(tag='model/encoder',
                             text_string=str(summary(self.model.encoder, eval(config['data']['img_shape']))))
        self.writer.add_text(tag='model/decoder',
                             text_string=str(summary(self.model.decoder, eval(config['data']['img_shape']))))
        self.writer.add_text(tag='model/keypointer',
                             text_string=str(summary(self.model.keypointer, eval(config['data']['img_shape']))))
        """

    def preprocess(self,
                   x: torch.Tensor,
                   label: torch.Tensor,
                   config: dict) -> (torch.Tensor, (torch.Tensor, torch.Tensor)):

        """ Uses the first frame of a sequence of images as source and the last frame as target.

        :param x: Torch image series tensor in (N, T, C, H, W)
        :param label: -
        :param config: Additional args
        :return: Source frame in (N, C, H, W), target frame in (N, C, H, W)
        """

        assert x.dim() == 5

        # Randomly sample target frame from within t=s+1 to t=s+19
        # t_diff = random.randint(a=1, b=19)
        t_diff = random.randint(a=1, b=config['model']['n_frames'] - 1)

        # sample_frame = x[:, 0, ...] - 0.5
        # sample_frame = x[:, 0, ...]
        # target_frame = x[:, -1, ...] - 0.5
        # target_frame = x[:, -1, ...]
        # target_frame = x[:, 0 + t_diff, ...]

        # Normalizing to [-1, 1] range
        # TODO: The Human36M Data seems to work better in [0, 1] range instead...
        # sample_frame = 2 * ((x[:, 0, ...] - x[:, 0, ...].min()) / (x[:, 0, ...].max() - x[:, 0, ...].min())) - 1
        # target_frame = 2 * ((x[:, 0 + t_diff, ...] - x[:, 0 + t_diff, ...].min()) /
        #                    (x[:, 0 + t_diff, ...].max() - x[:, 0 + t_diff, ...].min())) - 1

        #sample_frame = ((x[:, 0, ...] - x[:, 0, ...].min()) / (x[:, 0, ...].max() - x[:, 0, ...].min()))
        #target_frame = ((x[:, 0 + t_diff, ...] - x[:, 0 + t_diff, ...].min()) /
        #                (x[:, 0 + t_diff, ...].max() - x[:, 0 + t_diff, ...].min()))

        # Drawing target frame randomly from different sequence (from batch)
        """
        N = x.shape[0]
        sample_frame = x[:, 0, ...]
        target_frame = torch.empty_like(sample_frame)
        for n in range(x.shape[0]):
            random_target_idx = random.randint(0, x.shape[0]-1)
            target_frame[n, ...] = x[random_target_idx, 0 + t_diff, ...]

        t = torch.cat([sample_frame, target_frame], dim=0)
        t = Tran(t)

        sample_frame = t[0:N, ...]
        target_frame = t[N:, ...]
        """

        sample_frame = x[:, 0, ...]
        target_frame = x[:, 0 + t_diff, ...]

        # target_frame = Tran(x[:, 0 + t_diff, ...])

        return sample_frame, target_frame

    def loss_func(self, prediction: torch.Tensor, target: torch.Tensor, config: dict) -> torch.Tensor:
        if config['training']['loss_function'] in ['mse', 'l2', 'MSE']:
            return F.mse_loss(input=prediction, target=target, reduction='mean')
        elif config['training']['loss_function'] in ['sse', 'SSE']:
            # return F.mse_loss(input=prediction, target=target, reduction='sum')
            return F.mse_loss(prediction, target, reduction='sum') / \
                   (torch.var(target) * 2 * target.shape[0] * target.shape[1])
        elif config['training']['loss_function'] in ['bce', 'BCE']:
            # Map to [0, 1]
            # prediction = (prediction + 1.0) / 2.0
            # target = (target + 1.0) / 2.0
            return F.binary_cross_entropy(input=prediction, target=target)
        elif config['training']['loss_function'] in ['alexnet', 'AlexNet', 'ALEXNET',
                                                     'inception', 'Inception', 'INCEPTION']:
            # Map to [0, 1]
            # prediction = (prediction + 1.0) / 2.0
            # target = (target + 1.0) / 2.0
            loss_perception = perception_loss(perception_net=self.perception_net,
                                              prediction=prediction.unsqueeze(1),
                                              target=target.unsqueeze(1))
            # loss_pixel = F.binary_cross_entropy(input=prediction, target=target)
            loss_pixel = F.mse_loss(input=prediction, target=target, reduction='mean')
            return 0.5 * loss_perception + 0.5 * loss_pixel
        else:
            raise ValueError("Given loss function not implemented.")

    def step(self,
             sample: torch.Tensor,
             target: torch.Tensor,
             global_epoch_number: int,
             save_grad_flow_plot: bool,
             save_val_sample: bool,
             config: dict, mode: str) -> torch.Tensor:

        assert mode in ['training', 'validation']

        self.optim.zero_grad()

        reconstruction = self.model(sample, target)
        # reconstruction, source_p, target_p = self.model(sample, target)

        loss = self.loss_func(prediction=reconstruction, target=target, config=config)
        # map_p = (0.1*source_p + 0.5*target_p)
        # loss += map_p

        if mode == 'training':

            loss.backward()

            torch.nn.utils.clip_grad_norm_(self.model.parameters(), config['training']['grad_clip_max_norm'])

            self.optim.step()

            if save_grad_flow_plot:
                # self.writer.add_scalar(tag='train/p', scalar_value=map_p, global_step=global_epoch_number)

                plot_grad_flow(named_parameters=self.model.encoder.named_parameters(),
                               epoch=global_epoch_number,
                               tag_name='encoder',
                               summary_writer=self.writer)
                plot_grad_flow(named_parameters=self.model.keypointer.named_parameters(),
                               epoch=global_epoch_number,
                               tag_name='keypointer',
                               summary_writer=self.writer)
                plot_grad_flow(named_parameters=self.model.decoder.named_parameters(),
                               epoch=global_epoch_number,
                               tag_name='decoder',
                               summary_writer=self.writer)

        if mode == 'validation' and config['validation']['save_eval_examples'] and save_val_sample:

            with torch.no_grad():
                source_kpts, source_gmaps, source_fmaps = self.model.keypointer(sample)
                _N, _K, _H, _W = source_fmaps.shape
                target_kpts, target_gmaps, target_fmaps = self.model.keypointer(target)
                key_point_coordinates = torch.cat([source_kpts.unsqueeze(1), target_kpts.unsqueeze(1)], dim=1)

                # Adapt to visualization
                key_point_coordinates[..., 0] *= -1.0

                # Convert kpt coordinates from [-1, 1]x[-1, 1] to [0, H]x[0, W]
                img_coordinates = kpts_2_img_coordinates(key_point_coordinates, (_H, _W)).cpu()  # (N, 2, K, 2)
                key_point_coordinates[..., :2] *= -1.0

                _sample = torch.cat([sample.unsqueeze(1), target.unsqueeze(1)], dim=1)
                # _sample = ((_sample + 1) / 2.0).clip(0.0, 1.0)

                reconstruction = torch.cat([sample.unsqueeze(1), reconstruction.unsqueeze(1)], dim=1)
                # reconstruction = ((reconstruction + 1) / 2.0).clip(0.0, 1.0)

                torch_img_series_tensor = gen_eval_imgs(sample=_sample,
                                                        reconstruction=reconstruction,
                                                        key_points=key_point_coordinates)

                self.writer.add_video(tag='val/reconstruction_sample',
                                      vid_tensor=torch_img_series_tensor,
                                      global_step=global_epoch_number)

                cm = pylab.get_cmap('gist_rainbow')

                fig, ax = plt.subplots(2, _K, figsize=(_K * 4, 8))
                for _k in range(_K):
                    ax[0, _k].imshow(source_fmaps[0, _k, ...].cpu(), cmap='gray')
                    ax[0, _k].set_title(f'Source frame - Keypoint {_k}')
                    ax[0, _k].scatter(img_coordinates[0, 0, _k, 0], img_coordinates[0, 0, _k, 1],
                                      color=cm(1. * _k / _K), marker="^", s=150)

                    max_loc_source = torch.argmax(source_fmaps[0, _k, ...]).cpu()
                    max_loc_source_h = max_loc_source / source_fmaps[0, _k, ...].shape[-2]
                    max_loc_source_w = max_loc_source % source_fmaps[0, _k, ...].shape[-1]
                    ax[0, _k].scatter(max_loc_source_w, max_loc_source_h,
                                      color='red', marker="x", s=250)

                    ax[1, _k].imshow(target_fmaps[0, _k, ...].cpu(), cmap='gray')
                    ax[1, _k].set_title(f'Target frame - Keypoint {_k}')
                    ax[1, _k].scatter(img_coordinates[0, 1, _k, 0], img_coordinates[0, 1, _k, 1],
                                      color=cm(1. * _k / _K), marker="^", s=150)

                    max_loc_target = torch.argmax(target_fmaps[0, _k, ...]).cpu()
                    max_loc_target_h = max_loc_target / target_fmaps[0, _k, ...].shape[-2]
                    max_loc_target_w = max_loc_target % target_fmaps[0, _k, ...].shape[-1]
                    ax[1, _k].scatter(max_loc_target_w, max_loc_target_h,
                                      color='red', marker="x", s=250)

                    # np.save(f"source_map_k{_k}.npy", source_fmaps[0, _k, ...].cpu().numpy())
                    # np.save(f"target_map_k{_k}.npy", target_fmaps[0, _k, ...].cpu().numpy())
                # exit()

                plt.tight_layout()
                self.writer.add_figure(tag="val/feature_maps",
                                       figure=fig,
                                       global_step=global_epoch_number)

                fig, ax = plt.subplots(2, _K, figsize=(_K * 4, 8))
                for _k in range(_K):
                    ax[0, _k].imshow(source_gmaps[0, _k, ...].cpu(), cmap='gray')
                    ax[0, _k].set_title(f'Source frame - Keypoint {_k}')
                    ax[0, _k].scatter(img_coordinates[0, 0, _k, 0], img_coordinates[0, 0, _k, 1],
                                      color=cm(1. * _k / _K), marker="^", s=150)

                    ax[1, _k].imshow(target_gmaps[0, _k, ...].cpu(), cmap='gray')
                    ax[1, _k].set_title(f'Target frame - Keypoint {_k}')
                    ax[1, _k].scatter(img_coordinates[0, 1, _k, 0], img_coordinates[0, 1, _k, 1],
                                      color=cm(1. * _k / _K), marker="^", s=150)

                plt.tight_layout()
                self.writer.add_figure(tag="val/gaussian_maps",
                                       figure=fig,
                                       global_step=global_epoch_number)

                plt.close()

                self.writer.flush()

        return loss


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""NetCDF"""

import netCDF4
import numpy as np

def load_dataset(nc_file):
    '''Load nc-file into dataset'''
    return netCDF4.Dataset(nc_file, mode='r')

def print_metadata(filehandle):
    '''Print metadata'''
    # Metadata
    print(f'Metadata: {filehandle}')

    # Dictionaries
    print(f'Dictionaries: {filehandle.__dict__}')

def print_dimensions(filehandle):
    '''Print dimensions'''
    for dim in filehandle.dimensions.values():
        print(f'Dimension: {dim}')
    print(f'Dimensions: {filehandle.dimensions}')

def print_variables(filehandle):
    '''Print variables'''
    for var in filehandle.variables.values():
        print(f'Variable: {var}')
    print(f'Variables: {filehandle.variables.keys()}')

def get_coords(filehandle):
    '''Get coords'''
    #lons = fh.variables['lon'][80:100, 85:100]
    #lats = fh.variables['lat'][65:100, 90:100]
    lons = filehandle.variables['lon'][:]
    lats = filehandle.variables['lat'][:]
    return lons, lats

def get_values(filehandle, variable='prediction'):
    '''Get values from variable'''
    return filehandle.variables[variable][:]

def get_closest(lats,lons,lat_pt,lon_pt):

  dist_2 = (lats-lat_pt)**2 + (lons-lon_pt)**2
  min_dist = dist_2.argmin()

  return np.unravel_index(min_dist, lats.shape)

def main():
    ''' Test '''
    data = load_dataset('INTER_OPER_R___RD1NRT__L3__20140204T080000_20140205T080000_0001.nc')

    print('                                                     ')
    print('=====================================================')

    print_metadata(data)

    #print_dimensions(data)

    #print_variables(data)

    #print(get_coords(data))

    # Print all values
    #for variable in data.variables.keys():
    #    print(f'Variable: {variable}')
    #    #print(f'    type: {type(get_values(data,variable))}')
    #    print(f'    values: {get_values(data,variable).compressed()}')
    #prediction = data.variables['prediction']
    #print(prediction)

    #lons = data.variables['lon'][:]
    #lats = data.variables['lat'][:]
    #print(f'Lon: {lons.mean()}, lat: {lats.mean()}')
    #y_min,x_min = get_closest(lats, lons, 60,24)
    #print(y_min, x_min)

    data.close()

if __name__ == "__main__":
    main()


text:
# ******************************************************************************
# Copyright 2014-2018 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ******************************************************************************
# pylint: skip-file
import itertools
import numpy as np
import os
import pytest
import subprocess as subp

from neon import NervanaObject
from utils import call_func, gen_backend_tensors, tensors_allclose
try:
    from neon.backends.nervanagpu import NervanaGPU
except ImportError:
    # stub out the class
    class NervanaGPU(object):
        pass

class TestFuncs(object):

    """
    A collection of functions to be tested
    """
    @staticmethod
    def func_dot_reduction_mix(be, x0, x1, x2, x3, x4):
        f1 = be.std(be.var(x0, axis=0, keepdims=True), axis=1, keepdims=True)
        f2 = (be.max(x1, axis=0, keepdims=True) +
              be.min(x1, axis=0, keepdims=True))
        f3 = be.std(x2, keepdims=True)
        f4 = be.dot(1.0 / x3, x4 / x2)
        f5 = be.dot(x3, x4 - x0)
        f6 = be.dot(x2 / f4, f5 + x3)
        return f1 + f2 + f3 + f4 + 1.0 / (be.dot(f5, f6))

    @staticmethod
    def func_dot_reduction_transpose_mix(be, x0, x1, x2, x3, x4):
        f1 = be.std(be.var(x0, axis=0, keepdims=True), axis=1, keepdims=True)
        f2 = (be.max(x1, axis=0, keepdims=True) +
              be.min(x1, axis=0, keepdims=True))
        f3 = be.std(x2, keepdims=True).T
        f4 = be.dot(1.0 / x3, (x4 / x2).T).T
        f5 = be.dot(x3, (x4 - x0).T)
        f6 = be.dot(x2 / f4.T, f5 + x3).T
        return f1 + f2 + f3 + f4 + 1.0 / (be.dot(f5, f6))

def pytest_generate_tests(metafunc):
    """
    Test generator
    """
    # number of test to repeat
    test_indices = [0]

    # test params
    test_funcs = [
        TestFuncs.func_dot_reduction_mix,
        TestFuncs.func_dot_reduction_transpose_mix,
    ]
    test_tensor_flags = ['pos_rand', 'neg_rand', 'rand']
    test_tensor_dims = [(2, 2)]

    # generate params for testing
    if 'custom_args' in metafunc.fixturenames:
        fargs = itertools.product(test_indices,
                                  test_funcs,
                                  test_tensor_flags,
                                  test_tensor_dims)
        # parameterize test call
        metafunc.parametrize("custom_args", fargs)

def test_vs_numpy_mkl(backend_tests_mkl, custom_args):
    test_idx, f, flag, dim = custom_args

    # backend
    be = NervanaObject.be
    dtype = be.default_dtype

    # tensors
    tensors = gen_backend_tensors([np, be], [dim] * 5, [flag] * 5, dtype=dtype)

    # compare function value and gradient
    numpy_func_val = call_func(f, np, tensors[0])
    backend_func_val = call_func(f, be, tensors[1])

    assert tensors_allclose(numpy_func_val, backend_func_val, rtol=1e-2, atol=1e-2)

@pytest.mark.hasgpu
def test_vs_numpy(backend_tests, custom_args):
    test_idx, f, flag, dim = custom_args

    # backend
    be = NervanaObject.be
    dtype = be.default_dtype

    # tensors
    tensors = gen_backend_tensors([np, be], [dim] * 5, [flag] * 5, dtype=dtype)

    # compare function value and gradient
    numpy_func_val = call_func(f, np, tensors[0])
    backend_func_val = call_func(f, be, tensors[1])

    try:
        assert tensors_allclose(numpy_func_val, backend_func_val, rtol=1e-2, atol=1e-2)
    except AssertionError:
        # xfail for gpu backend on TITAN XP platforms
        if isinstance(NervanaObject.be, NervanaGPU):

            if os.getenv("PLATFORM"):
                platform = os.getenv("PLATFORM")
            else:
                if os.path.exists("/usr/bin/nvidia-smi"):
                    cmd = '/usr/bin/nvidia-smi -q | grep "Product Name" | tail -1 | cut -f 2 -d \':\' | \
                           cut -f 2,3 -d \' \''
                    gpu_info = subp.check_output(cmd, shell=True)
                else:
                    gpu_info = "unknown"

            if gpu_info == 'TITAN Xp\n':
                platform = "TITANXP"

            if platform == 'TITANXP':
                pytest.xfail(reason="xfail issue #854 with {} PLATFORM".format(platform))
            else:
                assert tensors_allclose(numpy_func_val, backend_func_val, rtol=1e-2, atol=1e-2)

max_stars_repo_name: rsketine/neon

max_issues_repo_name: EquifAI/neon

max_forks_repo_name: EquifAI/neon


text:
import cv2
import numpy as np
import pyttsx3
import speech_recognition as sr
import datetime
import wikipedia
import webbrowser
import os
import smtplib

engine= pyttsx3.init('sapi5')
voices= engine.getProperty('voices')
engine.setProperty('voice',voices[0].id)

#speak function
def speak(text):
    engine.say(text)
    engine.runAndWait()

#taking command
def takeCommand():
    r=sr.Recognizer()
    with sr.Microphone() as source:
        print("Listening...")
        audio=r.listen(source)
    try:
        print("Recognizing...")
        query=r.recognize_google(audio, language ='en-in')
        print(query)

    except Exception as e:
        print("Sorry please can you repeat again")
        query=None
    return query

speak('Video recording, picture click or web cam')
cam=takeCommand()
#0 for 1st webcam 1 for 2nd
cap=cv2.VideoCapture(0)
sve=cv2.VideoWriter_fourcc(*'XVID')

out=cv2.VideoWriter('output.avi',sve, 20.0, (640,480))
#must be 640,480 otherwise will not be supported

if 'video' or 'recording' in cam.lower():
    while True:
        ret, frame=cap.read()
        gray=cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
        cv2.imshow('color Camera',frame)
        cv2.imshow('gray frame',gray)
        out.write(frame)
        if cv2.waitKey(1) & 0xFF==ord('q'):
            break
elif 'picture' or 'photo' or 'click' or 'capture' in cam.lower():
    while True:
        ret, frame=cap.read()
        gray=cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
        cv2.imshow('color Camera',frame)
        cv2.imshow('gray frame',gray)
        speak('press     c        to capture')
        if cv2.waitKey(1) & 0xFF==ord('c'):
            out.write(frame)
            break
elif 'webcam' in cam.lower():
    while True:
        ret, frame=cap.read()
        gray=cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
        cv2.imshow('color Camera',frame)
        cv2.imshow('gray frame',gray)
        if cv2.waitKey(1) & 0xFF==ord('q'):
            break
cap.release()
out.release()
cv2.destroyAllWindows

max_stars_repo_name: subho-dev/jarvis

max_issues_repo_name: subho-dev/jarvis

max_forks_repo_name: subho-dev/jarvis