text:
import pickle
import nltk.stem
import re
import string
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score, precision_score, recall_score,f1_score
import pandas as pd
#from sklearn.cluster import KMeans
import numpy as np
#there are 2 model that were trained to identify the keys because the keys scam and fraud were too iterlinked to be correctely indentified in the same model
filename="model2.pkl"

QUOTES = re.compile(r'(writes in|writes:|wrote:|says:|said:|^In article|^Quoted from|^\||^>)')

try:
    #loading the pickle file
    data = pd.read_pickle('cleanDataset.pkl')
except:
    print("dataset Not found")

data['label'] = LabelEncoder().fit_transform(data['tag'])
#print(data)
data=data[['article','label']]
print(data)
## Full Model F1 score:  0.691239827668741

if( filename=="model1.pkl"):
    data=data[data['label']!=5] ##Model2 F1 score:  0.8933074684772065
    data=data[data['label']!=0]
    data=data[data['label']!=2]
else:
    data=data[data['label']!=3] ## Model1 F1 score:  0.8902527075812274
    data=data[data['label']!=1]

print(data)
data.article=data.article.astype(str)

X_train, X_test, y_train, y_test = train_test_split(data['article'], data['label'], test_size=0.2, random_state=1,shuffle=True)

cv = CountVectorizer(strip_accents='ascii', token_pattern=u'(?ui)\\b\\w*[a-z]+\\w*\\b', lowercase=True, stop_words='english')
X_train_cv = cv.fit_transform(X_train)
X_test_cv = cv.transform(X_test)

naive_bayes = MultinomialNB()
naive_bayes.fit(X_train_cv, y_train)
predictions = naive_bayes.predict(X_test_cv)
predictions_prob = naive_bayes.predict_proba(X_test_cv)
print(predictions)
print(predictions_prob)

pickle.dump(naive_bayes, open(filename, 'wb'))
pickle.dump(cv, open(filename.replace(".pkl","")+"cv.pkl", 'wb'))
print('Accuracy score: ', accuracy_score(y_test, predictions))
print('Precision score: ', precision_score(y_test, predictions,average='micro'))
print('Recall score: ', recall_score(y_test, predictions,average='micro'))
print('F1 score: ', f1_score(y_test, predictions,average='micro', labels=np.unique(y_test)))


text:
import numpy as np

from opfython.math import distance

def test_additive_symmetric_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.additive_symmetric_distance(x, y)

    assert dist == 0.4813445378151263

def test_average_euclidean_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.average_euclidean_distance(x, y)

    assert dist == 0.2236067977499792

def test_bhattacharyya_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.bhattacharyya_distance(x, y)

    assert dist == -2.3499617065547542

def test_bray_curtis_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.bray_curtis_distance(x, y)

    assert dist == 0.03809523809523813

def test_canberra_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.canberra_distance(x, y)

    assert dist == 0.33983837574300413

def test_chebyshev_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.chebyshev_distance(x, y)

    assert dist == 0.3000000000000007

def test_chi_squared_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.chi_squared_distance(x, y)

    assert dist == 0.029526480999131792

def test_chord_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.chord_distance(x, y)

    assert dist == 0.05642296490643823

def test_clark_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.clark_distance(x, y)

    assert dist == 0.22448075858553082

def test_cosine_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.cosine_distance(x, y)

    assert dist == 0.00159177548441658

def test_dice_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.dice_distance(x, y)

    assert dist == 0.0023820867079562547

def test_divergence_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.divergence_distance(x, y)

    assert dist == 0.10078322195027073

def test_euclidean_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.euclidean_distance(x, y)

    assert dist == 0.4472135954999584

def test_gaussian_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.gaussian_distance(x, y)

    assert dist == 0.6394073191618967

def test_gower_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.gower_distance(x, y)

    assert dist == 0.2000000000000002

def test_hamming_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.hamming_distance(x, y)

    assert dist == 4

def test_hassanat_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.hassanat_distance(x, y)

    assert dist == 0.2571314102564104

def test_hellinger_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.hellinger_distance(x, y)

    assert dist == 0.2435717239311041

def test_jaccard_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.jaccard_distance(x, y)

    assert dist == 0.004752851711026626

def test_jeffreys_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.jeffreys_distance(x, y)

    assert np.round(dist, 5) == 0.11884

def test_jensen_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.jensen_distance(x, y)

    assert np.round(dist, 5) == 0.00740

def test_jensen_shannon_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.jensen_shannon_distance(x, y)

    assert np.round(dist, 5) == 0.01481

def test_k_divergence_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.k_divergence_distance(x, y)

    assert dist == -0.18527516196697194

def test_kulczynski_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.kulczynski_distance(x, y)

    assert dist == 0.07920792079207929

def test_kullback_leibler_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.kullback_leibler_distance(x, y)

    assert np.round(dist, 5) == -0.34023

def test_log_euclidean_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.log_euclidean_distance(x, y)

    assert dist == 36964.004940249884

def test_log_squared_euclidean_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.log_squared_euclidean_distance(x, y)

    assert dist == 18232.155679395495

def test_lorentzian_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.lorentzian_distance(x, y)

    assert dist == 0.7153488885436325

def test_manhattan_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.manhattan_distance(x, y)

    assert dist == 0.8000000000000008

def test_matusita_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.matusita_distance(x, y)

    assert dist == 0.17223121769698138

def test_max_symmetric_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.max_symmetric_distance(x, y)

    assert dist == 0.12254901960784322

def test_mean_censored_euclidean_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.mean_censored_euclidean_distance(x, y)

    assert dist == 0.2236067977499792

def test_min_symmetric_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.min_symmetric_distance(x, y)

    assert dist == 0.11812324929971998

def test_neyman_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.neyman_distance(x, y)

    assert dist == 0.11812324929971998

def test_non_intersection_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.non_intersection_distance(x, y)

    assert dist == 0.4000000000000004

def test_pearson_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.pearson_distance(x, y)

    assert dist == 0.12254901960784322

def test_sangvi_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.sangvi_distance(x, y)

    assert dist == 0.11810592399652717

def test_soergel_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.soergel_distance(x, y)

    assert dist == 0.07339449541284411

def test_squared_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.squared_distance(x, y)

    assert dist == 0.059052961998263584

def test_squared_chord_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.squared_chord_distance(x, y)

    assert dist == 0.029663592349384996

def test_squared_euclidean_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.squared_euclidean_distance(x, y)

    assert dist == 0.20000000000000046

def test_statistic_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.statistic_distance(x, y)

    assert dist == 0.08914713150337261

def test_topsoe_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.topsoe_distance(x, y)

    assert np.round(dist, 5) == 0.02962

def test_vicis_symmetric1_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.vicis_symmetric1_distance(x, y)

    assert dist == 0.3002436268625096

def test_vicis_symmetric2_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.vicis_symmetric2_distance(x, y)

    assert dist == 0.134873949579832

def test_vicis_symmetric3_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.vicis_symmetric3_distance(x, y)

    assert dist == 0.10579831932773118

def test_vicis_wave_hedges_distance():
    x = np.asarray([5.1, 3.5, 1.4, 0.3])
    y = np.asarray([5.4, 3.4, 1.7, 0.2])

    dist = distance.vicis_wave_hedges_distance(x, y)

    assert dist == 0.8025210084033614

ext: py

max_stars_repo_name: gugarosa/opfython

max_issues_repo_name: gugarosa/opfython

max_forks_repo_name: gugarosa/opfython

reason: import numpy


text:
# coding: utf-8
###
 # @file   byzPS.py
 # @author Arsany Guirguis  <arsany.guirguis@epfl.ch>
 #
 # @section LICENSE
 #
 # Copyright (c) 2020 Arsany Guirguis.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
 #
 # The above copyright notice and this permission notice shall be included in all
 # copies or substantial portions of the Software.
 #
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 #
 # @section DESCRIPTION
 #
 # Garfield implementation. The parameter server implementation
###

#!/usr/bin/env python

import numpy as np
import argparse
import os
import pathlib
import signal
import sys
import time
import time
import random
import math
import all_pb2, all_pb2_grpc
import grpc
from grpc_service_impl import TrainMessageExchangeServicer
from concurrent import futures
import threading
from multiprocessing.dummy import Pool as ThreadPool

import collections
import builtins as __builtin__

import tensorflow as tf

root = pathlib.Path(__file__).resolve().parent
sys.path.append(str(root / "models"))

import aggregators
import native
import tools
import models
import experiments
import helper

def println(str):
  print(str)
  sys.stdout.flush()

device='gpu'
lengths = {"cnnet" : 1756426, "mnist" : 79510, "slim-cifarnet-cifar10" : 1756426, "slim-vgg_a-cifar10" : 128807306, "slim-inception_v1-cifar10" : 5602874, "slim-inception_v3-cifar10" : 24353332, "slim-resnet_v2_50-cifar10" : 23539850, "slim-resnet_v2_200-cifar10": 62697610}
MAX_MESSAGE_LENGTH = 600000000
# Argument parsing
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--ps_hosts',
type=str,
default="127.0.0.1:4523",
help='Comma-separated list of hostname:port pairs')
parser.add_argument('--worker_hosts',
type=str,
default="127.0.0.1:4524",
help='Comma-separated list of hostname:port pairs')
parser.add_argument('--log_dir',
type=str,
default='',
help='Directory where to write event logs and checkpoint.')
parser.add_argument('--job_name',
type=str,
default='',
help="One of 'ps', 'worker'")
parser.add_argument('--task_index',
type=int,
default=0,
help="Index of task. e.g., if 2 workers: \n1st worker-> task_index 0 & 2nd worker-> task_index 1")
parser.add_argument('--max_steps',
type=int,
default=3000,
help='Maximum number of steps to run.')
parser.add_argument('--eval_steps',
type=float,
default=10,
help='Save model checkpoints frequency.')
parser.add_argument('--log_frequency_steps',
type=int,
default=10,
help='How often to log results to the console.')
parser.add_argument('--cpus',
type=int,
default=0,
help='Number of CPU per worker (does not affect the chief). 0 for every available logical core.')
parser.add_argument('--rule',
type=str,
default="krum",
help='Rule to choose (one of "krum", "kardam", ...).')
parser.add_argument('--rule_m',
type=int,
default=0,
help='Parameter for some rules (currently only for "krum"); optional, 0 for default.')
parser.add_argument('--experiment',
type=str,
default="slim-cifarnet-cifar10",
help='Problem and associated, fixed model to choose (one of "mnist", "cifar10", ...).')
parser.add_argument('--batch',
type=int,
default=128,
help='Batch size to use.')
parser.add_argument('--rate',
type=float,
default=1e-3,
help='(Base) learning rate to use.')
parser.add_argument('--nbbyzwrk',
type=int,
default=0,
help='(Declared) number of byzantine workers.')
parser.add_argument('--nbbyzps',
type=int,
default=0,
help='(Declared) number of byzantine PSes.')
parser.add_argument('--vanilla',
type=bool,
default=False,
help='If True, the vanilla, non-byzantine case is executed. This is the baseline that we should compare with.')
parser.add_argument('--asyncr',
type=bool,
default=False,
help='If True, the network is assumed to be asynchronous. Thus, only qourum is waited before proceeding.')
parser.add_argument('--smart',
type=bool,
default=False,
help='If True, the smart byzPS algorithm runs.')
parser.add_argument('--log',
type=bool,
default=False,
help='If True, detailed steps are printed.')
parser.add_argument('--bench',
type=bool,
default=False,
help='If True, time elapsed in each step is printed.')
parser.add_argument('--time_save',
type=bool,
default=False,
help='If True, a BW aggressive but time-saver algorithm is applied.')
parser.add_argument('--less_grad',
type=bool,
default=False,
help='If True, PS collects only 2f+3 grads instead of asking all workers to send.')
parser.add_argument('--l',
type=int,
default=1,
help='Lipschtiz constant l. This should depend on the loss function used. For now, I will keep it to 1.')

FLAGS = parser.parse_args()
T = int(1/(3*FLAGS.l*FLAGS.rate))
grad_length = lengths[FLAGS.experiment]

print("T: ", T)
print("Using smart? ", FLAGS.smart)
print("Using batch: ", FLAGS.batch)
# User home for slim (suppose you downloaded the associated files)
homedir = pathlib.Path.home()

# Parsing command line arguments
ps_hosts = FLAGS.ps_hosts.split(",")
ps_addrs = [addr[:addr.index(":")] for addr in ps_hosts]
ps_ports = [int(addr[addr.index(":")+1:]) for addr in ps_hosts]
num_ps = len(ps_hosts)

worker_hosts = FLAGS.worker_hosts.split(",")
worker_addrs = [addr[:addr.index(":")] for addr in worker_hosts]
worker_ports = [int(addr[addr.index(":")+1:]) for addr in worker_hosts]
num_workers = len(worker_hosts)

num_byzwrks = FLAGS.nbbyzwrk
num_byzps = FLAGS.nbbyzps
if FLAGS.job_name == "ps":
  uid = -1*(FLAGS.task_index+1)
else:
  uid = FLAGS.task_index+1

#Initializing the service class which holds all information may be pulled by the client (workers for PS and vice versa)
service = TrainMessageExchangeServicer(num_workers, FLAGS.smart, uid)
def serve():
    server = grpc.server(futures.ThreadPoolExecutor(max_workers=1000))
    all_pb2_grpc.add_TrainMessageExchangeServicer_to_server(
        service, server)
    server.add_insecure_port(ps_hosts[int(FLAGS.task_index)])
    print("gRPC server now will start at PS index ", FLAGS.task_index)
    sys.stdout.flush()
    return server

server = serve()
server.start()
time.sleep(10)

#Connect to all other PSes
def connect_PS():
  for i in range(num_ps):
    if i != FLAGS.task_index:
      channel = grpc.insecure_channel(ps_hosts[i], options=[('grpc.max_send_message_length', MAX_MESSAGE_LENGTH), ('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH)])
      stub = all_pb2_grpc.TrainMessageExchangeStub(channel)
      ps_conns.append(stub)

#Connect to all workers
def connect_workers():
  for i in range(num_workers):
    channel = grpc.insecure_channel(worker_hosts[i], options=[('grpc.max_send_message_length', MAX_MESSAGE_LENGTH), ('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH)])
    stub = all_pb2_grpc.TrainMessageExchangeStub(channel)
    worker_conns.append(stub)

#Read gradient from given connection
def get_grad(conn):
  global request
  read = False
  counter = 0		#So that we do not loop forever
  while not read:
    try:
      proto_grad = conn.GetGradients(request)
      grad = np.frombuffer(proto_grad.gradients, np.float32)
      read = True
    except Exception as e:
      print("PS read_gradients exception: ")
      counter+=1
      if counter > 10:			#any reasonable large enough number
        exit(0)
  return grad

quorum = (2*num_byzwrks + 3) if FLAGS.asyncr else num_workers
quorum_ps = (2*num_byzps + 3) if FLAGS.asyncr else num_ps
grads = [tf.placeholder(dtype=tf.float32, shape=(grad_length)) for _ in range(quorum)]
grads_ps = [tf.placeholder(dtype=tf.float32, shape=(grad_length)) for _ in range(quorum_ps)]
grads_nt = []
grads_ps_nt = []
#Create pools for parallelizing reading gradients
pool_ps = ThreadPool(quorum_ps)
pool_wk = ThreadPool(quorum)
#Read gradients from workers (or PSes, in case of the asyncr protocol)
def read_gradients(PS=False):
  if FLAGS.log:
    println("[PS] trying to read gradients")
  global grads_nt
  global grads_ps_nt
  if PS:
    grads_ps_nt = []
  else:
    grads_nt = []

# Read gradients in a parallel fashion
  if PS:
    if FLAGS.log:
      println("[PS] Preparing to read from how many PS? " + str(quorum_ps))
    grads_ps_nt = pool_ps.map(get_grad, ps_conns)
  else:
    if FLAGS.log:
      println("[PS] Preparing to read from how many Worker? " + str(quorum))
    grads_nt = pool_wk.map(get_grad, worker_conns)

  if FLAGS.log:
    println("At PS, done reading gradients")

request = all_pb2.Request()
request.req_id = uid
ps_conns = []
connect_PS()
empty = all_pb2.Empty()
worker_conns = []
connect_workers()

iteration_num=-1
graph = tf.get_default_graph()

#Building the graph....
with tf.device('/'+device+':'+str(FLAGS.task_index%2)):
  optimizer = tf.train.RMSPropOptimizer(FLAGS.rate) #tf.train.AdadeltaOptimizer() #tf.train.AdamOptimizer() #tf.train.RMSPropOptimizer(FLAGS.rate) #tf.train.GradientDescentOptimizer(FLAGS.rate)
  experiment = experiments.instantiate(FLAGS.experiment, ['batch-size:'+str(FLAGS.batch), 'eval-batch-size:1'])
  with tf.device('/cpu:*'):
    acc_tn = experiment.accuracy('/cpu:0', ['/'+device+':'+str(FLAGS.task_index%2)], trace=False)
  #Initializing aggregators.....
  #Krum
  q_size = num_workers
  if FLAGS.asyncr:
    q_size = (2*num_byzwrks) + 3
  krum_op = aggregators.instantiate("krum", q_size, num_byzwrks, None)
  #Average
  avg_op = aggregators.instantiate("average", num_workers, num_byzwrks, None)
  #Median
  med_op = aggregators.instantiate("median", quorum_ps, num_byzps, None)
  #Bulyan
  bul_op = aggregators.instantiate("bulyan", q_size, num_byzwrks, None)

  #The 'Experiment' interface related definitions
  loss_tn = experiment.losses('/cpu:0', ['/'+device+':'+str(FLAGS.task_index%2)], trace=False)
  grad_vars = optimizer.compute_gradients(loss_tn[0])
  gradient_tn, flatmap = helper.flatten(grad_vars)

  #2) Read the gradients for the new iteration and #3) The logic of aggregation
  grads = [tf.placeholder(dtype=tf.float32, shape=(grad_length)) for _ in range(quorum)]
  if FLAGS.smart:
    last_grad = krum_op.aggregate(grads)		#This list is defined above
  elif FLAGS.asyncr:
    last_grad = krum_op.aggregate(grads)			#Apply the strongest GAR
  elif FLAGS.vanilla:
    last_grad = avg_op.aggregate(grads)                #This list is defined above
  if FLAGS.asyncr:		#Then more steps are required before applying this aggregated gradients
    grads_ps = [tf.placeholder(dtype=tf.float32, shape=(grad_length)) for _ in range(quorum_ps)]
    med_grad = med_op.aggregate(grads_ps)
    apply_op = optimizer.apply_gradients(helper.inflate(med_grad, helper.mapflat(flatmap)))
  elif FLAGS.smart:
    grads_ps = [tf.placeholder(dtype=tf.float32, shape=(grad_length)) for _ in range(num_ps)]
    med_grad = med_op.aggregate(grads_ps)
    apply_op_gather = optimizer.apply_gradients(helper.inflate(med_grad, helper.mapflat(flatmap)))
    apply_op = optimizer.apply_gradients(helper.inflate(last_grad, helper.mapflat(flatmap)))
  else:
    #4) finally, apply this aggregation
    apply_op = optimizer.apply_gradients(helper.inflate(last_grad, helper.mapflat(flatmap)))

#Define get accuracy function separately to run it in a different thread
def get_acc(sess,train_t, cur_t, iter_n):
  acc = sess.run(acc_tn)
  for key, val in acc.items():
    println(key+":"+str(val)+" elapsed_time: "+str(cur_t - train_t)+" iteration: "+str(iter_n))
  println("Throughput: " + str(iter_n/(cur_t - train_t)) + "steps/sec")

#get unified model in the beginning so that all PSes are on the same page....
def getUnifiedModel(sess):
  model = ps_conns[0].GetUnifiedModel(request)
  model = np.frombuffer(model.model, np.float32)
  model_variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
  cur = 0
  if FLAGS.log:
    println("PS will try to write the same model of PS, received model is of length "+str(len(model)))
  for var in model_variables:
    println("Fetching new variable...")
    next = cur + tf.size(var)
    next = sess.run(next)
    assign = tf.assign(var, tf.reshape(model[cur:next],tf.shape(var)))
    sess.run(assign)
    cur = next
  println("PS is done with writing the new model....now this PS has the same model like the master PS, model size: "+str(len(model)))
  return model

#Read the current state of the model
model_variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
mod_res = []
for var in model_variables:
  mod_res.append(tf.reshape(var, (-1,)))
mod_res = tf.concat(mod_res, 0)
def read_model(sess):
  return sess.run(mod_res)

config=tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(graph=graph, config=config)
with sess.as_default():
  sess.run(tf.global_variables_initializer())
  coord = tf.train.Coordinator()
  tf.train.start_queue_runners(sess=sess, coord=coord) # If there is any queue in the graph

  #First, read the model and distribute it to everybody so that all of the machines are on the same page
  if FLAGS.task_index == 0:
    model_np = read_model(sess)
    service.mod.model = model_np.tobytes()
    service.mod.init = True
  else:
    model_np = getUnifiedModel(sess)

  if FLAGS.log:
    println(model_np)
    println("The model is of size: "+str(len(model_np)))

  train_t = time.time()
  while iteration_num < FLAGS.max_steps:
    if FLAGS.bench:
      iter_t = time.time()
    iteration_num+=1
    if FLAGS.log:
      println("PS iteration_num: "+str(iteration_num))
    #Get the accuracy from time to time
    if iteration_num%FLAGS.eval_steps == 1:
      if FLAGS.bench:
        acc_t = time.time()
      acc_th = threading.Thread(target=get_acc, args=(sess,train_t,time.time(),iteration_num,))
      acc_th.start()

      if FLAGS.bench:
        println("[PS "+str(FLAGS.task_index)+"] Time elapsed to get accuracy: " + str(time.time() - acc_t))
    if iteration_num > 0:
      #1) Prepare gradients from the last iteartion
      if FLAGS.log:
        println("PS prepares gradient for the next iteration")
      if FLAGS.bench:
        np_t = time.time()
      service.pbuf_grad.gradients = aggr_res.tobytes()			#convert grad tensor to a protocol buffer
      service.pbuf_grad.iter = iteration_num
      if FLAGS.bench:
        println("[PS "+str(FLAGS.task_index)+"] time to convert gradient to bytes: " + str(time.time() - np_t))
    if FLAGS.bench:
      read_t = time.time()
    request.iter = iteration_num
    read_gradients()
    if FLAGS.bench:
      println("[PS "+str(FLAGS.task_index)+"] time to read gradients is: " + str(time.time() - read_t))
      feed_t = time.time()
    #Now, feed values to input of aggregate: grads (from grads_nt numpy array)
    if FLAGS.log:
      println("Number of collected grads: " + str(len(grads_nt)))
    try:
      feed_dict = dict([(grads[i], grads_nt[i]) for i in range(len(grads))])
    except Exception as e:
      println(e)
      exit(0)
    if FLAGS.log:
      println("[PS] done feed dict....")
      for k,v in feed_dict.items():
        println(str(len(v)))
    if FLAGS.bench:
      println("[PS "+str(FLAGS.task_index)+"] time to create feed_dict is: " + str(time.time() - feed_t))
      run_t = time.time()
    try:
      if FLAGS.asyncr:
        aggr_res = sess.run(last_grad, feed_dict=feed_dict)
      else:
        aggr_res,_ = sess.run([last_grad, apply_op], feed_dict=feed_dict)
      if FLAGS.bench:
        println("[PS "+str(FLAGS.task_index)+"] time to aggregate and apply gradients: " + str(time.time() - run_t))
    except Exception as e:
      println("problems with running last_grad and apply_op")
      println(e)
      exit(0)
    if FLAGS.log:
      println("[PS] Aggregation at PS...done!")
    #If the asyncr protocol, one more round of exchanging gradients between PSes is required
    if FLAGS.asyncr or (FLAGS.smart and iteration_num%T == 0):
      if FLAGS.log:
        println("[PS] Starting the additional asyncronous block")
      service.pbuf_grad.gradients = aggr_res.tobytes()
      service.pbuf_grad.iter = iteration_num+0.5
      if FLAGS.bench:
        read_ps = time.time()
      read_gradients(PS=True)
      if FLAGS.bench:
        println("[PS "+str(FLAGS.task_index)+"] Time to read from other PSes: " + str(time.time()-read_ps))
      if FLAGS.log:
        println("[PS] Read gradients from other PSes")
      #Add myself to the list
      grads_ps_nt.append(aggr_res)
      #Feed the tensor and do the median
      feed_dict = dict([(grads_ps[i], grads_ps_nt[i]) for i in range(len(grads_ps))])

      if FLAGS.log:
        println("[PS] Done feeding to tensor")
      if FLAGS.bench:
        final_up = time.time()
      if FLAGS.asyncr:
        aggr_res,_ = sess.run([med_grad, apply_op], feed_dict=feed_dict)
      else:
        aggr_res,_ = sess.run([med_grad, apply_op_gather], feed_dict=feed_dict)
      if FLAGS.bench:
        print("[PS "+str(FLAGS.task_index)+"] Time to aggregate PS responses and apply them: " + str(time.time()-final_up))
      if FLAGS.log:
        println("[PS] Done the additional asyncronous block")

    if FLAGS.bench:
      println("[PS "+str(FLAGS.task_index)+"] time of one complete iteration: " + str(time.time() - iter_t))
time.sleep(10)
server.stop(0)

max_stars_repo_name: sahareslami/Garfield

max_issues_repo_name: sahareslami/Garfield

max_forks_repo_name: sahareslami/Garfield


text:
#!/usr/bin/python
import sys
import numpy as np

import util
import class_crawler

def invest_simulation(tickers, weight, time_start, time_end):
	# create object web_spider to fetch data
	spider = class_crawler.web_spider()
	returns = spider.yahoo_price(tickers[0], time_start, time_end, 'd')
	dates = sorted(returns.keys())
	buy_hold_return = []

	# get the starting and ending time in this file
	date_start = dates[0]
	date_end = dates[len(dates) - 1]
	buy_hold_return.append(returns[date_end].adjClose / returns[date_start].adjClose - 1)

	for ticker in tickers[1:]:
		returns = spider.yahoo_price(ticker, time_start, time_end, 'd')
		buy_hold_return.append(returns[date_end].adjClose / returns[date_start].adjClose - 1)
	realized_return = np.dot(weight, buy_hold_return)

	# calc the market return, namely S&P 500
	market_returns = spider.yahoo_price("^GSPC", time_start, time_end, 'd')
	market_return = market_returns[date_end].adjClose / market_returns[date_start].adjClose - 1

	print " Realized return:", util.perc(realized_return, 2), "from", time_start, "to", time_end
	print "  S&P 500 return:", util.perc(market_return, 2)
	return realized_return

def main():
	invest_simulation(["BABA"], [1], "2016-05-30", "2016-06-30")

if __name__ == "__main__":
	sys.exit(main())


text:
# -*- coding: utf-8 -*
import numpy as np

import torch
import torch.nn.functional as F
from torch import nn

from ...module_base import ModuleBase
from ..loss_base import TRACK_LOSSES
from .utils import SafeLog

eps = np.finfo(np.float32).tiny

@TRACK_LOSSES.register
class SigmoidCrossEntropyRetina(ModuleBase):

    default_hyper_params = dict(
        name="focal_ce",
        background=0,
        ignore_label=-1,
        weight=1.0,
        alpha=0.5,
        gamma=0.0,
    )

    def __init__(self, ):
        super(SigmoidCrossEntropyRetina, self).__init__()
        self.safelog = SafeLog()
        self.register_buffer("t_one", torch.tensor(1., requires_grad=False))

    def update_params(self, ):
        self.background = self._hyper_params["background"]
        self.ignore_label = self._hyper_params["ignore_label"]
        self.weight = self._hyper_params["weight"]
        # focal loss coefficients
        self.register_buffer(
            "alpha",
            torch.tensor(float(self._hyper_params["alpha"]),
                         requires_grad=False))
        self.register_buffer(
            "gamma",
            torch.tensor(float(self._hyper_params["gamma"]),
                         requires_grad=False))

    # def forward(self, pred, label):
    def forward(self, pred_data, target_data):
        r"""
        Focal loss
        :param pred: shape=(B, HW, C), classification logits (BEFORE Sigmoid)
        :param label: shape=(B, HW)
        """
        r"""
        Focal loss
        Arguments
        ---------
        pred: torch.Tensor
            classification logits (BEFORE Sigmoid)
            format: (B, HW)
        label: torch.Tensor
            training label
            format: (B, HW)

        Returns
        -------
        torch.Tensor
            scalar loss
            format: (,)
        """
        # resolve prediction & target
        pred = pred_data["cls_pred"]
        label = target_data["cls_gt"]
        # mask
        mask = ~(label == self.ignore_label)
        mask = mask.type(torch.Tensor).to(label.device)
        vlabel = label * mask
        zero_mat = torch.zeros(pred.shape[0], pred.shape[1], pred.shape[2] + 1)

        one_mat = torch.ones(pred.shape[0], pred.shape[1], pred.shape[2] + 1)
        index_mat = vlabel.type(torch.LongTensor)

        onehot_ = zero_mat.scatter(2, index_mat, one_mat)
        onehot = onehot_[:, :, 1:].type(torch.Tensor).to(pred.device)

        pred = torch.sigmoid(pred)
        pos_part = (1 - pred)**self.gamma * onehot * self.safelog(pred)
        neg_part = pred**self.gamma * (1 - onehot) * self.safelog(1 - pred)
        loss = -(self.alpha * pos_part +
                 (1 - self.alpha) * neg_part).sum(dim=2) * mask.squeeze(2)

        positive_mask = (label > 0).type(torch.Tensor).to(pred.device)

        loss = loss.sum() / torch.max(positive_mask.sum(),
                                      self.t_one) * self._hyper_params["weight"]
        extra = dict()

        return loss, extra

if __name__ == '__main__':
    B = 16
    HW = 17 * 17
    pred_cls = pred_ctr = torch.tensor(
        np.random.rand(B, HW, 1).astype(np.float32))
    pred_reg = torch.tensor(np.random.rand(B, HW, 4).astype(np.float32))

    gt_cls = torch.tensor(np.random.randint(2, size=(B, HW, 1)),
                          dtype=torch.int8)
    gt_ctr = torch.tensor(np.random.rand(B, HW, 1).astype(np.float32))
    gt_reg = torch.tensor(np.random.rand(B, HW, 4).astype(np.float32))

    criterion_cls = SigmoidCrossEntropyRetina()
    loss_cls = criterion_cls(pred_cls, gt_cls)

    criterion_ctr = SigmoidCrossEntropyCenterness()
    loss_ctr = criterion_ctr(pred_ctr, gt_ctr, gt_cls)

    criterion_reg = IOULoss()
    loss_reg = criterion_reg(pred_reg, gt_reg, gt_cls)

    from IPython import embed
    embed()


text:
import sys

import tensorflow as tf

import lidbox.features as features
import lidbox.features.audio as audio_features

def tf_print(*args, **kwargs):
    if "summarize" not in kwargs:
        kwargs["summarize"] = -1
    if "output_stream" not in kwargs:
        kwargs["output_stream"] = sys.stdout
    return tf.print(*args, **kwargs)

def make_label2onehot(labels):
    """
    >>> labels = tf.constant(["one", "two", "three"], tf.string)
    >>> label2int, OH = make_label2onehot(labels)
    >>> for i in range(3):
    ...     (label2int.lookup(labels[i]).numpy(), tf.math.argmax(OH[i]).numpy())
    (0, 0)
    (1, 1)
    (2, 2)
    """
    num_labels = tf.size(labels)
    labels_enum = tf.range(num_labels)
    # Label to int or one past last one if not found
    # TODO slice index out of bounds is probably not a very informative error message
    out_of_bounds_result = tf.cast(num_labels, dtype=tf.int32)
    label2int = tf.lookup.StaticHashTable(
        tf.lookup.KeyValueTensorInitializer(labels, labels_enum),
        out_of_bounds_result)
    OH = tf.one_hot(labels_enum, num_labels)
    return label2int, OH

def matplotlib_colormap_to_tensor(colormap_key):
    """
    Given a matplotlib colormap name, extract all RGB values from its cmap numpy array into a tf.constant.
    """
    from matplotlib.cm import get_cmap
    from numpy import arange
    cmap = get_cmap(colormap_key)
    rgb = cmap(arange(cmap.N + 1))[:,:3]
    return tf.convert_to_tensor(rgb)

@tf.function
def tensors_to_rgb_images(inputs, colors, size_multiplier=1):
    """
    Map all values of 'inputs' between [0, 1] and then into RGB color indices.
    Gather colors from 'colors' using the indices.
    Based on https://gist.github.com/jimfleming/c1adfdb0f526465c99409cc143dea97b
    """
    tf.debugging.assert_rank(inputs, 3, message="tensors_to_rgb_images expects batches of 2 dimensional tensors with shape [batch, cols, rows].")
    tf.debugging.assert_rank(colors, 2, message="tensors_to_rgb_images expects a colormap of shape [color, component].")
    tf.debugging.assert_equal(tf.shape(colors)[1], 3, message="tensors_to_rgb_images expects an RGB colormap.")
    # Scale features between 0 and 1 to produce a grayscale image
    inputs = features.feature_scaling(inputs, tf.constant(0.0), tf.constant(1.0))
    # Map linear colormap over all grayscale values [0, 1] to produce an RGB image
    indices = tf.cast(tf.math.round(inputs * tf.cast(tf.shape(colors)[0] - 1, tf.float32)), tf.int32)
    tf.debugging.assert_non_negative(indices, message="Negative color indices")
    images = tf.gather(colors, indices, axis=0, batch_dims=0)
    tf.debugging.assert_rank(images, 4, message="Gathering colors failed, output images do not have a channel dimension. Make sure the inputs have a known shape.")
    # Here it is assumed the output images are going to Tensorboard
    images = tf.image.transpose(images)
    images = tf.image.flip_up_down(images)
    # Rows and columns from shape
    old_size = tf.cast(tf.shape(images)[1:3], tf.float32)
    new_size = tf.cast(size_multiplier * old_size, tf.int32)
    images = tf.image.resize(images, new_size)
    tf.debugging.assert_all_finite(images, message="Tensor conversion to RGB images failed, non-finite values in output")
    return images

@tf.function
def count_dim_sizes(ds, element_key=0, ndims=1):
    """
    Given a dataset 'ds' of 'ndims' dimensional tensors at element index 'element_key', accumulate the shape counts of all tensors.
    >>> batch, x, y = 10, 3, 4
    >>> data = tf.random.normal((batch, x, y))
    >>> meta = tf.random.normal((batch, 1))
    >>> ds = tf.data.Dataset.from_tensor_slices((data, meta))
    >>> data_sizes = count_dim_sizes(ds, 0, 2)
    >>> x_size = tf.squeeze(data_sizes[0], 0)
    >>> tf.math.reduce_all(x_size == [batch, x]).numpy()
    True
    >>> y_size = tf.squeeze(data_sizes[1], 0)
    >>> tf.math.reduce_all(y_size == [batch, y]).numpy()
    True
    >>> meta_size = tf.squeeze(count_dim_sizes(ds, 1, 1), [0, 1])
    >>> tf.math.reduce_all(meta_size == [batch, 1]).numpy()
    True
    """
    tf.debugging.assert_greater(ndims, 0)
    get_shape_at_index = lambda t: tf.shape(t[element_key])
    shapes_ds = ds.map(get_shape_at_index)
    ones = tf.ones(ndims, dtype=tf.int32)
    shape_indices = tf.range(ndims, dtype=tf.int32)
    def accumulate_dim_size_counts(counter, shape):
        enumerated_shape = tf.stack((shape_indices, shape), axis=1)
        return tf.tensor_scatter_nd_add(counter, enumerated_shape, ones)
    max_sizes = shapes_ds.reduce(
        tf.zeros(ndims, dtype=tf.int32),
        lambda acc, shape: tf.math.maximum(acc, shape))
    max_max_size = tf.math.reduce_max(max_sizes)
    size_counts = shapes_ds.reduce(
        tf.zeros((ndims, max_max_size + 1), dtype=tf.int32),
        accumulate_dim_size_counts)
    sorted_size_indices = tf.argsort(size_counts, direction="DESCENDING")
    sorted_size_counts = tf.gather(size_counts, sorted_size_indices, batch_dims=1)
    is_nonzero = sorted_size_counts > 0
    return tf.ragged.stack(
        (tf.ragged.boolean_mask(sorted_size_counts, is_nonzero),
         tf.ragged.boolean_mask(sorted_size_indices, is_nonzero)),
        axis=2)

def reduce_min_max_num_sum(ds, key, batch_size):
    """
    Given a dataset 'ds', reduce all tensors at element index 'element_key' over the whole dataset into scalar values: minimum, maximum, count, sum.
    Reductions can be computed in batches.
    """
    def _accumulate_batch(c, x):
        min = tf.math.minimum(c[0], tf.cast(tf.math.reduce_min(x[key]), tf.float64))
        max = tf.math.maximum(c[1], tf.cast(tf.math.reduce_max(x[key]), tf.float64))
        num = c[2] + tf.size(x[key], tf.int64)
        sum = c[3] + tf.cast(tf.math.reduce_sum(x[key]), tf.float64)
        return min, max, num, sum
    min = tf.constant(tf.float64.max)
    max = tf.constant(tf.float64.min)
    num = tf.constant(0, tf.int64)
    sum = tf.constant(0, tf.float64)
    return (ds.batch(batch_size)
              .reduce((min, max, num, sum), _accumulate_batch))

def compute_vad_decision_stats(original_ds, batch_size):
    def _accumulate_vad_batches(c, vad_decisions):
        num_speech = tf.math.reduce_sum(tf.cast(vad_decisions, tf.int64), axis=-1)
        num_not_speech = tf.math.reduce_sum(tf.cast(~vad_decisions, tf.int64), axis=-1)
        return c[0] + 1, c[1] + num_speech, c[2] + num_not_speech
    def _make_init_reduce_values(n):
        return tf.constant(0, tf.int64), tf.zeros([n], tf.int64), tf.zeros([n], tf.int64)
    # Drop all unnecessary data
    ds = original_ds.map(lambda x: x["vad_is_speech"])
    num_batches, num_speech_batch, num_not_speech_batch = (ds
            .batch(batch_size, drop_remainder=True)
            .reduce(_make_init_reduce_values(batch_size), _accumulate_vad_batches))
    num_batched = batch_size * num_batches
    num_remainder, num_speech_r, num_not_speech_r = (ds
            .skip(num_batched)
            .batch(1)
            .reduce(_make_init_reduce_values(1), _accumulate_vad_batches))
    num = num_batched + num_remainder
    num_speech = tf.math.reduce_sum(num_speech_batch) + tf.math.reduce_sum(num_speech_r)
    num_not_speech = tf.math.reduce_sum(num_not_speech_batch) + tf.math.reduce_sum(num_not_speech_r)
    speech_ratio = tf.math.divide_no_nan(
            tf.cast(num_speech, tf.float64),
            tf.cast(num_not_speech + num_speech, tf.float64))
    return num, num_speech, num_not_speech, speech_ratio

@tf.function
def extract_features(signals, sample_rates, feattype, spec_kwargs, melspec_kwargs, mfcc_kwargs, db_spec_kwargs, feat_scale_kwargs, window_norm_kwargs):
    tf.debugging.assert_rank(signals, 2, message="Input signals for feature extraction must be batches of mono signals without channels, i.e. of shape [B, N] where B is batch size and N number of samples.")
    tf.debugging.assert_equal(sample_rates, [sample_rates[0]], message="Different sample rates in a single batch not supported, all signals in the same batch should have the same sample rate.")
    #TODO batches with different sample rates (probably not worth the effort)
    sample_rate = sample_rates[0]
    X = audio_features.spectrograms(signals, sample_rate, **spec_kwargs)
    tf.debugging.assert_all_finite(X, "spectrogram failed")
    if feattype in ("melspectrogram", "logmelspectrogram", "mfcc"):
        X = audio_features.melspectrograms(X, sample_rate=sample_rate, **melspec_kwargs)
        tf.debugging.assert_all_finite(X, "melspectrogram failed")
        if feattype in ("logmelspectrogram", "mfcc"):
            X = tf.math.log(X + 1e-6)
            tf.debugging.assert_all_finite(X, "logmelspectrogram failed")
            if feattype == "mfcc":
                coef_begin = mfcc_kwargs.get("coef_begin", 1)
                coef_end = mfcc_kwargs.get("coef_end", 13)
                mfccs = tf.signal.mfccs_from_log_mel_spectrograms(X)
                X = mfccs[..., coef_begin:coef_end]
                tf.debugging.assert_all_finite(X, "mfcc failed")
    elif feattype in ("db_spectrogram",):
        X = audio_features.power_to_db(X, **db_spec_kwargs)
        tf.debugging.assert_all_finite(X, "db_spectrogram failed")
    if feat_scale_kwargs:
        X = features.feature_scaling(X, **feat_scale_kwargs)
        tf.debugging.assert_all_finite(X, "feature scaling failed")
    if window_norm_kwargs:
        X = features.window_normalization(X, **window_norm_kwargs)
        tf.debugging.assert_all_finite(X, "window normalization failed")
    return X

max_stars_repo_name: ishine/lidbox

max_issues_repo_name: AI-X-King/lidbox

max_forks_repo_name: AI-X-King/lidbox

reason: from numpy


text:
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# Copyright (c) 2011-2020, wradlib developers.
# Distributed under the MIT License. See LICENSE.txt for more info.

import numpy as np

from wradlib import classify, io, util

from . import has_data, requires_data

@requires_data
class TestHydrometeorClassification:
    if has_data:
        filename = util.get_wradlib_data_file("misc/msf_xband.gz")
        msf = io.get_membership_functions(filename)
        msf_idp = msf[0, 0, :, 0]
        msf_obs = msf[..., 1:]

        hmca = np.array(
            [
                [4.34960938, 15.68457031, 14.62988281],
                [7.78125, 5.49902344, 5.03808594],
                [0.49659729, 0.22286987, 0.86561584],
                [-9.11071777, -1.60217285, 11.15356445],
                [25.6, 25.6, 25.6],
            ]
        )

        msf_val = classify.msf_index_indep(msf_obs, msf_idp, hmca[0])
        fu = classify.fuzzyfi(msf_val, hmca)
        w = np.array([2.0, 1.0, 1.0, 1.0, 1.0])
        prob = classify.probability(fu, w)

    def test_msf_index_indep(self):
        tst = np.array([-20, 10, 110])
        res = np.array(
            [[[[0.0, 0.0, 0.0, 0.0], [5.0, 10.0, 35.0, 40.0], [0.0, 0.0, 0.0, 0.0]]]]
        )
        msf_val = classify.msf_index_indep(self.msf_obs[0:1, 0:1], self.msf_idp, tst)
        np.testing.assert_array_equal(msf_val, res)

    def test_fuzzify(self):
        res = np.array(
            [
                [0.0, 1.0, 1.0],
                [0.0, 0.0, 0.0],
                [0.0, 0.0, 1.0],
                [0.0, 0.0, 0.0],
                [1.0, 1.0, 1.0],
            ]
        )
        fu = classify.fuzzyfi(self.msf_val, self.hmca)
        np.testing.assert_array_equal(fu[0], res)

    def test_probability(self):
        res = np.array(
            [
                [0.16666667, 0.5, 0.66666667],
                [0.0, 0.0, 0.0],
                [0.0, 0.0, 0.0],
                [0.0, 0.21230469, 0.33333333],
                [0.0, 0.0, 0.0],
                [0.0, 0.0, 0.0],
                [0.0, 0.0, 0.0],
                [0.0, 0.33333333, 0.33333333],
                [0.0, 0.33333333, 0.47532552],
                [0.28997396, 0.5, 0.5],
                [0.28997396, 0.5, 0.33333333],
            ]
        )
        prob = classify.probability(self.fu, self.w)
        np.testing.assert_array_almost_equal(prob, res, decimal=8)

    def test_classify(self):
        res_idx = np.array(
            [
                [1, 1, 1],
                [2, 2, 2],
                [3, 4, 4],
                [4, 5, 5],
                [5, 6, 6],
                [6, 11, 11],
                [7, 3, 3],
                [8, 7, 7],
                [11, 8, 10],
                [0, 0, 8],
                [9, 9, 9],
                [10, 10, 0],
            ]
        )
        res_vals = np.array(
            [
                [0.0, 0.0, 0.0],
                [0.0, 0.0, 0.0],
                [0.0, 0.0, 0.0],
                [0.0, 0.0, 0.0],
                [0.0, 0.0, 0.0],
                [0.0, 0.0, 0.0],
                [0.0, 0.21230469, 0.33333333],
                [0.0, 0.33333333, 0.33333333],
                [0.0, 0.33333333, 0.33333333],
                [0.16666667, 0.5, 0.47532552],
                [0.28997396, 0.5, 0.5],
                [0.28997396, 0.5, 0.66666667],
            ]
        )

        hmc_idx, hmc_vals = classify.classify(self.prob, threshold=0.0)

        np.testing.assert_array_almost_equal(hmc_idx, res_idx)
        np.testing.assert_array_almost_equal(hmc_vals, res_vals)

max_stars_repo_name: syedhamidali/wradlib

max_issues_repo_name: syedhamidali/wradlib

max_forks_repo_name: syedhamidali/wradlib


text:
import numpy as np
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import LeaveOneOut
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report

def jho(feat, label, opts):
    k  = 5
    ho = 0.3   # ratio of testing set

    if 'k' in opts:
        k  = opts['k']
    if 'ho' in opts:
        ho = opts['ho']

    # number of instances
    num_data = np.size(feat, 0)
    label    = label.reshape(num_data)  # Solve bug

    # prepare data
    xtrain, xtest, ytrain, ytest = train_test_split(feat, label, test_size=ho, stratify=label)

    # train model
    mdl     = KNeighborsClassifier(n_neighbors = k)
    mdl.fit(xtrain, ytrain)
    # prediction
    ypred   = mdl.predict(xtest)

    # confusion matric
    uni     = np.unique(ytest)
    confmat = confusion_matrix(ytest, ypred, labels=uni)
    # report
    report  = classification_report(ytest, ypred)
    # accuracy
    acc     = np.sum(ytest == ypred) / np.size(xtest,0)

    print("Accuracy (KNN_HO):", 100 * acc)

    knn = {'acc': acc, 'con': confmat, 'r': report}

    return knn

def jkfold(feat, label, opts):
    k     = 5
    kfold = 10   # number of k in kfold

    if 'k' in opts:
        k     = opts['k']
    if 'kfold' in opts:
        kfold = opts['kfold']

    # number of instances
    num_data = np.size(feat, 0)
    # define selected features
    x_data   = feat
    y_data   = label.reshape(num_data)  # Solve bug

    fold     = StratifiedKFold(n_splits = kfold)
    fold.get_n_splits(x_data, y_data)

    ytest2 = []
    ypred2 = []
    Afold2 = []
    for train_idx, test_idx in fold.split(x_data, y_data):
        xtrain  = x_data[train_idx,:]
        ytrain  = y_data[train_idx]
        xtest   = x_data[test_idx,:]
        ytest   = y_data[test_idx]
        # train model
        mdl     = KNeighborsClassifier(n_neighbors = k)
        mdl.fit(xtrain, ytrain)
        # prediction
        ypred   = mdl.predict(xtest)
        # accuracy
        Afold   = np.sum(ytest == ypred) / np.size(xtest,0)

        ytest2  = np.concatenate((ytest2, ytest), axis=0)
        ypred2  = np.concatenate((ypred2, ypred), axis=0)
        Afold2.append(Afold)

    # average accuracy
    Afold2  = np.array(Afold2)
    acc     = np.mean(Afold2)
    # confusion matric
    uni     = np.unique(ytest2)
    confmat = confusion_matrix(ytest2, ypred2, labels=uni)
    # report
    report  = classification_report(ytest2, ypred2)

    print("Accuracy (KNN_K-fold):", 100 * acc)

    knn = {'acc': acc, 'con': confmat, 'r': report}

    return knn

def jloo(feat, label, opts):
    k = 5

    if 'k' in opts:
        k     = opts['k']

    # number of instances
    num_data = np.size(feat, 0)
    # define selected features
    x_data   = feat
    y_data   = label.reshape(num_data)  # Solve bug

    loo      = LeaveOneOut()
    loo.get_n_splits(x_data)

    ytest2 = []
    ypred2 = []
    Afold2 = []
    for train_idx, test_idx in loo.split(x_data):
        xtrain = x_data[train_idx,:]
        ytrain = y_data[train_idx]
        xtest  = x_data[test_idx,:]
        ytest  = y_data[test_idx]
        # train model
        mdl     = KNeighborsClassifier(n_neighbors = k)
        mdl.fit(xtrain, ytrain)
        # prediction
        ypred   = mdl.predict(xtest)
        # accuracy
        Afold   = np.sum(ytest == ypred) / np.size(xtest,0)

        ytest2  = np.concatenate((ytest2, ytest), axis=0)
        ypred2  = np.concatenate((ypred2, ypred), axis=0)
        Afold2.append(Afold)

    # average accuracy
    Afold2  = np.array(Afold2)
    acc     = np.mean(Afold2)
    # confusion matric
    uni     = np.unique(ytest2)
    confmat = confusion_matrix(ytest2, ypred2, labels=uni)
    # report
    report  = classification_report(ytest2, ypred2)

    print("Accuracy (KNN_LOO):", 100 * acc)

    knn = {'acc': acc, 'con': confmat, 'r': report}

    return knn


text:
"""Test is_povm."""
import numpy as np

from toqito.measurement_props import is_povm
from toqito.random import random_povm

def test_is_povm_true():
    """Test if valid measurement returns True."""
    dim, num_inputs, num_outputs = 2, 2, 2
    measurements = random_povm(dim, num_inputs, num_outputs)

    np.testing.assert_equal(is_povm([measurements[:, :, 0, 0], measurements[:, :, 0, 1]]), True)

def test_is_povm_false():
    """Test if invalid POVM returns False."""
    non_meas_1 = np.array([[1, 2], [3, 4]])
    non_meas_2 = np.array([[5, 6], [7, 8]])
    non_meas = [non_meas_1, non_meas_2]

    np.testing.assert_equal(is_povm(non_meas), False)

def test_is_povm_false_not_sum_identity():
    """Test if invalid POVM (does not sum to identity)."""
    non_meas_1 = np.array([[1, 0], [0, 1]])
    non_meas_2 = np.array([[1, 0], [0, 1]])
    non_meas = [non_meas_1, non_meas_2]

    np.testing.assert_equal(is_povm(non_meas), False)

if __name__ == "__main__":
    np.testing.run_module_suite()

max_stars_repo_name: paniash/toqito

max_issues_repo_name: paniash/toqito

max_forks_repo_name: paniash/toqito


text:
#!/usr/bin/env python2
'''
    Client code for collecting data with a robot in an episodic manner.
    Triggers model learning when new data is available, and policy learning
    whe new model is available
'''
import argparse
import dill
import numpy as np
import os
import pickle
import rospy
import requests
import threading
import traceback
import yaml

from collections import OrderedDict
from functools import partial
from Queue import Queue, Empty
from robot_learning.ros_plant import ROSPlant

from kusanagi.base import (apply_controller, train_dynamics,
                           preprocess_angles, ExperienceDataset)
from kusanagi.ghost.algorithms import mc_pilco
from kusanagi.ghost.control import RandPolicy
from kusanagi import utils

compile_lock = threading.RLock()

def numpy_code_constructor(loader, node):
    code_string = loader.construct_scalar(node)
    return eval(code_string)

def include_constructor(loader, node):
    filename = loader.construct_scalar(node)
    if not os.path.isabs(filename):
        root = os.path.dirname(loader.stream.name)
        filename = os.path.abspath(os.path.join(root, filename))
    data = {}
    with open(filename, 'r') as f:
        data = yaml.load(f)
    return data

def default_config():
    config = dict(
        initial_random_trials=4,
        output_directory='/data/robot_learning',
    )
    return config

def parse_config(config_path):
    '''
        loads configuration for learning tasks (policy and costs parameters)
        from a yaml file
    '''
    yaml.add_constructor('!include', include_constructor)
    yaml.add_constructor('!numpy', numpy_code_constructor)
    config = default_config()
    with open(config_path, 'r') as f:
        config = yaml.load(f)
    return config

def mc_pilco_polopt(task_name, task_spec, task_queue):
    '''
    executes one iteration of mc_pilco (model updating and policy optimization)
    '''
    # get task specific variables
    dyn = task_spec['transition_model']
    exp = task_spec['experience']
    pol = task_spec['policy']
    plant_params = task_spec['plant']
    immediate_cost = task_spec['cost']['graph']
    H = int(np.ceil(task_spec['horizon_secs']/plant_params['dt']))
    n_samples = task_spec.get('n_samples', 100)

    if task_state[task_name] != 'init':
        # train dynamics model. TODO block if training multiple tasks with
        # the same model
        if dyn.optimizer.loss_fn is None:
            with compile_lock:
                train_dynamics(
                    dyn, exp, pol.angle_dims,
                    wrap_angles=task_spec['wrap_angles'])
        else:
            train_dynamics(
                dyn, exp, pol.angle_dims,
                wrap_angles=task_spec['wrap_angles'])

        # init policy optimizer if needed
        optimizer = task_spec['optimizer']
        if optimizer.loss_fn is None:
            task_state[task_name] = 'compile_polopt'

            # get policy optimizer options
            split_H = task_spec.get('split_H', 1)
            mm_state = task_spec.get('mm_state', True)
            mm_cost = task_spec.get('mm_cost', True)
            noisy_policy_input = task_spec.get('noisy_policy_input', False)
            noisy_cost_input = task_spec.get('noisy_cost_input', False)
            truncate_gradient = task_spec.get('truncate_gradient', -1)
            learning_rate = task_spec.get('learning_rate', 1e-3)
            gradient_clip = task_spec.get('gradient_clip', 1.0)
            crn = task_spec.get('crn', 500)

            # get extra inputs, if needed
            import theano
            import theano.tensor as tt
            ex_in = OrderedDict(
                [(k, v) for k, v in immediate_cost.keywords.items()
                 if type(v) is tt.TensorVariable
                 and len(v.get_parents()) == 0])
            task_spec['extra_in'] = ex_in

            # build loss function
            outputs, inps, updts = mc_pilco.get_loss(
                pol, dyn, immediate_cost,
                n_samples=n_samples,
                mm_cost=mm_cost,
                mm_state=mm_state,
                noisy_cost_input=noisy_cost_input,
                noisy_policy_input=noisy_policy_input,
                split_H=split_H,
                truncate_gradient=truncate_gradient,
                crn=crn,
                intermediate_outs=True,
                **ex_in)
            loss = outputs[0]
            inps += ex_in.values()

            # add loss function as objective for optimizer
            with compile_lock:
                task_spec['rollout_fn'] = theano.function(inps, outputs, updates=updts)
                optimizer.set_objective(
                    loss, pol.get_params(symbolic=True), inps, updts,
                    clip=gradient_clip, learning_rate=learning_rate)
            task_spec['opt_iters'] = 0

        # train policy # TODO block if learning a multitask policy
        task_state[task_name] = 'update_polopt'
        # build inputs to optimizer
        p0 = plant_params['state0_dist']
        gamma = task_spec['discount']
        polopt_args = [p0.mean, p0.cov, H, gamma]
        extra_in = task_spec.get('extra_in', OrderedDict)
        if len(extra_in) > 0:
            polopt_args += [task_spec['cost']['params'][k] for k in extra_in]

        # update dyn and pol (resampling)
        def callback(*args, **kwargs):
            task_spec['opt_iters'] += 1
            crn = task_spec.get('crn', 500)
            if crn > 0  and task_spec['opt_iters'] % crn == 0:
                if hasattr(dyn, 'update'):
                    dyn.update(n_samples)
                if hasattr(pol, 'update'):
                    pol.update(n_samples)
        # call minimize
        callback()
        optimizer.minimize(
            *polopt_args, return_best=task_spec['return_best'],
            callback=callback)
        pol.save(output_filename='policy_%s_%d.zip'% (task_name, exp.n_episodes()))
        task_state[task_name] = 'ready'

    # check if task is done
    init_polopt_iter = task_spec.get('init_polopt_iter', 0)
    n_polopt_iters = len([p for p in exp.policy_parameters if len(p) > 0])
    if n_polopt_iters > task_spec['n_opt'] + init_polopt_iter:
        task_state[task_name] = 'done'
    # put task in the queue for execution
    task_queue.put((task_name, task_spec))

    return

def http_polopt(task_name, task_spec, task_queue):
    # TODO: Automate the harcoded url requests and responses
    url = "http://mc_pilco_server:8008/get_task_init_status/%s" % task_name

    # check if task id exists in server
    http_response = requests.get(url)
    rospy.loginfo(http_response.text)

    # if task_name doesn't exists then upload the task_spec for task_name
    if http_response.text == "get_task_init_status/%s: NOT FOUND" % task_name:
        url = "http://mc_pilco_server:8008/init_task/%s" % task_name
        tspec_pkl = pickle.dumps(task_spec, 2)
        http_response = requests.post(
            url, files={'tspec_file': ('task_spec.pkl', tspec_pkl)})
        rospy.loginfo(http_response.text)
        # TODO: Error Handling

    # TODO: Error handling if the task_spec upload fails
    # send latest experience for task_name
    url = "http://mc_pilco_server:8008/optimize/%s" % task_name
    exp_pkl = pickle.dumps(task_spec['experience'], 2)
    pol_params_pkl = pickle.dumps(
        task_spec['policy'].get_params(symbolic=False), 2)

    http_response = requests.post(
        url,
        files={
            'exp_file': ('experiance.pkl', exp_pkl),
            'pol_params_file': ('policy_params.pkl', pol_params_pkl)
        }
    )

    pol_params = pickle.loads(http_response.text)
    task_spec['policy'].set_params(pol_params)

    task_queue.put((task_name, task_spec))

if __name__ == '__main__':
    np.set_printoptions(linewidth=200, precision=3)
    rospy.init_node('kusanagi_ros', disable_signals=True)

    parser = argparse.ArgumentParser(
        'rosrun robot_learning task_client.py')
    parser.add_argument(
        'config_path', metavar='FILE',
        help='A YAML file containing the configuration for the learning task.',
        type=str)
    parser.add_argument(
        '-e', '--load_experience',
        help="load past experience if available",
        action="store_true")
    # args = parser.parse_args()
    args = parser.parse_args(rospy.myargv()[1:])
    load_experience = args.load_experience

    # import yaml
    config = parse_config(args.config_path)

    # init output dir
    output_directory = config['output_directory']
    utils.set_output_dir(output_directory)
    try:
        os.makedirs(output_directory)
    except Exception as e:
        if not load_experience:
            # move the old stuff
            dir_time = str(os.stat(output_directory).st_ctime)
            target_dir = os.path.dirname(output_directory)+'_'+dir_time
            os.rename(output_directory, target_dir)
            os.mkdir(output_directory)
            utils.print_with_stamp(
                'Moved old results from [%s] to [%s]' % (output_directory,
                                                         target_dir))

    utils.print_with_stamp('Results will be saved in [%s]' % output_directory)

    # init environment with first task params
    plant_params = config['tasks'].values()[0]['plant']
    env = ROSPlant(**plant_params)

    # init task queue and list of learning threads
    tasks = Queue()
    task_state = {}
    polopt_threads = []

    # populate task queue
    for task_name in config['tasks']:
        task_state[task_name] = 'init'
        spec = config['tasks'][task_name]
        exp = spec.get('experience', None)
        pol = spec['policy']
        pol(np.zeros(pol.D))
        random_exp_path = spec.get('random_exp_path', None)
        if exp is None:
            exp = ExperienceDataset(name=task_name)
            if not exp.load() and random_exp_path is not None:
                base_path, filename = os.path.split(
                    random_exp_path)
                fname = exp.filename
                exp.load(base_path, filename)
                # restore previous filename
                exp.filename = fname

            if exp.n_episodes() > 0:
                if len(exp.policy_parameters[-1]) > 0:
                    pol.set_params(exp.policy_parameters[-1])
                # exp.truncate(spec['initial_random_trials'])
                spec['initial_random_trials'] -= exp.n_episodes()
                # in case we are loading policy parameters, increase the n_opt counter
                n_polopt_iters = len([p for p in exp.policy_parameters if len(p) > 0])
                spec['init_polopt_iter'] = n_polopt_iters
                print n_polopt_iters
                task_state[task_name] = 'ready'
        spec['experience'] = exp
        # trigger policy init (for kusanagi only)

        # Optimize policy on the loaded experience
        if exp.n_episodes() > 0:
            polopt_fn = spec.get('polopt_fn',
                        config.get('default_polopt_fn',
                                mc_pilco_polopt))
            polopt_fn(task_name, spec, tasks)
        else:
            tasks.put((task_name, spec))
        with open(
            os.path.join(
                output_directory, '%s_spec.dill') % (task_name), 'wb') as f:
            dill.dump(spec, f)

    # while tasks are not done
    while not all([st == 'done' for st in task_state]):
        # get new task
        new_task_ready = False
        rospy.loginfo('Waiting for new task')
        while not new_task_ready:
            try:
                name, spec = tasks.get(timeout=5)
                new_task_ready = True
            except Empty:
                pass
        #utils.set_logfile("%s.log" % name, base_path="/localdata")
        # if task is done, pass
        exp = spec.get('experience')
        n_rnd = len([p for p in exp.policy_parameters if len(p) == 0])

        msg_ = '==== Executing %s task [iteration %d] ====' % (
            name, exp.n_episodes()+1-n_rnd)
        rospy.loginfo(msg_)
        #utils.print_with_stamp(msg_)

        # set plant parameters for current task
        plant_params = spec['plant']
        env.init_params(**plant_params)

        # load policy
        if task_state[name] == 'init' and spec['initial_random_trials'] > 0:
            # collect random experience
            pol = RandPolicy(maxU=spec['policy'].maxU,
                             minU=spec['policy'].minU,
                             random_walk=spec.get('random_walk', False))
            spec['initial_random_trials'] -= 1
            if spec['initial_random_trials'] < 1:
                task_state[name] = 'ready'
        else:
            # TODO load policy parameters from disk
            pol = spec['policy']

        polopt_fn = spec.get('polopt_fn',
                             config.get('default_polopt_fn',
                                        mc_pilco_polopt))

        # set task horizon
        H = int(np.ceil(spec['horizon_secs']/env.dt))

        # execute tasks and collect experience data
        preprocess = None
        if hasattr(pol, 'angle_dims'):
            preprocess = partial(
                preprocess_angles, angle_dims=pol.angle_dims)
        experience = apply_controller(env, pol, H, preprocess)
        # print experience[0]
        # append new experience to dataset
        states, actions, costs, infos = experience
        ts = [info.get('t', None) for info in infos]
        pol_params = (pol.get_params(symbolic=False)
                      if hasattr(pol, 'get_params') else [])

        exp.append_episode(
            states, actions, costs, infos, pol_params, ts)

        exp.save()
        if task_state[name] == 'init':
            # save random experience for later
            fname = exp.filename
            exp.save(base_path, filename)
            # restore previous filename
            exp.filename = fname

        if task_state[name] == 'done':
            rospy.loginfo(
                'Finished %s task [iteration %d]' % (
                    name, exp.n_episodes()-n_rnd))
            break # TODO don't break when running multiple tasks

        spec['experience'] = exp

        # launch learning in a separate thread
        #new_thread = threading.Thread(name=name, target=polopt_fn,
        #                              args=(name, spec, tasks))
        #polopt_threads.append(new_thread)
        #new_thread.start()
        polopt_fn(name, spec, tasks)
        # http_polopt(name, spec, tasks)

reason: import numpy,import theano


text:
import json
import os
from os.path import join as pjoin

import nibabel as nib
import numpy as np

from utils import DATA_DIRS

def calculate_and_save_normalization():
    projection_path = DATA_DIRS['projections']

    def normfunction(x):
        return np.percentile(x, 99)

    all_normvalues = []
    for fname in os.listdir(projection_path):
        print(fname)
        img = nib.load(pjoin(projection_path, fname)).get_fdata()
        all_normvalues.append(normfunction(img))

    with open('normalization.json', 'w') as json_file:
        json.dump({
            "projections_99": np.median(all_normvalues),
        }, json_file)

if __name__ == "__main__":
    calculate_and_save_normalization()