text:
import pyrogue
import rogue.interfaces.stream
import time
import numpy as n
import os

# load the multiprocessing library functions needed
from multiprocessing import Process, Lock, Pipe

## load for parsing of configuration file
#import configparser

# load the data writer
import rogueFreeStreamRaw_PyMod as freeStream

#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# get the home directory
""" temporarily disable the configuration read
configfile = os.path.join(os.environ['HOME'],'python/python-lib/lzrd_rogue/lzrd_phase1_config')    # temporarily hardcoded until I find a better way to do this

config = configparser.ConfigParser(converters={
    'iarr': lambda arrstr: n.fromstring(arrstr, dtype=n.int64, sep=','),
    'farr': lambda arrstr: n.fromstring(arrstr, dtype=n.float64, sep=','),
    'tupple': lambda arrstr: tuple(n.fromstring(arrstr, dtype=n.int64, sep=','))})
a=config.read(configfile)

# How many digitizers are there?
DIGI_BOARD_NUM = config['DIGITIZER'].getint('board_num')
# get the digitizer ID mapping
DIGI_BOARD_INDEX = {}
for i in range(DIGI_BOARD_NUM):
    digikey = 'DIGI_BOARD_%d' %(i)
    # get the IDs of the digitizers
    tid = config[digikey].getint('id')
    DIGI_BOARD_INDEX[tid] = i

"""
DIGI_BOARD_NUM = n.array(1, dtype=n.float64)
DIGI_BOARD_INDEX = {0:0}

UINT64_VERSION_OF_32 = n.uint64(32)

#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

# Stream processors
class StreamProc(rogue.interfaces.stream.Slave):

    def __init__(self, stime, sevts, realTimeSim = False):
        # initialize interfaces
        rogue.interfaces.stream.Slave.__init__(self)

        # store the setting used to indicate that we are simulating waveforms
        self.realTimeSim = realTimeSim
        # Keep track of the dead time
        self.running_deadtime = 0
        # Create locks and pipes
        self.lock = Lock()
        self.parent_conn, self.child_conn = Pipe()
        # pass the main running function to a separate thread
        self.processor = Process(target=sevts.run_subprocess, args=(self.lock, self.child_conn))
        self.processor.start()
        # keep track of times
        self.t0 = time.time()   # initial time for frame generation
        self.ft0 = stime

    def __del__(self):
        #
        rogue.interfaces.stream.Slave.__del__(self)

    def end(self):
        # send the command to end the execution
        self.parent_conn.send((True, None, 0))
        self.parent_conn.close()
        self.processor.join()   # rejoin the terminated tread
        self.processor.terminate()
        del self.processor

    def _acceptFrame(self,frame):
        cframe = bytearray(frame.getPayload())
        frame.read(cframe,0)
        #
        #rdata = n.frombuffer(cframe, dtype=n.uint32, count=4, offset=len(cframe)-16)
        #fstarttime = (n.uint64(rdata[0]) << UINT64_VERSION_OF_32) + n.uint64(rdata[1])
        #fendtime = (n.uint64(rdata[2]) << UINT64_VERSION_OF_32) + n.uint64(rdata[3])
        # temporarily disable the frame start and end reading since they don't exist yet.
        fstarttime = n.uint64(0)
        fendtime = n.uint64(0)
        framelen = n.int64(fendtime)-fstarttime
        #print(self.running_deadtime, framelen)
        if self.realTimeSim:
            # enable real time distribution of waveforms
            fcurrenttime = (fendtime-self.ft0)/1e9
            currenttime = (time.time()-self.t0)
            if currenttime < fcurrenttime:
                time.sleep(fcurrenttime-currenttime)
        #print('framelen', framelen)
        # add a frame to the streamer
        # make sure there is no data waiting to be picked up by the processor
        child_conn_stat = self.child_conn.poll()
        if not child_conn_stat:
            # pipe is empty. Check if the processor has the thread locked. If not, send the frame
            lockstatus = self.lock.acquire(block=False) # aquire lock if the processor isn't busy
            #print('loop lockstatus is', lockstatus, child_conn_stat)
            if lockstatus:
                # Send current frame to the processor along with the pre-frame dead time
                #print("Sending data")
                self.parent_conn.send((False, cframe, self.running_deadtime))
                self.lock.release()
                # wait until the child process confirms that it got the data. This is needed for synchronization since the processor tends to be slower than the parent process hence the parent process might jump to the next frame before the processor is finished reading the current one.
                # I know that it's not great since it means that this process can hang if the processor takes too long but I couldn't think of a way to give the processor enough time to finish reading the data (before the parent process tries to send another frame) without putting an artifical sleep in the processor or the parent or making the processor use 100% of the cpu (stuck in a full speed while True loop)
                self.parent_conn.recv_bytes()    # this was sent after the processor aquired a lock
                # reset the dead time
                self.running_deadtime = 0
            else:
                self.running_deadtime += framelen
        else:
            self.running_deadtime += framelen

ext: py

max_stars_repo_name: slaclab/lz-test-stand

max_issues_repo_name: slaclab/lz-test-stand

max_forks_repo_name: slaclab/lz-test-stand

reason: import numpy


text:
"""
@author: Christian Wieprecht
@license: Apache License, Version 2.0
"""

import numpy as np
from scipy.cluster.vq import kmeans2

def cluster(descriptors, n_centroids):
    """
    Cluster all feature vectors in descriptors into codebook of size n_centroids.

    @param descriptors: n_samples x n_features matrix
    @param n_centroids: number of clusters to be created
    @return: codebook, labels
    """
    print("Extracting codebook of size {} from {} descriptors".format(n_centroids, len(descriptors)))

    # needs vlfeat, faster for integer vectors, note that the input matrix n_features x n_samples
    # codebook, labels = vlfeat.vl_ikmeans(np.array(descriptors).T, n_centroids, max_niters=20, verbose=1)

    codebook, labels = kmeans2(np.array(descriptors), n_centroids, iter=20, minit='points')
    return codebook, labels

max_stars_repo_name: cwiep/query-by-online-handwriting

max_issues_repo_name: cwiep/query-by-online-handwriting

max_forks_repo_name: cwiep/query-by-online-handwriting

reason: import numpy,from scipy


text:
import os
import numpy as np

# 从一个文件夹中将所有的图片的路径保存下来
# ['E:\\MyPythonProjects\\facenet-master\\data\\images\\Anthony_Hopkins_0001.jpg',
#  'E:\\MyPythonProjects\\facenet-master\\data\\images\\Anthony_Hopkins_0002.jpg']
def get_image_paths(facedir):
    image_paths = []
    if os.path.isdir(facedir):
        images = os.listdir(facedir)
        image_paths = [os.path.join(facedir, img) for img in images]
    return image_paths

class ImageClass():
    "Stores the paths to images for a given class"

    def __init__(self, name, image_paths):
        self.name = name
        self.image_paths = image_paths

    def __str__(self):
        return self.name + ', ' + str(len(self.image_paths)) + ' images'

    def __len__(self):
        return len(self.image_paths)

def get_dataset(path, has_class_directories=True):
    dataset = []
    path_exp = os.path.expanduser(path)
    classes = [path for path in os.listdir(path_exp) \
               if os.path.isdir(os.path.join(path_exp, path))]
    classes.sort()
    nrof_classes = len(classes)
    for i in range(nrof_classes):
        class_name = classes[i]
        facedir = os.path.join(path_exp, class_name)
        image_paths = get_image_paths(facedir)
        dataset.append(ImageClass(class_name, image_paths))

    return dataset

#
# Aaron_Sorkin, 2 images
# 2 文件夹里面有几张图片
# ['E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Sorkin\\Aaron_Sorkin_0001.png',
#  'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Sorkin\\Aaron_Sorkin_0002.png']
# 列出所有的文件夹和图片的路径之后，就可以自己写pairs.txt文件了

def get_image_paths_and_labels(dataset):  # 获得所有图片的路径和类别
    image_paths_flat = []  # 所有图片的路径
    labels_flat = []  # 所有图片的类别： 0，1，2，3....按照这样的方式来标图片的类别
    for i in range(len(dataset)):
        image_paths_flat += dataset[i].image_paths  # 数据集中每一个子文件夹都是一个类比，不论里面有几张图片
        labels_flat += [i] * len(dataset[i].image_paths)
    return image_paths_flat, labels_flat

def shuffle_examples(image_paths, labels):  # 把数据集打乱
    shuffle_list = list(zip(image_paths, labels))
    np.random.shuffle(shuffle_list)
    image_paths_shuff, labels_shuff = zip(*shuffle_list)
    return image_paths_shuff, labels_shuff

'''
加载所有的类别

一个类别的图片（300对）
    1.把图片数量>=2的过滤出来。这里大于几可以自己设一下
    2.有放回的抽取300个类，这样可以保证有一定的重复率。
    2.每个类从中随机抽取2张图片作为一对
不同类别的图像
    1.随机选择600个类 ，先shuffle，再选择600个即可
    2.每个类中选择一个图片（组成300对）
这里采用的是10Ford交叉验证
要求，每一个Ford选择的图片尽可能的不重复
把每一次抽中的图片从待选择的文件夹中去掉即可
如果图片的类别数量不够，可以修改这里的每一折选择的图片的数量
可以使用remove函数试试
'''
# 这里的写法，对于数据量比较小的情况下，可以凑合使用。但是，也做不到全部测试数据的效果。
def gen_pairs_txt(path,number_of_images_in_a_ford=300,number_of_fords=10,filename=r'E:\MyPythonProjects\captchaOneChineseTrain/pairs.txt'):

    dataset = get_dataset(path)
    number_of_all_classes_in_original_dataset=len(dataset)
    all_image_index = [index for index in range(number_of_all_classes_in_original_dataset)]
    classes_with_images_more_than_two = []
    # 筛选图片数量>=2的class
    for i in range(number_of_all_classes_in_original_dataset):
        if len(dataset[i].image_paths) >= 2:  # 如果图片的数量>=2
            classes_with_images_more_than_two.append(i)
    print('classes_with_images_more_than_two',len(classes_with_images_more_than_two))

    f=open(filename,'w') # 打开文件
    f.write(str(number_of_fords)+' '+str(number_of_images_in_a_ford)+'\n')
    for fords in range(number_of_fords):
        print("*******************************************************************")
        print('************************     ',fords)
        print("*******************************************************************")

        same_used_classes = []
        diff_used_classes = []

        np.random.shuffle(classes_with_images_more_than_two)
        # 从里面选择300个类
        # print('len(classes_with_images_more_than_two)=',len(classes_with_images_more_than_two))
        for i in range(np.min([number_of_images_in_a_ford, len(classes_with_images_more_than_two)])):
            same_class_index = classes_with_images_more_than_two[i]
            same_used_classes.append(same_class_index)
            # 从每个类中随机选择2张图片
            class_name = dataset[same_class_index].name
            number_of_images_in_a_class = len(dataset[same_class_index])
            images_in_a_class = [index for index in range(number_of_images_in_a_class)]
            np.random.shuffle(images_in_a_class)
            # 在lfw数据集中图片的下标是从1开始的
            first_image = images_in_a_class[0] + 1
            second_image = images_in_a_class[1] + 1
            # print(class_name, first_image, second_image)  # 把这个保存到文件中去
            f.write(class_name+' '+str(first_image)+' '+str(second_image)+'\n')
        # 把用过的类从有>=2张图片的类的集合中去掉
        # for i in range(len(same_used_classes)):
        #     classes_with_images_more_than_two.remove(same_used_classes[i])

        # 选择不同类别的图片
        # 从全部的图片中随机选择600个类

        np.random.shuffle(all_image_index)
        for i in range(np.min([number_of_images_in_a_ford * 2, len(all_image_index)])):
            diff_used_classes.append(all_image_index[i])
        diff_first_image_indexs = diff_used_classes[0::2]
        diff_second_image_indexs = diff_used_classes[1::2]
        for i in range(len(diff_first_image_indexs)):
            # 在lfw数据集中图片的下标是从1开始的
            diff_first_image_name = dataset[diff_first_image_indexs[i]].name
            diff_first_image = np.random.randint(0, len(dataset[diff_first_image_indexs[i]].image_paths)) + 1

            diff_second_image_name = dataset[diff_second_image_indexs[i]].name
            diff_second_image = np.random.randint(0, len(dataset[diff_second_image_indexs[i]].image_paths)) + 1
            # print(diff_first_image_name, diff_first_image, diff_second_image_name, diff_second_image)  # 写到文件中去
            f.write(diff_first_image_name + ' ' + str(diff_first_image) + ' '+ diff_second_image_name+' '+str(diff_second_image)+'\n')
        # 将用过的从全部的里面去掉
        # for i in diff_used_classes:
        #     all_image_index.remove(i)
        # for i in same_used_classes:
        #     if all_image_index.count(i) > 0:
        #         all_image_index.remove(i)
        # 这里的写法好像和pairs.txt中的规律不一样，但是，应该不影响正确率的判断
    f.close()# 关闭文件

# 下面这段代码是针对手写汉字识别这样的具有大量样本的数据集
# 从里面得到可以充分利用数据集数据的抽样获得pairs.txt文件的方法
def gen_pairs_txt_from_chinese_handwrite_dataset(path, number_of_images_in_a_fold=370, number_of_folds=10, filename=''):
    dataset = get_dataset(path)
    number_of_all_classes_in_original_dataset = len(dataset)  # 一共拥有的类别的数量
    # 所有的类别的下标
    all_image_index = [index for index in range(number_of_all_classes_in_original_dataset)]
    for fords in range(number_of_folds):
        print("*******************************************************************")
        print('************************       ', fords)
        print("*******************************************************************")

        same_used_classes = []
        diff_used_classes = []

        # 先随机选择400个类别
        np.random.shuffle(all_image_index)
        selected_class_index=all_image_index[0:int(number_of_images_in_a_fold)]
        # 需要再想一想，要使用到 5% 左右的数据来进行测试。而且是使用 10 折交叉验证

#     same_used_classes=[]
#     diff_used_classes=[]
#
#
#
#     np.random.shuffle(classes_with_images_more_than_two)
#     # 从里面选择300个类
#     # print('len(classes_with_images_more_than_two)=',len(classes_with_images_more_than_two))
#     for i in range(np.min([number_of_images_in_a_ford,len(classes_with_images_more_than_two)])):
#         same_class_index=classes_with_images_more_than_two[i]
#         same_used_classes.append(same_class_index)
#         # 从每个类中随机选择2张图片
#         class_name=dataset[same_class_index].name
#         number_of_images_in_a_class=len(dataset[same_class_index])
#         images_in_a_class=[index for index in range(number_of_images_in_a_class)]
#         np.random.shuffle(images_in_a_class)
#         # 在lfw数据集中图片的下标是从1开始的
#         first_image=images_in_a_class[0]+1
#         second_image=images_in_a_class[1]+1
#         print(class_name,first_image,second_image)# 把这个保存到文件中去
#     # 把用过的类从有>=2张图片的类的集合中去掉
#     for i in range(len(same_used_classes)):
#         classes_with_images_more_than_two.remove(same_used_classes[i])
#
#     # 选择不同类别的图片
#     # 从全部的图片中随机选择600个类
#
#     np.random.shuffle(all_image_index)
#     for i in range(np.min([number_of_images_in_a_ford*2,number_of_all_classes_in_original_dataset])):
#         diff_used_classes.append(all_image_index[i])
#     diff_first_image_indexs =diff_used_classes[0::2]
#     diff_second_image_indexs=diff_used_classes[1::2]
#     for i in range(len(diff_first_image_indexs)):
#         # 在lfw数据集中图片的下标是从1开始的
#         diff_first_image_name=dataset[diff_first_image_indexs[i]].name
#         diff_first_image=np.random.randint(0,len(dataset[diff_first_image_indexs[i]].image_paths))+1
#
#         diff_second_image_name=dataset[diff_second_image_indexs[i]].name
#         diff_second_image=np.random.randint(0,len(dataset[diff_second_image_indexs[i]].image_paths))+1
#         print(diff_first_image_name,diff_first_image,diff_second_image_name,diff_second_image) # 写到文件中去
#     # 将用过的从全部的里面去掉
#     for i in diff_used_classes:
#         all_image_index.remove(i)
#     for i in same_used_classes:
#         if all_image_index.count(i)>0:
#             all_image_index.remove(i)
#     # 这里的写法好像和pairs.txt中的规律不一样，但是，应该不影响正确率的判断
#
#
### 测试代码
#
# path=r'E:\MyPythonProjects\facenet-master\datasets\lfw_mtcnnpy_160/'
# dataset=get_dataset(path)
# print(len(dataset))
# print(dataset[5])
# print(len(dataset[5]))
# print(dataset[5].image_paths)
# print("-------------------------------------------")
# image_paths_flat, labels_flat=get_image_paths_and_labels(dataset)
# print(image_paths_flat)
# print(labels_flat)
'''
['E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Eckhart\\Aaron_Eckhart_0001.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Guiel\\Aaron_Guiel_0001.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Patterson\\Aaron_Patterson_0001.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Peirsol\\Aaron_Peirsol_0001.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Peirsol\\Aaron_Peirsol_0002.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Peirsol\\Aaron_Peirsol_0003.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Peirsol\\Aaron_Peirsol_0004.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Pena\\Aaron_Pena_0001.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Sorkin\\Aaron_Sorkin_0001.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Sorkin\\Aaron_Sorkin_0002.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Tippin\\Aaron_Tippin_0001.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Abba_Eban\\Abba_Eban_0001.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Abbas_Kiarostami\\Abbas_Kiarostami_0001.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Abdel_Aziz_Al-Hakim\\Abdel_Aziz_Al-Hakim_0001.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Abdel_Madi_Shabneh\\Abdel_Madi_Shabneh_0001.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Abdel_Nasser_Assidi\\Abdel_Nasser_Assidi_0001.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Abdel_Nasser_Assidi\\Abdel_Nasser_Assidi_0002.png']
[0, 1, 2, 3, 3, 3, 3, 4, 5, 5, 6, 7, 8, 9, 10, 11, 11]
'''
# image_paths_shuff, labels_shuff=shuffle_examples(image_paths_flat,labels_flat)
# print(image_paths_shuff)
# print(labels_shuff)
'''
('E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Patterson\\Aaron_Patterson_0001.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Eckhart\\Aaron_Eckhart_0001.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Sorkin\\Aaron_Sorkin_0001.png',
 'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Guiel\\Aaron_Guiel_0001.png',
 'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Peirsol\\Aaron_Peirsol_0004.png',
'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Abdel_Nasser_Assidi\\Abdel_Nasser_Assidi_0001.png',
 'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Pena\\Aaron_Pena_0001.png',
 'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Tippin\\Aaron_Tippin_0001.png',
 'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Abdel_Madi_Shabneh\\Abdel_Madi_Shabneh_0001.png',
 'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Sorkin\\Aaron_Sorkin_0002.png',
 'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Peirsol\\Aaron_Peirsol_0001.png',
 'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Abbas_Kiarostami\\Abbas_Kiarostami_0001.png',
 'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Peirsol\\Aaron_Peirsol_0002.png',
 'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Aaron_Peirsol\\Aaron_Peirsol_0003.png',
 'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Abba_Eban\\Abba_Eban_0001.png',
 'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Abdel_Aziz_Al-Hakim\\Abdel_Aziz_Al-Hakim_0001.png',
 'E:\\MyPythonProjects\\facenet-master\\datasets\\val_data/Abdel_Nasser_Assidi\\Abdel_Nasser_Assidi_0002.png')
(2, 0, 5, 1, 3, 11, 4, 6, 10, 5, 3, 8, 3, 3, 7, 9, 11)
'''
print('*********************************************************************************')
print('*********************************************************************************')
print('*********************************************************************************')

# path=r'E:\MyPythonProjects\facenet-master\datasets\lfw_mtcnnpy_160/'
# 手写汉字图片的路径
path=r'E:\MyPythonProjects\captchaOneChineseTrain\dataset/'
dataset=get_dataset(path)
print(len(dataset))
# 写到的目标文件 E:\MyPythonProjects\captchaOneChineseTrain/pairs.txt
filename=r'E:\MyPythonProjects\captchaOneChineseTrain/pairs.txt'
#
gen_pairs_txt(path,number_of_images_in_a_ford=600,number_of_fords=10,filename=filename)

# 这里的代码解释都是从实际应用的概念出发进行解释的。但是，只要数据准备是靠谱的，那么，这里面数据的处理逻辑是一样的。

max_stars_repo_name: shuyuandeqipa/mycaptcha

max_issues_repo_name: shuyuandeqipa/mycaptcha

max_forks_repo_name: shuyuandeqipa/mycaptcha


text:
import numpy as np

def normalize(vector: np.ndarray) -> None:
    """
    normalize vector (in-place)
    :param vector: vector to normalize
    """
    norm = np.linalg.norm(vector)
    if norm != 0:
        vector /= np.linalg.norm(vector)

def normalized(vector: np.ndarray) -> np.ndarray:
    """
    :param vector: vector to normalize
    :return: normalized version of vector
    """
    norm = np.linalg.norm(vector)
    if norm != 0:
        return vector.copy() / np.linalg.norm(vector)
    else:
        return vector.copy()

max_stars_repo_name: adrianmgg/raymarch

max_issues_repo_name: adrianmgg/raymarch

max_forks_repo_name: adrianmgg/raymarch


text:
import socket
import cv2
import numpy

def recvall(sock, count):
    buf = b''
    while count:
        newbuf = sock.recv(count)
        if not newbuf: return None
        buf += newbuf
        count -= len(newbuf)
    return buf

TCP_IP = "127.0.0.1"
TCP_PORT = 8000

sock = socket.socket()
sock.connect((TCP_IP, TCP_PORT))
while 1:
    length = recvall(sock,16)
    try:
        stringData = recvall(sock, int(length))
        data = numpy.fromstring(stringData, dtype='uint8')
        decimg=cv2.imdecode(data,1)
        cv2.imshow('CLIENT2',decimg)
        cv2.waitKey(10)
    except ValueError:
        print("ValueError")

sock.close()
cv2.destroyAllWindows()

max_stars_repo_name: mskhrk/IoT-home-security

max_issues_repo_name: mskhrk/IoT-home-security

max_forks_repo_name: mskhrk/IoT-home-security


text:
# -*- coding: utf-8 -*-
"""Different tools to work with datasets in HDF5 file format.

Created on Thu Apr  2 21:12:46 2015

@author: akusok
"""

from __future__ import division

import numpy as np
from six import string_types
from six.moves import range
import csv
from tables import open_file, Atom, Filters
import os
import fasteners  # inter-process file lock

def _prepare_fHH(fHH, fHT, nnet, precision):
    """Prepares files for fHH, fHT if they are needed.

    Args:
        fHH (string): hdf5 filename to store HH, None for ignore disk storage
        fHT (string): hdf5 filename to store HT, None for ignore disk storage
        nnet (nnets Object): neural network implementation from HPELM
        precision (np.float32/64): precision
    """
    if (fHH is not None) and (fHT is not None):
        # reset accumulated data in ELM
        nnet.reset()
        L = nnet.L
        outputs = nnet.outputs
        norm = nnet.norm

        # process fHH
        if os.path.isfile(fHH):
            h5 = open_file(fHH, 'r')
            node = None
            for node in h5.walk_nodes():
                pass  # find a node with whatever name
            try:
                assert node is not None, "Matrix in %d does not exist" % fHH
                msg = "Matrix in {:d} has a wrong shape: ({:d}, {:d}) expected, ({:d}, {:d}) found"
                msg = msg.format(fHH, L, L, node.shape[0], node.shape[1])
                assert node is not None and node.shape[0] == L and node.shape[1] == L, msg
            except AssertionError:
                raise  # re-raise same error
            finally:
                h5.close()
        else:
            make_hdf5(np.eye(L, L, dtype=precision)*norm, fHH, precision)

        # process fHT
        if os.path.isfile(fHT):
            h5 = open_file(fHT, 'r')
            node = None
            for node in h5.walk_nodes():
                pass  # find a node with whatever name
            try:
                assert node is not None, "Matrix in %d does not exist" % fHT
                msg = "Matrix in {:d} has a wrong shape: ({:d}, {:d}) expected, ({:d}, {:d}) found"
                msg = msg.format(fHT, L, outputs, node.shape[0], node.shape[1])
                assert node is not None and node.shape[0] == L and node.shape[1] == outputs, msg
            except AssertionError:
                raise  # re-raise same error
            finally:
                h5.close()
        else:
            make_hdf5((L, outputs), fHT, precision)

def _write_fHH(fHH, fHT, HH, HT):
    """Writes HH,HT data into fHH,fHT files, multi-process safe with lock file.

    Lock file has the same name as fHH,fHT, but with '.lock' extension.
    """
    fHH_lock = fHH + ".lock"
    with fasteners.InterProcessLock(fHH_lock):
        h5 = open_file(fHH, "a")
        node = None
        for node in h5.walk_nodes():
            pass  # find a node with whatever name
        node[:] += HH
        h5.flush()
        h5.close()

    fHT_lock = fHT + ".lock"
    with fasteners.InterProcessLock(fHT_lock):
        h5 = open_file(fHT, "a")
        for node in h5.walk_nodes():
            pass  # find a node with whatever name
        node[:] += HT
        h5.flush()
        h5.close()

def normalize_hdf5(h5file, mean=None, std=None, batch=None):
    """Calculates and applies normalization to data in HDF5 file.

    :param h5file: - HDF5 format data file
    :param mean: - known vector of mean values
    :param std: - known vector of standard deviations
    :param batch: - number of rows to read at once, default is a native batch size
    """

    h5 = open_file(h5file, "a")
    node = None
    for node in h5.walk_nodes():
        pass  # find a node with whatever name
    dt = node.dtype
    N, d = node.shape  # HDF5 files are transposed, for Matlab compatibility
    if batch is None:
        batch = node.chunkshape[0]
    nb = N//batch
    if N > nb*batch:
        nb += 1  # add last incomplete step

    if mean is None or std is None:
        if node.attrs.mean is None:  # data was not normalized before
            print("calculating mean and standard deviation of data")
            E_x = np.zeros((d,), dtype=np.float64)
            E_x2 = np.zeros((d,), dtype=np.float64)
            for b in range(nb):
                start = b*batch
                step = min(batch, N-start)
                X1 = node[start: start+step, :].astype(np.float64)
                E_x += np.mean(X1, 0) * (1.0*step/N)
                E_x2 += np.mean(X1**2, 0) * (1.0*step/N)
            mean = E_x
            E2_x = E_x**2
            std = (E_x2 - E2_x)**0.5
            node.attrs.mean = mean
            node.attrs.std = std
            h5.close()
            return mean, std
        else:  # data is already normalized
            print("data was already normalized, returning 'mean', 'std' parameters")
            print("if you want to run normalization anyway, call the function with 'mean' and 'std' params")
            h5.close()
            return node.attrs.mean, node.attrs.std
    else:
        msg1 = "Incorrect lenght of a vector of means: %d expected, %d found" % (d, len(mean))
        assert len(mean) == d, msg1
        msg2 = "Incorrect lenght of a vector of standard deviations: %d expected, %d found" % (d, len(std))
        assert len(std) == d, msg2
        node.attrs.mean = mean
        node.attrs.std = std
    std[std == 0] = 1  # prevent division by zero for std=0

    print("applying normalization")
    for b in range(nb):
        start = b*batch
        step = min(batch, N-start)
        X = node[start: start+step].astype(np.float64)
        X = (X - mean) / std
        node[start: start+step] = X.astype(dt)

    h5.close()  # closing file
    return mean, std

def make_hdf5(data, h5file, dtype=np.float64, delimiter=" ", skiprows=0, comp_level=0):
    """Makes an HDF5 file from whatever given data.

    :param data: - input data in Numpy.ndarray or filename, or a shape tuple
    :param h5file: - name (and path) of the output HDF5 file
    :param dtype: - type of data in HDF5 file, from Numpy types
    :param delimiter: - data delimiter for text, csv files
    :param skiprows: - number of rows to skip, if input is read from a text file
    :param comp_level: - compression level of the HDF5 file
    """
    assert comp_level < 10, "Compression level must be 0-9 (0 for no compression)"
    fill = ""

    # open data file
    X = None
    if isinstance(data, np.ndarray):
        X = data
    elif isinstance(data, string_types) and data[-3:] in ['npy']:
        X = np.load(data)
    elif isinstance(data, string_types) and data[-3:] in ['.gz', 'bz2']:
        X = np.loadtxt(data, dtype=dtype, delimiter=delimiter, skiprows=skiprows)
    elif isinstance(data, string_types) and data[-3:] in ['txt', 'csv']:
        # iterative out-of-memory loader for huge .csv/.txt files
        fill = "iter"
        # check data dimensionality
        with open(data, "rU") as f:
            for _ in range(skiprows):
                f.readline()
            reader = csv.reader(f, delimiter=delimiter)
            for line in reader:
                X = np.fromiter(line, dtype=dtype)
                break
    elif isinstance(data, tuple) and len(data) == 2:
        X = np.empty((1, 1))
        fill = "empty"
    else:
        assert False, "Input data must be Numpy ndarray, .npy file, or .txt/.csv text file (compressed .gz/.bz2)"

    # process data
    if len(X.shape) == 1:
        X = X[:, np.newaxis]
    assert len(X.shape) == 2, "Data in Numpy ndarray must have 2 dimensions"
    # create hdf5 file
    if comp_level > 0:
        flt = Filters(complevel=comp_level, shuffle=True)
    else:
        flt = Filters(complevel=0)
    h5 = open_file(h5file, "w")
    a = Atom.from_dtype(np.dtype(dtype), dflt=0)
    # write data to hdf5 file
    if fill == "iter":  # iteratively fill the data
        h5data = h5.create_earray(h5.root, "data", a, (0, X.shape[0]), filters=flt)
        with open(data, "rU") as f:
            for _ in range(skiprows):
                f.readline()
            reader = csv.reader(f, delimiter=delimiter)
            for line in reader:
                row = np.fromiter(line, dtype=dtype)
                h5data.append(row[np.newaxis, :])
    elif fill == "empty":  # no fill at all
        h5data = h5.create_carray(h5.root, "data", a, data, filters=flt)
    else:  # write whole data matrix
        h5data = h5.create_carray(h5.root, "data", a, X.shape, filters=flt)
        h5data[:] = X
    # close the file
    h5data.attrs.mean = None
    h5data.attrs.std = None
    h5.flush()
    h5.close()

def _ireader(fX, q_in, q_out):
    """Asyncronous reader for an HDF5 file.

    q_in - a (start, stop) tuple of read indexes; if start >= stop then reader terminates
    q_out - a queue for chunks red from a disk
    """
    assert isinstance(fX, string_types), "Asyncronous I/O only supported with HDF5 data files"
    hX = open_file(fX, "r")
    X = None
    for X in hX.walk_nodes():
        pass  # find a node with whatever name

    while True:  # returning data chunks on demand
        start, stop = q_in.get()
        if start >= stop:
            break
        q_out.put(X[start:stop])
    hX.close()

def _iwriter(fX, q_in):
    """Asyncronous writer for an HDF5 file.

    q_in - a (Xbatch, start, stop) tuple of data to write indexes; if q_in is None then writer terminates
    """
    assert isinstance(fX, string_types), "Asyncronous I/O only supported with HDF5 data files"
    hX = open_file(fX, "a")
    X = None
    for X in hX.walk_nodes():
        pass  # find a node with whatever name

    while True:  # returning data chunks on demand
        data = q_in.get()
        if data is None:
            break
        Xb, start, stop = data
        X[start:stop] = Xb
    X.flush()
    hX.close()

if __name__ == "__main__":
    # def make_hdf5(data, h5file, dtype=np.float64, delimiter=" ", skiprows=0, comp_level=0):
    # make_hdf5("textfile.txt", "text.h5")
    # make_hdf5("textfile.txt", "textz.h5", comp_level=3)
    normalize_hdf5("text.h5")
    print("Done!")


text:
import urllib
from bs4 import BeautifulSoup
from collections import Counter
import numpy as np
import matplotlib.pyplot as plt

my_url = 'https://www.foxnews.com'

dt = urllib.request.urlopen(my_url, timeout=0.5)
soup = BeautifulSoup(dt, "lxml")

a = soup.find_all('h2', 'title')
news = []
for ref in a:
  news.append(ref.a.string)

cnt = Counter()
for item in news:
  for word in item.split(" "):
    if word.istitle():
      cnt[word] += 1
com = cnt.most_common(10)
freqs = []
people = []
for x,y in com:
  people.append(x)
  freqs.append(y)

y_pos = np.arange(len(people))
fig, ax = plt.subplots()
ax.barh(y_pos, freqs, align='center')
ax.set_yticks(y_pos)
ax.set_yticklabels(people)
ax.invert_yaxis()
ax.set_xlabel('Mentions')
ax.set_title(my_url)
plt.show()

max_stars_repo_name: matthew-buchanan/jabber-scraper

max_issues_repo_name: matthew-buchanan/jabber-scraper

max_forks_repo_name: matthew-buchanan/jabber-scraper


text:
"""
config.py

Configuration Variables for the Addition NPI Task => Stores Scratch-Pad Dimensions, Vector/Program
Embedding Information, etc.
"""
import numpy as np
import sys
import time

CONFIG = {
    "ENVIRONMENT_ROW": 4,         # Input 1, Input 2, Carry, Output
    "ENVIRONMENT_COL": 10,        # 10-Digit Maximum for Addition Task
    "ENVIRONMENT_DEPTH": 11,      # Size of each element vector => One-Hot, Options: 0-9

    "ARGUMENT_NUM": 3,            # Maximum Number of Program Arguments
    "ARGUMENT_DEPTH": 11,         # Size of Argument Vector => One-Hot, Options 0-9, Default (10)
    "DEFAULT_ARG_VALUE": 10,      # Default Argument Value

    "PROGRAM_NUM": 6,             # Maximum Number of Subroutines
    "PROGRAM_KEY_SIZE": 5,        # Size of the Program Keys
    "PROGRAM_EMBEDDING_SIZE": 10  # Size of the Program Embeddings
}

PROGRAM_SET = [
    ("MOVE_PTR", 4, 2),       # Moves Pointer (4 options) either left or right (2 options)
    ("WRITE", 2, 10),         # Given Carry/Out Pointer (2 options) writes digit (10 options)
    ("ADD",),                 # Top-Level Add Program (calls children routines)
    ("ADD1",),                # Single-Digit (Column) Add Operation
    ("CARRY",),               # Carry Operation
    ("LSHIFT",)               # Shifts all Pointers Left (after Single-Digit Add)
]

PROGRAM_ID = {x[0]: i for i, x in enumerate(PROGRAM_SET)}

class ScratchPad():           # Addition Environment
    def __init__(self, in1, in2, rows=CONFIG["ENVIRONMENT_ROW"], cols=CONFIG["ENVIRONMENT_COL"]):
        # Setup Internal ScratchPad
        self.rows, self.cols = rows, cols
        self.scratchpad = np.zeros((self.rows, self.cols), dtype=np.int8)

        # Initialize ScratchPad In1, In2
        self.init_scratchpad(in1, in2)

        # Pointers initially all start at the right
        self.in1_ptr, self.in2_ptr, self.carry_ptr, self.out_ptr = self.ptrs = \
            [(x, -1) for x in range(4)]

    def init_scratchpad(self, in1, in2):
        """
        Initialize the scratchpad with the given input numbers (to be added together).
        """
        lst = [str(in1), str(in2)]
        for inpt in range(len(lst)):
            for i in range(1, len(lst[inpt]) + 1):
                self.scratchpad[inpt, -i] = int(lst[inpt][-i])

    def done(self):
        if self.in1_ptr[1] < -self.cols:
            return True
        else:
            return False
            # lst = [x[1] for x in self.ptrs]
            # if len(set(lst)) == 1:
            #     return sum(sum([self[x[0], :min(x[1] + 1, -1)] for x in self.ptrs])) == 0
            # else:
            #     return False

    def add1(self):
        temp = self[self.in1_ptr] + self[self.in2_ptr] + self[self.carry_ptr]
        return temp % 10, int(temp / 10)

    def write_carry(self, carry_val, debug=False):
        carry_row, carry_col = self.carry_ptr
        self[(carry_row, carry_col - 1)] = carry_val
        if debug:
            self.pretty_print()

    def write_out(self, value, debug=False):
        self[self.out_ptr] = value
        if debug:
            self.pretty_print()

    def lshift(self):
        self.in1_ptr, self.in2_ptr, self.carry_ptr, self.out_ptr = self.ptrs = \
            [(x, y - 1) for (x, y) in self.ptrs]

    def pretty_print(self):
        new_strs = ["".join(map(str, self[i])) for i in range(4)]
        line_length = len('Input 1:' + " " * 5 + new_strs[0])
        print('Input 1:' + " " * 5 + new_strs[0])
        print('Input 2:' + " " * 5 + new_strs[1])
        print('Carry  :' + " " * 5 + new_strs[2])
        print('-' * line_length)
        print('Output :' + " " * 5 + new_strs[3])
        print('')
        time.sleep(.1)
        sys.stdout.flush()

    def get_env(self):
        env = np.zeros((CONFIG["ENVIRONMENT_ROW"], CONFIG["ENVIRONMENT_DEPTH"]), dtype=np.int32)
        if self.in1_ptr[1] < -CONFIG["ENVIRONMENT_COL"]:
            env[0][10] = 1
        else:
            env[0][self[self.in1_ptr]] = 1
        if self.in2_ptr[1] < -CONFIG["ENVIRONMENT_COL"]:
            env[1][10] = 1
        else:
            env[1][self[self.in2_ptr]] = 1
        if self.carry_ptr[1] < -CONFIG["ENVIRONMENT_COL"]:
            env[2][10] = 1
        else:
            env[2][self[self.carry_ptr]] = 1
        if self.out_ptr[1] < -CONFIG["ENVIRONMENT_COL"]:
            env[3][10] = 1
        else:
            env[3][self[self.out_ptr]] = 1
        return env.flatten()

    def execute(self, prog_id, args):
        if prog_id == 0:               # MOVE!
            ptr, lr = args
            lr = (lr * 2) - 1
            if ptr == 0:
                self.in1_ptr = (self.in1_ptr[0], self.in1_ptr[1] + lr)
            elif ptr == 1:
                self.in2_ptr = (self.in2_ptr[0], self.in2_ptr[1] + lr)
            elif ptr == 2:
                self.carry_ptr = (self.carry_ptr[0], self.carry_ptr[1] + lr)
            elif ptr == 3:
                self.out_ptr = (self.out_ptr[0], self.out_ptr[1] + lr)
            else:
                raise NotImplementedError
            self.ptrs = [self.in1_ptr, self.in2_ptr, self.carry_ptr, self.out_ptr]
        elif prog_id == 1:             # WRITE!
            ptr, val = args
            if ptr == 0:
                self[self.out_ptr] = val
            elif ptr == 1:
                self[self.carry_ptr] = val
            else:
                raise NotImplementedError

    def __getitem__(self, item):
        return self.scratchpad[item]

    def __setitem__(self, key, value):
        self.scratchpad[key] = value

class Arguments():             # Program Arguments
    def __init__(self, args, num_args=CONFIG["ARGUMENT_NUM"], arg_depth=CONFIG["ARGUMENT_DEPTH"]):
        self.args = args
        self.arg_vec = np.zeros((num_args, arg_depth), dtype=np.float32)

def get_args(args, arg_in=True):
    if arg_in:
        arg_vec = np.zeros((CONFIG["ARGUMENT_NUM"], CONFIG["ARGUMENT_DEPTH"]), dtype=np.int32)
    else:
        arg_vec = [np.zeros((CONFIG["ARGUMENT_DEPTH"]), dtype=np.int32) for _ in
                   range(CONFIG["ARGUMENT_NUM"])]
    if len(args) > 0:
        for i in range(CONFIG["ARGUMENT_NUM"]):
            if i >= len(args):
                arg_vec[i][CONFIG["DEFAULT_ARG_VALUE"]] = 1
            else:
                arg_vec[i][args[i]] = 1
    else:
        for i in range(CONFIG["ARGUMENT_NUM"]):
            arg_vec[i][CONFIG["DEFAULT_ARG_VALUE"]] = 1
    return arg_vec.flatten() if arg_in else arg_vec


text:
import sys
sys.path.insert(0, './python/')
import caffe
import numpy as np
from lcg_random import lcg_rand
import ncs
from easydict import EasyDict as edict
import time

start_time = time.time()
ncs_time = 0.
adjusting_time = 0.
retraining_time = 0.
# model files
proto='./models/caltech_caffenet/train_val_caltech.prototxt'
weights='/home/deepModels/caffe_models/bvlc_reference_caffenet/scratch_caltech_caffenet_train_iter_10000.caffemodel'
solver_path='./models/caltech_caffenet/caltech_solver.prototxt'
es_method='ncs'
# cpu/gpu
caffe.set_mode_gpu()
caffe.set_device(0)
# init solver
solver = caffe.SGDSolver(solver_path)
# basic parameters
#   accuracy constraint for pruning
acc_constrain=0.08
#   stop iteration count
niter = 15001
#   stop pruning iteration count
prune_stop_iter = 10000
# interval for
prune_interval = 250
# interval for std variate
std_interval = 7000
#   the list of layer names
layer_name = ['conv1','conv2','conv3','conv4','conv5','fc6','fc7','fc8*']
#   the dict of layer names to its arrary indices
layer_inds = {'conv1':0, 'conv2':1, 'conv3':2,'conv4':3,'conv5':4,'fc6':5,'fc7':6,'fc8*':7}
#   the dict of crates for each layer
crates = {'conv1':0.001, 'conv2':0.001, 'conv3':0.001,'conv4':0.001,'conv5':0.001,'fc6':0.001,'fc7':0.001,'fc8*':0.001}
#   the list of the crates
crates_list = 8*[0.001]
#   the gamma for each layer
gamma = {'conv1':0.00002, 'conv2':0.00002, 'conv3':0.00002,'conv4':0.00002,'conv5':0.00002,'fc6':0.0002,'fc7':0.0002,'fc8*':0.0002}
gamma_star = 0.0002
ncs_stepsize = 50
#   random see for numpy.random
seed=np.random.randint(1000000)
#seed = 217750
np.random.seed([seed])
#   the dict to store intermedia results
es_cache = {}
#retrieval_tag=[]
r_count=0
# load the pretrained caffe model
if weights:
  solver.net.copy_from(weights)

# definition of many axuliliary methods
#   run the network on its dataset
def test_net(thenet, _start='data', _count=1):
   '''
    thenet: the object of network
    _start: the layer to start from
    _count: the number of batches to run
   '''
   scores = 0
   for i in range(_count):
      thenet.forward(start=_start)
      scores += thenet.blobs['accuracy'].data
   return scores/_count

#   Set the crates of each layer, the pruning will happen in the next forward action
def apply_prune(thenet, _crates):
   '''
      thenet: the model to be pruned
      _crates: the list of crates for layers
   '''
   for _id in range(len(layer_name)):
         if _crates[_id] < 0:
           continue
         layer_id = layer_name[_id]
         mask0 = thenet.params[layer_id][2].data.ravel()[0]
         if mask0 == 0:
           thenet.params[layer_id][2].data.ravel()[0] = -_crates[_id]
         elif mask0 == 1:
           thenet.params[layer_id][2].data.ravel()[0] = 1+_crates[_id]
         else:
           pdb.set_trace()

#  calcuate the sparsity of a network model
def get_sparsity(thenet):
   '''
     thenet: the network for checking
   '''
   remain = 0
   total = 0
   for layer_id in layer_name:
      remain += len(np.where(thenet.params[layer_id][2].data != 0)[0])
      remain += len(np.where(thenet.params[layer_id][3].data != 0)[0])
      total += thenet.params[layer_id][0].data.size
      total += thenet.params[layer_id][1].data.size
   #return total*1./(100.*remain)
   return remain*1./total

#  evaluate the accuracy of a network with a set of crates respect to a original accuracy
def evaluate(thenet, x_set, batchcount=1, accuracy_ontrain=0.9988):
   nofit=False
   fitness=[]
   X=[]
   for x in x_set:
     x_fit = 1.1
     apply_prune(thenet,x)
     acc = test_net(thenet, _start='conv1', _count=batchcount)
     if acc >= accuracy_ontrain - acc_constrain:
       x_fit = get_sparsity(thenet)
       nofit=True
     fitness.append(x_fit)
     X.append(x)
   return (X, fitness, nofit)
#------mian--------------
solver.step(1)
#  Adaptive dynamic surgery
for itr in range(niter):
   #r = np.random.rand()
   #if itr%500==0 and solver.test_nets[0].blobs['accuracy'].data >= 0.9909:
   #  retrieval_tag.append(itr)
   tmp_crates=[]
   tmp_ind = []
   for ii in layer_name:
      #tmp_crates.append(crates[ii]*(np.power(1+gamma[ii]*itr, -1)>np.random.rand()))
      tmp_tag = np.power(1+gamma[ii]*itr, -1)>np.random.rand()
      if tmp_tag:
        tmp_ind.append(ii)
        tmp_crates.append(tmp_tag*crates[ii])
   if itr < prune_stop_iter and itr%std_interval == 0:
      ncs_stepsize = ncs_stepsize/10.
   if itr%500 == 0:
        print "Compression:{}, Accuracy:{}".format(1./get_sparsity(solver.net), test_net(solver.net, _count=1, _start="conv1"))
   if len(tmp_ind)>0 and itr < prune_stop_iter:
         _tmp_c = np.array(len(crates_list)*[-1.])
         for t_name in tmp_ind:
            _tmp_c[layer_inds[t_name]] = crates[t_name]
         apply_prune(solver.net, _tmp_c)
   #if len(tmp_ind)>1 and itr < prune_stop_iter:
   if itr%prune_interval==0 and len(tmp_ind)>1 and itr < prune_stop_iter:
         ncs_start_t = time.time()
         accuracy_ = test_net(solver.net, _count=1, _start="conv1")

         # make sure a worable son x
         es = {}
         if es_method == 'ncs':
           __C = edict()
           __C.parameters = {'reset_xl_to_pop':False,'init_value':tmp_crates, 'stepsize':ncs_stepsize, 'bounds':[0.0, 10.], 'ftarget':0, 'tmax':1600, 'popsize':8, 'best_k':1}
           es = ncs.NCS(__C.parameters)
           print '***************NCS initialization***************'
           tmp_x_ = np.array(crates_list)
           tmp_input_x = tmp_crates
           for _ii in range(len(tmp_ind)):
             tmp_x_[layer_inds[tmp_ind[_ii]]] = tmp_input_x[_ii]
           _,tmp_fit,_= evaluate(solver.net, [tmp_x_], 1, accuracy_)
           es.set_initFitness(es.popsize*tmp_fit)
           print 'fit:{}'.format(tmp_fit)
           print '***************NCS initialization***************'

         # evolution loop
         while not es.stop():
           x = es.ask()
           X = []
           for x_ in x:
            tmp_x_ = np.array(crates_list)
            for _ii in range(len(tmp_ind)):
             tmp_x_[layer_inds[tmp_ind[_ii]]] = x_[_ii]
            X.append(tmp_x_)

           X_arrange,fit,has_fit_x = evaluate(solver.net, X, 1, accuracy_)

           X = []
           for x_ in X_arrange:
            tmp_x_ = np.array(len(tmp_ind)*[0.])
            for _ii in range(len(tmp_ind)):
             tmp_x_[_ii]= x_[layer_inds[tmp_ind[_ii]]]
            X.append(tmp_x_)
           #print X,fit
           es.tell(X, fit)
           #es.disp(100)
           for _ii in range(len(tmp_ind)):
             crates_list[layer_inds[tmp_ind[_ii]]] = es.result()[0][_ii]
         for c_i in range(len(crates_list)):
            crates[layer_name[c_i]] = crates_list[c_i]
         es_cache[itr]={'compression':-es.result()[1], 'crates':crates_list[:]}
         _tmp_c = np.array(len(crates_list)*[-1.])
         for t_name in tmp_ind:
            _tmp_c[layer_inds[t_name]] = crates[t_name]
         apply_prune(solver.net, crates_list)

         ncs_end_t = time.time()
         ncs_time += (ncs_end_t - ncs_start_t)

   loop_start_t = time.time()
   # adjusting or retraining
   solver.step(1)

   loop_end_t = time.time()
   if itr < prune_stop_iter:
     adjusting_time += (loop_end_t - loop_start_t)
   else:
     retraining_time += (loop_end_t - loop_start_t)

# record
import datetime
now = datetime.datetime.now()
time_styled = now.strftime("%Y-%m-%d %H:%M:%S")
out_ = open('record_{}.txt'.format(time_styled), 'w')
for key,value in es_cache.items():
   out_.write("Iteration[{}]:\t{}x\t{}\n".format(key,value['compression'],value['crates']))
out_.close()
print 'random seed:{}'.format(seed)
end_time = time.time()
#print(ncs_time)
#print(adjusting_time)
#print(retraining_time)
print('NCS time: %.4f mins' % (ncs_time/60.))
print('adjusting time: %.4f mins' % (adjusting_time/60.))
print('retraining time: %.4f mins' % (retraining_time/60.))
print('Total time: %.4f mins' % ((end_time - start_time)/60.))
#print "Retrieval accuracy @ iteration {}".format(retrieval_tag)
# save final model
#solver.net.save('./models/letnet5/9_letnet5_iter_{}.caffemodel'.format(itr+1))

max_stars_repo_name: Eveneko/SUSTech-Courses

max_issues_repo_name: Eveneko/SUSTech-Courses

max_forks_repo_name: Eveneko/SUSTech-Courses


text:
import numpy as np

def softmax(predictions):
    copy_predictions = np.copy(predictions)
    if predictions.ndim == 1:
        copy_predictions -= np.max(predictions) # numerical stability
        probs = np.exp(copy_predictions)/np.sum(np.exp(copy_predictions))
    else:
        copy_predictions -= np.amax(predictions, axis=1, keepdims=True) # numerical stability
        probs = np.exp(copy_predictions)/np.sum(np.exp(copy_predictions), axis=1, keepdims=True)

    return probs

def cross_entropy_loss(probs, target_index):
    if probs.ndim == 1:
        loss = -np.log(probs[target_index])
    else:
        batch_size = probs.shape[0]
        batch_loss = -np.log(probs[np.arange(batch_size), target_index.flatten()]) # return a copy of the array collapsed into one dimension
        loss = np.sum(batch_loss)/batch_size
    return loss

def softmax_with_cross_entropy(predictions, target_index):
    dprediction = softmax(predictions)
    loss = cross_entropy_loss(dprediction, target_index)

    if predictions.ndim == 1:
        dprediction[target_index] -= 1
    else:
        batch_size = predictions.shape[0]
        dprediction[np.arange(batch_size), target_index.flatten()] -= 1
        dprediction /= batch_size

    return loss, dprediction

def l2_regularization(W, reg_strength):
    loss = reg_strength * np.sum(pow(W, 2))
    grad = 2 * reg_strength * W
    return loss, grad

def linear_softmax(X, W, target_index):
    predictions = np.dot(X, W)
    loss, dprediction = softmax_with_cross_entropy(predictions, target_index)
    dW = np.dot(X.T, dprediction)

    return loss, dW

class LinearSoftmaxClassifier():
    def __init__(self):
        self.W = None

    def fit(self, X, y, batch_size=100, learning_rate=1e-7, reg=1e-5,
            epochs=1):
        num_train = X.shape[0]
        num_features = X.shape[1]
        num_classes = np.max(y)+1
        if self.W is None:
            self.W = 0.001 * np.random.randn(num_features, num_classes)

        loss_history = []
        for epoch in range(epochs):
            shuffled_indices = np.arange(num_train)
            np.random.shuffle(shuffled_indices)
            sections = np.arange(batch_size, num_train, batch_size)
            batches_indices = np.array_split(shuffled_indices, sections)

            for indices in batches_indices:
                batch_X, batch_y = X[indices], y[indices]
                loss, dW = linear_softmax(batch_X, self.W, batch_y)
                l2_loss, l2_dW = l2_regularization(self.W, reg)

                loss += l2_loss
                dW += l2_dW

                self.W -= learning_rate * dW
                loss_history.append(loss)

            print("Epoch %i, loss: %f" % (epoch, loss))

        return loss_history

    def predict(self, X):
        predictions = np.dot(X, self.W)
        y_pred = np.where(predictions == np.amax(predictions, axis=1, keepdims=True))[1]
        return y_pred


text:
# multi-headed cnn model
import numpy as np
import csv
import tensorflow as tf

# load a single file as a numpy array
def load_file(filepath):
    data = []
    with open(filepath) as csvfile:
        reader = csv.reader(csvfile, delimiter=',')
        for row in reader:
            data.append(row)
    return np.array(data)

# load a list of files and return as a 3d numpy array
def load_group(filenames, prefix=''):
    loaded = list()
    for name in filenames:
        data = load_file(prefix + name)
        loaded.append(data)
    # stack group so that features are the 3rd dimension
    loaded = np.dstack(loaded)
    return loaded

# load a dataset group, such as train or test
def load_dataset_group(group, prefix=''):
    filepath = prefix + group
    # load all 9 files as a single array
    filenames = list()
    # total acceleration
    filenames = ['01_acc_x.csv', '02_acc_y.csv', '03_acc_z.csv',
                 '04_gyro_x.csv', '05_gyro_y.csv', '06_gyro_z.csv',
                 '07_euler_x.csv', '08_euler_y.csv', '09_euler_z.csv']

    # load input data
    X = load_group(filenames, filepath).astype(np.float64)
    # load class output
    y = load_file(prefix + group + '10_label.csv').astype(np.int)
    return X, y

# load the dataset, returns train and test X and y elements
def load_dataset(prefix=''):
    # load all train
    trainX, trainy = load_dataset_group('train/', prefix + 'data/Gestures/Groups/')
    # load all test
    testX, testy = load_dataset_group('test/', prefix + 'data/Gestures/Groups/')
    # zero-offset class values
    trainy = trainy - 1
    testy = testy - 1
    # one hot encode y
    trainy = tf.keras.utils.to_categorical(trainy)
    testy = tf.keras.utils.to_categorical(testy)
    return trainX, trainy, testX, testy

# fit and evaluate a model
def evaluate_model(trainX, trainy, testX, testy):
    verbose, epochs, batch_size = 0, 10, 32
    n_timesteps, n_features, n_outputs = trainX.shape[1], trainX.shape[2], trainy.shape[1]
    # head 1
    inputs1 = tf.keras.layers.Input(shape=(n_timesteps,n_features))
    conv1 = tf.keras.layers.Conv1D(64, 3, activation='relu')(inputs1)
    drop1 = tf.keras.layers.Dropout(0.5)(conv1)
    pool1 = tf.keras.layers.MaxPooling1D()(drop1)
    flat1 = tf.keras.layers.Flatten()(pool1)
    # head 2
    inputs2 = tf.keras.layers.Input(shape=(n_timesteps,n_features))
    conv2 = tf.keras.layers.Conv1D(64, 5, activation='relu')(inputs2)
    drop2 = tf.keras.layers.Dropout(0.5)(conv2)
    pool2 = tf.keras.layers.MaxPooling1D()(drop2)
    flat2 = tf.keras.layers.Flatten()(pool2)
    # head 3
    inputs3 = tf.keras.layers.Input(shape=(n_timesteps,n_features))
    conv3 = tf.keras.layers.Conv1D(64, 9, activation='relu')(inputs3)
    drop3 = tf.keras.layers.Dropout(0.5)(conv3)
    pool3 = tf.keras.layers.MaxPooling1D()(drop3)
    flat3 = tf.keras.layers.Flatten()(pool3)
    # merge
    merged = tf.keras.layers.concatenate([flat1, flat2, flat3])
    # interpretation
    dense1 = tf.keras.layers.Dense(100, activation='relu')(merged)
    outputs = tf.keras.layers.Dense(n_outputs, activation='softmax')(dense1)
    model = tf.keras.Model(inputs=[inputs1, inputs2, inputs3], outputs=outputs)
    # save a plot of the model
    tf.keras.utils.plot_model(model, show_shapes=False, show_layer_names=True, to_file='figures/multiheaded.png')
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    # fit network
    model.fit([trainX,trainX,trainX], trainy, epochs=epochs, batch_size=batch_size, verbose=verbose)
    # evaluate model
    _, accuracy = model.evaluate([testX,testX,testX], testy, batch_size=batch_size, verbose=0)
    return accuracy

# summarize scores
def summarize_results(scores):
    print(scores)
    m, s = np.mean(scores), np.std(scores)
    print('Accuracy: %.3f%% (+/-%.3f)' % (m, s))

# run an experiment
def run_experiment(repeats=10):
    # load data
    trainX, trainy, testX, testy = load_dataset()
    # repeat experiment
    scores = list()
    for r in range(repeats):
        score = evaluate_model(trainX, trainy, testX, testy)
        score = score * 100.0
        print('>#%d: %.3f' % (r+1, score))
        scores.append(score)
    # summarize results
    summarize_results(scores)

# run the experiment
run_experiment()


text:
import numpy as np
import os
import time
import shutil
import netifaces
import traceback
import logging

from hpbandster.core.nameserver import NameServer, nic_name_to_host
from hpbandster.core.result import logged_results_to_HBS_result

from autoPyTorch.pipeline.base.sub_pipeline_node import SubPipelineNode
from autoPyTorch.pipeline.base.pipeline import Pipeline
from autoPyTorch.pipeline.nodes import MetricSelector
from autoPyTorch.utils.config.config_option import ConfigOption, to_bool
from autoPyTorch.utils.config.config_condition import ConfigCondition

from autoPyTorch.core.hpbandster_extensions.bohb_ext import BOHBExt
from autoPyTorch.core.hpbandster_extensions.hyperband_ext import HyperBandExt
from autoPyTorch.core.worker import AutoNetWorker

from autoPyTorch.components.training.budget_types import BudgetTypeTime, BudgetTypeEpochs, BudgetTypeTrainingTime
import copy

class OptimizationAlgorithm(SubPipelineNode):
    def __init__(self, optimization_pipeline_nodes):
        """OptimizationAlgorithm pipeline node.
        It will run either the optimization algorithm (BOHB, Hyperband - defined in config) or start workers
        Each worker will run the provided optimization_pipeline and will return the output
        of the pipeline_result_node to the optimization algorithm

        Train:
        The optimization_pipeline will get the following inputs:
        {hyperparameter_config, pipeline_config, X_train, Y_train, X_valid, Y_valid, budget, budget_type}
        The pipeline_result_node has to provide the following outputs:
        - 'loss': the optimization value (minimize)
        - 'info': dict containing info for the respective training process

        Predict:
        The optimization_pipeline will get the following inputs:
        {pipeline_config, X}
        The pipeline_result_node has to provide the following outputs:
        - 'Y': result of prediction for 'X'
        Note: predict will not call the optimization algorithm

        Arguments:
            optimization_pipeline {Pipeline} -- pipeline that will be optimized (hyperparamter)
            pipeline_result_node {PipelineNode} -- pipeline node that provides the results of the optimization_pieline
        """

        super(OptimizationAlgorithm, self).__init__(optimization_pipeline_nodes)

        self.algorithms = {"bohb": BOHBExt,
                           "hyperband": HyperBandExt}

        self.budget_types = dict()
        self.budget_types["time"] = BudgetTypeTime
        self.budget_types["epochs"] = BudgetTypeEpochs
        self.budget_types["training_time"] = BudgetTypeTrainingTime

    def fit(self, pipeline_config, X_train, Y_train, X_valid, Y_valid, result_loggers, dataset_info, shutdownables, refit=None):
        """Run the optimization algorithm.

        Arguments:
            pipeline_config {dict} -- The configuration of the pipeline.
            X_train {array} -- The data
            Y_train {array} -- The data
            X_valid {array} -- The data
            Y_valid {array} -- The data
            result_loggers {list} -- List of loggers that log the result
            dataset_info {DatasetInfo} -- Object with information about the dataset
            shutdownables {list} -- List of objects that need to shutdown when optimization is finished.

        Keyword Arguments:
            refit {dict} -- dict containing information for refitting. None if optimization run should be started. (default: {None})

        Returns:
            dict -- Summary of optimization run.
        """
        logger = logging.getLogger('autonet')
        res = None

        run_id, task_id = pipeline_config['run_id'], pipeline_config['task_id']

        # Use tensorboard logger
        if pipeline_config['use_tensorboard_logger'] and not refit:
            import tensorboard_logger as tl
            directory = os.path.join(pipeline_config['result_logger_dir'], "worker_logs_" + str(task_id))
            os.makedirs(directory, exist_ok=True)
            tl.configure(directory, flush_secs=5)

        # Only do refitting
        if (refit is not None):
            logger.info("Start Refitting")

            loss_info_dict = self.sub_pipeline.fit_pipeline(
                                    hyperparameter_config=refit["hyperparameter_config"], pipeline_config=pipeline_config,
                                    X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid,
                                    budget=refit["budget"], rescore=refit["rescore"], budget_type=self.budget_types[pipeline_config['budget_type']],
                                    optimize_start_time=time.time(), refit=True, hyperparameter_config_id=None, dataset_info=dataset_info)
            logger.info("Done Refitting")

            return {'optimized_hyperparameter_config': refit["hyperparameter_config"],
                    'budget': refit['budget'],
                    'loss': loss_info_dict['loss'],
                    'info': loss_info_dict['info']}

        # Start Optimization Algorithm
        try:
            ns_credentials_dir, tmp_models_dir, network_interface_name = self.prepare_environment(pipeline_config)

            # start nameserver if not on cluster or on master node in cluster
            if task_id in [1, -1]:
                NS = self.get_nameserver(run_id, task_id, ns_credentials_dir, network_interface_name)
                ns_host, ns_port = NS.start()

            if task_id != 1 or pipeline_config["run_worker_on_master_node"]:
                self.run_worker(pipeline_config=pipeline_config, run_id=run_id, task_id=task_id, ns_credentials_dir=ns_credentials_dir,
                    network_interface_name=network_interface_name, X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid,
                    dataset_info=dataset_info, shutdownables=shutdownables)

            # start BOHB if not on cluster or on master node in cluster
            res = None
            if task_id in [1, -1]:
                self.run_optimization_algorithm(pipeline_config=pipeline_config, run_id=run_id, ns_host=ns_host,
                    ns_port=ns_port, nameserver=NS, task_id=task_id, result_loggers=result_loggers,
                    dataset_info=dataset_info, logger=logger)

                res = self.parse_results(pipeline_config)

        except Exception as e:
            print(e)
            traceback.print_exc()
        finally:
            self.clean_up(pipeline_config, ns_credentials_dir, tmp_models_dir)

        if res:
            return res
        return {'optimized_hyperparameter_config': dict(), 'budget': 0, 'loss': float('inf'), 'info': dict()}

    def predict(self, pipeline_config, X):
        """Run the predict pipeline.

        Arguments:
            pipeline_config {dict} -- The configuration of the pipeline
            X {array} -- The data

        Returns:
            dict -- The predicted values in a dictionary
        """
        result = self.sub_pipeline.predict_pipeline(pipeline_config=pipeline_config, X=X)
        return {'Y': result['Y']}

    # OVERRIDE
    def get_pipeline_config_options(self):
        options = [
            ConfigOption("run_id", default="0", type=str, info="Unique id for each run."),
            ConfigOption("task_id", default=-1, type=int, info="ID for each worker, if you run AutoNet on a cluster. Set to -1, if you run it locally."),
            ConfigOption("algorithm", default="bohb", type=str, choices=list(self.algorithms.keys()), info="Algorithm to use for config sampling."),
            ConfigOption("budget_type", default="time", type=str, choices=list(self.budget_types.keys())),
            ConfigOption("min_budget", default=lambda c: self.budget_types[c["budget_type"]].default_min_budget, type=float, depends=True, info="Min budget for fitting configurations."),
            ConfigOption("max_budget", default=lambda c: self.budget_types[c["budget_type"]].default_max_budget, type=float, depends=True, info="Max budget for fitting configurations."),
            ConfigOption("max_runtime",
                default=lambda c: ((-int(np.log(c["min_budget"] / c["max_budget"]) / np.log(c["eta"])) + 1) * c["max_budget"])
                        if c["budget_type"] == "time" else float("inf"),
                type=float, depends=True, info="Total time for the run."),
            ConfigOption("num_iterations",
                default=lambda c:  (-int(np.log(c["min_budget"] / c["max_budget"]) / np.log(c["eta"])) + 1)
                        if c["budget_type"] == "epochs" else float("inf"),
                type=float, depends=True, info="Number of successive halving iterations."),
            ConfigOption("eta", default=3, type=float, info='eta parameter of Hyperband.'),
            ConfigOption("min_workers", default=1, type=int),
            ConfigOption("working_dir", default=".", type="directory"),
            ConfigOption("network_interface_name", default=self.get_default_network_interface_name(), type=str),
            ConfigOption("memory_limit_mb", default=1000000, type=int),
            ConfigOption("use_tensorboard_logger", default=False, type=to_bool),
            ConfigOption("run_worker_on_master_node", default=True, type=to_bool),
            ConfigOption("use_pynisher", default=True, type=to_bool)
        ]
        return options

    # OVERRIDE
    def get_pipeline_config_conditions(self):
        def check_runtime(pipeline_config):
            return pipeline_config["budget_type"] != "time" or pipeline_config["max_runtime"] >= pipeline_config["max_budget"]

        return [
            ConfigCondition.get_larger_equals_condition("max budget must be greater than or equal to min budget", "max_budget", "min_budget"),
            ConfigCondition("When time is used as budget, the max_runtime must be larger than the max_budget", check_runtime)
        ]

    def get_default_network_interface_name(self):
        """Get the default network interface name

        Returns:
            str -- The default network interface name
        """
        try:
            return netifaces.gateways()['default'][netifaces.AF_INET][1]
        except:
            return 'lo'

    def prepare_environment(self, pipeline_config):
        """Create necessary folders and get network interface name

        Arguments:
            pipeline_config {dict} -- The configuration of the pipeline

        Returns:
            tuple -- path to created directories and network interface namei
        """
        if not os.path.exists(pipeline_config["working_dir"]) and pipeline_config['task_id'] in [1, -1]:
            try:
                os.mkdir(pipeline_config["working_dir"])
            except:
                pass
        tmp_models_dir = os.path.join(pipeline_config["working_dir"], "tmp_models_" + str(pipeline_config['run_id']))
        ns_credentials_dir = os.path.abspath(os.path.join(pipeline_config["working_dir"], "ns_credentials_" + str(pipeline_config['run_id'])))
        network_interface_name = self.get_nic_name(pipeline_config)

        if os.path.exists(tmp_models_dir) and pipeline_config['task_id'] in [1, -1]:
            shutil.rmtree(tmp_models_dir)  # not used right now
        if os.path.exists(ns_credentials_dir) and pipeline_config['task_id'] in [1, -1]:
            shutil.rmtree(ns_credentials_dir)
        return ns_credentials_dir, tmp_models_dir, network_interface_name

    def clean_up(self, pipeline_config, tmp_models_dir, ns_credentials_dir):
        """Remove created folders

        Arguments:
            pipeline_config {dict} -- The pipeline config
            tmp_models_dir {[type]} -- The path to the temporary models (not used right now)
            ns_credentials_dir {[type]} --  The path to the nameserver credentials
        """
        if pipeline_config['task_id'] in [1, -1]:
            # Delete temporary files
            if os.path.exists(tmp_models_dir):
                shutil.rmtree(tmp_models_dir)
            if os.path.exists(ns_credentials_dir):
                shutil.rmtree(ns_credentials_dir)

    def get_nameserver(self, run_id, task_id, ns_credentials_dir, network_interface_name):
        """Get the namesever object

        Arguments:
            run_id {str} -- The id of the run
            task_id {int} -- An id for the worker
            ns_credentials_dir {str} -- Path to ns credentials
            network_interface_name {str} -- The network interface name

        Returns:
            NameServer -- The NameServer object
        """
        if not os.path.isdir(ns_credentials_dir):
            try:
                os.mkdir(ns_credentials_dir)
            except:
                pass
        return NameServer(run_id=run_id, nic_name=network_interface_name, working_directory=ns_credentials_dir)

    def get_optimization_algorithm_instance(self, config_space, run_id, pipeline_config, ns_host, ns_port, loggers, previous_result=None):
        """Get an instance of the optimization algorithm

        Arguments:
            config_space {ConfigurationSpace} -- The config space to optimize.
            run_id {str} -- An Id for the current run.
            pipeline_config {dict} -- The configuration of the pipeline.
            ns_host {str} -- Nameserver host.
            ns_port {int} -- Nameserver port.
            loggers {list} -- Loggers to log the results.

        Keyword Arguments:
            previous_result {Result} -- A previous result to warmstart the search (default: {None})

        Returns:
            Master -- An optimization algorithm.
        """
        optimization_algorithm = self.algorithms[pipeline_config["algorithm"]]
        kwargs = {"configspace": config_space, "run_id": run_id,
                  "eta": pipeline_config["eta"], "min_budget": pipeline_config["min_budget"], "max_budget": pipeline_config["max_budget"],
                  "host": ns_host, "nameserver": ns_host, "nameserver_port": ns_port,
                  "result_logger": combined_logger(*loggers),
                  "ping_interval": 10**6,
                  "working_directory": pipeline_config["working_dir"],
                  "previous_result": previous_result}
        hb = optimization_algorithm(**kwargs)
        return hb

    def parse_results(self, pipeline_config):
        """Parse the results of the optimization run

        Arguments:
            pipeline_config {dict} -- The configuration of the pipeline.

        Raises:
            RuntimeError:  An Error occurred when parsing the results.

        Returns:
            dict -- Dictionary summarizing the results
        """
        try:
            res = logged_results_to_HBS_result(pipeline_config["result_logger_dir"])
            id2config = res.get_id2config_mapping()
            incumbent_trajectory = res.get_incumbent_trajectory(bigger_is_better=False, non_decreasing_budget=False)
        except Exception as e:
            raise RuntimeError("Error parsing results. Check results.json and output for more details. An empty results.json is usually caused by a misconfiguration of AutoNet.")

        if (len(incumbent_trajectory['config_ids']) == 0):
            return dict()

        final_config_id = incumbent_trajectory['config_ids'][-1]
        final_budget = incumbent_trajectory['budgets'][-1]
        best_run = [r for r in res.get_runs_by_id(final_config_id) if r.budget == final_budget][0]
        return {'optimized_hyperparameter_config': id2config[final_config_id]['config'],
                'budget': final_budget,
                'loss': best_run.loss,
                'info': best_run.info}

    def run_worker(self, pipeline_config, run_id, task_id, ns_credentials_dir, network_interface_name,
            X_train, Y_train, X_valid, Y_valid, dataset_info, shutdownables):
        """ Run the AutoNetWorker

        Arguments:
            pipeline_config {dict} -- The configuration of the pipeline
            run_id {str} -- An id for the run
            task_id {int} -- An id for the worker
            ns_credentials_dir {str} -- path to nameserver credentials
            network_interface_name {str} -- the name of the network interface
            X_train {array} -- The data
            Y_train {array} -- The data
            X_valid {array} -- The data
            Y_valid {array} -- The data
            dataset_info {DatasetInfo} -- Object describing the dataset
            shutdownables {list} -- A list of objects that need to shutdown when the optimization is finished
        """
        if not task_id == -1:
            time.sleep(5)
        while not os.path.isdir(ns_credentials_dir):
            time.sleep(5)
        host = nic_name_to_host(network_interface_name)

        worker = AutoNetWorker(pipeline=self.sub_pipeline, pipeline_config=pipeline_config,
                              X_train=X_train, Y_train=Y_train, X_valid=X_valid, Y_valid=Y_valid, dataset_info=dataset_info,
                              budget_type=self.budget_types[pipeline_config['budget_type']],
                              max_budget=pipeline_config["max_budget"],
                              host=host, run_id=run_id,
                              id=task_id, shutdownables=shutdownables,
                              use_pynisher=pipeline_config["use_pynisher"])
        worker.load_nameserver_credentials(ns_credentials_dir)
        # run in background if not on cluster
        worker.run(background=(task_id <= 1))

    def run_optimization_algorithm(self, pipeline_config, run_id, ns_host, ns_port, nameserver, task_id, result_loggers,
            dataset_info, logger):
        """

        Arguments:
            pipeline_config {dict} -- The configuration of the pipeline
            run_id {str} -- An id for the run
            ns_host {str} -- Nameserver host.
            ns_port {int} -- Nameserver port.
            nameserver {[type]} -- The nameserver.
            task_id {int} -- An id for the worker
            result_loggers {[type]} -- [description]
            dataset_info {DatasetInfo} -- Object describing the dataset
            logger {list} -- Loggers to log the results.
        """
        config_space = self.pipeline.get_hyperparameter_search_space(dataset_info=dataset_info, **pipeline_config)

        logger.info("[AutoNet] Start " + pipeline_config["algorithm"])

        # initialize optimization algorithm
        if pipeline_config['use_tensorboard_logger']:
            result_loggers.append(tensorboard_logger())

        HB = self.get_optimization_algorithm_instance(config_space=config_space, run_id=run_id,
            pipeline_config=pipeline_config, ns_host=ns_host, ns_port=ns_port, loggers=result_loggers)

        # start algorithm
        min_num_workers = pipeline_config["min_workers"] if task_id != -1 else 1

        reduce_runtime = pipeline_config["max_budget"] if pipeline_config["budget_type"] == "time" else 0
        HB.run_until(runtime=(pipeline_config["max_runtime"] - reduce_runtime),
                     n_iterations=pipeline_config["num_iterations"],
                     min_n_workers=min_num_workers)

        HB.shutdown(shutdown_workers=True)
        nameserver.shutdown()

    @staticmethod
    def get_nic_name(pipeline_config):
        """Get the nic name from the pipeline config"""
        return pipeline_config["network_interface_name"] or (netifaces.interfaces()[1] if len(netifaces.interfaces()) > 1 else "lo")

    def clean_fit_data(self):
        super(OptimizationAlgorithm, self).clean_fit_data()
        self.sub_pipeline.root.clean_fit_data()

class tensorboard_logger(object):
    def __init__(self):
        self.start_time = time.time()
        self.incumbent = float('inf')

    def new_config(self, config_id, config, config_info):
        pass

    def __call__(self, job):
        import tensorboard_logger as tl
        # id = job.id
        budget = job.kwargs['budget']
        # config = job.kwargs['config']
        timestamps = job.timestamps
        result = job.result
        exception = job.exception

        time_step = int(timestamps['finished'] - self.start_time)

        if result is not None:
            tl.log_value('BOHB/all_results', result['loss'] * -1, time_step)
            if result['loss'] < self.incumbent:
                self.incumbent = result['loss']
            tl.log_value('BOHB/incumbent_results', self.incumbent * -1, time_step)

class combined_logger(object):
    def __init__(self, *loggers):
        self.loggers = loggers

    def new_config(self, config_id, config, config_info):
        for logger in self.loggers:
            logger.new_config(config_id, config, config_info)

    def __call__(self, job):
        for logger in self.loggers:
            logger(job)

max_stars_repo_name: jmm-montiel/Auto-PyTorch

max_issues_repo_name: jmm-montiel/Auto-PyTorch

max_forks_repo_name: jmm-montiel/Auto-PyTorch