text:
import numpy as np
import argparse
import itertools
import pandas as pd
from sklearn.metrics import mean_squared_error
from vowpalwabbit import pyvw
import pylibvw

from sklearn.preprocessing import PolynomialFeatures
import itertools
import logging
import random
from .openml_data_helper import OpenML2VWData
RANDOMSEED = 8888
LOG_TRANSFORMATION_THRESHOLD = 100

logger = logging.getLogger(__name__)
class DataSimulator:
    def __init__(self, iter_num, parameter=None):
        self.Y = None
        self.raw_ns = ['a', 'b', 'c', 'd', 'e']
        #key is namespace id, and value is the dim of the namespace
        self.raw_ns_dic = {'a':3, 'b':3, 'c':3, 'd':3, 'e':3}
        self.ground_truth_ns = ['a', 'b', 'c', 'd', 'e','ab', 'ac', 'cd',]  #'ac', 'cd'
        self._random_feature = np.random.RandomState(RANDOMSEED)
        self._random_param = np.random.RandomState(RANDOMSEED+100)
        self._random_noise = np.random.RandomState(RANDOMSEED+1234)
        self._generate_raw_X(iter_num)
        self._generate_parameter()
        self._generate_reward(self.vw_x_dic_list)
        self._construct_vw_example()

    def _generate_raw_X(self, iter_num):
        self.vw_x_dic_list = []
        for i in range(iter_num):
            vw_x_dic = {}
            count = 0
            for raw_ns, ns_dim in self.raw_ns_dic.items():
                feature = self._random_feature.uniform(0,1,ns_dim)
                # feature = feature/np.linalg.norm(feature)
                vw_x_dic[raw_ns] = feature
                count +=1
            for ns in self.ground_truth_ns:
                if ns not in vw_x_dic and len(ns) >1:
                    inter_feature = vw_x_dic[str(ns)[0]]
                    for i in range(1, len(str(ns))):
                        inter_feature = np.outer(inter_feature, vw_x_dic[str(ns)[i]]).flatten()
                    vw_x_dic[ns] = inter_feature
            self.vw_x_dic_list.append(vw_x_dic)

    def _generate_parameter(self):
        #generate reward paramter dictionary
        vw_parameter_dic = {}
        count =0
        for ns_int in self.ground_truth_ns:
            ns_dim = 1
            for ns in ns_int:
                ns_dim *= self.raw_ns_dic[ns]
            parameter = self._random_param.uniform(0,1,ns_dim)
            # parameter = parameter/np.linalg.norm(parameter)
            vw_parameter_dic[ns_int] = parameter

            count +=1
        self.vw_parameter_dic = vw_parameter_dic

    def _generate_reward(self, vw_x_list):
        # input is a dictionary of feature in vw format
        y_list = []
        for vw_x_dic in vw_x_list:
            reward_fs = 0
            for ns, fs in vw_x_dic.items():
                if ns in self.vw_parameter_dic:
                    # print(ns, fs, self.vw_parameter_dic[ns])
                    reward_fs += np.dot(vw_x_dic[ns], self.vw_parameter_dic[ns])
            noise = self._random_noise.normal(0, 0.1, 1)[0]
            r = reward_fs  + noise
            log_r = 1/(1 + np.exp(-r))
            if log_r >0.5:
                label = 1
            else: label = 0
            y_list.append(r)
            # y_list.append(label)
        self.Y = y_list

    def _construct_vw_example(self):
        # construct a list of vw example
        self.vw_examples = []
        for i, x_dic in enumerate(self.vw_x_dic_list):
            raw_vw_example = str(self.Y[i]) + ' '
            for ns, ns_x in x_dic.items():
                raw_vw_example = raw_vw_example + '|' + str(ns) + ' ' + ' '.join([str(s) for s in ns_x]) + ' '
            # pyvw_example = pyvw.example(raw_vw_example)
            # print(raw_vw_example)
            self.vw_examples.append(raw_vw_example)

def get_data(iter_num=None, data_source='simulation', vw_format=True,
             max_ns_num=10, shuffle=False, use_log=True):
    logging.info('generating data')
    # get data from simulation
    # TODO: rename data_source
    vw_examples = None
    if isinstance(data_source, str) and 'simu' in data_source:
        # get simulation data
        data = DataSimulator(iter_num)
    else:
        # get openml data
        data_id = int(data_source)
        data = OpenML2VWData(data_id, max_ns_num, 'regression')
    Y = data.Y
    if vw_format:
        vw_examples = data.vw_examples
    logger.debug('first data %s', vw_examples[0])
    # TODO: what is this doing?
    if isinstance(data_source, str) and 'simu' in data_source:
        return vw_examples, Y
    # do data shuffling or log transformation for oml data when needed
    if shuffle:
        random.seed(54321)
        random.shuffle(vw_examples)
    # do log transformation
    unique_y = set(Y)
    min_y = min(unique_y)
    max_y = max(unique_y)
    if use_log and max((max_y - min_y), max_y) >= LOG_TRANSFORMATION_THRESHOLD:
        log_vw_examples = []
        for v in vw_examples:
            org_y = v.split('|')[0]
            y = float(v.split('|')[0])
            # shift y to ensure all y are positive
            if min_y <= 0:
                y = y + abs(min_y) + 1
            log_y = np.log(y)
            log_vw = v.replace(org_y + '|', str(log_y) + ' |')
            log_vw_examples.append(log_vw)
        logger.info('log_vw_examples %s', log_vw_examples[0:2])
        if log_vw_examples:
            return log_vw_examples, Y
    return vw_examples, Y

max_stars_repo_name: varun-suryan/FLAML

max_issues_repo_name: varun-suryan/FLAML

max_forks_repo_name: varun-suryan/FLAML


text:
# Copyright (c) 2020, Xilinx
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of FINN nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import numpy as np
import time
from finn.custom_op.multithreshold import multithreshold

def compare(x, y):
    """Comparison helper function for multithresholding.

    Gets two values and returns 1.0 if x>=y otherwise 0.0."""
    if x >= y:
        return 1.0
    else:
        return 0.0

# naive implementation of thresholding for performance comparison
def multithreshold_elementwise(v, thresholds, out_scale=None, out_bias=None):
    """Given a set of threshold values t={t_0, t_1 ... t_n} the successive
    thresholding maps any real number x to an integer in the interval [0, n],
    where the returned integer is the number of thresholds x is greater than
    or equal to.

    The output tensor will be scaled by out_scale and biased by out_bias."""
    # the inputs are expected to be in the shape (N,C,H,W) or (N, C)
    # the MultiThreshold node supports a data_layout attribute that can be set
    # to 'NHWC' to support (N,H,W,C) data layout mode for in-out as well
    # N : Batch size
    # C : Number of channels
    # H : Heigth of the input images
    # W : Width of the input images
    #
    # the thresholds are expected to be in the shape (C, B)
    # C : Number of channels (must be the same value as C in input tensor
    #     or 1 if all channels use the same threshold value)
    # B : Desired activation steps => i.e. for 4-bit activation,
    #     B=7 (2^(n)-1 and n=4)
    # the output tensor will be scaled by out_scale and biased by out_bias
    # assert threshold shape
    is_global_threshold = thresholds.shape[0] == 1
    assert (
        v.shape[1] == thresholds.shape[0]
    ) or is_global_threshold, """"Threshold
    shape incorrect"""
    # save the required shape sizes for the loops (N, C and B)
    num_batch = v.shape[0]
    num_channel = v.shape[1]
    num_act = thresholds.shape[1]
    # reshape inputs to enable channel-wise reading
    vr = v.reshape((v.shape[0], v.shape[1], -1))
    # save the new shape size of the images
    num_img_elem = vr.shape[2]
    # initiate output tensor
    ret = np.zeros_like(vr)
    # iterate over thresholds channel-wise
    for t in range(num_channel):
        channel_thresh = thresholds[0] if is_global_threshold else thresholds[t]
        # iterate over batches
        for b in range(num_batch):
            # iterate over image elements on which the thresholds will be applied
            for elem in range(num_img_elem):
                # iterate over the different thresholds for one channel
                for a in range(num_act):
                    # apply successive thresholding to every element
                    ret[b][t][elem] += compare(vr[b][t][elem], channel_thresh[a])
    if out_scale is None:
        out_scale = 1.0
    if out_bias is None:
        out_bias = 0.0
    return out_scale * ret.reshape(v.shape) + out_bias

def test_multithreshold():

    inputs = np.ndarray(
        shape=(6, 3, 2, 2),
        buffer=np.array(
            [
                4.8,
                3.2,
                1.2,
                4.9,
                7.8,
                2.4,
                3.1,
                4.7,
                6.2,
                5.1,
                4.9,
                2.2,
                6.2,
                0.0,
                0.8,
                4.7,
                0.2,
                5.6,
                8.9,
                9.2,
                9.1,
                4.0,
                3.3,
                4.9,
                2.3,
                1.7,
                1.3,
                2.2,
                4.6,
                3.4,
                3.7,
                9.8,
                4.7,
                4.9,
                2.8,
                2.7,
                8.3,
                6.7,
                4.2,
                7.1,
                2.8,
                3.1,
                0.8,
                0.6,
                4.4,
                2.7,
                6.3,
                6.1,
                1.4,
                5.3,
                2.3,
                1.9,
                4.7,
                8.1,
                9.3,
                3.7,
                2.7,
                5.1,
                4.2,
                1.8,
                4.1,
                7.3,
                7.1,
                0.4,
                0.2,
                1.3,
                4.3,
                8.9,
                1.4,
                1.6,
                8.3,
                9.4,
            ]
        ),
    )

    thresholds = np.ndarray(
        shape=(3, 7),
        buffer=np.array(
            [
                0.8,
                1.4,
                1.7,
                3.5,
                5.2,
                6.8,
                8.2,
                0.2,
                2.2,
                3.5,
                4.5,
                6.6,
                8.6,
                9.2,
                1.3,
                4.1,
                4.5,
                6.5,
                7.8,
                8.1,
                8.9,
            ]
        ),
    )

    outputs = np.ndarray(
        shape=(6, 3, 2, 2),
        buffer=np.array(
            [
                4.0,
                3.0,
                1.0,
                4.0,
                5.0,
                2.0,
                2.0,
                4.0,
                3.0,
                3.0,
                3.0,
                1.0,
                5.0,
                0.0,
                1.0,
                4.0,
                1.0,
                4.0,
                6.0,
                7.0,
                7.0,
                1.0,
                1.0,
                3.0,
                3.0,
                3.0,
                1.0,
                3.0,
                4.0,
                2.0,
                3.0,
                7.0,
                3.0,
                3.0,
                1.0,
                1.0,
                7.0,
                5.0,
                4.0,
                6.0,
                2.0,
                2.0,
                1.0,
                1.0,
                2.0,
                1.0,
                3.0,
                3.0,
                2.0,
                5.0,
                3.0,
                3.0,
                4.0,
                5.0,
                7.0,
                3.0,
                1.0,
                3.0,
                2.0,
                1.0,
                4.0,
                6.0,
                6.0,
                0.0,
                1.0,
                1.0,
                3.0,
                6.0,
                1.0,
                1.0,
                6.0,
                7.0,
            ]
        ),
    )

    results = multithreshold(inputs, thresholds)
    assert (results == outputs).all()

    results_scaled = multithreshold(inputs, thresholds, 2.0, -1.0)
    outputs_scaled = 2.0 * outputs - 1.0
    assert (results_scaled == outputs_scaled).all()

    # performance and random test
    np.random.seed(0)
    inputs = np.random.random((1, 256, 64, 64))
    thresholds = (np.array([[1, 2, 3, 4, 5, 6]]) - 0.5) / 6

    before = time.time()
    vec_results = multithreshold(inputs, thresholds)
    after = time.time()
    vector_runtime = after - before

    before = time.time()
    nonvec_results = multithreshold_elementwise(inputs, thresholds)
    after = time.time()
    non_vector_runtime = after - before

    assert (vec_results == nonvec_results).all()

    return vector_runtime, non_vector_runtime

if __name__ == "__main__":
    vector_runtime, non_vector_runtime = test_multithreshold()

    print("Runtime non-vectorized: ", non_vector_runtime, "s")
    print("Runtime vectorized: ", vector_runtime, "s")
    print("Speed-up: ", non_vector_runtime / vector_runtime)

ext: py

max_stars_repo_name: alinavalinav/finn

max_issues_repo_name: alinavalinav/finn

max_forks_repo_name: alinavalinav/finn

reason: import numpy


text:
"""
unit test for OutlierRemover

@TODO add test for inplace parameter
"""
import pandas as pd
import numpy as np
import os

from numpy.testing import assert_array_equal
from clintk.utils import outliers

curpath = os.path.dirname(__file__)
# print('CURPATH IS: \n', curpath)

values = {'col1': np.arange(-2, 2, step=0.5),
          'col2': np.arange(0, 16, step=2),
          'col3': np.arange(1, 9, step=1)}

df = pd.DataFrame(values)

# col1  col2  col3
# 0  -2.0     0     1
# 1  -1.5     2     2
# 2  -1.0     4     3
# 3  -0.5     6     4
# 4   0.0     8     5
# 5   0.5    10     6
# 6   1.0    12     7
# 7   1.5    14     8

mean1, mean2 = np.mean(df['col1']), np.mean(df['col2'])

class TestOutliers(object):
    def setUp(self):
        print(curpath)
        return outliers.OutlierRemover(curpath + '/bounds_test.txt', False)

    def test_remove(self):
        remover = self.setUp()

        cleansed_df = remover.transform(df)

        true_col1 = np.array([mean1, mean1, mean1, mean1, 0, 0.5, 1, mean1])
        true_col2 = np.array([mean2, mean2, mean2, 6, 8, 10, 12, 14])
        true_col3 = np.array([6, 6, 6, 6, 5, 6, 7, 8])

        assert_array_equal(true_col1, cleansed_df['col1'].values)
        assert_array_equal(true_col2, cleansed_df['col2'].values)
        assert_array_equal(true_col3, cleansed_df['col3'].values)

max_stars_repo_name: DITEP/db-cleansing

max_issues_repo_name: DITEP/db-cleansing

max_forks_repo_name: DITEP/db-cleansing

reason: import numpy,from numpy


text:
import pandas as pd
import numpy as np
import torch
import torchvision.transforms as transforms
from torch.autograd import Variable
from PIL import Image
import os

class ImagePredictor(object):
    def __init__(self, model_root):
        self.model_classification = torch.load(os.path.join(model_root, 'model_classification.pt'))
        self.topics = ['food', 'others', 'pet', 'selfie', 'view']
        self.models = [torch.load(os.path.join(model_root, 'model_%s.pt'%topic)) for topic in self.topics]

        self.transform = transforms.Compose(
            [transforms.CenterCrop(256),
        #      transforms.Resize(224),
             transforms.ToTensor(),
             transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

        self.use_gpu = torch.cuda.is_available()

    def predict(self, image_path, return_classes=False):
        """Given a image path, return the predicted score of this image"""
        image = self.load_image(image_path)
        class_scores = self.model_classification(image)
        _, preds = torch.max(class_scores.data, 1)
        pred_topic = self.topics[preds.item()]

        model = self.models[preds]
        scores = model(image).data.numpy()[0][0]
        # scores = scores.exp() / (scores.exp().sum())
        if return_classes:
            return (scores, pred_topic)
        return scores

    def rank(self, image_path_list):
        """
        Given a list of path, return a list of indices that can sort the array in descending order
        See: https://docs.scipy.org/doc/numpy/reference/generated/numpy.argsort.html
        """
        return np.argsort(np.array([-self.predict(x) for x in image_path_list]))

    def load_image(self, image_path):
        """load image, returns tensor"""
        image = Image.open(image_path).convert('RGB')
        image = self.transform(image).float()
        image = Variable(image, requires_grad=True)
        image = image.unsqueeze(0)  #this is for VGG, may not be needed for ResNet
        if self.use_gpu:
            return image.cuda()
        return image

if __name__ == '__main__':
    m = ImagePredictor('trained_models')
    print('Single Prediction: ', m.predict('images/pet/0.jpg', return_classes=True))
    image_list = ['images/pet/0.jpg', 'images/selfie/1.jpg', 'images/food/11.jpg',
    'images/view/18.jpg']
    print('Ranking Prediction: ', m.rank(image_list))


text:
import numpy as np
import tensorflow as tf
from tensorflow.keras.layers import Dense,Conv2D, MaxPool2D, Flatten
from tensorflow.keras.models import Model
from sklearn.decomposition import PCA

"""
USAGE:
This file is meant to be used as a library/module which is imported into other programs in order to test the algorithm.
You can do data processing/evaluation elsewhere, this is just where raw model code is kept.
The commented out code at the bottom just tests the functions above on dummy data
"""

"""Syntax for model layer definition:
If mode is "dense", just input a list of layer sizes where the last layer size is the hash size
If mode is "conv", input a list of alternating 3-tuple and integer where the tuple is:
(num filters, filter size, stride) and the number is max pool size
At the end append one number to the list that is the hash size
"""

EPOCH_LOSS_BLOCK_SIZE = 4
CONV_ERROR = 0.01

# Original paper: https://www.cv-foundation.org/openaccess/content_cvpr_2015/papers/Liong_Deep_Hashing_for_2015_CVPR_paper.pdf

# There are M+1 layers in the network
# activation for every layer is tanh, layers are just dense
# DATA IS ALWAYS ASSUMED TO BE IN ROW VECTORS FOR THIS WHOLE PROGRAM

# W for the first layer is supposed to be initialized to the first [layer_1_size] eigenvectors of the covariance matrix
# this function just finds those eigenvectors where n_dims is the number to get
def initialize_W(data, n_dims):
    pca = PCA(n_dims).fit(data)
    return pca.components_.T

class DeepHash(Model):
    def __init__(self, layer_sizes, l2_, l3_, l1_, initial_W, mode='dense'):
        super(DeepHash, self).__init__()
        self.layer_sizes = layer_sizes
        self.l1 = l1_
        self.l2 = l2_
        self.l3 = l3_
        self.initial_W = initial_W
        self.mode = mode

    def custom_reg(self, W):
        # NOTE THAT WE DO THE REVERSE OF THE PAPER WITH OUR TRANSPOSE HERE, SINCE TENSORFLOW STORES WEIGHT MATRICES TRANSPOSED
        # W is [this_layer_nodes, next_layer_nodes] in shape
        part1 = self.l2 * tf.square(tf.norm(tf.matmul(tf.transpose(W), W) - tf.eye(W.shape[1])))
        part2 = self.l3 / 2 * tf.square(tf.norm(W))
        #print(part1+part2)
        return part1 + part2

    def custom_bias_reg(self, b):
        return self.l3 / 2 * tf.square(tf.norm(b))

    def custom_W_init(self, shape, dtype=None):
        return self.initial_W

    def build(self, input_shape):
        self.fc_layers = []
        if self.mode=='dense':
            self.fc_layers.append(Dense(self.layer_sizes[0], activation='tanh', input_shape=input_shape,
                                    kernel_regularizer=self.custom_reg, bias_regularizer=self.custom_bias_reg,
                                    bias_initializer='ones', kernel_initializer=self.custom_W_init))
        elif self.mode=='conv':
            self.fc_layers.append(Conv2D(self.layer_sizes[0][0],self.layer_sizes[0][1],self.layer_sizes[0][2],activation='tanh',padding='same'))
            self.fc_layers.append(MaxPool2D(self.layer_sizes[1]))
        for n, layer in enumerate(self.layer_sizes[1:]):
            if self.mode=='dense':
                self.fc_layers.append(Dense(layer, activation='tanh', kernel_regularizer=self.custom_reg,
                                        bias_regularizer=self.custom_bias_reg, bias_initializer='ones',
                                        kernel_initializer='identity'))
            elif self.mode=='conv':
                if n==0:
                    continue
                if n==len(self.layer_sizes)-2:
                    break
                if n%2==0:
                    self.fc_layers.append(MaxPool2D(layer))
                else:
                    self.fc_layers.append(
                        Conv2D(layer[0], layer[1], layer[2], activation='tanh',padding='same'))

        if self.mode=='conv':
            self.fc_layers.append(Flatten())
            self.fc_layers.append(Dense(self.layer_sizes[-1],activation='tanh'))

    def call(self, inputs):
        h = inputs
        for layer in self.fc_layers:
            h = layer(h)
        quantized = tf.math.sign(h)
        # NOTE WE PUT THE TRANSPOSE FIRST BECAUSE WE HAVE ROW VECTORS, NOT COLUMN VECTORS HERE
        # Trace is not affected by transposing which is why we don't need another transpose on the whole thing

        #h_tilde = h-tf.expand_dims(tf.reduce_mean(h,axis=1),1)
        h_tilde = h-tf.expand_dims(tf.reduce_mean(h,axis=0),0)

        N = tf.cast(tf.shape(inputs)[0], tf.float32)
        J2 = tf.math.negative(self.l1 / (2 * N) * tf.linalg.trace(tf.matmul(tf.transpose(h_tilde), h_tilde)))
        print(J2)
        self.add_loss(J2)

        J1 = .5 * tf.square(tf.norm(quantized - h))/N
        print(J1)
        self.add_loss(J1)  # both are row vectors, so it's fine

        return h, quantized

# This trains a DeepHash model for the specified number of epochs given an optimizer object and training data
def train_unsupervised(model, epochs, data, optimizer, conv_error):
    old_loss = 0
    epoch_loss = 0
    print("Epoch block size: ", EPOCH_LOSS_BLOCK_SIZE)
    for i in range(epochs):
        # GradientTape creates an environment that makes it easier to find gradients.
        with tf.GradientTape() as tape:
            out = model(data)
            #print(out)
            loss = tf.reduce_sum(model.losses)

            # Finds the gradient of the loss w.r.t. the trainable variables (parameters W, c)
            grad = tape.gradient(loss, model.trainable_variables)

            #print("GRAD:")
            #print(np.mean([tf.reduce_mean(g) for g in grad]))
            # Updates W^m, c^m
            optimizer.apply_gradients(zip(grad, model.trainable_variables))

            # Printing loss
            if i % EPOCH_LOSS_BLOCK_SIZE == 0 and i is not 0:
                print("Epoch block", i/4, "avg. loss:", epoch_loss / EPOCH_LOSS_BLOCK_SIZE)
                epoch_loss = 0
            else:
                epoch_loss += loss

            # Stop condition
            if i > 1 and abs(loss-old_loss) < conv_error and conv_error!=0:
                print("loss flattened below convergence error of", conv_error)
                return
        old_loss = loss

# Here pos_pairs and neg_pairs are assumed to be of shape [num_pairs, 2, data_dimension]
# alpha is the coefficient for the supervised loss
def train_supervised(model, epochs, pos_pairs, neg_pairs, data, optimizer, alpha, conv_error):
    old_loss = 0
    epoch_loss = 0
    print("Epoch block size: ", EPOCH_LOSS_BLOCK_SIZE)
    for i in range(epochs):
        with tf.GradientTape() as tape:
            out = model(data)
            # this is the unsupervised loss
            loss_unsup = tf.reduce_sum(model.losses)

            # Here is the supervised loss calculation as specified in the paper
            out_pos1 = model(pos_pairs[:, 0])[0]  # we only need H here, not B, which is why we take the 0 index at the end
            out_pos2 = model(pos_pairs[:, 1])[0]
            out_neg1 = model(neg_pairs[:, 0])[0]
            out_neg2 = model(neg_pairs[:, 1])[0]
            pos_diff = out_pos1 - out_pos2
            neg_diff = out_neg1 - out_neg2
            N_pos = tf.cast(tf.shape(pos_pairs)[0], tf.float32)
            N_neg = tf.cast(tf.shape(neg_pairs)[0], tf.float32)
            sigma_W = 1 / N_pos * tf.matmul(tf.transpose(pos_diff), pos_diff)
            sigma_B = 1 / N_neg * tf.matmul(tf.transpose(neg_diff), neg_diff)
            loss_sup = alpha * tf.linalg.trace(sigma_W - sigma_B)
            print("sup loss: " + str(loss_sup))
            loss = loss_unsup + loss_sup

            # taking the gradient and applying the optimizer
            grad = tape.gradient(loss, model.trainable_variables)
            optimizer.apply_gradients(zip(grad, model.trainable_variables))

            # Printing loss
            if i % EPOCH_LOSS_BLOCK_SIZE == 0 and i is not 0:
                print("Epoch block", i/4, "avg. loss:", epoch_loss / EPOCH_LOSS_BLOCK_SIZE)
                epoch_loss = 0
            else:
                epoch_loss += loss

            # Stop condition
            if i > 1 and abs(loss-old_loss) < conv_error:
                print("loss flattened below convergence error of", conv_error)
                return
        old_loss = loss

# if __name__ == '__main__':
#     d = 100
#     N = 1000
#     hash_size = 20
#     sample_data = np.random.random(size=[N,d])
#     CONV_ERROR = 0.01
#
#     model = DeepHash([64,32,20],1,.001,.001, initialize_W(sample_data,64))
#     y = model(sample_data)
#     print(y[0].shape)
#     print(y[1].shape)
#     model.summary()
#     opt = tf.keras.optimizers.Adam(.001)
#     train_unsupervised(model,50,sample_data,opt, CONV_ERROR)
#     # pos_samples, neg_samples = np.random.random(size=[N,2,d]), np.random.random(size=[N,2,d])
#     # train_supervised(model,100,pos_samples,neg_samples,sample_data,opt,.01, CONV_ERROR)
#
#     #conv version
#     imd = 32
#     chan = 3
#     N = 1000
#     hash_size = 20
#     sample_data = np.random.random(size=[N,imd, imd, chan])
#     CONV_ERROR = 0.01
#
#     model = DeepHash([(16,6,1),2,(32,4,2),2,(64,4,2),2,hash_size],1,.001,.001, None,'conv')
#     y = model(sample_data)
#     print(y[0].shape)
#     print(y[1].shape)
#     model.summary()
#     opt = tf.keras.optimizers.Adam(.001)
#     train_unsupervised(model,50,sample_data,opt, CONV_ERROR)

max_stars_repo_name: ZovcIfzm/DeepHashing

max_issues_repo_name: ZovcIfzm/DeepHashing

max_forks_repo_name: ZovcIfzm/DeepHashing


text:
from typing import Any, Dict, List, Optional, Tuple, Union

import h5py
import numpy as np

from tianshou.data import Batch
from tianshou.data.batch import _alloc_by_keys_diff, _create_value
from tianshou.data.utils.converter import from_hdf5, to_hdf5

class ReplayBuffer:
    """:class:`~tianshou.data.ReplayBuffer` stores data generated from interaction \
    between the policy and environment.

    ReplayBuffer can be considered as a specialized form (or management) of Batch. It
    stores all the data in a batch with circular-queue style.

    For the example usage of ReplayBuffer, please check out Section Buffer in
    :doc:`/tutorials/concepts`.

    :param int size: the maximum size of replay buffer.
    :param int stack_num: the frame-stack sampling argument, should be greater than or
        equal to 1. Default to 1 (no stacking).
    :param bool ignore_obs_next: whether to store obs_next. Default to False.
    :param bool save_only_last_obs: only save the last obs/obs_next when it has a shape
        of (timestep, ...) because of temporal stacking. Default to False.
    :param bool sample_avail: the parameter indicating sampling only available index
        when using frame-stack sampling method. Default to False.
    """

    _reserved_keys = ("obs", "act", "rew", "done", "obs_next", "info", "policy")

    def __init__(
        self,
        size: int,
        stack_num: int = 1,
        ignore_obs_next: bool = False,
        save_only_last_obs: bool = False,
        sample_avail: bool = False,
        **kwargs: Any,  # otherwise PrioritizedVectorReplayBuffer will cause TypeError
    ) -> None:
        self.options: Dict[str, Any] = {
            "stack_num": stack_num,
            "ignore_obs_next": ignore_obs_next,
            "save_only_last_obs": save_only_last_obs,
            "sample_avail": sample_avail,
        }
        super().__init__()
        self.maxsize = int(size)
        assert stack_num > 0, "stack_num should be greater than 0"
        self.stack_num = stack_num
        self._indices = np.arange(size)
        self._save_obs_next = not ignore_obs_next
        self._save_only_last_obs = save_only_last_obs
        self._sample_avail = sample_avail
        self._meta: Batch = Batch()
        self._ep_rew: Union[float, np.ndarray]
        self.reset()

    def __len__(self) -> int:
        """Return len(self)."""
        return self._size

    def __repr__(self) -> str:
        """Return str(self)."""
        return self.__class__.__name__ + self._meta.__repr__()[5:]

    def __getattr__(self, key: str) -> Any:
        """Return self.key."""
        try:
            return self._meta[key]
        except KeyError as exception:
            raise AttributeError from exception

    def __setstate__(self, state: Dict[str, Any]) -> None:
        """Unpickling interface.

        We need it because pickling buffer does not work out-of-the-box
        ("buffer.__getattr__" is customized).
        """
        self.__dict__.update(state)

    def __setattr__(self, key: str, value: Any) -> None:
        """Set self.key = value."""
        assert (key not in self._reserved_keys
                ), "key '{}' is reserved and cannot be assigned".format(key)
        super().__setattr__(key, value)

    def save_hdf5(self, path: str) -> None:
        """Save replay buffer to HDF5 file."""
        with h5py.File(path, "w") as f:
            to_hdf5(self.__dict__, f)

    @classmethod
    def load_hdf5(cls, path: str, device: Optional[str] = None) -> "ReplayBuffer":
        """Load replay buffer from HDF5 file."""
        with h5py.File(path, "r") as f:
            buf = cls.__new__(cls)
            buf.__setstate__(from_hdf5(f, device=device))  # type: ignore
        return buf

    def reset(self, keep_statistics: bool = False) -> None:
        """Clear all the data in replay buffer and episode statistics."""
        self.last_index = np.array([0])
        self._index = self._size = 0
        if not keep_statistics:
            self._ep_rew, self._ep_len, self._ep_idx = 0.0, 0, 0

    def set_batch(self, batch: Batch) -> None:
        """Manually choose the batch you want the ReplayBuffer to manage."""
        assert len(batch) == self.maxsize and set(batch.keys()).issubset(
            self._reserved_keys
        ), "Input batch doesn't meet ReplayBuffer's data form requirement."
        self._meta = batch

    def unfinished_index(self) -> np.ndarray:
        """Return the index of unfinished episode."""
        last = (self._index - 1) % self._size if self._size else 0
        return np.array([last] if not self.done[last] and self._size else [], int)

    def prev(self, index: Union[int, np.ndarray]) -> np.ndarray:
        """Return the index of previous transition.

        The index won't be modified if it is the beginning of an episode.
        """
        index = (index - 1) % self._size
        end_flag = self.done[index] | (index == self.last_index[0])
        return (index + end_flag) % self._size

    def next(self, index: Union[int, np.ndarray]) -> np.ndarray:
        """Return the index of next transition.

        The index won't be modified if it is the end of an episode.
        """
        end_flag = self.done[index] | (index == self.last_index[0])
        return (index + (1 - end_flag)) % self._size

    def update(self, buffer: "ReplayBuffer") -> np.ndarray:
        """Move the data from the given buffer to current buffer.

        Return the updated indices. If update fails, return an empty array.
        """
        if len(buffer) == 0 or self.maxsize == 0:
            return np.array([], int)
        stack_num, buffer.stack_num = buffer.stack_num, 1
        from_indices = buffer.sample_indices(0)  # get all available indices
        buffer.stack_num = stack_num
        if len(from_indices) == 0:
            return np.array([], int)
        to_indices = []
        for _ in range(len(from_indices)):
            to_indices.append(self._index)
            self.last_index[0] = self._index
            self._index = (self._index + 1) % self.maxsize
            self._size = min(self._size + 1, self.maxsize)
        to_indices = np.array(to_indices)
        if self._meta.is_empty():
            self._meta = _create_value(  # type: ignore
                buffer._meta, self.maxsize, stack=False)
        self._meta[to_indices] = buffer._meta[from_indices]
        return to_indices

    def _add_index(self, rew: Union[float, np.ndarray],
                   done: bool) -> Tuple[int, Union[float, np.ndarray], int, int]:
        """Maintain the buffer's state after adding one data batch.

        Return (index_to_be_modified, episode_reward, episode_length,
        episode_start_index).
        """
        self.last_index[0] = ptr = self._index
        self._size = min(self._size + 1, self.maxsize)
        self._index = (self._index + 1) % self.maxsize

        self._ep_rew += rew
        self._ep_len += 1

        if done:
            result = ptr, self._ep_rew, self._ep_len, self._ep_idx
            self._ep_rew, self._ep_len, self._ep_idx = 0.0, 0, self._index
            return result
        else:
            return ptr, self._ep_rew * 0.0, 0, self._ep_idx

    def add(
        self,
        batch: Batch,
        buffer_ids: Optional[Union[np.ndarray, List[int]]] = None
    ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
        """Add a batch of data into replay buffer.

        :param Batch batch: the input data batch. Its keys must belong to the 7
            reserved keys, and "obs", "act", "rew", "done" is required.
        :param buffer_ids: to make consistent with other buffer's add function; if it
            is not None, we assume the input batch's first dimension is always 1.

        Return (current_index, episode_reward, episode_length, episode_start_index). If
        the episode is not finished, the return value of episode_length and
        episode_reward is 0.
        """
        # preprocess batch
        new_batch = Batch()
        for key in set(self._reserved_keys).intersection(batch.keys()):
            new_batch.__dict__[key] = batch[key]
        batch = new_batch
        assert set(["obs", "act", "rew", "done"]).issubset(batch.keys())
        stacked_batch = buffer_ids is not None
        if stacked_batch:
            assert len(batch) == 1
        if self._save_only_last_obs:
            batch.obs = batch.obs[:, -1] if stacked_batch else batch.obs[-1]
        if not self._save_obs_next:
            batch.pop("obs_next", None)
        elif self._save_only_last_obs:
            batch.obs_next = (
                batch.obs_next[:, -1] if stacked_batch else batch.obs_next[-1]
            )
        # get ptr
        if stacked_batch:
            rew, done = batch.rew[0], batch.done[0]
        else:
            rew, done = batch.rew, batch.done
        ptr, ep_rew, ep_len, ep_idx = list(
            map(lambda x: np.array([x]), self._add_index(rew, done))
        )
        try:
            self._meta[ptr] = batch
        except ValueError:
            stack = not stacked_batch
            batch.rew = batch.rew.astype(float)
            batch.done = batch.done.astype(bool)
            if self._meta.is_empty():
                self._meta = _create_value(  # type: ignore
                    batch, self.maxsize, stack)
            else:  # dynamic key pops up in batch
                _alloc_by_keys_diff(self._meta, batch, self.maxsize, stack)
            self._meta[ptr] = batch
        return ptr, ep_rew, ep_len, ep_idx

    def sample_indices(self, batch_size: int) -> np.ndarray:
        """Get a random sample of index with size = batch_size.

        Return all available indices in the buffer if batch_size is 0; return an empty
        numpy array if batch_size < 0 or no available index can be sampled.
        """
        if self.stack_num == 1 or not self._sample_avail:  # most often case
            if batch_size > 0:
                return np.random.choice(self._size, batch_size)
            elif batch_size == 0:  # construct current available indices
                return np.concatenate(
                    [np.arange(self._index, self._size),
                     np.arange(self._index)]
                )
            else:
                return np.array([], int)
        else:
            if batch_size < 0:
                return np.array([], int)
            all_indices = prev_indices = np.concatenate(
                [np.arange(self._index, self._size),
                 np.arange(self._index)]
            )
            for _ in range(self.stack_num - 2):
                prev_indices = self.prev(prev_indices)
            all_indices = all_indices[prev_indices != self.prev(prev_indices)]
            if batch_size > 0:
                return np.random.choice(all_indices, batch_size)
            else:
                return all_indices

    def sample(self, batch_size: int) -> Tuple[Batch, np.ndarray]:
        """Get a random sample from buffer with size = batch_size.

        Return all the data in the buffer if batch_size is 0.

        :return: Sample data and its corresponding index inside the buffer.
        """
        indices = self.sample_indices(batch_size)
        return self[indices], indices

    def get(
        self,
        index: Union[int, List[int], np.ndarray],
        key: str,
        default_value: Any = None,
        stack_num: Optional[int] = None,
    ) -> Union[Batch, np.ndarray]:
        """Return the stacked result.

        E.g., if you set ``key = "obs", stack_num = 4, index = t``, it returns the
        stacked result as ``[obs[t-3], obs[t-2], obs[t-1], obs[t]]``.

        :param index: the index for getting stacked data.
        :param str key: the key to get, should be one of the reserved_keys.
        :param default_value: if the given key's data is not found and default_value is
            set, return this default_value.
        :param int stack_num: Default to self.stack_num.
        """
        if key not in self._meta and default_value is not None:
            return default_value
        val = self._meta[key]
        if stack_num is None:
            stack_num = self.stack_num
        try:
            if stack_num == 1:  # the most often case
                return val[index]
            stack: List[Any] = []
            if isinstance(index, list):
                indices = np.array(index)
            else:
                indices = index  # type: ignore
            for _ in range(stack_num):
                stack = [val[indices]] + stack
                indices = self.prev(indices)
            if isinstance(val, Batch):
                return Batch.stack(stack, axis=indices.ndim)
            else:
                return np.stack(stack, axis=indices.ndim)
        except IndexError as exception:
            if not (isinstance(val, Batch) and val.is_empty()):
                raise exception  # val != Batch()
            return Batch()

    def __getitem__(self, index: Union[slice, int, List[int], np.ndarray]) -> Batch:
        """Return a data batch: self[index].

        If stack_num is larger than 1, return the stacked obs and obs_next with shape
        (batch, len, ...).
        """
        if isinstance(index, slice):  # change slice to np array
            # buffer[:] will get all available data
            indices = self.sample_indices(0) if index == slice(None) \
                else self._indices[:len(self)][index]
        else:
            indices = index
        # raise KeyError first instead of AttributeError,
        # to support np.array([ReplayBuffer()])
        obs = self.get(indices, "obs")
        if self._save_obs_next:
            obs_next = self.get(indices, "obs_next", Batch())
        else:
            obs_next = self.get(self.next(indices), "obs", Batch())
        return Batch(
            obs=obs,
            act=self.act[indices],
            rew=self.rew[indices],
            done=self.done[indices],
            obs_next=obs_next,
            info=self.get(indices, "info", Batch()),
            policy=self.get(indices, "policy", Batch()),
        )

max_stars_repo_name: ClarenceYC/tianshou

max_issues_repo_name: ClarenceYC/tianshou

max_forks_repo_name: ClarenceYC/tianshou


text:
""" Run this module to generate the dataset for testing, randomly. """

import numpy as np
import copy
import csv
import os
from datetime import datetime
from dependencies import *
from basics import *
import random as rd
from time import time
from shutil import copy as shucopy

__author__ = 'Duy Cao'
__version__ = '2020.5.5'

TEST_DATASET_SIZE = 100 # notice that the final dataset may be smaller
                         # after duplicate_removal
AVAILABLE_ROTATIONS = ['l', 'u', 'r', 'b', 'f', 'd', \
                        'l2', 'u2', 'r2', 'b2', 'f2', 'd2', \
                        'lr', 'ur', 'rr', 'br', 'fr', 'dr']

def datetime_(): # date and time will be the name of the dataset
    t = datetime.now().strftime("%y%m%d%H%M")
    return t

def touch(path, path2, filename): # path = 'test' or 'train'
    if not os.path.exists('datasets'):
        os.mkdir('datasets')
    if not os.path.exists('datasets/' + path):
        os.mkdir('datasets/' + path)
    if not os.path.exists('datasets/' + path + '/' + path2):
        os.mkdir('datasets/' + path + '/' + path2)
    if os.path.exists('datasets/' + path + '/' + path2 + '/' + filename + '.csv'):
        os.remove('datasets/' + path + '/' + path2 + '/' + filename + '.csv')
        f = open('datasets/' + path + '/' + path2 + '/' + filename + '.csv', 'x')
        f.close()
    else:
        f = open('datasets/' + path + '/' + path2 + '/' + filename + '.csv', 'x')
        f.close()

def generator(path, path2, filename): # rotating randomly & write combinations into dataset
    touch(path, path2, filename)
    cube = init_cube(RUBIK_SIZE)
    path = 'datasets/' + path + '/' + path2 + '/' + filename + '.csv'
    with open(path, 'w', newline='') as dataset:
        generator = csv.writer(dataset)
        for i in range(TEST_DATASET_SIZE):
            move = rd.choice(AVAILABLE_ROTATIONS)
            cube = turn(move, cube)
            row = rubik_to_array(cube, RUBIK_SIZE)
            row = np.append(row, rotating_notations(move))
            generator.writerow(row)

def duplicate_removal(path, path2, filename): # remove duplicate in datasets
    old_path = 'datasets/' + path + '/' + path2 + '/' + filename + '.csv'
    new_path = 'datasets/' + path + '/' + path2 + '/' + filename + '_cached.csv'
    shucopy(old_path, new_path)
    os.remove(old_path)
    touch(path, path2, filename)
    with open(new_path, 'r') as clipboard, open (old_path, 'w') as final:
        cached = set()
        counter = 0
        for line in clipboard:
            if line in cached:
                counter+=1
            if line not in cached:
                cached.add(line)
                final.write(line)
    os.remove(new_path)
    return(counter)

def duplicate_removal_v2(path, path2, filename, size): # check only combinations, not full row
    old_path = 'datasets/' + path + '/' + path2 + '/' + filename + '.csv'
    new_path = 'datasets/' + path + '/' + path2 + '/' + filename + '_cached.csv'
    num_of_cols = size**2*6
    shucopy(old_path, new_path)
    os.remove(old_path)
    touch(path, path2, filename)
    with open(new_path, 'r') as clipboard, open (old_path, 'w') as final:
        cached = set()
        counter = 0
        for line in clipboard:
            if line[:num_of_cols] in cached:
                counter+=1
            if line[:num_of_cols] not in cached:
                cached.add(line[:num_of_cols])
                final.write(line)
    os.remove(new_path)
    return(counter)

if __name__ == '__main__':
    start = time()
    filename = datetime_()
    dir = 'test'
    dir2 = str(RUBIK_SIZE)
    generator(dir, dir2, filename)
    print("Done! \nTime elapsed = " + str(round((time() - start), 3)) + " seconds")
    duplicates = duplicate_removal_v2(dir, dir2, filename, RUBIK_SIZE)
    print(str(duplicates) + "/" + str(TEST_DATASET_SIZE) + " were duplicates " + \
        "and automatically removed.")

max_stars_repo_name: caominhduy/DeepRuby

max_issues_repo_name: caominhduy/DeepRuby

max_forks_repo_name: caominhduy/DeepRuby


text:
import click
from selenium import webdriver, common
from PIL import Image
import hashlib
import time
import requests
import io
from typing import List, Dict, Tuple
import os
from pathlib import Path
import numpy as np
import pickle
from threading import Thread
import logging

def fetch_image_urls(
    query: str,
    max_links_to_fetch: int,
    wd: webdriver,
    sleep_between_interactions: float = 0.3,
) -> List[str]:
    def scroll_to_end(wd):
        wd.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(sleep_between_interactions)

    # build the google query
    search_url = "https://www.google.com/search?safe=off&site=&tbm=isch&source=hp&q={q}&oq={q}&gs_l=img"

    # load the page
    wd.get(search_url.format(q=query))

    image_urls = set()
    image_count = 0
    results_start = 0
    while image_count < max_links_to_fetch:
        scroll_to_end(wd)

        # get all image thumbnail results
        try:
            thumbnail_results = wd.find_elements_by_css_selector("img.rg_ic")
        except Exception:
            thumbnail_results = wd.find_elements_by_css_selector("img.rg_i")
        number_results = len(thumbnail_results)

        for img in thumbnail_results[results_start:number_results]:
            # try to click every thumbnail such that we can get the real image behind it
            try:
                img.click()
                time.sleep(sleep_between_interactions)
            except Exception:
                continue

            # extract image urls
            try:
                actual_images = wd.find_elements_by_css_selector("img.irc_mi")
            except Exception:
                actual_images = wd.find_elements_by_css_selector("img.n3NVCb")
            for actual_image in actual_images:
                if actual_image.get_attribute("src"):
                    image_urls.add(actual_image.get_attribute("src"))

            image_count = len(image_urls)

            if len(image_urls) >= max_links_to_fetch:
                print(f"Found: {len(image_urls)} image links, done!")
                break

        else:
            time.sleep(0.5)
            load_more_button = wd.find_element_by_css_selector(".ksb")
            if load_more_button:
                wd.execute_script("document.querySelector('.ksb').click();")

        # move the result startpoint further down
        results_start = len(thumbnail_results)

    return list(image_urls)

def hash_urls(img_urls: List[str]):
    hashed_urls = dict()
    for url in img_urls:
        key = str(abs(hash(url)) % (10 ** 10))
        hashed_urls[key] = url

    return hashed_urls

def resize_img(img: Image):
    basewidth = 300
    if img.size[0] <= basewidth:
        return img
    else:
        wpercent = basewidth / float(img.size[0])
        hsize = int((float(img.size[1]) * float(wpercent)))
        img = img.resize((basewidth, hsize), Image.ANTIALIAS)
        return img

def dirfinder(dirs: List[str]):
    """Finds existing categories, including plurals, to make sure
    categories aren't scraped twice"""

    master_list = []
    for dir in dirs:
        for f in os.scandir(dir):
            if f.is_dir():
                master_list.append(f.name)

    for name in master_list:
        if not name.endswith('s'):
            master_list.append(name + 's')

    return master_list

def pre_prediction(img: Image, model_name: Path):
    """
    NOT IMPLEMENTED YET

    Function to help validation during curation process.
    Theoretically, first CNN should label all images as
    recycleable
    """
    clf = load_model(model_name)
    x = image.img_to_array(img)
    x = np.expand_dims(x, axis=0)
    valid = clf.predict_classes(x)
    return valid

def dict_chunker(result_dict: Dict[str, List[str]], n: int) -> List[List[str]]:
    flat_keys, flat_vals = list(result_dict.keys()), list(result_dict.values())
    keys = [flat_keys[i : i + n] for i in range(0, len(flat_keys), n)]
    vals = [flat_vals[i : i + n] for i in range(0, len(flat_vals), n)]
    return (keys, vals)

def download_image(folder_path: str, names: str, urls: str):
    for name, url in zip(names, urls):
        try:
            image_content = requests.get(url).content

        except Exception as e:
            print(f"ERROR - Could not download {url} - {e}")

        try:
            image_file = io.BytesIO(image_content)
            raw_image = Image.open(image_file).convert("RGB")
            image = resize_img(raw_image)
            file_path = os.path.join(folder_path, name + ".jpg")
            with open(file_path, "wb") as f:
                image.save(f, "JPEG", quality=85)

        except Exception as e:
            print(f"ERROR - Could not save {url} - {e}")

def multithreaded_save(
    chunks: Tuple[List[List[str]], List[List[str]]], target_path: Path
):
    """Takes a chunked dictionary and saves it in a mulithreaded manner."""
    for names, urls in zip(chunks[0], chunks[1]):
        Thread(target=download_image, args=(target_path, names, urls)).start()

@click.group()
@click.option("--data_path", type=click.Path(), default="/home/dal/CIf3R/data/interim")
@click.option("--result_count", default=500)
@click.option("--model", default="2019-08-28 08:03:49.h5")
@click.pass_context
def cli(ctx, data_path, result_count, model):
    log_fmt = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
    logdir = Path(data_path).parents[1] / "reports"
    logging.basicConfig(
        filename=logdir / "data.log", level=logging.INFO, format=log_fmt
    )
    logger = logging.getLogger(__name__)
    # instantiation and path definitions go here
    data_path = Path(data_path)

    ctx.obj = {
        'data_path': data_path,
        'result_count': result_count,
        'model': model,
        'logger': logger
        }

@cli.command()
@click.option("--dict_name", prompt=True)
@click.option(
    "--interrupted_on",
    help="If scrape_multiple is interrupted, this is the last item it scraped",
    default=False
)
@click.pass_context
def scrape_multiple(
    ctx,
    dict_name,
    interrupted_on
):
    """
    Main scraping function that iterates through dict of university guidelines, scraping each google
    images for each individual item, hasing t hem, and then saving them.
    """
    data_path = ctx.obj['data_path']
    result_count = ctx.obj['result_count']
    logger = ctx.obj['logger']
    # getting the recycling guidelines
    guideline_path = data_path.parents[0] / "external"
    with open(guideline_path / f"{dict_name}.pickle", "rb") as f:
        guideline_dict = pickle.load(f)

    #finding all existing categories
    existing = dirfinder([str(data_path / 'O'), str(data_path/'R')])
    print(guideline_dict)
    for broad_category, _dict in guideline_dict.items():
        for primary_category, queries in _dict.items():
                # accounting for discrepancy between Ubuntu 16.04 and 18.04
            try:
                wd = webdriver.Chrome("/home/dal/chromedriver/chromedriver")
            except NotADirectoryError:
                wd = webdriver.Chrome("/home/dal/chromedriver")
            for query in queries:
                print(query)
               #search for value that was stopped at
                if interrupted_on and query != interrupted_on:
                    pass
                elif query == interrupted_on:
                    interrupted_on = False
                    pass
                else:
                    clean_query = query.replace(" ", "_")
                    if clean_query in existing:
                        continue
                    target_path = data_path / f"{broad_category}/{clean_query}"
                    target_path.mkdir(parents=False, exist_ok=True)
                    logger.info(f"Starting scraping for {query}")

                    try:
                        google_img_result = fetch_image_urls(query, int(result_count), wd)
                    except common.exceptions.NoSuchElementException:
                        logger.info(f"Scraping failed for {query}")
                        continue

                    logger.info("Image URLs obtained! Hashing URLs...")
                    hashed_results = hash_urls(google_img_result)
                    logger.info("Saving images and metadata...")
                    chunks = dict_chunker(hashed_results, 5)
                    multithreaded_save(chunks, target_path)
                    logger.info(f"Finished saving images for {query}")

@cli.command()
@click.option("--query", prompt=True)
@click.option("--metadata", type=(str, str, str), prompt=True)
@click.pass_context
def scrape_single(ctx, query, metadata):
    """
    Scrapes single class of item. broad_category can be 'R' for recyclable or 'O'
    for not. Example is a piece of paper:

    query = 'construction paper'
    metadata = ('R', 'paper', 'piece_of_paper')*

    *Note that this will vary by university. This will be addressed post-scraping by the creation
    of a database with university recycling guidelines
    """
    broad_category, primary_category, folder_name = metadata
    valid_categories = ["O", "R"]
    if broad_category not in valid_categories:
        raise ValueError(
            f"broad_category must be one of {valid_categories} with R for recyclable"
        )

    # accounting for discrepancy between Ubuntu 16.04 and 18.04
    try:
        wd = webdriver.Chrome("/home/dal/chromedriver/chromedriver")
    except NotADirectoryError:
        wd = webdriver.Chrome("/home/dal/chromedriver")

    clean_query = query.replace(" ", "_")
    target_path = ctx.obj["data_path"] / f"{broad_category}/{folder_name}"
    target_path.mkdir(parents=False, exist_ok=True)
    google_img_result = fetch_image_urls(query, int(ctx.obj["result_count"]), wd)
    hashed_results = hash_urls(google_img_result)
    chunks = dict_chunker(hashed_results, 5)
    multithreaded_save(chunks, target_path)

if __name__ == "__main__":
    cli()

max_stars_repo_name: dendrondal/Recyclops

max_forks_repo_name: dendrondal/Recyclops


text:
import numpy as np
import pandas as pd
import jax.numpy as jnp
import numpyro.distributions as dists
from numpyro.primitives import sample, plate
from twinify.model_loading import TModelFunction
import argparse
from typing import Iterable

def model_factory(twinify_args: argparse.Namespace, unparsed_args: Iterable[str], orig_data: pd.DataFrame) -> TModelFunction:
    model_args_parser = argparse.ArgumentParser()
    model_args_parser.add_argument('--prior_mu', type=float, default=0.)
    args = model_args_parser.parse_args(unparsed_args, twinify_args)

    d = orig_data.shape[-1]
    print(f"Model using prior mu = {args.prior_mu}")
    print(f"Privacy parameter epsilon is {args.epsilon}")

    def model(z = None, num_obs_total = None) -> None:
        batch_size = 1
        if z is not None:
            batch_size = z.shape[0]
        if num_obs_total is None:
            num_obs_total = batch_size

        mu = sample('mu', dists.Normal(args.prior_mu).expand_by((d,)).to_event(1))
        sigma = sample('sigma', dists.InverseGamma(1.).expand_by((d,)).to_event(1))
        with plate('batch', num_obs_total, batch_size):
            sample('x', dists.Normal(mu, sigma).to_event(1), obs=z)

    return model

max_stars_repo_name: DPBayes/twinify

max_issues_repo_name: DPBayes/twinify

max_forks_repo_name: DPBayes/twinify

reason: import numpy,from numpy,import jax


text:
import sys
import os
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
sys.path.append(
        os.path.dirname(
                os.path.dirname(
                        os.path.abspath(__file__))))
from common_func import get_disc, get_benchmark_disc    # noqa: E402

# Data
aa = pd.read_csv('nts_csv_aa_0002.csv')
benchmark_aa = pd.read_csv('fiss_dens_AA', delim_whitespace=True, header=None)

# Fission cross section data
sigmaf = [1.11309e-03, 1.08682e-03, 1.52219e-03,
          2.58190e-03, 5.36326e-03, 1.44917e-02]

# Pre-multiply with appropriate group fission XS and sum at each position
fiss = np.zeros(201)
for i in range(6):
    group = 'group' + str(i+1)
    fiss += np.array(aa[group] * sigmaf[i]) * 1e6

# Calculate % discrepancy
disc_aa = get_disc(fiss, benchmark_aa)

ave_aa, std_aa = get_benchmark_disc(benchmark_aa)

f = open('fission-rate.txt', 'w')
f.write("Discrepancy in fission rate along AA' = " +
        str(disc_aa*100) + " %\n")
f.write("Benchmark average discrepancy in fission rate along AA' = " +
        str(ave_aa*100) + " %\n")
f.write("Benchmark discrepancy std dev in fission rate along AA' = " +
        str(std_aa*100) + " %\n")
f.close()

# %% Plot
x = np.linspace(0, 2, 201)
code = ['CNRS-SP1', 'CNRS-SP3', 'Polimi-Diff', 'PSI-Diff', 'TUD-S2', 'TUD-S6']

plt.rc('font', size=12)          # controls default text sizes
plt.rc('axes', titlesize=12)     # fontsize of the axes title
plt.rc('axes', labelsize=12)    # fontsize of the x and y labels
plt.rc('xtick', labelsize=11)    # fontsize of the tick labels
plt.rc('ytick', labelsize=11)    # fontsize of the tick labels
plt.rc('legend', fontsize=12)    # legend fontsize
plt.rc('figure', titlesize=12)  # fontsize of the figure title

fig, ax = plt.subplots(figsize=[6, 5])
ax.plot(x, fiss, label='Moltres', color='tab:blue')
ax.plot(x, benchmark_aa[:][2], label='CNRS-SP$_3$', color='tab:orange',
        linestyle='--')
ax.plot(x, benchmark_aa[:][3], label='Polimi', color='tab:green',
        linestyle=':')
ax.plot(x, benchmark_aa[:][4], label='PSI', color='tab:red',
        linestyle='-.')
ax.plot(x, benchmark_aa[:][6], label='TUD-S$_6$', color='tab:cyan',
        linestyle=(0, (3, 1, 1, 1, 1, 1)))
ax.legend()
ax.grid(which='both')
ax.set_xlim(0, 2)
ax.set_ylim(0, 2e19)
ax.set_xlabel(r'$x$ [m]')
ax.set_ylabel(r'Fission rate density [m$^{-3}\cdot$s$^{-1}$]')
plt.savefig('0-2-fiss-plot.png', dpi=400)

# %% Write tsv

coords = np.linspace(0, 2, 201)
aa_df = pd.DataFrame({'x (m)': np.around(coords, decimals=2),
                      'fiss dens (1/m3s)': fiss})
aa_df.to_csv('moltres_0.2_AA', index=False, sep='\t')