text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"]="0"

import numpy as np
np.random.seed(12345)

from keras.backend.tensorflow_backend import set_session
import tensorflow as tf
tf.set_random_seed(12345)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
#config.log_device_placement = True
sess = tf.Session(config=config)
set_session(sess)

import keras
import keras.backend as K
import argparse
import h5py
import pandas as pd
import math
import pickle

model = None

def create_parser():
	parser = argparse.ArgumentParser(description='Predict Binding Affinity between a compound and kinase')

	parser.add_argument('--input', '-i', required=True, type=str,
		help='A hdf file with compound and kinase')
	parser.add_argument('--output', '-o', type=str, default='predictions.csv',
		help="Output file's name")
	parser.add_argument('--csv-file', '-c', type=str, default='',
		help='CSV file')
	parser.add_argument('--error', '-e', action='store_true', default=False,
		help='Get error of predicted affinity')
	parser.add_argument('--model', '-m', type=str, default='best.hdf5',
		help='Path to model')
	parser.add_argument('--verbose', action='store_true', default=False,
		help='Increase output verbosity')

	return parser

def make_box(coords, features):
	box_size = 21
	grid_coords = coords + 10
	grid_coords = grid_coords.round().astype(int)

	in_box = ((grid_coords >= 0) & (grid_coords < box_size)).all(axis=1)
	grid = np.zeros((box_size, box_size, box_size, features.shape[1]), dtype=np.float32)

	for (x, y, z), f in zip(grid_coords[in_box], features[in_box]):
		grid[x, y, z] += f

	return grid

def create_model(dropout, rate):
	global model
	#
	#Create Model
	#
	model = keras.Sequential()

	#Conv_1
	model.add(keras.layers.Conv3D(64, kernel_size=(5, 5, 5), activation='relu',
		input_shape=(21, 21, 21, 19), padding='same'))
	model.add(keras.layers.Conv3D(64, kernel_size=(5, 5, 5), activation='relu',
		padding='same'))
	model.add(keras.layers.MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2)))
	#Conv_2
	model.add(keras.layers.Conv3D(128, kernel_size=(5, 5, 5), activation='relu',
		padding='same'))
	model.add(keras.layers.Conv3D(128, kernel_size=(5, 5, 5), activation='relu',
		padding='same'))
	model.add(keras.layers.MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2)))
	#Conv_3
	model.add(keras.layers.Conv3D(256, kernel_size=(5, 5, 5), activation='relu',
		padding='same'))
	model.add(keras.layers.Conv3D(256, kernel_size=(5, 5, 5), activation='relu',
		padding='same'))
	model.add(keras.layers.MaxPooling3D(pool_size=(2, 2, 2), strides=(2, 2, 2)))

	#Dense_1
	model.add(keras.layers.Flatten())
	model.add(keras.layers.Dense(1000, activation='relu'))
	model.add(keras.layers.Dropout(dropout))
	#Dense_2
	model.add(keras.layers.Dense(500, activation='relu'))
	model.add(keras.layers.Dropout(dropout))
	#Dense_3
	model.add(keras.layers.Dense(200, activation='relu'))
	model.add(keras.layers.Dropout(dropout))

	#Output
	model.add(keras.layers.Dense(1, activation='relu'))
	model.add(keras.layers.Dropout(dropout))

	model.compile(loss=loss_function, optimizer=keras.optimizers.Adam(lr=rate), metrics=['mse', 'mae', loss_function])

	return model

def main(argv):
	global model

	if argv.verbose:
		print('''
Input File:    {}
Ouput File:    {}
CSV File:      {}
Model Path:    {}
Validation:    {}
			'''.format(argv.input, argv.output, argv.csv_file, argv.model, argv.error))
	#
	#Get Data
	#
	if argv.verbose:
		print("Reading dataset and creating 20 A boxes")

	data = []
	affinity = []
	with h5py.File(argv.input, 'r') as f:
		keys = np.array(list(f.keys()))
		rot = np.array(pickle.load(open('rotations.pkl', 'rb')))

		rand_rot = [0] if not argv.error else np.random.choice(rot.shape[0], (4), replace=False)

		for key in keys:
			complex = f[key]

			for rotation in rand_rot:
				coords = np.dot(complex[:,:3], rot[rotation])
				features = complex[:,3:]

				data.append(make_box(coords, features))
				if argv.error:
					try:
						affinity.append(-math.log10(float(complex.attrs['affinity']) / 1e9))
					except:
						affinity.append(0)

	if argv.verbose:
		print('Loading model and predicting')
	data = np.array(data)
	if argv.error:
		affinity = np.array(affinity).flatten()

	model = create_model(0.5, 1e-5)
	model = keras.models.load_model(argv.model, custom_objects={'loss_function': loss_function})

	y_pred = model.predict(data, batch_size=5)
	y_pred = np.array(y_pred).flatten()

	if len(argv.csv_file) > 0:
		df = pd.read_csv(argv.csv_file)
		df = df.assign(pred=y_pred)
		df.rename(columns={'pred': 'pKd_[M]_pred'}, inplace=True)

		if not os.path.isfile(argv.output):
			open(argv.output, 'w')

		df.to_csv(argv.output, index=False, encoding='utf-8')

	if argv.error:
		#RMSE
		rmse = np.sqrt(np.mean(np.square(affinity - y_pred)))

		#MAE
		mae = np.mean(np.absolute(affinity - y_pred))

		#Pearson correlation
		R = np.corrcoef(affinity, y_pred)[0, 1]

		print('RMSE: ', rmse)
		print('MAE: ', mae)
		print('R: ', R)

def loss_function(y_true, y_pred):
	l2 = 0
	for w in model.layers:
		if type(w) is keras.layers.Conv3D or type(w) is keras.layers.Dense:
			weight = np.array(w.get_weights())
			l2 += np.sum(np.square(weight[0]))
	return K.mean(K.square(y_true - y_pred)) + (l2 * 1e-3)
	#return K.sqrt(K.mean(K.square(y_pred - y_true), axis=0))

if __name__ == '__main__':
	parser = create_parser()

	argv = parser.parse_args()

	main(argv)


text:
from styx_msgs.msg import TrafficLight
import rospy
import yaml
import os
import cv2
import numpy as np
import tensorflow as tf

# for debug
DEBUG = False

class TLClassifier(object):
    def __init__(self):
        #TODO load classifier
        self.classes = {1: TrafficLight.RED,
                        2: TrafficLight.YELLOW,
                        3: TrafficLight.GREEN,
                        4: TrafficLight.UNKNOWN}

        self.session = None
        self.model_graph = None

        conf = rospy.get_param("/traffic_light_config")
        self.config = yaml.load(conf)

        model = os.path.dirname(os.path.realpath(__file__)) + self.config['model']
        self.load_model(model)

        self.img_counter = 0

    def get_classification(self, image):
        """Determines the color of the traffic light in the image

        Args:
            image (cv::Mat): image containing the traffic light

        Returns:
            int: ID of traffic light color (specified in styx_msgs/TrafficLight)

        """
        #TODO implement light color prediction

        traffic_light_id, prob = self.predict(image)

        if traffic_light_id is not None:
            rospy.logdebug("Class: %d, Probability: %f", traffic_light_id, prob)

        return traffic_light_id

    def load_model(self, model):
        config = tf.ConfigProto()
        config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1

        self.model_graph = tf.Graph()
        with tf.Session(graph=self.model_graph, config=config) as sess:
            self.session = sess
            od_graph_def = tf.GraphDef()
            with tf.gfile.GFile(model, 'rb') as fid:
                serialized_graph = fid.read()
                od_graph_def.ParseFromString(serialized_graph)
                tf.import_graph_def(od_graph_def, name='')

    def predict(self, image, threshold=0.5):
        # definite input and output tensors for detection graph
        image_tensor = self.model_graph.get_tensor_by_name('image_tensor:0')

        # each box represents a part of the image where a particular object was detected
        detection_boxes = self.model_graph.get_tensor_by_name('detection_boxes:0')

        # score represent the probability of each class
        detection_scores = self.model_graph.get_tensor_by_name('detection_scores:0')
        detection_classes = self.model_graph.get_tensor_by_name('detection_classes:0')

        image = self.process_img(image)

        # actual detection
        (boxes, scores, classes) = self.session.run([detection_boxes, detection_scores, detection_classes],
                                                    feed_dict={image_tensor: np.expand_dims(image, axis=0)})

        boxes = np.squeeze(boxes)
        scores = np.squeeze(scores)
        classes = np.squeeze(classes).astype(np.int32)

        for i, box in enumerate(boxes):
            if scores[i] > threshold:
                traffic_light_id = self.classes[classes[i]]

                if DEBUG:
                    self.detected_img(image, traffic_light_id)

                return traffic_light_id, scores[i]
            else:
                self.detected_img(image, TrafficLight.UNKNOWN)

        return None, None

    def process_img(self, image):
        image_w , image_h = image.shape[:2]
        size_max = 300

        if image_h > size_max or image_w > size_max:
            scaling_factor = size_max / float(image_h)
            if scaling_factor > size_max / float(image_w):
                scaling_factor = size_max / float(image_w)
            # resize
            image = cv2.resize(image, None, fx=scaling_factor, fy=scaling_factor, interpolation=cv2.INTER_AREA)

        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

        return image

    def detected_img(self, image, traffic_light_id):
        detected_img_site = os.path.dirname(os.path.realpath(__file__)) + '/../../../../detected_img/site/'
        detected_img_sim = os.path.dirname(os.path.realpath(__file__)) + '/../../../../detected_img/sim/'

        # reconvert the image
        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
        if self.config['is_site']:
            cv2.imwrite(os.path.join(detected_img_site, "image_%04i_%d.jpg" % (self.img_counter, traffic_light_id)),
                        image)
        else:
            cv2.imwrite(os.path.join(detected_img_sim, "image_%04i_%d.jpg" % (self.img_counter, traffic_light_id)),
                        image)
        self.img_counter += 1

ext: py

reason: import numpy


text:
#
# BSD 3-Clause License
#
# Copyright (c) 2020, Jonathan Bac
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
import pytest
import numpy as np
import skdim

# import rpy2.robjects.packages as rpackages
# import rpy2.robjects.numpy2ri
# utils = rpackages.importr('utils')
# utils.install_packages('intrinsicDimension')
# utils.install_packages('ider')
# intdimr = rpackages.importr('intrinsicDimension')
# ider = intdimr = rpackages.importr('ider')
# rpy2.robjects.numpy2ri.activate()

@pytest.fixture
def data():
    X = np.zeros((30, 10))
    X[:, :5] = skdim.datasets.hyperBall(n=30, d=5, radius=1, random_state=0)
    return X

# test default and non-default parameters
def test_ess_results(data):
    x = skdim.id.ESS().fit_once(data)
    x2 = skdim.id.ESS(ver="b").fit_once(data)
    x3 = skdim.id.ESS(d=2).fit_once(data)

    #     rx=intdimr.essLocalDimEst(data)
    #     rx2=intdimr.essLocalDimEst(data,ver='b')
    #     rx3=intdimr.essLocalDimEst(data,d=2)
    #     rx_results = np.array([rx[0],rx[1],rx2[0],rx2[1],rx3[0],rx3[1]]).T
    rx_results = np.array(
        [[5.08681575, 0.88546915, 5.11274497, 0.371001, 5.06259738, 0.66653896,]]
    )

    assert np.allclose(
        np.array(
            [
                x.dimension_,
                x.essval_,
                x2.dimension_,
                x2.essval_,
                x3.dimension_,
                x3.essval_,
            ]
        ),
        rx_results,
    )

def test_mind_ml_results(data):
    #     all((skdim.id.MiND_ML(ver='MLk').fit(data).dimension_ == np.array(intdimr.dancoDimEst(data,k=20,D=10,ver='MIND_MLi')[0]),
    #          skdim.id.MiND_ML(ver='MLi').fit(data).dimension_ == np.array(intdimr.dancoDimEst(data,k=10,D=5,ver='MIND_MLk')[0])))
    assert all(
        (
            skdim.id.MiND_ML(ver="MLk").fit(data).dimension_ == 3.696083548872364,
            skdim.id.MiND_ML(ver="MLi").fit(data).dimension_ == 4,
        )
    )

# def test_mom_results(data):
#    x = skdim.id.MOM()
#    x = skdim.id.MOM(k=5)

def test_lpca_results(data):
    #     assert all((skdim.id.lPCA().fit(data).dimension_         == intdimr.pcaLocalDimEst(data,ver='FO')[0][0],
    #            skdim.id.lPCA(ver='fan').fit(data).dimension_    == intdimr.pcaLocalDimEst(data,ver='fan')[0][0],
    #            skdim.id.lPCA(ver='maxgap').fit(data).dimension_ == intdimr.pcaLocalDimEst(data,ver='maxgap')[0][0]
    #           ))
    assert all(
        (
            skdim.id.lPCA().fit(data).dimension_ == 5,
            skdim.id.lPCA(ver="Fan").fit(data).dimension_ == 3,
            skdim.id.lPCA(ver="maxgap").fit(data).dimension_ == 5,
        )
    )

def test_corrint_results(data):
    #     assert np.isclose(skdim.id.CorrInt().fit(data).dimension_,np.array(ider.corint(data,k1=10,k2=20)))
    assert np.isclose(skdim.id.CorrInt().fit(data).dimension_, 2.9309171335492548)

def test_danco_results():
    x = np.zeros(9)
    for i in range(2, 11):
        print(i, end="\r")
        X = skdim.datasets.hyperBall(n=100, d=i, radius=1, random_state=0)
        x[i - 2] = skdim.id.DANCo(D=11, fractal=False, random_state=0).fit(X).dimension_

    assert np.all(x == np.array([2, 3, 4, 5, 6, 7, 8, 9, 10]))

# def test_knn_results(data):
#    x = skdim.id.KNN().fit(data)
#    x = skdim.id.KNN(k=5).fit(data)
#    x = skdim.id.KNN(ps=np.arange(30,32)).fit(data)
#    x = skdim.id.KNN(M=2).fit(data)
#    x = skdim.id.KNN(gamma=3).fit(data)

def test_mada_results(data):
    #     assert np.allclose(skdim.id.Mada(local=True,k=20).fit(data).dimension_,np.array(ider.mada(data,local=True,k=20)))
    #
    assert np.allclose(
        skdim.id.MADA().fit(data).dimension_pw_,
        np.array(
            [
                2.73873109,
                2.1619804,
                2.29577097,
                3.13300883,
                2.68340134,
                2.7744833,
                2.16936429,
                2.63976628,
                3.73803225,
                2.68782875,
                3.3468158,
                3.3375878,
                2.58747076,
                3.49101585,
                2.84130906,
                3.68365681,
                3.5694458,
                3.50015857,
                2.78155042,
                4.47995493,
                4.0897491,
                3.79781342,
                2.62101409,
                3.03665971,
                2.84263582,
                3.82414291,
                2.65283788,
                3.46869821,
                3.851816,
                3.13222988,
            ]
        ),
    )

def test_mle_results(data):
    #     assert np.allclose(
    #         (skdim.id.MLE().fit(data).dimension_,
    #         skdim.id.MLE(k=5).fit(data).dimension_,
    #         skdim.id.MLE(n = 20, sigma=.1, dnoise='dnoiseGaussH').fit(data).dimension_,
    #         skdim.id.MLE(unbiased=True).fit(data).dimension_,
    #         skdim.id.MLE(K=10, neighborhood_based=False).fit(data).dimension_,
    #         skdim.id.MLE(neighborhood_aggregation='mean').fit(data).dimension_,
    #         ),
    #         (intdimr.maxLikGlobalDimEst(data,k=20)[0][0],
    #         intdimr.maxLikGlobalDimEst(data,k=5)[0][0],
    #         intdimr.maxLikGlobalDimEst(data,k=20,n = 20, sigma=.1, dnoise='dnoiseGaussH')[0][0],
    #         intdimr.maxLikGlobalDimEst(data,k=20,unbiased=True)[0][0],
    #         intdimr.maxLikGlobalDimEst(data,k=20,K=10, neighborhood_based=False)[0][0],
    #         intdimr.maxLikGlobalDimEst(data,k=20,neighborhood_aggregation='mean')[0][0]
    #         )
    #     )
    assert np.allclose(
        (
            skdim.id.MLE().fit(data).dimension_,
            skdim.id.MLE().fit(data, n_neighbors=5).dimension_,
            skdim.id.MLE(n=20, sigma=0.1, dnoise="dnoiseGaussH").fit(data).dimension_,
            skdim.id.MLE(unbiased=True).fit(data).dimension_,
            skdim.id.MLE(K=10, neighborhood_based=False).fit(data).dimension_,
            skdim.id.MLE().fit(data, comb="mean").dimension_,
        ),
        (
            3.341161830518472,
            4.350132038093711,
            3.2755228756927455,
            3.1653112078596046,
            5.312292926569348,
            3.4389424541367437,
        ),
    )

def test_tle_results(data):
    #     assert np.allclose(np.round(skdim.id.TLE().fit(data).dimension_,4),radovanovic_estimators_matlab(data,k=20)['id_tle'])
    assert np.allclose(
        np.round(skdim.id.TLE().fit(data).dimension_pw_, 4),
        np.array(
            [
                3.1975,
                3.0745,
                3.0447,
                3.3422,
                2.9231,
                3.7637,
                2.9517,
                3.2285,
                3.8731,
                2.9633,
                3.2595,
                4.1404,
                3.1353,
                3.671,
                3.6847,
                3.9358,
                3.9902,
                3.7741,
                3.5728,
                4.2938,
                4.2238,
                3.9431,
                3.0891,
                3.4052,
                4.0238,
                4.7747,
                3.9094,
                4.9064,
                4.0431,
                3.4192,
            ]
        ),
    )

def test_twonn_results(data):
    # to trigger the "n_features>25 condition"
    test_high_dim = np.zeros((len(data), 30))
    test_high_dim[:, : data.shape[1]] = data
    assert all(
        (
            np.round(skdim.id.TwoNN().fit(test_high_dim).dimension_, 5) == 4.05496,
            np.round(skdim.id.TwoNN(discard_fraction=0.05).fit(data).dimension_, 5)
            == 4.11323,
        )
    )

max_stars_repo_name: j-bac/skcontrib-id-estimators

max_issues_repo_name: j-bac/skcontrib-id-estimators

max_forks_repo_name: j-bac/skcontrib-id-estimators


text:
import argparse
import numpy as np
import os
from sklearn.model_selection import train_test_split

from utils import Audio

def load_wav_audio(audio_dir):
    audio = Audio()
    print(f'Download audio data from "{audio_dir}" folder')
    data = {}
    for cls in os.listdir(audio_dir):
        cls_dir = os.path.join(audio_dir, cls)

        data[cls] = []
        for fname in os.listdir(cls_dir):
            if not fname.endswith('.wav'):
                continue

            print('->', os.path.join(cls_dir, fname))

            # convert audio to mel
            mel = audio.audio_to_mel(os.path.join(cls_dir, fname))
            print('\tconvert to mel: done!')

            # get samples from mel-spectrogram of width 3 * 128 = 384
            k = mel.shape[1] // 25
            mels = audio.mel_sample(mel, width=384, k=k)
            print('\tsample mels: done!')

            # append samples
            if mels is not None:
                data[cls].append(mels)

        data[cls] = np.concatenate(data[cls], axis=0)
    return data

def create_dataset(data):
    os.makedirs('./data/train', exist_ok=True)
    os.makedirs('./data/test', exist_ok=True)
    for cls in data:
        train, test = train_test_split(data[cls], test_size=0.1, random_state=101)
        np.save(f'./data/train/{cls}.npy', train)
        np.save(f'./data/test/{cls}.npy', test)

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument('--audio_dir', action='store', type=str, default='data/music')
    args = parser.parse_args()

    # 1. load wav audio
    data = load_wav_audio(args.audio_dir)

    # 2. create dataset
    create_dataset(data)


text:
import os, sys

#find better way to do this.
projectDir = os.path.join(os.path.dirname(__file__), '../')
test = os.path.dirname(__file__)
displaySources = os.path.join(projectDir, "../thirdparty/e-Paper/RaspberryPi&JetsonNano/python/lib/waveshare_epd")
sys.path.append(displaySources)

import epd4in2 as display
import PIL.Image as Image
import numpy as np

class EInkDisplay:

    width = 0
    heigth = 0
    edp = None

    def __init__(self, width=400, heigth=300):
        self.width = width
        self.heigth = heigth
        self.InitDisplay()

    def ResizeTag(self, image):
        factor = float(self.heigth) / float(self.width)

        image = image.resize((self.width, self.heigth), Image.NEAREST)

        #produce aspect ratio we want
        padNumber = int(self.width * factor)
        pix = np.array(image)
        pix = np.pad(pix, ((0, 0), (padNumber / 2, 0), (0, 0)), 'constant')

        pix = np.roll(pix, -padNumber/4, axis=1)
        image = Image.fromarray(pix)

        image = image.resize((self.width, self.heigth), Image.NEAREST)
        return image

    def InitDisplay(self):
        self.epd = display.EPD()
        self.epd.init()

    def ShowImage(self, image):

        image = self.ResizeTag(image)
        self.epd.display_frame(self.epd.get_frame_buffer(image))


text:
import gym
import gym.spaces
import numpy as np
import random

def taxiagent(verbose=False):
    # List of all scores
    scores = []

    # Creating the gym environment
    env = gym.make("Taxi-v3")

    # Initializing the Q-table of size state-space x action-space with zeros
    Q = np.zeros((env.observation_space.n, env.action_space.n))

    # Set the hyper-parameters
    epsilon = 1.0
    epsilon_min = 0.005
    epsilon_decay = 0.99993
    episodes = 50000
    max_steps = 100
    learning_rate = 0.65
    gamma = 0.65

    for episode in range(episodes):
        # Reset the game-state, done and score before every episode
        state = env.reset()
        done = False
        score = 0

        for _ in range(max_steps):
            # With the probabilty of (1 - epsilon) take the best action in the Q-table
            if random.uniform(0, 1) > epsilon:
                action = np.argmax(Q[state, :])
            # Else take a random action
            else:
                action = env.action_space.sample()

            # Step the game forward
            next_state, reward, done, _ = env.step(action)

            # Add up the score
            score += reward

            # Update the Q-table with the Q-function
            Q[state, action] = (1 - learning_rate) * Q[state, action] + learning_rate * (reward + gamma * np.max(Q[next_state,:]))

            # Set the next state as the current state
            state = next_state

            if done:
                break

        # Reducing the epsilon each episode (Exploration-Exploitation trade-off)
        if epsilon >= epsilon_min:
            epsilon *= epsilon_decay

        scores.append(score)

        if verbose:
            print("Episode: {}/{}, score: {}".format(episode+1, episodes, score))

    return scores

if __name__ == "__main__":
    taxiagent(verbose=True)


text:
from PIL import Image
import scipy
import scipy.cluster
from pprint import pprint

image = Image.open('logo_newsblur_512.png')
NUM_CLUSTERS = 15

# Convert image into array of values for each point.
ar = scipy.misc.fromimage(image)
shape = ar.shape

# Reshape array of values to merge color bands.
if len(shape) > 2:
    ar = ar.reshape(scipy.product(shape[:2]), shape[2])

# Get NUM_CLUSTERS worth of centroids.
codes, _ = scipy.cluster.vq.kmeans(ar, NUM_CLUSTERS)

# Pare centroids, removing blacks and whites and shades of really dark and really light.
original_codes = codes
for low, hi in [(60, 200), (35, 230), (10, 250)]:
    codes = scipy.array([code for code in codes
                         if not ((code[0] < low and code[1] < low and code[2] < low) or
                                 (code[0] > hi and code[1] > hi and code[2] > hi))])
    if not len(codes): codes = original_codes
    else: break

# Assign codes (vector quantization). Each vector is compared to the centroids
# and assigned the nearest one.
vecs, _ = scipy.cluster.vq.vq(ar, codes)

# Count occurences of each clustered vector.
counts, bins = scipy.histogram(vecs, len(codes))

# Show colors for each code in its hex value.
colors = [''.join(chr(c) for c in code).encode('hex') for code in codes]
total = scipy.sum(counts)
color_dist = dict(zip(colors, [count/float(total) for count in counts]))
pprint(color_dist)

# Find the most frequent color, based on the counts.
index_max = scipy.argmax(counts)
peak = codes[index_max]
color = ''.join(chr(c) for c in peak).encode('hex')

max_stars_repo_name: jjhenkel/dockerizeme

max_issues_repo_name: jjhenkel/dockerizeme

max_forks_repo_name: jjhenkel/dockerizeme

reason: import scipy


text:
#function plot3D_currentloops( CurrentLoops, nSamplePoints, FigureID )
import matplotlib.pyplot as plt
import numpy as np
import config as cfg
import roto as roto
from mpl_toolkits import mplot3d

def plot3D_currentloops(nSamplePoints,FigureID):

    plt.figure( FigureID )
    ax = plt.axes(projection='3d')

    for ii in range(0 ,cfg.CurrentLoops.shape[0]):
        oclab   =   cfg.CurrentLoops[ii][0:3]
        nh      =   cfg.CurrentLoops[ii][3:6]
        I0      =   cfg.CurrentLoops[ii][6]
        Ra      =   cfg.CurrentLoops[ii][7]
        Nw      =   cfg.CurrentLoops[ii][8]

        dz  =   2.0*np.pi/nSamplePoints
        cz  =   np.cos(np.arange(0.0,dz*nSamplePoints,dz))
        sz  =   np.sin(np.arange(0.0,dz*nSamplePoints,dz))

        LoopX   =   Ra * cz
        LoopY   =   Ra * sz
        LoopZ   =  0.0 * cz

        ROT_LAB_LOOP    =   roto.ROT( nh )
        CurrentLoopX=np.zeros(sz.size)
        CurrentLoopY=np.zeros(sz.size)
        CurrentLoopZ=np.zeros(sz.size)

        for jj in range(0,sz.size):
            multiply=[ LoopX[jj], LoopY[jj], LoopZ[jj] ]
            CurrentLoopX[jj] = oclab[0] + ROT_LAB_LOOP[0].dot(multiply)
            CurrentLoopY[jj] = oclab[1] + ROT_LAB_LOOP[1].dot(multiply)
            CurrentLoopZ[jj] = oclab[2] + ROT_LAB_LOOP[2].dot(multiply)

#           % plot 3D loop center
        ax.scatter3D(  oclab[0], oclab[1], oclab[2])
        ax.hold(True)

        #% plot 3D loop coil
        ax.plot3D( CurrentLoopX, CurrentLoopY, CurrentLoopZ )
        plt.grid()
        ax.set_xlabel("X")
        ax.set_ylabel("Y")
        ax.set_zlabel("Z")

max_forks_repo_name: lcpp-org/PlasmaEDU


text:
import pandas as pd
import numpy as np
from datetime import datetime

def create_output(predictions):
    """
    Function that creates output csv submission file. Returns the submission dataframe.
    Input:
    - predictions: an array with shape (n_predictions, 1) with the ARRIVAL_DELAY outputs (ordered).
    Return:
    - submission: dataframe with id and ARRIVAL_DELAY fields.
    """
    submission = pd.DataFrame(data={"id": np.arange(len(predictions)),
                                    "ARRIVAL_DELAY": np.reshape(predictions, (len(predictions)))})
    filename = f"../submissions/submission_{datetime.now().strftime('%Y_%m_%d-%I_%M_%S_%p')}.csv"
    submission.to_csv(filename, index=False)
    return submission


text:
import os
import time
import argparse
import json
import numpy as np
import shutil

from collections import Counter

# paths and arguments
blend_files_path = '../blend_files'

param_json_path = 'data_generation_parameters.json'
obj_json_path = 'obj_dict.json'

with open(param_json_path, 'r') as load_file:
    data_gen_params = json.load(load_file)

with open(obj_json_path, 'r') as json_file:
    obj_dict = json.load(json_file)

blender_path = data_gen_params['paths']['blender_path']

blender_script_path = os.path.abspath('generate.py')
blendfile_path = os.path.abspath(os.path.join(blend_files_path, 'empty_scene.blend'))

parser = argparse.ArgumentParser(description='Range of Objects')
parser.add_argument('-start', type=int, help='start point in data list', default=0)
parser.add_argument('-end', type=int, help='end point in data list', default=200)
parser.add_argument('-out_file', type=str, help='file to output progress to', required=True)
parser.add_argument('-gpu', type=int, help='gpu index to use', required=True)
parser.add_argument('-v', dest='v', action='store_true', default=False)

args, unknown = parser.parse_known_args()
out_file = args.out_file

global_time = time.time()
i = 0

synset_list_13 = ['02691156', '02828884', '02933112',
                  '02958343', '03001627', '03211117',
                  '03636649', '03691459', '04090263',
                  '04256520', '04379243', '04401088',
                  '04530566']

synset_list_42 = ['02747177', '02801938', '02818832', '02871439', '02880940',
                  '02942699', '02954340', '03046257', '03207941', '03325088',
                  '03467517', '03593526', '03642806', '03759954', '03790512',
                  '03928116', '03948459', '04004475', '04099429', '04330267',
                  '04468005', '02773838', '02808440', '02843684', '02876657',
                  '02924116', '02946921', '02992529', '03085013', '03261776',
                  '03337140', '03513137', '03624134', '03710193', '03761084',
                  '03797390', '03938244', '03991062', '04074963', '04225987',
                  '04460130', '04554684']

new_dict = {}

obj_dict = {key:obj_dict[key] for key  in sorted(obj_dict.keys()) if obj_dict[key][0] in synset_list_13}
obj_dict = {i:obj_dict[key] for i, key  in enumerate(sorted(obj_dict.keys())) }

### Creating output directory + Moving the data_generation_params config file
output_dir = data_gen_params['paths']['output_path']

if not os.path.exists(output_dir):
    os.makedirs(output_dir)

shutil.copy(param_json_path,
            os.path.join(output_dir,
                         os.path.split(param_json_path)[1]
                )
            )

for i in range(args.start, args.end):
    synset, obj = obj_dict[i]
    start_time = time.time()

    #objects that stall the generating process
    if obj in ['3975b2350688e38c65552c4ac8607d25',
               'c5c4e6110fbbf5d3d83578ca09f86027']:
        continue

    if args.v:
         cmd = "{} -noaudio --background {} --python {} -- " \
                "--obj_n {} " \
                "--synset {} " \
                "--gpu {}".format(
                        blender_path,
                        blendfile_path,
                        blender_script_path,
                        obj,
                        synset,
                        args.gpu)

    else:
        cmd = "{} -noaudio --background {} --python {} -- "  \
                "--obj_n {} " \
                "--synset {} " \
                "--gpu {} 1>/tmp/out.txt".format(
                        blender_path,
                        blendfile_path,
                        blender_script_path,
                        obj,
                        synset,
                        args.gpu)

    os.system(cmd)
    with open(out_file, 'a') as f:
        out_str = "--- {:.2f} seconds for obj {} [{}/{}]---".format(time.time() - start_time,
                                                                obj,
                                                                i,
                                                                args.end)
        out_str += '\n'
        if i % 50 == 0:
            out_str += "Time since start is {:.2f} minutes".format(
                                                    (time.time() - global_time)/60)
            out_str += '\n'

        f.write(out_str)

total_time = time.time() - global_time

with open(out_file, 'a') as f:
    out_str = "Total time {:.2f}".format(total_time/60)
    out_str += '\n'
    f.write(out_str)


text:
import numpy as np
import matplotlib.pyplot as plt
import bz2
import scipy.optimize as opt
import cmath
import sys
import warnings

warnings.simplefilter(action='ignore', category=FutureWarning)

id = "24_5_2000_p300"
L = int(24) # lattice size
R_half = int(L / 2) # half lattice size
R_min = 6 # min R value for fit
R_max = 12 # max R value for fit

def corr_fit(r, m, z):
    a = z * np.exp(-m * r)
    b = z * np.exp(-m * (L - r))
    return a + b

def pion_mass(corr):
    n = len(corr[0])
    r = np.empty(n)
    corr_bar = np.empty(n)
    d_corr = np.empty(n)

    for i in range(0, R_half + 1):
        r[i] = i;
        corr_bar[i] = np.mean(corr[i]);
        d_corr[i] = np.std(corr[i]) / np.sqrt(n);

    popt, pcov = opt.curve_fit(corr_fit, r[R_min-1:R_max], corr_bar[R_min-1:R_max], [0.05, 1.0], sigma=d_corr[R_min-1:R_max])
    m = popt[0]
    d_m = np.sqrt(pcov[0][0])
    z = popt[1]
    d_z = np.sqrt(pcov[1][1])
    return m, d_m, z, d_z

def jackknife_pion_mass(corr, n_sub):
	if (n_sub == 0):
		n_sub = len(corr[0])
	f_sub = float(n_sub) # number of data subsets
	start = 0 # start of current block of data
	count = len(corr[0]) # number of data points
	size = count / n_sub # size of each block

	m_bar, _, z_bar, _ = pion_mass(corr)
	d_m = 0.0
	d_z = 0.0

	for n in range(0, n_sub):
		end = np.minimum(start + size, count)
		r = range(start, end)

		# copy the array and delete the current subset
		corr_d = np.delete(corr, r, 1)

		# calculate the creutz ratio and add to the error
        m_d, _, z_d, _ = pion_mass(corr_d)
        d_m += (m_d - m_bar)**2.0
        d_z += (z_d - z_bar)**2.0

        start = end

	d_m = np.sqrt((f_sub - 1) / f_sub * d_m)
	d_z = np.sqrt((f_sub - 1) / f_sub * d_z)
	return m_bar, d_m, z_bar, d_z

def parse_data_file(file):
    lines = file.readlines()
    a = np.empty((R_half+1,len(lines)))
    i = 0
    for l, line in enumerate(lines):
        tokens = line.split()
        for t, token in enumerate(tokens):
            if (t == 0):
                continue # skip trajectory id
            a[t-1,l] = float(token)
    return a

filename = "../batch/schw_" + id + "/data/pion/pion.dat"
pion_file = open(filename, "r")
C_pi = parse_data_file(pion_file)
m_bar, d_m, z_bar, d_z = jackknife_pion_mass(C_pi, 10)

print("m = %.12f (%.12f)" % (m_bar, d_m))
print("z = %.12f (%.12f)" % (z_bar, d_z))

n = R_half + 1
R = np.empty(n)
corr_bar = np.empty(n)
d_corr = np.empty(n)

for i in range(0, n):
    R[i] = i;
    corr_bar[i] = np.mean(C_pi[i]);
    d_corr[i] = np.std(C_pi[i]) / np.sqrt(n);

# calculate best fit curves
R_A = np.linspace(0, R_half, 1000)
corr_A = np.zeros(len(R_A))
for r in range(0, len(R_A)):
	corr_A[r] = corr_fit(R_A[r], m_bar, z_bar)

plt.rcParams.update({
    "text.usetex": False,
    "font.family": "sans-serif"})

# plot C vs r
plt.figure()
plt.xlim(0, R_half)
plt.yscale("log")
# plt.ylim(0.0, chi[1] + 0.1)
plt.errorbar(R, corr_bar, yerr=d_corr, color="blue", marker='o', ms=5, mew=0.5, mfc='none', linestyle='none', linewidth=0.5, capsize=2.5, capthick=0.5)
plt.plot(R_A, corr_A, color="blue", linewidth=0.5)
plt.xlabel("$R$")
plt.ylabel("$C_{\pi}(R)$")
plt.savefig("../plots/" + id + "_pi_r.pdf")
plt.close()

reason: import numpy,import scipy


text:
# Utilized Modules
import numpy as np
from orbital_analyses import u
# from parts_list import Parts_List
from orbital_analyses.Transform_State import Gregorian2JD

# TODO: Add uncertainties in with pint
# import uncertainties

# Scripts contains all variables that correspond to various requirements

# TODO: Iterations
# Initial iteration should be completly with user defined variables aside from any user added components.
# Following iterations should aim to use more part data as the iterations continue.
# Continue iterating until a solution is found where all requirements are set by data from real parts.

# %% Initial Iteration

###############################################################################
# Testing values (ITRF)
Rad = np.matrix([[-1033.4793830],
                 [7901.2952754],
                 [6380.3565958]])
Vel = np.matrix([[-3.225636520],
                 [-2.872451450],
                 [5.531924446]])
Acc = np.matrix([[-0.00191078482442021],
                 [-0.0022929137926414],
                 [-0.00239148524154372]])
# Initial Gregorian Date ([year; month; day; hour; minute; second])
GD_UTC = np.zeros((6, 1), dtype=np.float64)
GD_UTC[0, 0] = (2004)
GD_UTC[1, 0] = (4)
GD_UTC[2, 0] = (6)
GD_UTC[3, 0] = (7)
GD_UTC[4, 0] = (51)
GD_UTC[5, 0] = (28.386009)
GD_UTC = np.asmatrix(GD_UTC)

# Mission Requirements
###############################################################################

## Initial Radius and Velocity Vectors (INERTIAL) (J2000)
#Rad = np.matrix([[-2374.333753542197],
#                 [5963.296226761164],
#                 [3105.173918117055]]) * u.km
#Vel = np.matrix([[-5.658163685962852],
#                 [-3.49897296995646],
#                 [4.064517100619342]]) * (u.km * u.s)
#
## Initial Gregorian Date ([year; month; day; hour; minute; second])
#GD_UTC = np.zeros((6, 1), dtype=np.float64)
#GD_UTC[0, 0] = (2004)
#GD_UTC[1, 0] = (4)
#GD_UTC[2, 0] = (6)
#GD_UTC[3, 0] = (0)
#GD_UTC[4, 0] = (0)
#GD_UTC[5, 0] = (0)
#GD_UTC = np.asmatrix(GD_UTC)
#
#JD_UTC = Gregorian2JD(GD_UTC)

# Propagation info
Ssize = 1.0 * u.min
Steps = 600
Stime = (Steps * Ssize)
# Initial step size
# Max and min step size, max step attempts, accuracy

# Unchanging values

# Ground Station Requirements
# deg/deg/m - Madrid, Spain - For DSN
gs_ll = np.matrix([[40.416775], [356.29621]]) * u.deg
gs_a = np.matrix([[648.0]]) * u.m
GS = np.matrix([[gs_ll], [gs_a]])
GS_repeat = 4 * u.day

# EXAMPLE Ground Station Requirements
# deg/deg/m - Sacramento, CA
ex_gs_ll = np.matrix([[38.575764], [-121.478851]]) * u.deg
ex_gs_a = np.matrix([[7]]) * u.m
ex_GS = np.matrix([[ex_gs_ll], [ex_gs_a]])

# Target Requirements
# Define path that encircles california through lat/lon coordinates
Target = np.load('orbital_analyses/CA_Coords.npy')

# Subsystem Requirements
###############################################################################

# General
Mass = 1.33 * u.kg
Cost = (500000)
Vol = 1000000 * (u.mm ** 3)

# Power System
Batt_min_percent = 0.2
Batt_max_capacity = 38.0 * u.Wh

# C&DH System
Data_max_capacity = 2.0 * u.Gbyte
Data_min_store = 20.0 * u.Mbyte  # minimum data threshold to downlink
Data_max_pass = 0.0 * u.byte  # We don't know yet

# %% 2nd Iteration and on...

# Mission Requirements
###############################################################################
# TODO: Machine Learning :o

# Subsystem Requirements
###############################################################################
# Replaced by part data
# General
#IVol = Parts_List['Chassis']['volume']

#Data_max_capacity = Parts_List['Payload']['storage'] + Parts_List['Processor']['storage']

max_stars_repo_name: voyager-tech/faux-sat

max_issues_repo_name: voyager-tech/faux-sat

max_forks_repo_name: voyager-tech/faux-sat


text:
from __future__ import annotations

import contextlib
import json
import logging
import os
import time
import warnings
from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Sequence, Union

import numpy as np

from qcodes.dataset.data_set_protocol import (
    SPECS,
    BaseDataSet,
    CompletedError,
    ParameterData,
)
from qcodes.dataset.descriptions.dependencies import InterDependencies_
from qcodes.dataset.descriptions.param_spec import ParamSpec, ParamSpecBase
from qcodes.dataset.descriptions.rundescriber import RunDescriber
from qcodes.dataset.descriptions.versioning.converters import new_to_old
from qcodes.dataset.descriptions.versioning.rundescribertypes import Shapes
from qcodes.dataset.guids import generate_guid
from qcodes.dataset.linked_datasets.links import Link, links_to_str
from qcodes.dataset.sqlite.connection import ConnectionPlus, atomic
from qcodes.dataset.sqlite.database import conn_from_dbpath_or_conn
from qcodes.dataset.sqlite.queries import (
    RUNS_TABLE_COLUMNS,
    add_data_to_dynamic_columns,
    add_parameter,
    create_run,
    get_experiment_name_from_experiment_id,
    get_raw_run_attributes,
    get_runid_from_guid,
    get_sample_name_from_experiment_id,
    mark_run_complete,
    set_run_timestamp,
    update_parent_datasets,
    update_run_description,
)
from qcodes.utils.helpers import NumpyJSONEncoder

from .data_set_cache import DataSetCacheInMem
from .dataset_helpers import _add_run_to_runs_table
from .descriptions.versioning import serialization as serial
from .experiment_settings import get_default_experiment_id
from .exporters.export_info import ExportInfo
from .linked_datasets.links import str_to_links

if TYPE_CHECKING:
    import pandas as pd
    import xarray as xr

    from ..instrument.parameter import _BaseParameter

log = logging.getLogger(__name__)

class DataSetInMem(BaseDataSet):
    def __init__(
        self,
        run_id: int,
        captured_run_id: int,
        counter: int,
        captured_counter: int,
        name: str,
        exp_id: int,
        exp_name: str,
        sample_name: str,
        guid: str,
        path_to_db: Optional[str],
        run_timestamp_raw: Optional[float],
        completed_timestamp_raw: Optional[float],
        snapshot: Optional[str] = None,
        metadata: Optional[Mapping[str, Any]] = None,
        rundescriber: Optional[RunDescriber] = None,
        parent_dataset_links: Optional[Sequence[Link]] = None,
        export_info: Optional[ExportInfo] = None,
    ) -> None:
        """Note that the constructor is considered private.

        A ``DataSetInMem``
        should be constructed either using one of the load functions
        (``load_by_run_spec``, ``load_from_netcdf`` ...)
        or using the measurement context manager.
        """

        self._run_id = run_id
        self._captured_run_id = captured_run_id
        self._counter = counter
        self._captured_counter = captured_counter
        self._name = name
        self._exp_id = exp_id
        self._exp_name = exp_name
        self._sample_name = sample_name
        self._guid = guid
        self._cache = DataSetCacheInMem(self)
        self._run_timestamp_raw = run_timestamp_raw
        self._completed_timestamp_raw = completed_timestamp_raw
        self._path_to_db = path_to_db
        if metadata is None:
            self._metadata = {}
        else:
            self._metadata = dict(metadata)
        if rundescriber is None:
            interdeps = InterDependencies_()
            rundescriber = RunDescriber(interdeps, shapes=None)

        self._rundescriber = rundescriber
        if parent_dataset_links is not None:
            self._parent_dataset_links = list(parent_dataset_links)
        else:
            self._parent_dataset_links = []
        if export_info is not None:
            self._export_info = export_info
        else:
            self._export_info = ExportInfo({})
        self._metadata["export_info"] = self._export_info.to_str()
        self._snapshot_raw_data = snapshot

    def _dataset_is_in_runs_table(
        self, path_to_db: Optional[Union[str, Path]] = None
    ) -> bool:
        """
        Does this run exist in the given db

        """
        if isinstance(path_to_db, Path):
            path_to_db = str(path_to_db)

        with contextlib.closing(
            conn_from_dbpath_or_conn(conn=None, path_to_db=path_to_db)
        ) as conn:
            run_id = get_runid_from_guid(conn, self.guid)
        return run_id is not None

    def write_metadata_to_db(
        self, path_to_db: Optional[Union[str, Path]] = None
    ) -> None:
        from .experiment_container import load_or_create_experiment

        if self._dataset_is_in_runs_table(path_to_db=path_to_db):
            return
        if isinstance(path_to_db, Path):
            path_to_db = str(path_to_db)

        with contextlib.closing(
            conn_from_dbpath_or_conn(conn=None, path_to_db=path_to_db)
        ) as conn:
            with atomic(conn) as aconn:
                exp = load_or_create_experiment(
                    conn=aconn,
                    experiment_name=self.exp_name,
                    sample_name=self.sample_name,
                    load_last_duplicate=True,
                )
                _add_run_to_runs_table(self, aconn, exp.exp_id, create_run_table=False)
            self._path_to_db = conn.path_to_dbfile

    @classmethod
    def _create_new_run(
        cls,
        name: str,
        path_to_db: Optional[Union[Path, str]] = None,
        exp_id: Optional[int] = None,
    ) -> DataSetInMem:
        if path_to_db is not None:
            path_to_db = str(path_to_db)

        with contextlib.closing(
            conn_from_dbpath_or_conn(conn=None, path_to_db=path_to_db)
        ) as conn:
            if exp_id is None:
                exp_id = get_default_experiment_id(conn)
            name = name or "dataset"
            sample_name = get_sample_name_from_experiment_id(conn, exp_id)
            exp_name = get_experiment_name_from_experiment_id(conn, exp_id)
            guid = generate_guid()

            run_counter, run_id, _ = create_run(
                conn, exp_id, name, guid=guid, parameters=None, create_run_table=False
            )

            ds = cls(
                run_id=run_id,
                captured_run_id=run_id,
                counter=run_counter,
                captured_counter=run_counter,
                name=name,
                exp_id=exp_id,
                exp_name=exp_name,
                sample_name=sample_name,
                guid=guid,
                path_to_db=conn.path_to_dbfile,
                run_timestamp_raw=None,
                completed_timestamp_raw=None,
                metadata=None,
            )

        return ds

    @classmethod
    def _load_from_netcdf(
        cls, path: Union[Path, str], path_to_db: Optional[Union[Path, str]] = None
    ) -> DataSetInMem:
        """
        Create a in memory dataset from a netcdf file.
        The netcdf file is expected to contain a QCoDeS dataset that
        has been exported using the QCoDeS netcdf export functions.

        Args:
            path: Path to the netcdf file to import.
            path_to_db: Optional path to a database where this dataset may be
                exported to. If not supplied the path can be given at export time
                or the dataset exported to the default db as set in the QCoDeS config.

        Returns:
            The loaded dataset.
        """
        # in the code below floats and ints loaded from attributes are explicitly casted
        # this is due to some older versions of qcodes writing them with a different backend
        # reading them back results in a numpy array of one element

        import xarray as xr

        loaded_data = xr.load_dataset(path, engine="h5netcdf")

        parent_dataset_links = str_to_links(
            loaded_data.attrs.get("parent_dataset_links", "[]")
        )
        if path_to_db is not None:
            path_to_db = str(path_to_db)

        with contextlib.closing(
            conn_from_dbpath_or_conn(conn=None, path_to_db=path_to_db)
        ) as conn:
            run_data = get_raw_run_attributes(conn, guid=loaded_data.guid)
            path_to_db = conn.path_to_dbfile

        if run_data is not None:
            run_id = run_data["run_id"]
            counter = run_data["counter"]
        else:
            run_id = int(loaded_data.captured_run_id)
            counter = int(loaded_data.captured_counter)

        path = str(path)
        path = os.path.abspath(path)

        export_info = ExportInfo.from_str(loaded_data.attrs.get("export_info", ""))
        export_info.export_paths["nc"] = path
        non_metadata = {
            "run_timestamp_raw",
            "completed_timestamp_raw",
            "ds_name",
            "exp_name",
            "sample_name",
            "export_info",
            "parent_dataset_links",
        }

        metadata_keys = (
            set(loaded_data.attrs.keys()) - set(RUNS_TABLE_COLUMNS) - non_metadata
        )
        metadata = {}
        for key in metadata_keys:
            data = loaded_data.attrs[key]
            if isinstance(data, np.ndarray) and data.size == 1:
                data = data[0]
            metadata[key] = data

        ds = cls(
            run_id=run_id,
            captured_run_id=int(loaded_data.captured_run_id),
            counter=counter,
            captured_counter=int(loaded_data.captured_counter),
            name=loaded_data.ds_name,
            exp_id=0,
            exp_name=loaded_data.exp_name,
            sample_name=loaded_data.sample_name,
            guid=loaded_data.guid,
            path_to_db=path_to_db,
            run_timestamp_raw=float(loaded_data.run_timestamp_raw),
            completed_timestamp_raw=float(loaded_data.completed_timestamp_raw),
            metadata=metadata,
            rundescriber=serial.from_json_to_current(loaded_data.run_description),
            parent_dataset_links=parent_dataset_links,
            export_info=export_info,
            snapshot=loaded_data.snapshot,
        )
        ds._cache = DataSetCacheInMem(ds)
        ds._cache._data = cls._from_xarray_dataset_to_qcodes_raw_data(loaded_data)

        return ds

    @classmethod
    def _load_from_db(cls, conn: ConnectionPlus, guid: str) -> DataSetInMem:

        run_attributes = get_raw_run_attributes(conn, guid)
        if run_attributes is None:
            raise RuntimeError(
                f"Could not find the requested run with GUID: {guid} in the db"
            )

        metadata = run_attributes["metadata"]

        export_info_str = metadata.get("export_info", "")
        export_info = ExportInfo.from_str(export_info_str)

        ds = cls(
            run_id=run_attributes["run_id"],
            captured_run_id=run_attributes["captured_run_id"],
            counter=run_attributes["counter"],
            captured_counter=run_attributes["captured_counter"],
            name=run_attributes["name"],
            exp_id=run_attributes["experiment"]["exp_id"],
            exp_name=run_attributes["experiment"]["name"],
            sample_name=run_attributes["experiment"]["sample_name"],
            guid=guid,
            path_to_db=conn.path_to_dbfile,
            run_timestamp_raw=run_attributes["run_timestamp"],
            completed_timestamp_raw=run_attributes["completed_timestamp"],
            metadata=metadata,
            rundescriber=serial.from_json_to_current(run_attributes["run_description"]),
            parent_dataset_links=str_to_links(run_attributes["parent_dataset_links"]),
            export_info=export_info,
            snapshot=run_attributes["snapshot"],
        )
        xr_path = export_info.export_paths.get("nc")

        cls._set_cache_from_netcdf(ds, xr_path)
        return ds

    @classmethod
    def _set_cache_from_netcdf(cls, ds: DataSetInMem, xr_path: Optional[str]) -> bool:
        import xarray as xr

        success = True
        if xr_path is not None:
            try:
                loaded_data = xr.load_dataset(xr_path, engine="h5netcdf")
                ds._cache = DataSetCacheInMem(ds)
                ds._cache._data = cls._from_xarray_dataset_to_qcodes_raw_data(
                    loaded_data
                )
            except (
                FileNotFoundError,
                OSError,
            ):  # older versions of h5py may throw a OSError here
                success = False
                warnings.warn(
                    "Could not load raw data for dataset with guid :"
                    f"{ds.guid} from location {xr_path}"
                )
        else:
            warnings.warn(f"No raw data stored for dataset with guid : {ds.guid}")
            success = False
        return success

    def set_netcdf_location(self, path: Union[str, Path]) -> None:
        """
        Change the location that a DataSetInMem refers to and
        load the raw data into the cache from this location.

        This may be useful if loading the dataset from a database raises a warning
        since the location of the raw data has moved. If this is the case you may
        be able to use this method to update the metadata in the database to refer to
        the new location.
        """
        if isinstance(path, Path):
            path = str(path)
        data_loaded = self._set_cache_from_netcdf(self, path)
        if data_loaded:
            export_info = self.export_info
            export_info.export_paths["nc"] = path
            self._set_export_info(export_info)
        else:
            raise FileNotFoundError(f"Could not load a netcdf file from {path}")

    @staticmethod
    def _from_xarray_dataset_to_qcodes_raw_data(
        xr_data: xr.Dataset,
    ) -> Dict[str, Dict[str, np.ndarray]]:
        output: Dict[str, Dict[str, np.ndarray]] = {}
        for datavar in xr_data.data_vars:
            output[str(datavar)] = {}
            data = xr_data[datavar]
            output[str(datavar)][str(datavar)] = data.data
            coords_unexpanded = []
            for coord in data.coords:
                coords_unexpanded.append(xr_data[coord].data)
            coords_arrays = np.meshgrid(*coords_unexpanded, indexing="ij")
            for coord_name, coord_array in zip(data.coords, coords_arrays):
                output[str(datavar)][str(coord_name)] = coord_array
        return output

    def prepare(
        self,
        *,
        snapshot: Mapping[Any, Any],
        interdeps: InterDependencies_,
        shapes: Shapes = None,
        parent_datasets: Sequence[Mapping[Any, Any]] = (),
        write_in_background: bool = False,
    ) -> None:
        if not self.pristine:
            raise RuntimeError("Cannot prepare a dataset that is not pristine.")

        self.add_snapshot(json.dumps({"station": snapshot}, cls=NumpyJSONEncoder))

        if interdeps == InterDependencies_():
            raise RuntimeError("No parameters supplied")

        self._set_interdependencies(interdeps, shapes)
        links = [Link(head=self.guid, **pdict) for pdict in parent_datasets]
        self._set_parent_dataset_links(links)

        if self.pristine:
            self._perform_start_actions()
            self.cache.prepare()

    @property
    def pristine(self) -> bool:
        """Is this :class:`.DataSetInMem` pristine?

        A pristine :class:`.DataSetInMem` has not yet been started,
        meaning that parameters can still be added and removed, but results
        can not be added.
        """
        return self._run_timestamp_raw is None and self._completed_timestamp_raw is None

    @property
    def running(self) -> bool:
        """
        Is this :class:`.DataSetInMem` currently running?

        A running :class:`.DataSetInMem` has been started,
        but not yet completed.
        """
        return (
            self._run_timestamp_raw is not None
            and self._completed_timestamp_raw is None
        )

    @property
    def completed(self) -> bool:
        """
        Is this :class:`.DataSetInMem` completed?

        A completed :class:`.DataSetInMem` may not be modified in
        any way.
        """
        return self._completed_timestamp_raw is not None

    def mark_completed(self) -> None:
        """
        Mark :class:`.DataSetInMem` as complete and thus read only and notify the subscribers
        """
        if self.completed:
            return
        if self.pristine:
            raise RuntimeError(
                "Can not mark a dataset as complete before it "
                "has been marked as started."
            )

        self._complete(True)

    @property
    def run_id(self) -> int:
        return self._run_id

    @property
    def captured_run_id(self) -> int:
        return self._captured_run_id

    @property
    def counter(self) -> int:
        return self._counter

    @property
    def captured_counter(self) -> int:
        return self._captured_counter

    @property
    def guid(self) -> str:
        return self._guid

    @property
    def number_of_results(self) -> int:
        return self.__len__()

    @property
    def name(self) -> str:
        return self._name

    @property
    def exp_name(self) -> str:
        return self._exp_name

    @property
    def exp_id(self) -> int:
        return self._exp_id

    @property
    def sample_name(self) -> str:
        return self._sample_name

    @property
    def path_to_db(self) -> Optional[str]:
        return self._path_to_db

    @property
    def run_timestamp_raw(self) -> Optional[float]:
        """
        Returns run timestamp as number of seconds since the Epoch

        The run timestamp is the moment when the measurement for this run
        started.
        """
        return self._run_timestamp_raw

    @property
    def completed_timestamp_raw(self) -> Optional[float]:
        """
        Returns timestamp when measurement run was completed
        as number of seconds since the Epoch

        If the run (or the dataset) is not completed, then returns None.
        """
        return self._completed_timestamp_raw

    @property
    def snapshot(self) -> Optional[Dict[str, Any]]:
        """Snapshot of the run as dictionary (or None)."""
        snapshot_json = self._snapshot_raw
        if snapshot_json is not None:
            return json.loads(snapshot_json)
        else:
            return None

    def add_snapshot(self, snapshot: str, overwrite: bool = False) -> None:
        """
        Adds a snapshot to this run.

        Args:
            snapshot: the raw JSON dump of the snapshot
            overwrite: force overwrite an existing snapshot
        """
        if self.snapshot is None or overwrite:
            self._add_to_dyn_column_if_in_db("snapshot", snapshot)
            self._snapshot_raw_data = snapshot
        elif self.snapshot is not None and not overwrite:
            log.warning(
                "This dataset already has a snapshot. Use overwrite"
                "=True to overwrite that"
            )

    @property
    def _snapshot_raw(self) -> Optional[str]:
        """Snapshot of the run as a JSON-formatted string (or None)."""
        return self._snapshot_raw_data

    def add_metadata(self, tag: str, metadata: Any) -> None:
        """
        Adds metadata to the :class:`.DataSet`.

        The metadata is stored under the provided tag.
        Note that None is not allowed as a metadata value.

        Args:
            tag: represents the key in the metadata dictionary
            metadata: actual metadata
        """

        self._metadata[tag] = metadata
        self._add_to_dyn_column_if_in_db(tag, metadata)
        self._add_metadata_to_netcdf_if_nc_exported(tag, metadata)

    def _add_to_dyn_column_if_in_db(self, tag: str, data: Any) -> None:
        if self._dataset_is_in_runs_table():
            with contextlib.closing(
                conn_from_dbpath_or_conn(conn=None, path_to_db=self._path_to_db)
            ) as conn:
                with atomic(conn) as aconn:
                    add_data_to_dynamic_columns(aconn, self.run_id, {tag: data})

    @property
    def metadata(self) -> Dict[str, Any]:
        return self._metadata

    @property
    def paramspecs(self) -> Dict[str, ParamSpec]:
        return {ps.name: ps for ps in self._get_paramspecs()}

    @property
    def description(self) -> RunDescriber:
        return self._rundescriber

    @property
    def parent_dataset_links(self) -> List[Link]:
        """
        Return a list of Link objects. Each Link object describes a link from
        this dataset to one of its parent datasets
        """
        return self._parent_dataset_links

    @property
    def cache(self) -> DataSetCacheInMem:
        return self._cache

    @property
    def export_info(self) -> ExportInfo:
        return self._export_info

    def _set_export_info(self, export_info: ExportInfo) -> None:
        tag = "export_info"
        metadata = export_info.to_str()

        self._metadata[tag] = metadata
        self._add_to_dyn_column_if_in_db(tag, metadata)

        self._export_info = export_info

    def _enqueue_results(self, result_dict: Mapping[ParamSpecBase, np.ndarray]) -> None:
        """
        Enqueue the results, for this dataset directly into cache

        Before we can enqueue the results, all values of the results dict
        must have the same length. We enqueue each parameter tree separately,
        effectively mimicking making one call to add_results per parameter
        tree.

        Deal with 'numeric' type parameters. If a 'numeric' top level parameter
        has non-scalar shape, it must be unrolled into a list of dicts of
        single values (database).
        """
        self._raise_if_not_writable()
        interdeps = self._rundescriber.interdeps

        toplevel_params = set(interdeps.dependencies).intersection(set(result_dict))
        new_results: Dict[str, Dict[str, np.ndarray]] = {}
        for toplevel_param in toplevel_params:
            inff_params = set(interdeps.inferences.get(toplevel_param, ()))
            deps_params = set(interdeps.dependencies.get(toplevel_param, ()))
            all_params = inff_params.union(deps_params).union({toplevel_param})

            new_results[toplevel_param.name] = {}
            new_results[toplevel_param.name][
                toplevel_param.name
            ] = self._reshape_array_for_cache(
                toplevel_param, result_dict[toplevel_param]
            )
            for param in all_params:
                if param is not toplevel_param:
                    new_results[toplevel_param.name][
                        param.name
                    ] = self._reshape_array_for_cache(param, result_dict[param])

        # Finally, handle standalone parameters

        standalones = set(interdeps.standalones).intersection(set(result_dict))

        if standalones:
            for st in standalones:
                new_results[st.name] = {
                    st.name: self._reshape_array_for_cache(st, result_dict[st])
                }

        self.cache.add_data(new_results)

    def _flush_data_to_database(self, block: bool = False) -> None:
        pass

    # not part of the protocol specified api

    def _set_parent_dataset_links(self, links: List[Link]) -> None:
        """
        Assign one or more links to parent datasets to this dataset. It is an
        error to assign links to a non-pristine dataset

        Args:
            links: The links to assign to this dataset
        """
        if not self.pristine:
            raise RuntimeError(
                "Can not set parent dataset links on a dataset "
                "that has been started."
            )

        if not all(isinstance(link, Link) for link in links):
            raise ValueError("Invalid input. Did not receive a list of Links")

        for link in links:
            if link.head != self.guid:
                raise ValueError(
                    "Invalid input. All links must point to this dataset. "
                    "Got link(s) with head(s) pointing to another dataset."
                )

        self._parent_dataset_links = links

    def _set_interdependencies(
        self, interdeps: InterDependencies_, shapes: Shapes = None
    ) -> None:
        """
        Set the interdependencies object (which holds all added
        parameters and their relationships) of this dataset and
        optionally the shapes object that holds information about
        the shape of the data to be measured.
        """
        if not isinstance(interdeps, InterDependencies_):
            raise TypeError(
                "Wrong input type. Expected InterDepencies_, " f"got {type(interdeps)}"
            )

        if not self.pristine:
            mssg = "Can not set interdependencies on a DataSet that has been started."
            raise RuntimeError(mssg)
        self._rundescriber = RunDescriber(interdeps, shapes=shapes)

    def _get_paramspecs(self) -> SPECS:
        old_interdeps = new_to_old(self.description.interdeps)
        return list(old_interdeps.paramspecs)

    def _complete(self, value: bool) -> None:

        with contextlib.closing(
            conn_from_dbpath_or_conn(conn=None, path_to_db=self._path_to_db)
        ) as conn:
            if value:
                self._completed_timestamp_raw = time.time()
                mark_run_complete(conn, self.run_id, self._completed_timestamp_raw)

    def _perform_start_actions(self) -> None:
        """
        Perform the actions that must take place once the run has been started
        """

        with contextlib.closing(
            conn_from_dbpath_or_conn(conn=None, path_to_db=self._path_to_db)
        ) as conn:
            paramspecs = new_to_old(self.description.interdeps).paramspecs

            for spec in paramspecs:
                add_parameter(
                    spec, conn=conn, run_id=self.run_id, insert_into_results_table=False
                )

            desc_str = serial.to_json_for_storage(self.description)

            update_run_description(conn, self.run_id, desc_str)
            self._run_timestamp_raw = time.time()
            set_run_timestamp(conn, self.run_id, self._run_timestamp_raw)

            pdl_str = links_to_str(self._parent_dataset_links)
            update_parent_datasets(conn, self.run_id, pdl_str)

    def _raise_if_not_writable(self) -> None:
        if self.pristine:
            raise RuntimeError(
                "This DataSet has not been marked as started. "
                "Please mark the DataSet as started before "
                "adding results to it."
            )
        if self.completed:
            raise CompletedError(
                "This DataSet is complete, no further " "results can be added to it."
            )

    def __len__(self) -> int:
        """
        The in memory dataset does not have a concept of sqlite rows
        so the length is represented by the number of all datapoints,
        summing across parameter trees.
        """
        values: List[int] = []
        for sub_dataset in self.cache.data().values():
            subvals = tuple(val.size for val in sub_dataset.values() if val is not None)
            values.extend(subvals)

        if len(values):
            return sum(values)
        else:
            return 0

    def __repr__(self) -> str:
        out = []
        heading = f"{self.name} #{self.run_id}@memory"
        out.append(heading)
        out.append("-" * len(heading))
        ps = self.description.interdeps.paramspecs
        if len(ps) > 0:
            for p in ps:
                out.append(f"{p.name} - {p.type}")

        return "\n".join(out)

    @property
    def _parameters(self) -> Optional[str]:
        psnames = [ps.name for ps in self.description.interdeps.paramspecs]
        if len(psnames) > 0:
            return ",".join(psnames)
        else:
            return None

    def to_xarray_dataarray_dict(
        self,
        *params: Union[str, ParamSpec, _BaseParameter],
        start: Optional[int] = None,
        end: Optional[int] = None,
    ) -> Dict[str, xr.DataArray]:
        self._warn_if_set(*params, start=start, end=end)
        return self.cache.to_xarray_dataarray_dict()

    def to_xarray_dataset(
        self,
        *params: Union[str, ParamSpec, _BaseParameter],
        start: Optional[int] = None,
        end: Optional[int] = None,
    ) -> xr.Dataset:
        self._warn_if_set(*params, start=start, end=end)
        return self.cache.to_xarray_dataset()

    def to_pandas_dataframe_dict(
        self,
        *params: Union[str, ParamSpec, _BaseParameter],
        start: Optional[int] = None,
        end: Optional[int] = None,
    ) -> Dict[str, pd.DataFrame]:
        self._warn_if_set(*params, start=start, end=end)
        return self.cache.to_pandas_dataframe_dict()

    def to_pandas_dataframe(
        self,
        *params: Union[str, ParamSpec, _BaseParameter],
        start: Optional[int] = None,
        end: Optional[int] = None,
    ) -> pd.DataFrame:
        self._warn_if_set(*params, start=start, end=end)
        return self.cache.to_pandas_dataframe()

    def get_parameter_data(
        self,
        *params: Union[str, ParamSpec, _BaseParameter],
        start: Optional[int] = None,
        end: Optional[int] = None,
    ) -> ParameterData:
        self._warn_if_set(*params, start=start, end=end)
        return self.cache.data()

    @staticmethod
    def _warn_if_set(
        *params: Union[str, ParamSpec, _BaseParameter],
        start: Optional[int] = None,
        end: Optional[int],
    ) -> None:
        if len(params) > 0 or start is not None or end is not None:
            warnings.warn(
                "Passing params, start or stop to to_xarray_... and "
                "to_pandas_... methods has no effect for DataSetInMem. "
                "This will be an error in the future."
            )

def load_from_netcdf(
    path: Union[Path, str], path_to_db: Optional[Union[Path, str]] = None
) -> DataSetInMem:
    """
    Create a in memory dataset from a netcdf file.
    The netcdf file is expected to contain a QCoDeS dataset that
    has been exported using the QCoDeS netcdf export functions.

    Args:
        path: Path to the netcdf file to import.
        path_to_db: Optional path to a database where this dataset may be
            exported to. If not supplied the path can be given at export time
            or the dataset exported to the default db as set in the QCoDeS config.

    Returns:
        The loaded dataset.
    """
    return DataSetInMem._load_from_netcdf(path=path, path_to_db=path_to_db)

max_stars_repo_name: edumur/Qcodes

max_issues_repo_name: edumur/Qcodes

max_forks_repo_name: edumur/Qcodes


text:
#!/usr/bin/env python3
# coding: utf-8
import config as config
import constants as constants
import pandas as pd
import numpy as np
import json
import concurrent.futures
import requests
import time

output = []
CONNECTIONS = 60
TIMEOUT = 8

tmdb_id_list = pd.read_csv("../data/processed/dataset_builder/tmdb_ids.csv", engine="python")

urls = [constants.BASE_URL + 'movie/' + str(i) + '?api_key=' +  config.tmdb_api_key + '&append_to_response=credits' for i in tmdb_id_list.tmdb_id]

def request_tmdb(url, timeout):
    request = requests.get(url, timeout=timeout)
    return request.json()

with concurrent.futures.ThreadPoolExecutor(max_workers=CONNECTIONS) as executor:
    future_to_url = (executor.submit(request_tmdb, url, TIMEOUT) for url in urls)
    time_1 = time.time()
    print(future_to_url)
    for future in concurrent.futures.as_completed(future_to_url):
        try:
            data = future.result()
        except Exception as exc:
            data = str(type(exc))
        finally:
            output.append(data)
            print(str(len(output)),end="\r")
    time_2 = time.time()

print(f'Took {time_2-time_1:.2f} s')

import json
with open('../data/processed/json/tmdb_movie_list.json', 'w', encoding='utf-8') as f:
    json.dump(output, f, ensure_ascii=False, indent=4)

max_stars_repo_name: sideround/predict-revenue-new-releases

max_issues_repo_name: sideround/predict-revenue-new-releases

max_forks_repo_name: sideround/predict-revenue-new-releases


text:
# -*- coding: utf-8 -*-
###############################################################################
# Copyright (c), Forschungszentrum Jülich GmbH, IAS-1/PGI-1, Germany.         #
#                All rights reserved.                                         #
# This file is part of the AiiDA-FLEUR package.                               #
#                                                                             #
# The code is hosted on GitHub at https://github.com/JuDFTteam/aiida-fleur    #
# For further information on the license, see the LICENSE.txt file            #
# For further information please visit http://www.flapw.de or                 #
# http://aiida-fleur.readthedocs.io/en/develop/                               #
###############################################################################

"""
In this module are plot routines collected to create default plots out of certain
AiiDA output nodes from certain workflows with matplot lib.

Comment: This makes plot_methods shorter to use for a Fleur, AiiDA user.
Be aware that requirements are the aiida-fleur plugin and aiida-fleur-based
Since we have a dependence to AiiDA here, it might be better to make a separate repo,
if this evolves.
"""
# TODO but allow to optional parse information for saving and title,
#  (that user can put pks or structure formulas in there)

from __future__ import absolute_import
from __future__ import print_function
from pprint import pprint
import six
from six.moves import range
import numpy as np
#import matplotlib.pyplot as pp
#from masci_tools.vis.plot_methods import *
from aiida.plugins import DataFactory
from aiida.orm import load_node
from aiida.orm import WorkChainNode
from aiida.orm import Node

###########################
## general plot routine  ##
###########################

def plot_fleur(*args, **kwargs):
    """
    This methods takes any amount of AiiDA node and starts
    the standard visualisation either as single or together visualisation.
    (if they are provided as list)
    i.e plot_fleur(123, [124,125], uuid, save=False)

    Some general parameters of plot methods can be given as
    keyword arguments.
    example: save: should the plots be saved automatically

    """

    '''
    def set_plot_defaults(title_fontsize = 16,
                      linewidth = 2.0,
                      markersize = 4.0,
                      labelfonstsize = 15,
                      ticklabelsize = 14,
                      tick_params = {'size' : 4.0, 'width' : 1.0,
                                     'labelsize' : ticklabelsize_g,
                                     'length' : 5},
                      save_plots = False, #True,
                      save_format = 'pdf'):
    '''
    from masci_tools.vis.plot_methods import set_plot_defaults

    save = False
    show_dict = False
    for key, val in six.iteritems(kwargs):
        if key == 'save':
            save=val
        if key == 'show_dict':
            show_dict = val
    #    # the rest we ignore for know
    #Just call set plot defaults
    # TODO, or rather parse it onto plot functions...?
    set_plot_defaults(**kwargs)

    for arg in args:
        if isinstance(arg, list):
            # try plot together
            plot_fleur_mn(arg, save=save)
        else:
            #print(arg)
            # plot alone
            plot_fleur_sn(arg, show_dict=show_dict, save=save)

def plot_fleur_sn(node, show_dict=False, save=False):
    """
    This methods takes any single AiiDA node and starts the standard visualisation for
    if it finds one
    """
    #show_dic = show_dic
    ParameterData = DataFactory('dict')
    if isinstance(node, int):#pk
        node = load_node(node)

    if isinstance(node, (str, six.text_type)): #uuid
        node = load_node(node) #try

    if isinstance(node, Node):
        if isinstance(node, WorkChainNode):
            output_list = node.get_outgoing().all()
            for out_link in output_list:
                if 'output_' in out_link.link_label:
                    if 'wc' in out_link.link_label or 'wf' in out_link.link_label:
                        if 'para' in out_link.link_label:# We are just looking for parameter
                            #nodes, structures, bands, dos and so on we tread different
                            node = out_link.node# we only visualize last output node
        if isinstance(node, ParameterData):
            p_dict = node.get_dict()
            workflow_name = p_dict.get('workflow_name', None)
            try:
                plotf = FUNCTIONS_DICT[workflow_name]
            except KeyError:
                print(('Sorry, I do not know how to visualize this workflow: {}, node {}.'
                       'Please implement me in plot_fleur_aiida!'.format(workflow_name, node)))
                if show_dict:
                    pprint(p_dict)
                return
            plotf(node)
        else:
            print('I do not know how to visualize this node: {}, type {}'.format(node, type(node)))
    else:
        print(('The node provided: {}, type {} is not an AiiDA object'.format(node, type(node))))
    # check if AiiDa node
    #check what type of node
    # if calcfunction, get certain output node
    #if parameterData, output node check if workflow name tag
    # if routine known plot,
    #else say I do not know

def plot_fleur_mn(nodelist, save=False):
    """
    This methods takes any amount of AiiDA node as a list and starts
    the standard visualisation for it, if it finds one.

    Some nodes types it tries to display together if it knows how to.
    and if they are given as a list.

    param: save showed the plots be saved automatically

    """
    ###
    # Things to plot together
    all_nodes = {}
    ###
    ParameterData = DataFactory('dict')

    if not isinstance(nodelist, list):
        print(('The nodelist provided: {}, type {} is not a list. '
               'I abort'.format(nodelist, type(nodelist))))
        return None

    node_labels = []
    for node in nodelist:
        # first find out what we have then how to visualize
        if isinstance(node, int):#pk
            node = load_node(node)
        if isinstance(node, (str, six.text_type)): #uuid
            node = load_node(node) #try

        if isinstance(node, Node):
            node_labels.append(node.label)
            if isinstance(node, WorkChainNode):
                output_list = node.get_outgoing().all()
                for out_link in output_list:
                    if 'output_' in out_link.link_label:
                        if 'wc' in out_link.link_label or 'wf' in out_link.link_label:
                            if 'para' in out_link.link_label:# We are just looking for parameter
                                #nodes, structures, bands, dos and so on we tread different
                                node = out_link.node# we only visualize last output node
            if isinstance(node, ParameterData):
                p_dict = node.get_dict()
                workflow_name = p_dict.get('workflow_name', None)
                cur_list = all_nodes.get(workflow_name, [])
                cur_list.append(node)
                all_nodes[workflow_name] = cur_list
            else:
                print(('I do not know how to visualize this node: {}, '
                       'type {} from the nodelist {}'.format(node, type(node), nodelist)))
        else:
            print(('The node provided: {} of type {} in the nodelist {}'
                   ' is not an AiiDA object'.format(node, type(node), nodelist)))

    #print(all_nodes)
    for node_key, nodelist in six.iteritems(all_nodes):
        try:
            plotf = FUNCTIONS_DICT[node_key]
        except KeyError:
            print(('Sorry, I do not know how to visualize'
                   ' these nodes (multiplot): {} {}'.format(node_key, nodelist)))
            continue
        plot_res = plotf(nodelist, labels=node_labels)

###########################
## general plot routine  ##
###########################

def plot_fleur_scf_wc(nodes, labels=None):
    """
    This methods takes an AiiDA output parameter node or a list from a scf workchain and
    plots number of iteration over distance and total energy
    """
    from masci_tools.vis.plot_methods import plot_convergence_results_m

    if labels is None:
        labels = []

    if isinstance(nodes, list):
        if len(nodes) >= 2:
            #return # TODO
            pass
        else:
            nodes = [nodes[0]]
    else:
        nodes = [nodes]#[0]]
    #scf_wf = load_node(6513)

    iterations = []
    distance_all_n = []
    total_energy_n = []
    modes = []

    for node in nodes:
        iteration = []
        output_d = node.get_dict()
        total_energy = output_d.get('total_energy_all')
        distance_all = output_d.get('distance_charge_all')
        iteration_total = output_d.get('iterations_total')
        mode = output_d.get('conv_mode')
        for i in range(1, len(total_energy)+1):
            iteration.append(iteration_total - len(total_energy) + i)
        iterations.append(iteration)
        distance_all_n.append(distance_all)
        total_energy_n.append(total_energy)
        modes.append(mode)
    #plot_convergence_results(distance_all, total_energy, iteration)
    if labels:
        plot_convergence_results_m(distance_all_n, total_energy_n, iterations,
                                   plot_labels=labels, modes=modes)
    else:
        plot_convergence_results_m(distance_all_n, total_energy_n, iterations, modes=modes)

def plot_fleur_dos_wc(node, labels=None):
    """
    This methods takes an AiiDA output parameter node from a density of states
    workchain and plots a simple density of states
    """
    from masci_tools.vis.plot_methods import plot_dos

    if labels is None:
        labels = []

    if isinstance(node, list):
        if len(node) > 2:
            return # TODO
        else:
            node = node[0]

    output_d = node.get_dict()
    path_to_dosfile = output_d.get('dosfile', None)
    print(path_to_dosfile)
    if path_to_dosfile:
        plot_dos(path_to_dosfile, only_total=False)
    else:
        print('Could not retrieve dos file path from output node')

def plot_fleur_eos_wc(node, labels=None):
    """
    This methods takes an AiiDA output parameter node from a density of states
    workchain and plots a simple density of states
    """
    from masci_tools.vis.plot_methods import plot_lattice_constant

    if labels is None:
        labels = []

    if isinstance(node, list):
        if len(node) > 2:
            Total_energy = []
            scaling = []
            plotlables = []

            for i, nd in enumerate(node):
                outpara = nd.get_dict()
                volume_gs = outpara.get('volume_gs')
                scale_gs = outpara.get(u'scaling_gs')
                total_e = outpara.get('total_energy')
                total_e_norm = np.array(total_e) - total_e[0]
                Total_energy.append(total_e_norm)
                scaling.append(outpara.get('scaling'))
                plotlables.append((r'gs_vol: {:.3} A^3, gs_scale {:.3}, data {}'
                                   ''.format(volume_gs, scale_gs, i)))
                plotlables.append(r'fit results {}'.format(i))
            plot_lattice_constant(Total_energy, scaling, multi=True, plotlables=plotlables)
            return # TODO
        else:
            node = node[0]

    outpara = node.get_dict()
    Total_energy = outpara.get('total_energy')
    scaling = outpara.get('scaling')
    #fit = outpara.get('fitresults')
    #fit = outpara.get('fit')

    #def parabola(x, a, b, c):
    #    return a*x**2 + b*x + c

    #fit_y = []
    #fit_y = [parabola(scale2, fit[0], fit[1], fit[2]) for scale2 in scaling]
    plot_lattice_constant(Total_energy, scaling)#, fit_y)
    return

def plot_fleur_band_wc(node, labels=None):
    """
    This methods takes an AiiDA output parameter node from a band structure
    workchain and plots a simple band structure
    """
    from masci_tools.vis.plot_methods import plot_bands

    if labels is None:
        labels = []

    if isinstance(node, list):
        if len(node) > 2:
            return # TODO
        else:
            node = node[0]

    output_d = node.get_dict()
    path_to_bands_file = output_d.get('bandfile', None)
    print(path_to_bands_file)
    kpath = output_d.get('kpath', {})#r"$\Gamma$": 0.00000, r"$H$" : 1.04590,
    #    r"$N$" : 1.78546, r"$P$": 2.30841, r"$\Gamma1$" : 3.21419, r"$N1$" : 3.95375} )

    if path_to_bands_file:
        plot_bands(path_to_bands_file, kpath)
    else:
        print('Could not retrieve dos file path from output node')

def plot_fleur_relax_wc(node, labels=None):
    """
    This methods takes an AiiDA output parameter node from a relaxation
    workchain and plots some information about atom movements and forces
    """
    if labels is None:
        labels = []

    pass

    #plot_relaxation_results

def plot_fleur_corehole_wc(nodes, labels=None):
    """
    This methods takes AiiDA output parameter nodes from a corehole
    workchain and plots some information about Binding energies
    """

    if labels is None:
        labels = []

    pass

def plot_fleur_initial_cls_wc(nodes, labels=None):
    """
    This methods takes AiiDA output parameter nodes from a initial_cls
    workchain and plots some information about corelevel shifts.
    (Spectra)
    """
    if labels is None:
        labels = []

    pass

FUNCTIONS_DICT = {
        'FleurScfWorkChain' : plot_fleur_scf_wc,
        'FleurEosWorkChain' : plot_fleur_eos_wc,
        'fleur_dos_wc' : plot_fleur_dos_wc,
        'fleur_band_wc' : plot_fleur_band_wc,
        #'fleur_corehole_wc' : plot_fleur_corehole_wc,
        #'fleur_initial_cls_wc' : plot_fleur_initial_cls_wc
        }

def clear_dict_empty_lists(to_clear_dict):
    """
    Removes entries from a nested dictionary which are empty lists.

    param to_clear_dict dict: python dictionary which should be 'compressed'
    return new_dict dict: compressed python dict version of to_clear_dict

    Hints: recursive
    """
    new_dict = {}
    if not to_clear_dict:
        return new_dict

    if not isinstance(to_clear_dict, dict):
        return to_clear_dict

    for key, value in six.iteritems(to_clear_dict):
        if value:
            new_value = clear_dict_empty_lists(value)
            if new_value:
                new_dict[key] = new_value
    return new_dict

max_stars_repo_name: anoopkcn/aiida-fleur

max_issues_repo_name: anoopkcn/aiida-fleur

max_forks_repo_name: anoopkcn/aiida-fleur


text:
import networkx as nx
from xarray import Dataset
from ...typing import DataMapping
from ...dispatch import dispatches_from, NetworkxBackend
from .. import core

def _maximal_independent_set(idi, idj, cmp):
    G = nx.Graph()
    G.add_edges_from(zip(idi, idj))
    index_to_keep = nx.algorithms.mis.maximal_independent_set(G)
    return list(set(G.nodes) - set(index_to_keep))

@dispatches_from(core.maximal_independent_set)
def maximal_independent_set(ds: DataMapping) -> DataMapping:
    """Networkx MIS

    Note: `cmp` is ignored by this backend.

    Returns
    -------
    Dataset
    """
    return core._maximal_independent_set(_maximal_independent_set, ds)

core.register_backend_function(NetworkxBackend)(maximal_independent_set)

max_stars_repo_name: tomwhite/gwas-analysis

max_issues_repo_name: tomwhite/gwas-analysis

max_forks_repo_name: tomwhite/gwas-analysis

reason: import networkx


text:
from dataset.mnist import load_mnist
from practice.layers import Affine, Relu, SoftmaxWithLoss
from practice.batch_normalization import BatchNormalization
import numpy as np
import re

class MultiLayerNetL2Norm:
    def __init__(self, weight_init_std, input_size, node_size, output_size, layer_size, l2_norm_lambda,
                 use_batch_norm=True):
        self.layer_size = layer_size
        self.use_batch_norm = use_batch_norm
        self.weight_init_std = weight_init_std
        self.input_size = input_size
        self.node_size = node_size
        self.output_size = output_size
        self.l2_norm_lambda = l2_norm_lambda
        self.params = {}
        self.layers = {}
        if use_batch_norm:
            self.__init_params_and_layers_with_batch_norm()
        else:
            self.__init_params_and_layers_without_batch_norm()

    def __init_params_and_layers_without_batch_norm(self):
        self.params["W1"] = self.weight_init_std * np.random.randn(self.input_size, self.node_size)
        self.params["b1"] = np.zeros(self.node_size)
        self.layers["affine1"] = Affine(self.params["W1"], self.params["b1"])
        self.layers["relu1"] = Relu()
        for i in range(1, self.layer_size):
            self.params[f'W{i + 1}'] = self.weight_init_std * np.random.randn(self.node_size, self.node_size)
            self.params[f'b{i + 1}'] = np.zeros(self.node_size)
            self.layers[f'affine{i + 1}'] = Affine(self.params[f'W{i + 1}'], self.params[f'b{i + 1}'])
            self.layers[f'relu{i + 1}'] = Relu()
        self.params[f"W{self.layer_size + 1}"] = self.weight_init_std * np.random.randn(self.node_size,
                                                                                self.output_size)
        self.params[f"b{self.layer_size + 1}"] = np.zeros(self.output_size)
        self.layers[f"affine{self.layer_size + 1}"] = Affine(self.params[f"W{self.layer_size + 1}"],
                                                             self.params[f"b{self.layer_size + 1}"])
        self.lastLayer = SoftmaxWithLoss()

    def __init_params_and_layers_with_batch_norm(self):
        self.params["W1"] = self.weight_init_std * np.random.randn(self.input_size, self.node_size)
        self.params["b1"] = np.zeros(self.node_size)
        self.layers["affine1"] = Affine(self.params["W1"], self.params["b1"])

        # initialize gamma and beta params
        self.params["gamma1"] = np.ones(self.node_size)
        self.params["beta1"] = np.zeros(self.node_size)
        self.layers['batch_norm1'] = BatchNormalization(self.params['gamma1'], self.params['beta1'], 0.9)
        self.layers["relu1"] = Relu()
        for i in range(1, self.layer_size):
            self.params[f'W{i + 1}'] = self.weight_init_std * np.random.randn(self.node_size, self.node_size)
            self.params[f'b{i + 1}'] = np.zeros(self.node_size)
            self.layers[f'affine{i + 1}'] = Affine(self.params[f'W{i + 1}'], self.params[f'b{i + 1}'])

            # initialize gamma and beta params
            self.params[f"gamma{i + 1}"] = np.ones(self.node_size)
            self.params[f"beta{i + 1}"] = np.zeros(self.node_size)
            self.layers[f'batch_norm{i + 1}'] = BatchNormalization(self.params[f'gamma{i + 1}'],
                                                                   self.params[f'beta{i + 1}'], 0.9)
            self.layers[f'relu{i + 1}'] = Relu()
        self.params[f"W{self.layer_size + 1}"] = self.weight_init_std * np.random.randn(self.node_size,
                                                                                self.output_size)
        self.params[f"b{self.layer_size + 1}"] = np.zeros(self.output_size)
        self.layers[f"affine{self.layer_size + 1}"] = Affine(self.params[f"W{self.layer_size + 1}"],
                                                             self.params[f"b{self.layer_size + 1}"])
        self.lastLayer = SoftmaxWithLoss()

    def predict(self, x, training_flg=True):
        for key, layer in self.layers.items():
            if re.search('batch_norm', key):
                x = layer.forward(x, training_flg=training_flg)
            else:
                x = layer.forward(x)
        return x

    def loss(self, x, t):
        y = self.predict(x)
        main_loss = self.lastLayer.forward(y, t)
        l2_norm = 0

        # for each W, we calculate sum of squares of its all elements, and add up all these
        for i in range(1, self.layer_size + 2):
            l2_norm += np.sum(self.params[f"W{i}"] ** 2)
        main_loss += 0.5 * self.l2_norm_lambda * l2_norm

    def accuracy(self, x, t):
        y = self.predict(x, training_flg=False)
        y_labels = np.argmax(y, axis=1)
        t_labels = np.argmax(t, axis=1)
        return np.mean(y_labels == t_labels)

    def gradient(self, x, t):
        self.loss(x, t)

        layers = list(self.layers.values())
        layers.reverse()

        dout = 1
        dout = self.lastLayer.backward(dout)
        for layer in layers:
            dout = layer.backward(dout)
        grads = {}
        for i in range(1, self.layer_size + 2):
            grads[f"W{i}"] = self.layers[f"affine{i}"].dW + self.l2_norm_lambda * self.layers[f"affine{i}"].W
            grads[f"b{i}"] = self.layers[f"affine{i}"].db
        if self.use_batch_norm:
            for i in range(self.layer_size):
                grads[f"gamma{i + 1}"] = self.layers[f"batch_norm{i + 1}"].dgamma
                grads[f"beta{i + 1}"] = self.layers[f"batch_norm{i + 1}"].dbeta
        return grads

max_stars_repo_name: samrullo/deep-learning-from-scratch

max_issues_repo_name: samrullo/deep-learning-from-scratch

max_forks_repo_name: samrullo/deep-learning-from-scratch


text:
import os
import pathlib
import zipfile

import click
import numpy as np
import tensorflow as tf
from tensorflow.keras.preprocessing.sequence import pad_sequences

from gbqa import GBQA
from gbqa.configs import MODEL_PATH, PRETRAIN_MODEL_DOWNLOAD_URL, DATASET_CONFIG
from gbqa.file_utils import download_file, load_preprocessor, get_model_paths
from gbqa.models.preprocess import EnglishSequenceProcessor, SparqlSequenceProcessor
from gbqa.models import NMTModel, TemplateBasedModel

@click.group()
def main():
    pass

@main.command()
@click.option("-l", "--list",
              is_flag=True,
              help="Show all available datasets for download.")
@click.option("-d", "--download",
              type=str,
              help="Download dataset.")
def dataset(list, download):
    print("This feature has not been implemented yet")

@main.command()
@click.option("-l", "--lists",
              is_flag=True,
              help="Show all installed models.")
@click.option("-t", "--train",
              type=str,
              help="Train and Save new model.\nUsage: `gbqa model --train MODELNAME --MODEL_TYPE`")
@click.option("-e", "--evaluate",
              type=str,
              help="Evaluate model.\nUsage: `gbqa model --evaluate MODELNAME --MODEL_TYPE`")
@click.option("--template/--nmt", default="True",
              help="Model type to use. There are two type available, TEMPLATE_BASED and NMT. \nDefault: TEMPLATE_BASED")
@click.option("-p", "--pretrain",
              is_flag=True,
              help="Download pretrain model.")
@click.option("-i", "--install",
              type=str,
              help="Install your model. \nUsage: `gbqa model -i path_to_your_model/MODELNAME_MODELTYPE.hdf5`")
def model(lists, train, evaluate, template, pretrain, install):
    if lists or train or evaluate:
        print("This feature has not been implemented yet")

    if pretrain:
        downloaded_file = os.path.join(MODEL_PATH, "pretrain.zip")
        download_file(PRETRAIN_MODEL_DOWNLOAD_URL, downloaded_file)
        zip = zipfile.ZipFile(downloaded_file)
        zip.extractall(MODEL_PATH)
        os.remove(downloaded_file)

        print("Download Pretrain models completed.")

@main.command()
@click.option("--stdin/--file", default="True",
              help="Predict from stdin or file.")
@click.option("-m", "model",
              default="PRETRAIN",
              help="Model name to use. Use \"gbqa model --list\" to see all available model to use.\nDefault: PRETRAIN")
@click.option("--template/--nmt", default="True",
              help="Model type to use. There are two type available, TEMPLATE_BASED and NMT. \nDefault: TEMPLATE_BASED")
def predict(stdin, model, template):
    if stdin == False:
        print("File prediction is not supported at the moment. Stdin will be used")

    if np.sum([os.path.exists(get_model_paths("pretrain", "ENGLISH_PROCESSOR")),
              os.path.exists(get_model_paths("pretrain", "SPARQL_PROCESSOR")),
              os.path.exists(get_model_paths("pretrain", "ENTITY_PROCESSOR"))]) < 3:
        print("Pre-train model can not be found on your disk. Please download it using `gbqa model --pretrain` command.")

    gbqa_instance = GBQA(model, "TEMPLATE_BASED" if template else "NMT")

    while True:
        question = input("Ask me: \n")

        predicted = gbqa_instance.predict_sparql(question)

        if predicted["status"] == 0:
            print(">>>", predicted["sparql"])
        else:
            print("ERROR:", predicted["msg"])

        print()

if __name__ == "__main__":
    main()

# What is the name of the capital of America?