text:
import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.utils.data as Data
from torch.autograd import Variable
import matplotlib.pyplot as plt
import numpy as np
import os
import torch.nn.functional as F
from prettytable import PrettyTable

# predefine some const
EPOCH = 2
BATCH_SIZE = 200
LEARNING_RATE = 0.001
#train_npy_FILE = "D:/TestWithMFCC39/NN for 0Health-1Dementia/npy/totalNPY for train.npy"
train_npy_FILE = "D:/TestWithMFCC39/small data sets in the form of npy/train/trainDS.npy"
testDIR = "D:/TestWithMFCC39/small data sets in the form of npy/test"
modelDIR = "D:/TestWithMFCC39/small data sets in the form of npy"
modelFILE = "reload_model.pth"

#用于打印分割线
def print_line(char,string):
    print(char*33,string,char*32)

# define the neural network
class myCNN1D(nn.Module):

    def num_flat_features(self,x):
        size=x.size()[1:] # all dimensions except the batch dimension
        num_features=1
        for s in size:
            num_features*=s
        return num_features

    def __init__(self):
        super(myCNN1D, self).__init__()
        self.layer1_conv = nn.Sequential(
            nn.Conv1d(in_channels=1, out_channels=2, kernel_size=5, stride=1, padding=0, dilation=1, groups=1,
                      bias=True),
            nn.ReLU(),
            # nn.MaxPool1d(2)
        )
        self.layer2_conv = nn.Sequential(
            nn.Conv1d(in_channels=2, out_channels=3, kernel_size=6, stride=1, padding=0, dilation=1, groups=1,
                      bias=True),
            nn.ReLU()
        )
        self.layer3_fc1 = nn.Linear(3*5*6, 100)
        self.layer4_output = nn.Linear(100, 1)

    def forward(self, x):
        out = self.layer1_conv(x)
        out = self.layer2_conv(out)
        out = out.view(-1, self.num_flat_features(out))
        out = self.layer3_fc1(out)
        out = self.layer4_output(out)
        return out

# build a CNN
myConv = myCNN1D()

# Define Loss and Optimizer
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(myConv.parameters(), lr=LEARNING_RATE)

# 制作训练集
trainDS = np.load(train_npy_FILE)
print("The shape of trainDS is {}".format(trainDS.shape))
trainDS = trainDS[:,np.newaxis]
print("The shape of trainDS is {}".format(trainDS.shape))

x_train = trainDS[:,:,1:]
y_train = trainDS[:,:,0]
y_train = torch.from_numpy(y_train).float()
x_train = torch.from_numpy(x_train).float()

trainDataSet = Data.TensorDataset(x_train, y_train)

trainLoader = Data.DataLoader(
    dataset=trainDataSet,  # torch TensorDataset format
    batch_size=BATCH_SIZE,  # mini batch size
    shuffle=True,  # 要不要打乱数据 (打乱比较好)
    #num_workers=1,  # 多线程来读数据
)

# Train the Model
print_line('*','Training Begin')
for epoch in range(EPOCH):
    for i, (x_train, y_train) in enumerate(trainLoader):
        x_train = Variable(x_train)
        y_train = Variable(y_train)

        # Forward + Backward + Optimize
        optimizer.zero_grad()
        outputs = myConv(x_train)

        # y_train = y_train.type(torch.LongTensor)
        loss = criterion(outputs, y_train)
        loss.backward()
        optimizer.step()

    print('Epoch [{}/{}] Loss: {:.8f}'.format(epoch + 1, EPOCH, loss.data))

torch.save(myConv, modelDIR+'/'+modelFILE)
reload_model = torch.load(modelDIR+'/'+modelFILE)

# Test the Model
print_line('*', 'Testing Begin')

testFILES = os.listdir(testDIR)

for f in testFILES:
    if f[0] == '.':
        continue
    correct = 0
    try:
        tempFILE = testDIR + '/' + f
        tempMatrix = np.load(tempFILE)
        #print("The shape of {0} is {1}".format(f, tempMatrix.shape))
        testMatrix = tempMatrix[:, np.newaxis]
        #print("The shape of testMatrix is {}".format(testMatrix.shape))

        x_test = testMatrix[:, :, 1:]
        y_test = testMatrix[:, :, 0]
        y_test = torch.from_numpy(y_test).float()
        x_test = torch.from_numpy(x_test).float()

        testDataSet = Data.TensorDataset(x_test, y_test)

        testLoader = Data.DataLoader(
            dataset=testDataSet,  # torch TensorDataset format
            batch_size=BATCH_SIZE,  # mini batch size
            shuffle=True  # 要不要打乱数据
            # num_workers=1,  # 多线程来读数据
        )

        for x_test, y_test in testLoader:
            x_test = Variable(x_test)
            outputs = reload_model(x_test)
            outputs = outputs.data.squeeze()

        is_sick = 0
        in_one_file_total = 0
        for output in outputs:
            in_one_file_total += 1
            if (output - 0) ** 2 - (output - 1) ** 2 < 0:
                predicted = 0
            else:
                predicted = 1
                is_sick += 1
            if (predicted - y_test[0])**2<0.00001:
                correct+=1
        print('This guy may be {} percentage in sick.'.format(100*is_sick / in_one_file_total))

    except:
        print("There are something wrong with this file.")

    print('Test Accuracy of for file {0} is {1}%.'.format(f, 100 * correct / in_one_file_total))

ext: py

max_stars_repo_path: 1DCNN by PyTorch v8.py

max_issues_repo_path: 1DCNN by PyTorch v8.py

max_forks_repo_path: 1DCNN by PyTorch v8.py

reason: import numpy


text:
import tvm
from tvm import relay

import numpy as np

from tvm.contrib.download import download_testdata
from tvm.relay.frontend.pytorch import get_graph_input_names

# PyTorch imports
import torch
import torchvision

model = torch.load("model/net_008.pth")

for k, v in model.items():
    print(k)
    print(v)

input_shape = [1, 24, 24]
input_data = torch.randn(input_shape)
scripted_model = torch.jit.trace(model, input_data)

#input_shape = [1, 28, 28]
#input_data = torch.randn(input_shape)
#scripted_model = torch.jit.trace(model, input_data).eval()

#from torchvision import transforms

#transform = transforms.ToTensor()


text:
import pandas as pd
import requests
from bs4 import BeautifulSoup as s
import numpy
import re

r = requests.get('https://en.wikipedia.org/wiki/World_War_II_casualties')
soup = s(r.content,'html.parser')

cols=[]

dt = soup.select('table')[0]
for i in dt.select('th'):
    cols.append(i.get_text().strip().replace('\xa0%',''))

country=[]
c = [i.name for i in countries]
for i in dt.select('tr b'):
    country.append(i.get_text().strip().replace('\xa0',''))

df = pd.read_html('https://en.wikipedia.org/wiki/World_War_II_casualties')[0]
df = df.drop(61)
df['Country'] = country
df = df.iloc[:-1,:-1]
df = df.drop(['Civilian deaths due towar-related famine and disease'],axis=1)

for i in range(len(df.columns)):
    df.iloc[:,i]=df.iloc[:,i].str.replace(r"[\(\[].*?[\)\]]", "",regex=True)

df = df.rename(columns={'Total population1/1/1939': 'Total population as of 1/1/1939'})

df.to_csv('ww2_dataset.csv')


text:
import sys
sys.path.insert(0, '..')

from utils import data
import os
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import censusdata

pop_data = censusdata.download('acs5', 2015, censusdata.censusgeo([('state', '*')]), ['B01003_001E'])
df = pd.DataFrame(pop_data)
df.columns = ['Population']

new_indices = []
state_names = []

for index in df.index.tolist():
    new_index = index.geo[0][1]
    new_indices.append(new_index)
    state_names.append(index.name)
df.index = new_indices
df.index = state_names
df.sort_index()

plt.style.use('fivethirtyeight')

# ------------ HYPERPARAMETERS -------------
BASE_PATH = '../COVID-19/csse_covid_19_data/'
MIN_CASES = 1000
# ------------------------------------------

confirmed = os.path.join(
    BASE_PATH,
    'csse_covid_19_time_series',
    'time_series_covid19_confirmed_US.csv')
confirmed = data.load_csv_data(confirmed)
features = []
targets = []

red = ['Alabama', 'Alaska', 'Arizona', 'Arkansas', 'Florida', 'Georgia', 'Idaho', 'Indiana', 'Iowa', 'Mississippi',
       'Massachusetts', 'Maryland', 'Missouri', 'Montana', 'Nebraska', 'New Hampshire', 'North Dakota', 'Ohio',
       'Oklahoma', 'South Carolina', 'South Dakota', 'Tennessee', 'Texas', 'Utah', 'Vermont', 'West Virginia', 'Wyoming']

blue = ['California', 'Colorado', 'Connecticut', 'Delaware', 'Hawaii', 'Illinois', 'Kansas',
        'Kentucky', 'Louisiana', 'Maine', 'Michigan', 'Minnesota', 'Nevada', 'New Jersey', 'New Mexico', 'New York',
        'North Carolina', 'Oregon', 'Pennsylvania', 'Rhode Island',  'Virginia', 'Washington',  'Wisconsin']

red_pop = 0
blue_pop = 0

for state in red:
    red_pop += df.loc[state, 'Population']
for state in blue:
    blue_pop += df.loc[state, 'Population']

fig = plt.figure(figsize=(12, 12))
ax = fig.add_subplot(111)
cm = plt.get_cmap('jet')
NUM_COLORS = 0
LINE_STYLES = ['solid', 'dashed', 'dotted']
NUM_STYLES = len(LINE_STYLES)

states = np.unique(confirmed["Province_State"])
red_cases = np.ones((498,))
blue_cases = np.ones((498,))

for val in np.unique(confirmed["Province_State"]):
    df = data.filter_by_attribute(
        confirmed, "Province_State", val)
    cases, labels = data.get_cases_chronologically_state(df)
    cases = cases
    cases = cases.sum(axis=0)
    if val in red:
        red_cases = np.add(red_cases, cases)
    if val in blue:
        blue_cases = np.add(blue_cases, cases)

red_cases = (red_cases / red_pop) * 1000
blue_cases = (blue_cases / blue_pop) * 1000

red, = ax.plot(red_cases, label='Red', color='red')
blue, = ax.plot(blue_cases, label='Blue', color='blue')
ax.legend([red, blue], ["Republican", 'Democrat'], loc=3, ncol=4)
ax.set_ylabel('Cases per 1000 People')
ax.set_xlabel("Time (days since Jan 22, 2020)")
plt.title("Cases Divided by Blue or Red Governors")
plt.tight_layout()
plt.savefig('results/cases_by_gov.png')


text:
import qcdanalysistools.fitting as fitting
from . import variance
from . import Blocking,AnalysisParam
import matplotlib.pyplot as plt
import numpy as np

def analyse_numblock_dependency(t_data, t_Nbl_list, t_fn = "./blocking_analysis.pdf", t_var_factor = 1):
    r"""

    """
    color_list=['tab:blue','tab:orange','tab:green','tab:red','tab:purple',"tab:brown","tab:pink","tab:gray","tab:olive",'tab:cyan'] # 10 colors

    t_Nbl_list = np.array(t_Nbl_list)
    block_sizes = t_data.shape[0]//t_Nbl_list

    var_per_Nbl = np.zeros(shape=(t_Nbl_list.shape[0],*t_data.shape[1:]))

    for i_bl,N_bl in enumerate(t_Nbl_list):
        bl_params = AnalysisParam(Blocking,
            data_size  = t_data.shape[0],
            N_blk      = N_bl
        )
        var_per_Nbl[i_bl,:] = variance(bl_params,t_data,t_axis=0)*t_var_factor

    # Fit the variance to A/num_blocks
    fit_model = fitting.PropInversModel(t_A0=1.7)
    fit_results = []
    for i in range(len(t_Nbl_list) - 2):

        fitter = fitting.DiagonalLeastSquare(
            t_model=fit_model,
            t_abscissa=block_sizes[i:],
            t_ordinate=var_per_Nbl[i:,0],
            t_ordinate_var=np.ones_like(var_per_Nbl[i:,0])
        )

        fit_results.append(fitter.fit())
        print(f"Fitting from {t_Nbl_list[i]} to {t_Nbl_list[-1]}")
        fitter.print_result()

    fig = plt.figure(figsize=(11.7,8.3))

    ax_var = fig.add_subplot(2,2,1)
    ax_err = fig.add_subplot(2,2,2)
    ax_chi = fig.add_subplot(2,2,3)
    ax_pva = fig.add_subplot(2,2,4)

    dat_err = np.zeros(len(fit_results))
    dat_chi = np.zeros(len(fit_results))
    dat_pva = np.zeros(len(fit_results))

    # ==========================================================================
    # Variance by N_blocks
    # ==========================================================================
    for i_fr,fit_res in enumerate(fit_results):
        if i_fr == 0:
            ax_var.plot(block_sizes,fit_res['Best fit'],':',c=color_list[0],label="Best fits")
        else:
            ax_var.plot(block_sizes[i_fr:],fit_res['Best fit'],':',c=color_list[0])
            #ax_var.errorbar(t_Nbl_list[i_fr:],fit_res['Best fit'],yerr=np.ones_like(fit_res['Best fit'])*fit_res['Fit error'],fmt=':',c=color_list[0])

        dat_err[i_fr] = fit_res['Fit error']
        dat_chi[i_fr] = fit_res['red chisq']
        dat_pva[i_fr] = fit_res['p-value']
    ax_var.plot(block_sizes,var_per_Nbl[:,0],'o:',c=color_list[3],label="Var")

    ax_var.set_xlabel(r"Block Size")
    if t_var_factor == 1:
        ax_var.set_ylabel(r"$Var$")
    else:
        ax_var.set_ylabel(fr"$Var\cdot {t_var_factor:g}$")
    ax_var.legend()
    #ax_var.set_xticks(block_sizes[::4])

    # ==========================================================================
    # Fit error by N_blocks
    # ==========================================================================
    ax_err.plot(block_sizes[:-2],dat_err,'x:')
    ax_err.set_xlabel(r"Block Size")
    ax_err.set_ylabel(r"Fit error")
    #ax_err.set_xticks(block_sizes[::4])
    # ==========================================================================
    # Chisq/dof by N_blocks
    # ==========================================================================
    ax_chi.plot(block_sizes[:-2],dat_chi,'x:')
    ax_chi.set_yscale('log')
    ax_chi.set_xlabel(r"Block Size")
    ax_chi.set_ylabel(r"$\frac{\chi^2}{dof}$")
    #ax_chi.set_xticks(block_sizes[::4])

    # ==========================================================================
    # p-value by N_blocks
    # ==========================================================================
    ax_pva.plot(block_sizes[:-2],dat_pva,'x:')
    ax_pva.set_xlabel(r"Block Size")
    ax_pva.set_ylabel(r"p-value")
    #ax_pva.set_xticks(block_sizes[::4])

    fig.tight_layout()
    plt.savefig(t_fn)
    plt.clf()
    plt.close()

max_stars_repo_name: Marcel-Rodekamp/qcdanalysistools

max_issues_repo_name: Marcel-Rodekamp/qcdanalysistools

max_forks_repo_name: Marcel-Rodekamp/qcdanalysistools


text:
# -*- coding: utf-8 -*-
"""
Created on Fri Dec 15 17:49:56 2017

@author: Vladislav Sláma
"""

from QChemTool.QuantumChem.Classes.structure import Structure
from QChemTool.QuantumChem.Fluorographene.fluorographene import deleteFG, constrainsFG
from QChemTool.General.UnitsManager import position_units
import QChemTool.QuantumChem.output as out
import numpy as np

dir="C:/PhD/Ab-initio-META/Fluorographane/Excited_states/Distance-dependence/"
xyz_file="".join([dir,"FGrph_2perylene3_3dist_par_TDDFT-wB97XD_geom_BLYP-landl2dz_symm_9x14.xyz"])
gjf_file="".join([dir,"FGrph_1bisanthrene_5dist_ser_symm_18x9.gjf"])
struc=Structure()
struc.load_xyz(xyz_file)
#struc.load_gjf(gjf_file)

#constrain = constrainsFG(struc,border=True,defect=True)
#for ii in constrain:
#    print("X",ii+1,"F")

#def constrainsFG(struc,border=True):
#    if struc.bo

## Create gjf file for H optimization
#defect_struc=deleteFG(struc,add_hydrogen=True)
#defect_struc.output_to_xyz("Defect_test.xyz")
#
#Fragments=deleteFG(struc,add_hydrogen=True,Fragmentize=True)
#for ii in range(len(Fragments)):
#    structure=Fragments[ii]
#    structure.output_to_xyz("".join(["Defect_",str(ii),".xyz"]))
#
## Greate Gaussian input file for hydrogen optimization
#constrain=np.where(np.array(defect_struc.at_type)!='H')[0]
#with position_units('Angstrom'):
#    out.OutputToGauss(defect_struc.coor.value, defect_struc.at_type,
#                  'opt_restricted', 'BLYP', 'LANL2DZ', filename='Defects_Hopt.gjf',
#                  namechk='Defects_Hopt.chk', constrain=constrain, MEM_GB=2,
#                  Tight=True,verbose=False)

max_stars_repo_name: slamavl/QChemTool

max_issues_repo_name: slamavl/QChemTool

max_forks_repo_name: slamavl/QChemTool


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on 7:39 2018/3/30

@author:    Liu Jinbao
@mail:      liu.jinbao@outlook.com
@project:   PlasmaChemistry
@IDE:       PyCharm
"""
from __future__ import division, print_function, absolute_import
from .widgets import *
from .spectra_widget import *

__all__ = [s for s in dir() if not s.startswith('_')]

from numpy.testing import Tester

test = Tester().test

max_stars_repo_name: LiuPhyMan/SpecFit

max_issues_repo_name: LiuPhyMan/SpecFit

max_forks_repo_name: LiuPhyMan/SpecFit

reason: from numpy


text:
import random
import re
from copy import copy
from typing import Dict, List, Optional, Sequence, Tuple

import hypothesis.strategies as hst
import numpy as np
import pytest
from hypothesis import HealthCheck, given, settings

import qcodes as qc
from qcodes import (
    experiments,
    load_by_counter,
    load_by_id,
    new_data_set,
    new_experiment,
)
from qcodes.dataset.data_set import CompletedError, DataSet
from qcodes.dataset.descriptions.dependencies import InterDependencies_
from qcodes.dataset.descriptions.param_spec import ParamSpecBase
from qcodes.dataset.descriptions.rundescriber import RunDescriber
from qcodes.dataset.guids import parse_guid
from qcodes.dataset.sqlite.connection import path_to_dbfile
from qcodes.dataset.sqlite.database import get_DB_location
from qcodes.dataset.sqlite.queries import _unicode_categories
from qcodes.tests.common import error_caused_by
from qcodes.tests.dataset.helper_functions import verify_data_dict
from qcodes.tests.dataset.test_links import generate_some_links
from qcodes.utils.types import numpy_floats, numpy_ints

n_experiments = 0

def make_shadow_dataset(dataset: DataSet):
    """
    Creates a new DataSet object that points to the same run_id in the same
    database file as the given dataset object. Note that for a pristine run,
    the shadow dataset may be out of sync with its input dataset.

    Note that in order to achieve it `path_to_db` because this will create a
    new sqlite3 connection object behind the scenes. This is very useful for
    situations where one needs to assert the underlying modifications to the
    database file.
    """

    return DataSet(path_to_db=dataset.path_to_db, run_id=dataset.run_id)

@pytest.mark.usefixtures("experiment")
def test_has_attributes_after_init():
    """
    Ensure that all attributes are populated after __init__ in BOTH cases
    (run_id is None / run_id is not None)
    """

    attrs = ['path_to_db', 'conn', '_run_id', 'run_id',
             '_debug', 'subscribers', '_completed', 'name', 'table_name',
             'guid', 'number_of_results', 'counter', 'parameters',
             'paramspecs', 'exp_id', 'exp_name', 'sample_name',
             'run_timestamp_raw', 'completed_timestamp_raw', 'completed',
             'snapshot', 'snapshot_raw']

    path_to_db = get_DB_location()
    ds = DataSet(path_to_db, run_id=None)

    for attr in attrs:
        assert hasattr(ds, attr)
        getattr(ds, attr)

    ds = DataSet(path_to_db, run_id=1)

    for attr in attrs:
        assert hasattr(ds, attr)
        getattr(ds, attr)

@pytest.mark.usefixtures("experiment")
def test_dataset_length():

    path_to_db = get_DB_location()
    ds = DataSet(path_to_db, run_id=None)

    assert len(ds) == 0

    parameter = ParamSpecBase(name='single', paramtype='numeric',
                              label='', unit='N/A')
    idps = InterDependencies_(standalones=(parameter,))
    ds.set_interdependencies(idps)

    ds.mark_started()
    ds.add_results([{parameter.name: 1}])
    ds.mark_completed()

    assert len(ds) == 1

def test_dataset_location(empty_temp_db_connection):
    """
    Test that an dataset and experiment points to the correct db file when
    a connection is supplied.
    """
    exp = new_experiment("test", "test1", conn=empty_temp_db_connection)
    ds = DataSet(conn=empty_temp_db_connection)
    assert path_to_dbfile(empty_temp_db_connection) == \
        empty_temp_db_connection.path_to_dbfile
    assert exp.path_to_db == empty_temp_db_connection.path_to_dbfile
    assert ds.path_to_db == empty_temp_db_connection.path_to_dbfile

@pytest.mark.usefixtures("experiment")
def test_dataset_states():
    """
    Test the interplay between pristine, started, running, and completed
    """

    ds = DataSet()

    assert ds.pristine is True
    assert ds.running is False
    assert ds.started is False
    assert ds.completed is False

    with pytest.raises(RuntimeError, match='Can not mark DataSet as complete '
                                           'before it has '
                                           'been marked as started.'):
        ds.mark_completed()

    match = ('This DataSet has not been marked as started. '
             'Please mark the DataSet as started before '
             'adding results to it.')
    with pytest.raises(RuntimeError, match=match):
        ds.add_results([{'x': 1}])

    parameter = ParamSpecBase(name='single', paramtype='numeric',
                              label='', unit='N/A')
    idps = InterDependencies_(standalones=(parameter,))
    ds.set_interdependencies(idps)

    ds.mark_started()

    assert ds.pristine is False
    assert ds.running is True
    assert ds.started is True
    assert ds.completed is False

    match = ('Can not set interdependencies on a DataSet that has '
             'been started.')

    with pytest.raises(RuntimeError, match=match):
        ds.set_interdependencies(idps)

    ds.add_results([{parameter.name: 1}])

    ds.mark_completed()

    assert ds.pristine is False
    assert ds.running is False
    assert ds.started is True
    assert ds.completed is True

    match = ('Can not set interdependencies on a DataSet that has '
             'been started.')

    with pytest.raises(RuntimeError, match=match):
        ds.set_interdependencies(idps)

    match = ('This DataSet is complete, no further '
             'results can be added to it.')

    with pytest.raises(CompletedError, match=match):
        ds.add_results([{parameter.name: 1}])

@pytest.mark.parametrize("start_bg_writer", (True, False))
@pytest.mark.usefixtures('experiment')
def test_mark_completed_twice(start_bg_writer):
    """
    Ensure that its not an error to call mark_completed
    on an already completed dataset
    """
    ds = DataSet()
    ds.mark_started(start_bg_writer=start_bg_writer)
    ds.mark_completed()
    ds.mark_completed()

@pytest.mark.usefixtures('experiment')
def test_timestamps_are_none():
    ds = DataSet()

    assert ds.run_timestamp_raw is None
    assert ds.run_timestamp() is None

    ds.mark_started()

    assert isinstance(ds.run_timestamp_raw, float)
    assert isinstance(ds.run_timestamp(), str)

def test_dataset_read_only_properties(dataset):
    read_only_props = ['run_id', 'path_to_db', 'name', 'table_name', 'guid',
                       'number_of_results', 'counter', 'parameters',
                       'paramspecs', 'exp_id', 'exp_name', 'sample_name',
                       'run_timestamp_raw', 'completed_timestamp_raw',
                       'snapshot', 'snapshot_raw', 'dependent_parameters']

    # It is not expected to be possible to set readonly properties
    for prop in read_only_props:
        with pytest.raises(AttributeError, match="can't set attribute"):
            setattr(dataset, prop, True)

@pytest.mark.usefixtures("experiment")
@pytest.mark.parametrize("non_existing_run_id", (1, 0, -1, 'number#42'))
def test_create_dataset_from_non_existing_run_id(non_existing_run_id):
    with pytest.raises(ValueError, match=f"Run with run_id "
                                         f"{non_existing_run_id} does not "
                                         f"exist in the database"):
        _ = DataSet(run_id=non_existing_run_id)

def test_create_dataset_pass_both_connection_and_path_to_db(experiment):
    with pytest.raises(ValueError, match="Received BOTH conn and path_to_db. "
                                         "Please provide only one or "
                                         "the other."):
        some_valid_connection = experiment.conn
        _ = DataSet(path_to_db="some valid path", conn=some_valid_connection)

def test_load_by_id(dataset):
    ds = load_by_id(dataset.run_id)
    assert dataset.run_id == ds.run_id
    assert dataset.path_to_db == ds.path_to_db

@pytest.mark.usefixtures('experiment')
@pytest.mark.parametrize('non_existing_run_id', (1, 0, -1))
def test_load_by_id_for_nonexisting_run_id(non_existing_run_id):
    with pytest.raises(ValueError, match=f'Run with run_id '
                                         f'{non_existing_run_id} does not '
                                         f'exist in the database'):
        _ = load_by_id(non_existing_run_id)

@pytest.mark.usefixtures('experiment')
def test_load_by_id_for_none():
    with pytest.raises(ValueError, match='run_id has to be a positive integer, '
                                         'not None.'):
        _ = load_by_id(None)

@settings(deadline=None, max_examples=6)
@given(experiment_name=hst.text(min_size=1),
       sample_name=hst.text(min_size=1),
       dataset_name=hst.text(hst.characters(whitelist_categories=_unicode_categories),
                             min_size=1))
@pytest.mark.usefixtures("empty_temp_db")
def test_add_experiments(experiment_name,
                         sample_name, dataset_name):
    global n_experiments
    n_experiments += 1

    _ = new_experiment(experiment_name, sample_name=sample_name)
    exps = experiments()
    assert len(exps) == n_experiments
    exp = exps[-1]
    assert exp.name == experiment_name
    assert exp.sample_name == sample_name
    assert exp.last_counter == 0

    dataset = new_data_set(dataset_name)
    dsid = dataset.run_id
    loaded_dataset = load_by_id(dsid)
    expected_ds_counter = 1
    assert loaded_dataset.name == dataset_name
    assert loaded_dataset.counter == expected_ds_counter
    assert loaded_dataset.table_name == "{}-{}-{}".format("results",
                                                          exp.exp_id,
                                                          loaded_dataset.counter)
    expected_ds_counter += 1
    dataset = new_data_set(dataset_name)
    dsid = dataset.run_id
    loaded_dataset = load_by_id(dsid)
    assert loaded_dataset.name == dataset_name
    assert loaded_dataset.counter == expected_ds_counter
    assert loaded_dataset.table_name == "{}-{}-{}".format("results",
                                                          exp.exp_id,
                                                          loaded_dataset.counter)

@pytest.mark.usefixtures("experiment")
def test_dependent_parameters():

    pss: List[ParamSpecBase] = []

    for n in range(5):
        pss.append(ParamSpecBase(f'ps{n}', paramtype='numeric'))

    idps = InterDependencies_(dependencies={pss[0]: (pss[1], pss[2])})
    ds = DataSet(specs=idps)
    assert ds.dependent_parameters == (pss[0],)

    idps = InterDependencies_(dependencies={pss[0]: (pss[1], pss[2])},
                              standalones=(pss[3], pss[4]))
    ds = DataSet(specs=idps)
    assert ds.dependent_parameters == (pss[0],)

    idps = InterDependencies_(dependencies={pss[0]: (pss[1], pss[2]),
                                            pss[3]: (pss[4],)})

    ds = DataSet(specs=idps)
    assert ds.dependent_parameters == (pss[0], pss[3])

    idps = InterDependencies_(dependencies={pss[3]: (pss[1], pss[2]),
                                            pss[0]: (pss[4],)})

    ds = DataSet(specs=idps)
    assert ds.dependent_parameters == (pss[3], pss[0])

def test_set_interdependencies(dataset):
    exps = experiments()
    assert len(exps) == 1
    exp = exps[0]
    assert exp.name == "test-experiment"
    assert exp.sample_name == "test-sample"
    assert exp.last_counter == 1

    parameter_a = ParamSpecBase("a_param", "NUMERIC")
    parameter_b = ParamSpecBase("b_param", "NUMERIC")
    parameter_c = ParamSpecBase("c_param", "array")

    idps = InterDependencies_(
        inferences={parameter_c: (parameter_a, parameter_b)})

    dataset.set_interdependencies(idps)

    # write the parameters to disk
    dataset.mark_started()

    # Now retrieve the paramspecs

    shadow_ds = make_shadow_dataset(dataset)

    paramspecs = shadow_ds.paramspecs

    expected_keys = ['a_param', 'b_param', 'c_param']
    keys = sorted(list(paramspecs.keys()))
    assert keys == expected_keys
    for expected_param_name in expected_keys:
        ps = paramspecs[expected_param_name]
        assert ps.name == expected_param_name

    assert paramspecs == dataset.paramspecs

@pytest.mark.usefixtures("experiment")
def test_add_data_1d():
    exps = experiments()
    assert len(exps) == 1
    exp = exps[0]
    assert exp.name == "test-experiment"
    assert exp.sample_name == "test-sample"
    assert exp.last_counter == 0

    psx = ParamSpecBase("x", "numeric")
    psy = ParamSpecBase("y", "numeric")

    idps = InterDependencies_(dependencies={psy: (psx,)})

    mydataset = new_data_set("test-dataset")
    mydataset.set_interdependencies(idps)
    mydataset.mark_started()

    expected_x = []
    expected_y = []
    for x in range(100):
        expected_x.append(x)
        y = 3 * x + 10
        expected_y.append(y)
        mydataset.add_results([{"x": x, "y": y}])

    shadow_ds = make_shadow_dataset(mydataset)

    np.testing.assert_array_equal(
        mydataset.get_parameter_data()['y']['x'], expected_x)
    np.testing.assert_array_equal(
        mydataset.get_parameter_data()['y']['y'], expected_y)
    np.testing.assert_array_equal(
        shadow_ds.get_parameter_data()['y']['x'], expected_x)
    np.testing.assert_array_equal(
        shadow_ds.get_parameter_data()['y']['y'], expected_y)

    assert mydataset.completed is False
    mydataset.mark_completed()
    assert mydataset.completed is True

    with pytest.raises(CompletedError):
        mydataset.add_results([{'y': 500}])

    with pytest.raises(CompletedError):
        mydataset.add_results([{'x': 5}])

@pytest.mark.usefixtures("experiment")
def test_add_data_array():
    exps = experiments()
    assert len(exps) == 1
    exp = exps[0]
    assert exp.name == "test-experiment"
    assert exp.sample_name == "test-sample"
    assert exp.last_counter == 0

    idps = InterDependencies_(
        standalones=(ParamSpecBase("x", "numeric"),
                     ParamSpecBase("y", "array")))
    mydataset = new_data_set("test")
    mydataset.set_interdependencies(idps)
    mydataset.mark_started()

    expected_x = np.arange(100)
    expected_y = []
    for x in range(100):
        y = np.random.random_sample(10)
        expected_y.append(y)
        mydataset.add_results([{"x": x, "y": y}])

    shadow_ds = make_shadow_dataset(mydataset)

    np.testing.assert_array_equal(mydataset.get_parameter_data()[
                                  'x']['x'], np.array(expected_x))
    np.testing.assert_array_equal(shadow_ds.get_parameter_data()[
                                  'x']['x'], np.array(expected_x))

    y_data = mydataset.get_parameter_data()['y']['y']
    np.testing.assert_allclose(y_data, expected_y)
    y_data = shadow_ds.get_parameter_data()['y']['y']
    np.testing.assert_allclose(y_data, expected_y)

@pytest.mark.usefixtures("experiment")
def test_adding_too_many_results():
    """
    This test really tests the "chunking" functionality of the
    insert_many_values function of the sqlite.query_helpers module
    """
    dataset = new_data_set("test_adding_too_many_results")
    xparam = ParamSpecBase("x", "numeric", label="x parameter",
                           unit='V')
    yparam = ParamSpecBase("y", 'numeric', label='y parameter',
                           unit='Hz')
    idps = InterDependencies_(dependencies={yparam: (xparam,)})
    dataset.set_interdependencies(idps)
    dataset.mark_started()
    n_max = qc.SQLiteSettings.limits['MAX_VARIABLE_NUMBER']

    vals = np.linspace(0, 1, int(n_max/2)+2)
    results = [{'x': val} for val in vals]
    dataset.add_results(results)

    vals = np.linspace(0, 1, int(n_max/2)+1)
    results = [{'x': val, 'y': val} for val in vals]
    dataset.add_results(results)

    vals = np.linspace(0, 1, n_max*3)
    results = [{'x': val} for val in vals]
    dataset.add_results(results)

@pytest.mark.usefixtures("dataset")
def test_load_by_counter():
    exps = experiments()
    assert len(exps) == 1
    exp = exps[0]
    assert exp.name == "test-experiment"
    assert exp.sample_name == "test-sample"
    assert exp.last_counter == 1

    dataset = load_by_counter(1, 1)

    assert "test-dataset" == dataset.name
    assert exp.sample_name == dataset.sample_name
    assert exp.name == dataset.exp_name

@pytest.mark.usefixtures("experiment")
@pytest.mark.parametrize('nonexisting_counter', (-1, 0, 1, None))
def test_load_by_counter_for_nonexisting_counter(nonexisting_counter):
    exp_id = 1
    with pytest.raises(RuntimeError, match='Expected one row'):
        _ = load_by_counter(exp_id, nonexisting_counter)

@pytest.mark.usefixtures("empty_temp_db")
@pytest.mark.parametrize('nonexisting_exp_id', (-1, 0, 1, None))
def test_load_by_counter_for_nonexisting_experiment(nonexisting_exp_id):
    with pytest.raises(RuntimeError, match='Expected one row'):
        _ = load_by_counter(nonexisting_exp_id, 1)

@pytest.mark.usefixtures("empty_temp_db")
def test_dataset_with_no_experiment_raises():
    with pytest.raises(ValueError):
        new_data_set("test-dataset")

def test_guid(dataset):
    guid = dataset.guid
    assert len(guid) == 36
    parse_guid(guid)

def test_numpy_ints(dataset):
    """
     Test that we can insert numpy integers in the data set
    """
    xparam = ParamSpecBase('x', 'numeric')
    idps = InterDependencies_(standalones=(xparam,))
    dataset.set_interdependencies(idps)
    dataset.mark_started()

    results = [{"x": tp(1)} for tp in numpy_ints]
    dataset.add_results(results)
    expected_result = np.ones(len(numpy_ints))
    np.testing.assert_array_equal(dataset.get_parameter_data()[
                                  "x"]["x"], expected_result)

def test_numpy_floats(dataset):
    """
    Test that we can insert numpy floats in the data set
    """
    float_param = ParamSpecBase('y', 'numeric')
    idps = InterDependencies_(standalones=(float_param,))
    dataset.set_interdependencies(idps)
    dataset.mark_started()

    results = [{"y": tp(1.2)} for tp in numpy_floats]
    dataset.add_results(results)
    expected_result = np.array([tp(1.2) for tp in numpy_floats])
    data = dataset.get_parameter_data()["y"]["y"]
    assert np.allclose(data, expected_result, atol=1E-8)

def test_numpy_nan(dataset):
    parameter_m = ParamSpecBase("m", "numeric")
    idps = InterDependencies_(standalones=(parameter_m,))
    dataset.set_interdependencies(idps)
    dataset.mark_started()

    data_dict = [{"m": value} for value in [0.0, np.nan, 1.0]]
    dataset.add_results(data_dict)
    retrieved = dataset.get_parameter_data()["m"]["m"]
    assert np.isnan(retrieved[1])

def test_numpy_inf(dataset):
    """
    Test that we can insert and retrieve numpy inf in the data set
    """
    parameter_m = ParamSpecBase("m", "numeric")
    idps = InterDependencies_(standalones=(parameter_m,))
    dataset.set_interdependencies(idps)
    dataset.mark_started()

    data_dict = [{"m": value} for value in [-np.inf, np.inf]]
    dataset.add_results(data_dict)
    retrieved = dataset.get_parameter_data()["m"]["m"]
    assert np.isinf(retrieved).all()

def test_missing_keys(dataset):
    """
    Test that we can now have partial results with keys missing. This is for
    example handy when having an interleaved 1D and 2D sweep.
    """

    x = ParamSpecBase("x", paramtype='numeric')
    y = ParamSpecBase("y", paramtype='numeric')
    a = ParamSpecBase("a", paramtype='numeric')
    b = ParamSpecBase("b", paramtype='numeric')

    idps = InterDependencies_(dependencies={a: (x,), b: (x, y)})
    dataset.set_interdependencies(idps)
    dataset.mark_started()

    def fa(xv):
        return xv + 1

    def fb(xv, yv):
        return xv + 2 - yv * 3

    results = []
    xvals = [1, 2, 3]
    yvals = [2, 3, 4]

    for xv in xvals:
        results.append({"x": xv, "a": fa(xv)})
        for yv in yvals:
            results.append({"x": xv, "y": yv, "b": fb(xv, yv)})

    dataset.add_results(results)

    loaded_data = dataset.get_parameter_data()

    np.testing.assert_array_equal(loaded_data['a']['x'],
                                  np.array(xvals))
    np.testing.assert_array_equal(loaded_data['a']['a'],
                                  np.array([fa(xv) for xv in xvals]))

    np.testing.assert_array_equal(loaded_data['b']['x'],
                                  np.repeat(np.array(xvals), 3))
    np.testing.assert_array_equal(loaded_data['b']['y'],
                                  np.tile(np.array(yvals), 3))
    np.testing.assert_array_equal(loaded_data['b']['b'],
                                  np.array([fb(xv, yv) for xv in xvals for yv in yvals]))

def test_get_description(experiment, some_interdeps):

    ds = DataSet()

    assert ds.run_id == 1

    desc = ds.description
    assert desc == RunDescriber(InterDependencies_())

    ds.set_interdependencies(some_interdeps[1])

    assert ds.description.interdeps == some_interdeps[1]

    # the run description gets written as the dataset is marked as started,
    # so now no description should be stored in the database
    prematurely_loaded_ds = DataSet(run_id=1)
    assert prematurely_loaded_ds.description == RunDescriber(
        InterDependencies_())

    ds.mark_started()

    loaded_ds = DataSet(run_id=1)

    expected_desc = RunDescriber(some_interdeps[1])

    assert loaded_ds.description == expected_desc

def test_metadata(experiment, request):

    metadata1 = {'number': 1, "string": "Once upon a time..."}
    metadata2 = {'more': 'meta'}

    ds1 = DataSet(metadata=metadata1)
    request.addfinalizer(ds1.conn.close)
    ds2 = DataSet(metadata=metadata2)
    request.addfinalizer(ds2.conn.close)

    assert ds1.run_id == 1
    assert ds1.metadata == metadata1
    assert ds2.run_id == 2
    assert ds2.metadata == metadata2

    loaded_ds1 = DataSet(run_id=1)
    request.addfinalizer(loaded_ds1.conn.close)
    assert loaded_ds1.metadata == metadata1
    loaded_ds2 = DataSet(run_id=2)
    request.addfinalizer(loaded_ds2.conn.close)
    assert loaded_ds2.metadata == metadata2

    bad_tag = "lex luthor"
    bad_tag_msg = (
      f"Tag {bad_tag} is not a valid tag. "
      "Use only alphanumeric characters and underscores!"
    )
    with pytest.raises(RuntimeError,
                       match="Rolling back due to unhandled exception") as e1:
        ds1.add_metadata(bad_tag, "value")
    assert error_caused_by(e1, bad_tag_msg)

    good_tag = "tag"
    none_value_msg = (
      f"Tag {good_tag} has value None. "
      "That is not a valid metadata value!"
    )
    with pytest.raises(RuntimeError,
                       match="Rolling back due to unhandled exception") as e2:
        ds1.add_metadata(good_tag, None)
    assert error_caused_by(e2, none_value_msg)

def test_the_same_dataset_as(some_interdeps, experiment):

    ds = DataSet()
    ds.set_interdependencies(some_interdeps[1])
    ds.mark_started()
    ds.add_results([{'ps1': 1, 'ps2': 2}])

    same_ds_from_load = DataSet(run_id=ds.run_id)
    assert ds.the_same_dataset_as(same_ds_from_load)

    new_ds = DataSet()
    assert not ds.the_same_dataset_as(new_ds)

@pytest.mark.usefixtures("experiment")
def test_parent_dataset_links_invalid_input():
    """
    Test that invalid input is rejected
    """
    links = generate_some_links(3)

    ds = DataSet()

    for link in links:
        assert link.head != ds.guid

    match = re.escape('Invalid input. Did not receive a list of Links')
    with pytest.raises(ValueError, match=match):
        ds.parent_dataset_links = [ds.guid]

    match = re.escape('Invalid input. All links must point to this dataset. '
                      'Got link(s) with head(s) pointing to another dataset.')
    with pytest.raises(ValueError, match=match):
        ds.parent_dataset_links = links

@pytest.mark.usefixtures("experiment")
def test_parent_dataset_links(some_interdeps):
    """
    Test that we can set links and retrieve them when loading the dataset
    """
    links = generate_some_links(3)

    ds = DataSet()

    for link in links:
        link.head = ds.guid

    ds.set_interdependencies(some_interdeps[1])

    ds.parent_dataset_links = links[:2]
    # setting it again/overwriting it should be okay
    ds.parent_dataset_links = links

    ds.mark_started()

    match = re.escape('Can not set parent dataset links on a dataset '
                      'that has been started.')
    with pytest.raises(RuntimeError, match=match):
        ds.parent_dataset_links = links

    ds.add_results([{'ps1': 1, 'ps2': 2}])
    run_id = ds.run_id

    ds_loaded = DataSet(run_id=run_id)

    assert ds_loaded.parent_dataset_links == links

class TestGetData:
    x = ParamSpecBase("x", paramtype='numeric')
    n_vals = 5
    xvals = list(range(n_vals))
    # this is the format of how data is returned by DataSet.get_data
    # which means "a list of table rows"
    xdata = np.array(xvals)

    @pytest.fixture(autouse=True)
    def ds_with_vals(self, dataset):
        """
        This fixture creates a DataSet with values that is to be used by all
        the tests in this class
        """
        idps = InterDependencies_(standalones=(self.x,))
        dataset.set_interdependencies(idps)
        dataset.mark_started()
        for xv in self.xvals:
            dataset.add_results([{self.x.name: xv}])

        return dataset

    @pytest.mark.parametrize(
        ("start", "end", "expected"),
        [
            # test without start and end
            (None, None, xdata),

            # test for start only
            (0, None, xdata),
            (2, None, xdata[(2-1):]),
            (-2, None, xdata),
            (n_vals, None, xdata[(n_vals-1):]),
            (n_vals + 1, None, []),
            (n_vals + 2, None, []),

            # test for end only
            (None, 0, []),
            (None, 2, xdata[:2]),
            (None, -2, []),
            (None, n_vals, xdata),
            (None, n_vals + 1, xdata),
            (None, n_vals + 2, xdata),

            # test for start and end
            (0, 0, []),
            (1, 1, [xdata[1-1]]),
            (2, 1, []),
            (2, 0, []),
            (1, 0, []),
            (n_vals, n_vals, [xdata[n_vals-1]]),
            (n_vals, n_vals - 1, []),
            (2, 4, xdata[(2-1):4]),
        ],
    )
    def test_get_data_with_start_and_end_args(self, ds_with_vals,
                                              start, end, expected):
        data = ds_with_vals.get_parameter_data(
            self.x, start=start, end=end)['x']
        if len(expected) == 0:
            assert data == {}
        else:
            data = data['x']
            np.testing.assert_array_equal(data, expected)

@settings(deadline=600, suppress_health_check=(HealthCheck.function_scoped_fixture,))
@given(start=hst.one_of(hst.integers(1, 10**3), hst.none()),
       end=hst.one_of(hst.integers(1, 10**3), hst.none()))
def test_get_parameter_data(scalar_dataset, start, end):
    input_names = ['param_3']

    expected_names = {}
    expected_names['param_3'] = ['param_3', 'param_0', 'param_1', 'param_2']
    expected_shapes = {}
    expected_shapes['param_3'] = [(10**3, )]*4
    expected_values = {}
    expected_values['param_3'] = [np.arange(30000, 31000)] + \
                                 [np.arange(10000*a, 10000*a+1000)
                                  for a in range(3)]

    start, end = limit_data_to_start_end(start, end, input_names,
                                         expected_names, expected_shapes,
                                         expected_values)

    parameter_test_helper(scalar_dataset,
                          input_names,
                          expected_names,
                          expected_shapes,
                          expected_values,
                          start,
                          end)

def test_get_scalar_parameter_data_no_nulls(scalar_dataset_with_nulls):

    expected_names = {}
    expected_names['first_value'] = ['first_value', 'setpoint']
    expected_names['second_value'] = ['second_value', 'setpoint']
    expected_shapes = {}
    expected_shapes['first_value'] = [(1, ), (1,)]
    expected_shapes['second_value'] = [(1, ), (1,)]
    expected_values = {}
    expected_values['first_value'] = [np.array([1]), np.array([0])]
    expected_values['second_value'] = [np.array([2]), np.array([0])]

    parameter_test_helper(scalar_dataset_with_nulls,
                          list(expected_names.keys()),
                          expected_names,
                          expected_shapes,
                          expected_values)

def test_get_array_parameter_data_no_nulls(array_dataset_with_nulls):

    types = [p.type for p in array_dataset_with_nulls.paramspecs.values()]

    expected_names = {}
    expected_names['val1'] = ['val1', 'sp1', 'sp2']
    expected_names['val2'] = ['val2', 'sp1']
    expected_shapes = {}
    expected_values = {}

    if 'array' in types:
        shape = (1, 5)
    else:
        shape = (5,)

    expected_shapes['val1'] = [shape] * 3
    expected_shapes['val2'] = [shape] * 2
    expected_values['val1'] = [np.ones(shape),
                               np.arange(0, 5).reshape(shape),
                               np.arange(5, 10).reshape(shape)]
    expected_values['val2'] = [np.zeros(shape),
                               np.arange(0, 5).reshape(shape)]

    parameter_test_helper(array_dataset_with_nulls,
                          list(expected_names.keys()),
                          expected_names,
                          expected_shapes,
                          expected_values)

def test_get_array_parameter_data(array_dataset):
    paramspecs = array_dataset.paramspecs
    types = [param.type for param in paramspecs.values()]
    par_name = "array_setpoint_param"
    setpoint_name = "array_setpoint_param_this_setpoint"

    input_names = [par_name]

    expected_names = {}
    expected_names[par_name] = [par_name, setpoint_name]
    expected_shapes = {}
    expected_len = 5
    expected_shapes[par_name] = [(expected_len,), (expected_len,)]
    expected_values = {}
    expected_values[par_name] = [np.ones(expected_len) + 1,
                                 np.linspace(5, 9, expected_len)]
    if 'array' in types:
        expected_shapes[par_name] = [(1, expected_len),
                                     (1, expected_len)]
        for i in range(len(expected_values[par_name])):
            expected_values[par_name][i] = expected_values[par_name][i].reshape(
                1, expected_len)
    parameter_test_helper(array_dataset,
                          input_names,
                          expected_names,
                          expected_shapes,
                          expected_values)

def test_get_multi_parameter_data(multi_dataset):
    paramspecs = multi_dataset.paramspecs
    types = [param.type for param in paramspecs.values()]

    input_names = ['this', 'that']
    sp_names = ['multi_2d_setpoint_param_this_setpoint',
                'multi_2d_setpoint_param_that_setpoint']

    expected_names = {}
    expected_names['this'] = ['this'] + sp_names
    expected_names['that'] = ['that'] + sp_names
    expected_shapes = {}
    expected_values = {}
    shape_1 = 5
    shape_2 = 3

    this_data = np.zeros((shape_1, shape_2))
    that_data = np.ones((shape_1, shape_2))
    sp_1_data = np.tile(np.linspace(5, 9, shape_1).reshape(shape_1, 1),
                        (1, shape_2))
    sp_2_data = np.tile(np.linspace(9, 11, shape_2), (shape_1, 1))
    if 'array' in types:
        expected_shapes['this'] = [
            (1, shape_1, shape_2), (1, shape_1, shape_2)]
        expected_shapes['that'] = [
            (1, shape_1, shape_2), (1, shape_1, shape_2)]
        expected_values['this'] = [this_data.reshape(1, shape_1, shape_2),
                                   sp_1_data.reshape(1, shape_1, shape_2),
                                   sp_2_data.reshape(1, shape_1, shape_2)]
        expected_values['that'] = [that_data.reshape(1, shape_1, shape_2),
                                   sp_1_data.reshape(1, shape_1, shape_2),
                                   sp_2_data.reshape(1, shape_1, shape_2)]

    else:
        expected_shapes['this'] = [(15,), (15,)]
        expected_shapes['that'] = [(15,), (15,)]
        expected_values['this'] = [this_data.ravel(),
                                   sp_1_data.ravel(),
                                   sp_2_data.ravel()]
        expected_values['that'] = [that_data.ravel(),
                                   sp_1_data.ravel(),
                                   sp_2_data.ravel()]
    parameter_test_helper(multi_dataset,
                          input_names,
                          expected_names,
                          expected_shapes,
                          expected_values)

@settings(suppress_health_check=(HealthCheck.function_scoped_fixture,))
@given(start=hst.one_of(hst.integers(1, 9), hst.none()),
       end=hst.one_of(hst.integers(1, 9), hst.none()))
def test_get_array_in_scalar_param_data(array_in_scalar_dataset,
                                        start, end):
    par_name = "array_setpoint_param"
    setpoint_name = "array_setpoint_param_this_setpoint"

    input_names = [par_name]

    expected_names = {}
    expected_names[par_name] = [par_name, 'scalarparam', setpoint_name]
    expected_shapes = {}

    shape_1 = 9
    shape_2 = 5

    test_parameter_values = np.tile((np.ones(shape_2) + 1).reshape(1, shape_2),
                                    (shape_1, 1))
    scalar_param_values = np.tile(np.arange(1, 10).reshape(shape_1, 1),
                                  (1, shape_2))
    setpoint_param_values = np.tile((np.linspace(5, 9, shape_2)).reshape(1, shape_2),
                                    (shape_1, 1))
    expected_shapes[par_name] = {}
    expected_shapes[par_name] = [(shape_1, shape_2), (shape_1, shape_2)]
    expected_values = {}
    expected_values[par_name] = [
        test_parameter_values,
        scalar_param_values,
        setpoint_param_values]

    start, end = limit_data_to_start_end(start, end, input_names,
                                         expected_names, expected_shapes,
                                         expected_values)
    parameter_test_helper(array_in_scalar_dataset,
                          input_names,
                          expected_names,
                          expected_shapes,
                          expected_values,
                          start,
                          end)

def test_get_varlen_array_in_scalar_param_data(varlen_array_in_scalar_dataset):
    par_name = "array_setpoint_param"
    setpoint_name = "array_setpoint_param_this_setpoint"

    input_names = [par_name]

    expected_names = {}
    expected_names[par_name] = [par_name, 'scalarparam', setpoint_name]
    expected_shapes = {}

    n = 9
    n_points = (n*(n+1))//2

    scalar_param_values = []
    setpoint_param_values = []
    for i in range(1, n + 1):
        for j in range(i):
            setpoint_param_values.append(j)
            scalar_param_values.append(i)

    np.random.seed(0)
    test_parameter_values = np.random.rand(n_points)
    scalar_param_values = np.array(scalar_param_values)
    setpoint_param_values = np.array(setpoint_param_values)

    expected_shapes[par_name] = [(n_points,), (n_points,)]
    expected_values = {}
    expected_values[par_name] = [
        test_parameter_values.ravel(),
        scalar_param_values.ravel(),
        setpoint_param_values.ravel()]

    parameter_test_helper(varlen_array_in_scalar_dataset,
                          input_names,
                          expected_names,
                          expected_shapes,
                          expected_values)

@settings(suppress_health_check=(HealthCheck.function_scoped_fixture,))
@given(start=hst.one_of(hst.integers(1, 45), hst.none()),
       end=hst.one_of(hst.integers(1, 45), hst.none()))
def test_get_array_in_scalar_param_unrolled(array_in_scalar_dataset_unrolled,
                                            start, end):
    par_name = "array_setpoint_param"
    setpoint_name = "array_setpoint_param_this_setpoint"
    input_names = [par_name]

    expected_names = {}
    expected_names[par_name] = [par_name, 'scalarparam', setpoint_name]
    expected_shapes = {}

    shape_1 = 9
    shape_2 = 5

    test_parameter_values = np.tile((np.ones(shape_2) + 1).reshape(1, shape_2),
                                    (shape_1, 1))
    scalar_param_values = np.tile(np.arange(1, 10).reshape(shape_1, 1),
                                  (1, shape_2))
    setpoint_param_values = np.tile((np.linspace(5, 9, shape_2)).reshape(1, shape_2),
                                    (shape_1, 1))
    expected_shapes[par_name] = {}
    expected_shapes[par_name] = [(shape_1*shape_2,), (shape_1*shape_2,)]
    expected_values = {}
    expected_values[par_name] = [
        test_parameter_values.ravel(),
        scalar_param_values.ravel(),
        setpoint_param_values.ravel()]

    start, end = limit_data_to_start_end(start, end, input_names,
                                         expected_names, expected_shapes,
                                         expected_values)
    parameter_test_helper(array_in_scalar_dataset_unrolled,
                          input_names,
                          expected_names,
                          expected_shapes,
                          expected_values,
                          start,
                          end)

def test_get_array_in_str_param_data(array_in_str_dataset):
    paramspecs = array_in_str_dataset.paramspecs
    types = [param.type for param in paramspecs.values()]

    par_name = "array_setpoint_param"
    setpoint_name = "array_setpoint_param_this_setpoint"

    input_names = [par_name]

    expected_names = {}
    expected_names[par_name] = [par_name, 'textparam', setpoint_name]
    expected_shapes = {}

    shape_1 = 3
    shape_2 = 5

    test_parameter_values = np.tile((np.ones(shape_2) + 1).reshape(1, shape_2),
                                    (shape_1, 1))
    scalar_param_values = np.tile(np.array(['A', 'B', 'C']).reshape(shape_1, 1),
                                  (1, shape_2))
    setpoint_param_values = np.tile((np.linspace(5, 9, shape_2)).reshape(1, shape_2),
                                    (shape_1, 1))
    expected_shapes['array_setpoint_param'] = {}
    expected_values = {}

    if 'array' in types:
        expected_shapes[par_name] = [(3, 5), (3, 5)]
        expected_values[par_name] = [
            test_parameter_values,
            scalar_param_values,
            setpoint_param_values]
    else:
        expected_shapes[par_name] = [(15,), (15,)]
        expected_values[par_name] = [
            test_parameter_values.ravel(),
            scalar_param_values.ravel(),
            setpoint_param_values.ravel()]
    parameter_test_helper(array_in_str_dataset,
                          input_names,
                          expected_names,
                          expected_shapes,
                          expected_values)

def test_get_parameter_data_independent_parameters(standalone_parameters_dataset):
    ds = standalone_parameters_dataset

    paramspecs = ds.description.interdeps.non_dependencies
    params = [ps.name for ps in paramspecs]

    expected_toplevel_params = ['param_1', 'param_2', 'param_3']
    assert params == expected_toplevel_params

    expected_names = {}
    expected_names['param_1'] = ['param_1']
    expected_names['param_2'] = ['param_2']
    expected_names['param_3'] = ['param_3', 'param_0']

    expected_shapes = {}
    expected_shapes['param_1'] = [(10 ** 3,)]
    expected_shapes['param_2'] = [(10 ** 3,)]
    expected_shapes['param_3'] = [(10**3, )]*2

    expected_values = {}
    expected_values['param_1'] = [np.arange(10000, 10000 + 1000)]
    expected_values['param_2'] = [np.arange(20000, 20000 + 1000)]
    expected_values['param_3'] = [np.arange(30000, 30000 + 1000),
                                  np.arange(0, 1000)]

    parameter_test_helper(ds,
                          expected_toplevel_params,
                          expected_names,
                          expected_shapes,
                          expected_values)

def parameter_test_helper(ds: DataSet,
                          toplevel_names: Sequence[str],
                          expected_names: Dict[str, Sequence[str]],
                          expected_shapes: Dict[str, Sequence[Tuple[int, ...]]],
                          expected_values: Dict[str, Sequence[np.ndarray]],
                          start: Optional[int] = None,
                          end: Optional[int] = None):
    """
    A helper function to compare the data we actually read out of a given
    dataset with the expected data.

    Args:
        ds: the dataset in question
        toplevel_names: names of the toplevel parameters of the dataset
        expected_names: names of the parameters expected to be loaded for a
            given parameter as a sequence indexed by the parameter.
        expected_shapes: expected shapes of the parameters loaded. The shapes
            should be stored as a tuple per parameter in a sequence containing
            all the loaded parameters for a given requested parameter.
        expected_values: expected content of the data arrays stored in a
            sequenceexpected_names:

    """

    data = ds.get_parameter_data(*toplevel_names, start=start, end=end)
    dataframe = ds.to_pandas_dataframe_dict(*toplevel_names,
                                            start=start,
                                            end=end)

    all_data = ds.get_parameter_data(start=start, end=end)
    all_dataframe = ds.to_pandas_dataframe_dict(start=start, end=end)

    all_parameters = list(all_data.keys())
    assert set(data.keys()).issubset(set(all_parameters))
    assert list(data.keys()) == list(dataframe.keys())
    assert len(data.keys()) == len(toplevel_names)
    assert len(dataframe.keys()) == len(toplevel_names)

    verify_data_dict(data, dataframe, toplevel_names, expected_names,
                     expected_shapes, expected_values)
    verify_data_dict(all_data, all_dataframe, toplevel_names, expected_names,
                     expected_shapes, expected_values)

    # Now lets remove a random element from the list
    # We do this one by one until there is only one element in the list
    subset_names = copy(all_parameters)
    while len(subset_names) > 1:
        elem_to_remove = random.randint(0, len(subset_names) - 1)
        name_removed = subset_names.pop(elem_to_remove)
        expected_names.pop(name_removed)
        expected_shapes.pop(name_removed)
        expected_values.pop(name_removed)

        subset_data = ds.get_parameter_data(*subset_names,
                                            start=start, end=end)
        subset_dataframe = ds.to_pandas_dataframe_dict(*subset_names,
                                                       start=start,
                                                       end=end)
        verify_data_dict(subset_data, subset_dataframe, subset_names,
                         expected_names, expected_shapes, expected_values)

def limit_data_to_start_end(start, end, input_names, expected_names,
                            expected_shapes, expected_values):
    if not (start is None and end is None):
        if start is None:
            start = 1
        elif end is None:
            # all the shapes are the same so pick the first one
            end = expected_shapes[input_names[0]][0][0]
        if end < start:
            for name in input_names:
                expected_names[name] = []
                expected_shapes[name] = ()
                expected_values[name] = {}
        else:
            for name in input_names:
                new_shapes = []
                for shape in expected_shapes[name]:
                    shape_list = list(shape)
                    shape_list[0] = end - start + 1
                    new_shapes.append(tuple(shape_list))
                expected_shapes[name] = new_shapes
                for i in range(len(expected_values[name])):
                    expected_values[name][i] = \
                        expected_values[name][i][start - 1:end]
    return start, end

@pytest.mark.usefixtures("experiment")
def test_empty_ds_parameters():

    ds = new_data_set("mydataset")
    assert ds.parameters is None
    ds.mark_started()
    assert ds.parameters is None
    ds.mark_completed()
    assert ds.parameters is None

max_forks_repo_name: jenshnielsen/Qcodes


text:
# Copyright 2021 Fedlearn authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os,sys
import numpy as np
import time
from typing import Callable, List, Dict, Union

root_path = os.getcwd()
BERT_ROOT = os.path.join(root_path,'demos/HFL/example/pytorch/hugging_face')

from demos.HFL.example.pytorch.hugging_face.bert_client import BertClient
from demos.HFL.example.pytorch.hugging_face.local_bert_text_classifier.model import (
    BertTextClassifier,
    DataConfig,
    TrainConfig
)

_TRAIN_DATA_FILE_='data/demo_train_data.txt'

def get_train_args():
        train_data_file = f'{BERT_ROOT}/{_TRAIN_DATA_FILE_}'
        #extension = train_data_file.split(".")[-1]
        data_config = DataConfig(
            train_file=train_data_file,
            val_file=train_data_file,
            mlm_probability=0.15,
            max_train_samples=50000, #4500,
            max_eval_samples=5000, #1000,
            pad_to_max_length=True,
            line_by_line=True
        )
        train_config = TrainConfig(
            output_dir=f'{BERT_ROOT}/saved_text_classifier_models',
            do_train=True,
            do_eval=True,
            report_to='tensorboard',
            logging_dir=f'{BERT_ROOT}/log',
            num_train_epochs=1,
            #num_train_epochs=10,
            #per_device_train_batch_size=6
            per_device_train_batch_size=10,
            per_device_eval_batch_size=20,   # batch size for evaluation
            warmup_steps=500,                # number of warmup steps for learning rate scheduler
            weight_decay=0.01,               # strength of weight decay
            logging_steps=100,               # log & save weights each logging_steps
            logging_strategy="steps",
            #evaluation_strategy="steps",      # evaluate each `logging_steps`
            eval_steps=100,
            evaluation_strategy="steps",      # evaluate each `logging_steps`
            #evaluation_strategy="epoch",      # evaluate each `logging_steps`
        )

        return data_config, train_config,

class BertClientTextClassifier(BertClient):

    def __init__(self,
                config:Dict[str,Union[str,float,int]]=None):

        pretrained_model_name = \
            config.get('pretrained_model_name','bert-base-chinese') if config is not None else 'bert-base-chinese'

        self.model = BertTextClassifier(
            *get_train_args(),
            pretrained_model_name=pretrained_model_name)
        print('\t', self.model)

max_stars_repo_name: monadyn/fedlearn-algo

max_issues_repo_name: fedlearnAI/fedlearnalgo

max_forks_repo_name: fedlearnAI/fedlearnalgo


text:
import numpy as np

class AdaGrad:
    """ Performs the AdaGrad optimization procedure from Duchi, Hazan, and Singer.

    Parameters
    ----------
    params : Iterable
        The parameters over which to optimize.

    learning_rate : float, optional (default=0.01)
        The step size; eta in the paper.

    Extended Description
    --------------------
    This optimizer implements the AdaGrad procedure described in the paper
      "Adaptive Subgradient Methods for Online Learning and Stochastic Optimization"
      John Duchi, Elad Hazan, and Yoram Singer
    http://jmlr.org/papers/volume12/duchi11a/duchi11a.pdf
    """

    def __init__(self, params, *, learning_rate=0.01):
        raise NotImplementedError

max_stars_repo_name: davidmascharka/MyNN

max_issues_repo_name: davidmascharka/MyNN

max_forks_repo_name: davidmascharka/MyNN


text:
import warnings
warnings.filterwarnings("ignore")
from mmdet.apis import init_detector, inference_detector, show_result_pyplot
import mmcv
import os.path
from mmdet.models.detectors import BaseDetector
#BaseDetector.show_result instead of show_result
#import os.path as osp
import pickle
import shutil
import tempfile
import time
from PIL import Image

import mmcv
import torch
import torch.distributed as dist
from mmcv.runner import get_dist_info
from numpy import asarray
from mmcv.utils import is_str
from tqdm import tqdm

#from mmdet.core import encode_mask_results, tensor2imgs
config_fname = 'work_dirs/htc_x101_dcn/htc_x101_dcn.py'
checkpoint_fname = 'work_dirs/htc_x101_dcn/best_bbox_mAP.pth'
model = init_detector(config_fname, checkpoint_fname)

import os
imgFolderPath = "UIT-DODV/images/test/"

outputFolderPath = "dcn/"
print(outputFolderPath)
if not os.path.exists(outputFolderPath):
    os.makedirs(outputFolderPath)

for file in tqdm(os.listdir(imgFolderPath)):
  if file.endswith(".jpg") or file.endswith(".jpeg") or file.endswith(".JPG"):
    img = os.path.join(imgFolderPath, file)
    result = inference_detector(model, img)
    model.show_result(img, result, out_file= outputFolderPath + file)


text:
import numpy as np

def veclen(vectors):
    """ return L2 norm (vector length) along the last axis, for example to compute the length of an array of vectors """
    return np.sqrt(np.sum(vectors**2, axis=-1))

def normalized(vectors):
    """ normalize vector(s) such that ||vectors|| == 1 """
    lengths = veclen(vectors)
    return vectors / lengths[..., np.newaxis]

def filter_reindex(condition, target):
    """
    Filtering of index arrays.

    To explain this, let me give an example. Let's say you have the following data:
    >>> data = np.array(['a', 'b', 'c', 'd'])

    You also have another array that consists of indices into this array
    >>> indices = np.array([0, 3, 3, 0])
    >>> data[indices].tolist()
    ['a', 'd', 'd', 'a']

    Now, say you are filtering some elements in your data array
    >>> condition = (data == 'a') | (data == 'd')
    >>> filtered_data = data[condition]
    >>> filtered_data.tolist()
    ['a', 'd']

    The problem is that your index array doesn't correctly reference the new filtered data array
    >>> filtered_data[indices]
    Traceback (most recent call last):
        ...
    IndexError: index 3 is out of bounds for size 2

    Based on an old index array (target), this method returns a new index array
    that re-indices into the data array as if condition was applied to this array, so
    >>> filtered_indices = filter_reindex(condition, indices)
    >>> filtered_indices.tolist()
    [0, 1, 1, 0]
    >>> filtered_data[filtered_indices].tolist()
    ['a', 'd', 'd', 'a']

    >>> indices = np.array([1, 4, 1, 4])
    >>> condition = np.array([False, True, False, False, True])
    >>> filter_reindex(condition, indices).tolist()
    [0, 1, 0, 1]
    """
    if condition.dtype != np.bool:
        raise ValueError("condition must be a binary array")
    reindex = np.cumsum(condition) - 1
    return reindex[target]

def compute_average_edge_length(verts, tris):
    edges = tris[:, [0, 1, 1, 2, 2, 0]].reshape(-1, 2)
    edges = np.sort(edges, axis=1)
    ij_dtype = [('i', edges.dtype), ('j', edges.dtype)]
    edges_uniq = np.unique(edges.view(dtype=ij_dtype))
    edges_uniq = edges_uniq.view(dtype=edges.dtype).reshape(-1, 2)
    return veclen(verts[edges_uniq[:, 0]] - verts[edges_uniq[:, 1]]).mean()

max_stars_repo_name: YuePengUSTC/AADR

max_issues_repo_name: YuePengUSTC/AADR

max_forks_repo_name: YuePengUSTC/AADR


text:
# Copyright 2018/2019 The RLgraph authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import re

from rlgraph.utils.rlgraph_errors import RLGraphError
from rlgraph.utils.ops import DataOpDict, DataOpTuple, FLAT_TUPLE_OPEN, FLAT_TUPLE_CLOSE
from rlgraph.spaces.space import Space

class ContainerSpace(Space):
    """
    A simple placeholder class for Spaces that contain other Spaces.
    """
    def sample(self, size=None, horizontal=False):
        """
        Child classes must overwrite this one again with support for the `horizontal` parameter.

        Args:
            horizontal (bool): False: Within this container, sample each child-space `size` times.
                True: Produce `size` single containers in an np.array of len `size`.
        """
        raise NotImplementedError

class Dict(ContainerSpace, dict):
    """
    A Dict space (an ordered and keyed combination of n other spaces).
    Supports nesting of other Dict/Tuple spaces (or any other Space types) inside itself.
    """
    def __init__(self, spec=None, **kwargs):
        add_batch_rank = kwargs.pop("add_batch_rank", False)
        add_time_rank = kwargs.pop("add_time_rank", False)
        time_major = kwargs.pop("time_major", False)

        ContainerSpace.__init__(self, add_batch_rank=add_batch_rank, add_time_rank=add_time_rank, time_major=time_major)

        # Allow for any spec or already constructed Space to be passed in as values in the python-dict.
        # Spec may be part of kwargs.
        if spec is None:
            spec = kwargs

        space_dict = {}
        for key in sorted(spec.keys()):
            # Keys must be strings.
            if not isinstance(key, str):
                raise RLGraphError("ERROR: No non-str keys allowed in a Dict-Space!")
            # Prohibit reserved characters (for flattened syntax).
            if re.search(r'/|{}\d+{}'.format(FLAT_TUPLE_OPEN, FLAT_TUPLE_CLOSE), key):
                raise RLGraphError("ERROR: Key to Dict must not contain '/' or '{}\d+{}'! Is {}.".
                                   format(FLAT_TUPLE_OPEN, FLAT_TUPLE_CLOSE, key))
            value = spec[key]
            # Value is already a Space: Copy it (to not affect original Space) and maybe add/remove batch/time-ranks.
            if isinstance(value, Space):
                w_batch_w_time = value.with_extra_ranks(add_batch_rank, add_time_rank, time_major)
                space_dict[key] = w_batch_w_time
            # Value is a list/tuple -> treat as Tuple space.
            elif isinstance(value, (list, tuple)):
                space_dict[key] = Tuple(
                    *value, add_batch_rank=add_batch_rank, add_time_rank=add_time_rank, time_major=time_major
                )
            # Value is a spec (or a spec-dict with "type" field) -> produce via `from_spec`.
            elif (isinstance(value, dict) and "type" in value) or not isinstance(value, dict):
                space_dict[key] = Space.from_spec(
                    value, add_batch_rank=add_batch_rank, add_time_rank=add_time_rank, time_major=time_major
                )
            # Value is a simple dict -> recursively construct another Dict Space as a sub-space of this one.
            else:
                space_dict[key] = Dict(
                    value, add_batch_rank=add_batch_rank, add_time_rank=add_time_rank, time_major=time_major
                )

        dict.__init__(self, space_dict)

    def _add_batch_rank(self, add_batch_rank=False):
        super(Dict, self)._add_batch_rank(add_batch_rank)
        for v in self.values():
            v._add_batch_rank(add_batch_rank)

    def _add_time_rank(self, add_time_rank=False, time_major=False):
        super(Dict, self)._add_time_rank(add_time_rank, time_major)
        for v in self.values():
            v._add_time_rank(add_time_rank, time_major)

    def force_batch(self, samples):
        return dict([(key, self[key].force_batch(samples[key])) for key in sorted(self.keys())])

    @property
    def shape(self):
        return tuple([self[key].shape for key in sorted(self.keys())])

    def get_shape(self, with_batch_rank=False, with_time_rank=False, time_major=None, with_category_rank=False):
        return tuple([self[key].get_shape(
            with_batch_rank=with_batch_rank, with_time_rank=with_time_rank, time_major=time_major,
            with_category_rank=with_category_rank
        ) for key in sorted(self.keys())])

    @property
    def rank(self):
        return tuple([self[key].rank for key in sorted(self.keys())])

    @property
    def flat_dim(self):
        return int(np.sum([c.flat_dim for c in self.values()]))

    @property
    def dtype(self):
        return DataOpDict([(key, subspace.dtype) for key, subspace in self.items()])

    def get_variable(self, name, is_input_feed=False, add_batch_rank=None, add_time_rank=None, time_major=None,
                     **kwargs):
        return DataOpDict(
            [(key, subspace.get_variable(
                name + "/" + key, is_input_feed=is_input_feed, add_batch_rank=add_batch_rank,
                add_time_rank=add_time_rank, time_major=time_major, **kwargs
            )) for key, subspace in self.items()]
        )

    def _flatten(self, mapping, custom_scope_separator, scope_separator_at_start, scope_, list_):
        # Iterate through this Dict.
        scope_ += custom_scope_separator if len(scope_) > 0 or scope_separator_at_start else ""
        for key in sorted(self.keys()):
            self[key].flatten(mapping, custom_scope_separator, scope_separator_at_start, scope_ + key, list_)

    def sample(self, size=None, horizontal=False):
        if horizontal:
            return np.array([{key: self[key].sample() for key in sorted(self.keys())}] * (size or 1))
        else:
            return {key: self[key].sample(size=size) for key in sorted(self.keys())}

    def zeros(self, size=None):
        return DataOpDict([(key, subspace.zeros(size=size)) for key, subspace in self.items()])

    def contains(self, sample):
        return isinstance(sample, dict) and all(self[key].contains(sample[key]) for key in self.keys())

    def __repr__(self):
        return "Dict({})".format([(key, self[key].__repr__()) for key in self.keys()])

    def __eq__(self, other):
        if not isinstance(other, Dict):
            return False
        return dict(self) == dict(other)

class Tuple(ContainerSpace, tuple):
    """
    A Tuple space (an ordered sequence of n other spaces).
    Supports nesting of other container (Dict/Tuple) spaces inside itself.
    """
    def __new__(cls, *components, **kwargs):
        if isinstance(components[0], (list, tuple)) and not isinstance(components[0], Tuple):
            assert len(components) == 1
            components = components[0]

        add_batch_rank = kwargs.get("add_batch_rank", False)
        add_time_rank = kwargs.get("add_time_rank", False)
        time_major = kwargs.get("time_major", False)

        # Allow for any spec or already constructed Space to be passed in as values in the python-list/tuple.
        list_ = list()
        for value in components:
            # Value is already a Space: Copy it (to not affect original Space) and maybe add/remove batch-rank.
            if isinstance(value, Space):
                list_.append(value.with_extra_ranks(add_batch_rank, add_time_rank, time_major))
            # Value is a list/tuple -> treat as Tuple space.
            elif isinstance(value, (list, tuple)):
                list_.append(
                    Tuple(*value, add_batch_rank=add_batch_rank, add_time_rank=add_time_rank, time_major=time_major)
                )
            # Value is a spec (or a spec-dict with "type" field) -> produce via `from_spec`.
            elif (isinstance(value, dict) and "type" in value) or not isinstance(value, dict):
                list_.append(Space.from_spec(
                    value, add_batch_rank=add_batch_rank, add_time_rank=add_time_rank, time_major=time_major
                ))
            # Value is a simple dict -> recursively construct another Dict Space as a sub-space of this one.
            else:
                list_.append(Dict(
                    value, add_batch_rank=add_batch_rank, add_time_rank=add_time_rank, time_major=time_major
                ))

        return tuple.__new__(cls, list_)

    def __init__(self, *components, **kwargs):
        add_batch_rank = kwargs.get("add_batch_rank", False)
        add_time_rank = kwargs.get("add_time_rank", False)
        time_major = kwargs.get("time_major", False)
        super(Tuple, self).__init__(add_batch_rank=add_batch_rank, add_time_rank=add_time_rank, time_major=time_major)

    def _add_batch_rank(self, add_batch_rank=False):
        super(Tuple, self)._add_batch_rank(add_batch_rank)
        for v in self:
            v._add_batch_rank(add_batch_rank)

    def _add_time_rank(self, add_time_rank=False, time_major=False):
        super(Tuple, self)._add_time_rank(add_time_rank, time_major)
        for v in self:
            v._add_time_rank(add_time_rank, time_major)

    def force_batch(self, samples):
        return tuple([c.force_batch(samples[i]) for i, c in enumerate(self)])

    @property
    def shape(self):
        return tuple([c.shape for c in self])

    def get_shape(self, with_batch_rank=False, with_time_rank=False, time_major=None, with_category_rank=False):
        return tuple([c.get_shape(
            with_batch_rank=with_batch_rank, with_time_rank=with_time_rank, time_major=time_major,
            with_category_rank=with_category_rank
        ) for c in self])

    @property
    def rank(self):
        return tuple([c.rank for c in self])

    @property
    def flat_dim(self):
        return np.sum([c.flat_dim for c in self])

    @property
    def dtype(self):
        return DataOpTuple([c.dtype for c in self])

    def get_variable(self, name, is_input_feed=False, add_batch_rank=None, add_time_rank=None, time_major=None,
                     **kwargs):
        return DataOpTuple(
            [subspace.get_variable(
                name+"/"+str(i), is_input_feed=is_input_feed, add_batch_rank=add_batch_rank,
                add_time_rank=add_time_rank, time_major=time_major, **kwargs
            ) for i, subspace in enumerate(self)]
        )

    def _flatten(self, mapping, custom_scope_separator, scope_separator_at_start, scope_, list_):
        # Iterate through this Tuple.
        scope_ += (custom_scope_separator if len(scope_) > 0 or scope_separator_at_start else "") + FLAT_TUPLE_OPEN
        for i, component in enumerate(self):
            component.flatten(
                mapping, custom_scope_separator, scope_separator_at_start, scope_ + str(i) + FLAT_TUPLE_CLOSE, list_
            )

    def sample(self, size=None, horizontal=False):
        if horizontal:
            return np.array([tuple(subspace.sample() for subspace in self)] * (size or 1))
        else:
            return tuple(x.sample(size=size) for x in self)

    def zeros(self, size=None):
        return tuple([c.zeros(size=size) for i, c in enumerate(self)])

    def contains(self, sample):
        return isinstance(sample, (tuple, list, np.ndarray)) and len(self) == len(sample) and \
               all(c.contains(xi) for c, xi in zip(self, sample))

    def __repr__(self):
        return "Tuple({})".format(tuple([cmp.__repr__() for cmp in self]))

    def __eq__(self, other):
        return tuple.__eq__(self, other)

max_stars_repo_name: theSoenke/rlgraph

max_issues_repo_name: theSoenke/rlgraph

max_forks_repo_name: theSoenke/rlgraph


text:
"""Define the PETSc Vector classe."""
from __future__ import division

import numpy as np
from petsc4py import PETSc

from six import iteritems, itervalues
from six.moves import range

from openmdao.vectors.default_vector import DefaultVector, INT_DTYPE
from openmdao.vectors.petsc_transfer import PETScTransfer
from openmdao.utils.mpi import MPI

CITATION = '''@InProceedings{petsc-efficient,
    Author = "Satish Balay and William D. Gropp and Lois Curfman McInnes and Barry F. Smith",
    Title = "Efficient Management of Parallelism in Object Oriented Numerical Software Libraries",
    Booktitle = "Modern Software Tools in Scientific Computing",
    Editor = "E. Arge and A. M. Bruaset and H. P. Langtangen",
    Pages = "163--202",
    Publisher = "Birkh{\"{a}}user Press",
    Year = "1997"
}'''

class PETScVector(DefaultVector):
    """
    PETSc Vector implementation for running in parallel.

    Most methods use the DefaultVector's implementation.
    """

    TRANSFER = PETScTransfer
    cite = CITATION

    def _initialize_data(self, root_vector):
        """
        Internally allocate vectors.

        Parameters
        ----------
        root_vector : Vector or None
            the root's vector instance or None, if we are at the root.
        """
        super(PETScVector, self)._initialize_data(root_vector)

        self._petsc = {}
        self._imag_petsc = {}
        data = self._data

        if self._ncol == 1:
            if self._alloc_complex:
                self._petsc = PETSc.Vec().createWithArray(data.copy(), comm=self._system.comm)
            else:
                self._petsc = PETSc.Vec().createWithArray(data, comm=self._system.comm)
        else:
            # for now the petsc array is only the size of one column and we do separate
            # transfers for each column.
            if data.size == 0:
                self._petsc = PETSc.Vec().createWithArray(data.copy(), comm=self._system.comm)
            else:
                self._petsc = PETSc.Vec().createWithArray(data[:, 0].copy(),
                                                          comm=self._system.comm)

        # Allocate imaginary for complex step
        if self._alloc_complex:
            data = self._cplx_data.imag
            if self._ncol == 1:
                self._imag_petsc = PETSc.Vec().createWithArray(data, comm=self._system.comm)
            else:
                if data.size == 0:
                    self._imag_petsc = PETSc.Vec().createWithArray(data.copy(),
                                                                   comm=self._system.comm)
                else:
                    self._imag_petsc = PETSc.Vec().createWithArray(data[:, 0].copy(),
                                                                   comm=self._system.comm)

    def get_norm(self):
        """
        Return the norm of this vector.

        Returns
        -------
        float
            norm of this vector.
        """
        return self._system.comm.allreduce(np.linalg.norm(self._data))

    def dot(self, vec):
        """
        Compute the dot product of the real parts of the current vec and the incoming vec.

        Parameters
        ----------
        vec : <Vector>
            The incoming vector being dotted with self.

        Returns
        -------
        float
            The computed dot product value.
        """
        return self._system.comm.allreduce(np.dot(self._data, vec._data))

max_stars_repo_name: fzahle/OpenMDAO

max_issues_repo_name: gsoxley/OpenMDAO

max_forks_repo_name: gsoxley/OpenMDAO


text:
import sys
import unittest

import numpy as np

sys.path.insert(0, '..')
from src.smahat.encode import Encoder
from src.smahat.decode import Decoder

class TestSmahat(unittest.TestCase):
    def test_random(self):
        np.random.seed(0)

        sizes = [np.random.randint(0, 1000) for _ in range(10)]

        # booleans, percentages, negatives
        mins = [0, 0, -50]
        maxs = [1, 100, 50]

        for size in sizes:
            for min_v, max_v in zip(mins, maxs):
                values = np.random.randint(min_v, max_v + 1, size)
                values = list(map(lambda x: x.item(), values))

                encoder = Encoder(min_value=min_v, max_value=max_v)

                for v in values:
                    encoder.encode_next(v)

                content = encoder.get_encoded()

                self.assertEqual(
                    Encoder.encode_all(values,
                                       min_value=min_v,
                                       max_value=max_v), content)
                self.assertEqual(Decoder.decode_all(content), values)


text:
import csv
import pandas as pd
import logging
import os
import xml.etree.ElementTree as ET
from PIL import Image
import numpy as np
from torchvision import transforms
import torch.utils.data as td
import torch

import utils.utils as utils

# To incdude a new Dataset, inherit from Dataset and add all the Dataset specific parameters here.
# Goal : Remove any data specific parameters from the rest of the code

logger = logging.getLogger('iCARL')

class DatasetBase():
    ''' Base class to represent a Dataset
    data - list of strings. paths to images. N is the number of files.
    labels - np array Nx8 (topleft_x, topleft_y, topright_x, topright_y, botleft_x, botleft_y, botright_x, botright_y). normalized coordinated.
    '''

    def __init__(self, name):
        self.name = name
        self.data = []
        self.labels = []

    def __repr__(self) -> None:
        logger.debug("Ground Truth Shape: %s", str(self.labels.shape))
        logger.debug("Data shape %s", str(len(self.data)))

class SmartDoc(DatasetBase):
    '''
    '''
    def __init__(self, directories=[]):
        super().__init__("smartdoc")
        data = []
        labels = []
        self.train_transform = transforms.Compose([transforms.Resize([32, 32]),
                                                    transforms.ColorJitter(1.5, 1.5, 0.9, 0.5),
                                                    transforms.ToTensor()])

        self.test_transform = transforms.Compose([transforms.Resize([32, 32]),
                                                    transforms.ToTensor()])
        logger.info("Pass train/test data paths here")
        directories = [directories] if isinstance(directories, str) else directories
        for d in directories:
            print (d, "gt.csv")
            with open(os.path.join(d, "gt.csv"), 'r') as csvfile:
                spamreader = csv.reader(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
                import ast
                for row in spamreader:
                    data.append(os.path.join(d, row[0]))
                    test = row[1].replace("array", "")
                    labels.append(ast.literal_eval(test))
        labels = np.reshape(np.array(labels), (-1, 8))
        self.myData = [data, labels]

        self.data = data #TODO: delete this once get rid of the those redundant members
        self.labels = labels

        self.__repr__()

class MyDatasetDoc(td.Dataset):
    '''
    data - list of strings. path to images. N is the number of files.
    labels - np array Nx8 (topleft_x, topleft_y, topright_x, topright_y, botleft_x, botleft_y, botright_x, botright_y). normalized coordinated.
    '''
    def __init__(self, d):
        gt_colunms = ['img_name', 'topleft_x', 'topleft_y', 'topright_x', 'topright_y', 'botright_x', 'botright_y', 'botleft_x', 'botleft_y', 'w', 'h', 'ignore']
        self.train_transform = transforms.Compose([transforms.Resize([32, 32]),
                                                    transforms.ColorJitter(1.5, 1.5, 0.9, 0.5),
                                                    transforms.ToTensor()])

        self.test_transform = transforms.Compose([transforms.Resize([32, 32]),
                                                    transforms.ToTensor()])
        df = pd.read_csv(f'{d}/gt.csv', names=gt_colunms)
        df = df.drop(df[df.ignore == 1].index)
        self.data = list(d +'/'+ df.img_name)
        df[['topleft_x', 'topright_x', 'botright_x', 'botleft_x']] = df[['topleft_x', 'topright_x', 'botright_x', 'botleft_x']].div(df.w, axis=0)
        df[['topleft_y', 'topright_y', 'botright_y', 'botleft_y']] = df[['topleft_y', 'topright_y', 'botright_y', 'botleft_y']].div(df.h, axis=0)
        self.target = df.drop(['img_name', 'w', 'h', 'ignore'],axis=1).to_numpy()

    def __len__(self):
        return len(self.data)

    def random_split(self, train_cutoff = 0.8, seed=42):
        N = len(self)
        N_train = int(train_cutoff*N)
        N_test = N - N_train
        train_dataset, test_dataset = td.random_split(self, (N_train, N_test), generator=torch.Generator().manual_seed(seed))
        return train_dataset, test_dataset

    def __getitem__(self, index):
        img = Image.open(self.data[index])
        img = img.convert('RGB') if img.mode == 'L' else img
        if self.train_transform is not None:
            img = self.train_transform(img)
        target = self.target[index]
        return img, target

class MyDatasetCorner(td.Dataset):
    '''
    data - list of strings. path to images. N is the number of files.
    target - np array Nx2 (x, y). normalized coordinated.
    '''
    def __init__(self, df, is_rotating=False) -> None:
        self.train_transform = transforms.Compose([transforms.Resize([32, 32]),
                                                   transforms.ColorJitter(0.5, 0.5, 0.5, 0.5),
                                                   transforms.ToTensor()])

        self.test_transform = transforms.Compose([transforms.Resize([32, 32]),
                                                  transforms.ToTensor()])

        self.df = df
        self.data = list(df.directory +'/'+ df.img_name)
        self.target = df.drop(['directory', 'img_name', 'corner_type', 'w', 'h', 'ignore'],axis=1).to_numpy()

        self.is_rotating = is_rotating
        self.rotation_angles_to_be_topleft_corner= {'topleft': 0, 'topright':90, 'botright': 180, 'botleft':-90}

    @classmethod
    def from_directory(cls, d, is_rotating=False):
        """
        d - path to directory containing images and a csv gt file.
        is_rotating(bool) - if True rotate all samples to be topleft corners
        """
        gt_colunms = ['img_name', 'corner_type', 'x', 'y', 'w', 'h', 'ignore']
        df = pd.read_csv(f'{d}/gt.csv', names=gt_colunms)
        df['directory'] = d
        df = df.drop(df[df.ignore == 1].index)
        # df = df.drop(df[df.corner_type != 'topleft'].index) #topleft, topright, botright, botleft
        df[['x']] = df[['x']].div(df.w, axis=0)
        df[['y']] = df[['y']].div(df.h, axis=0)

        return cls(df, is_rotating)

    def random_split(self, train_cutoff = 0.8, seed=42):
        N = len(self)
        N_train = int(train_cutoff*N)
        N_test = N - N_train
        train_dataset, test_dataset = td.random_split(self, (N_train, N_test), generator=torch.Generator().manual_seed(seed))
        return train_dataset, test_dataset

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index):
        img = self.get_pil_image(index)
        img = img.convert('RGB') if img.mode == 'L' else img
        target = self.target[index]
        if self.is_rotating:
            angle = self.rotation_angles_to_be_topleft_corner[self.get_corner_type(index)]
            new_target_in_pixels = utils.rotate_translate_point(target * img.size, angle, img.size)
            img = img.rotate(angle, expand=True)
            target = np.array(new_target_in_pixels) / img.size
        if self.train_transform is not None:
            img = self.train_transform(img)
        return img, target

    def get_name(self, index):
        return self.data[index]

    def get_pil_image(self, index):
        return Image.open(self.get_name(index))

    def get_corner_type(self, index):
        return self.df.iloc[index]['corner_type']

class SmartDocDirectories(DatasetBase):
    '''
    Class to include MNIST specific details
    '''

    def __init__(self, directory="data"):
        super().__init__("smartdoc")
        self.data = []
        self.labels = []

        for folder in os.listdir(directory):
            if (os.path.isdir(directory + "/" + folder)):
                for file in os.listdir(directory + "/" + folder):
                    images_dir = directory + "/" + folder + "/" + file
                    if (os.path.isdir(images_dir)):

                        list_gt = []
                        tree = ET.parse(images_dir + "/" + file + ".gt")
                        root = tree.getroot()
                        for a in root.iter("frame"):
                            list_gt.append(a)

                        im_no = 0
                        for image in os.listdir(images_dir):
                            if image.endswith(".jpg"):
                                # print(im_no)
                                im_no += 1

                                # Now we have opened the file and GT. Write code to create multiple files and scale gt
                                list_of_points = {}

                                # img = cv2.imread(images_dir + "/" + image)
                                self.data.append(os.path.join(images_dir, image))

                                for point in list_gt[int(float(image[0:-4])) - 1].iter("point"):
                                    myDict = point.attrib

                                    list_of_points[myDict["name"]] = (
                                        int(float(myDict['x'])), int(float(myDict['y'])))

                                ground_truth = np.asarray(
                                    (list_of_points["tl"], list_of_points["tr"], list_of_points["br"],
                                     list_of_points["bl"]))
                                ground_truth = utils.sort_gt(ground_truth)
                                self.labels.append(ground_truth)

        self.labels = np.array(self.labels)
        self.labels = np.reshape(self.labels, (-1, 8))
        self.myData = []
        for a in range(len(self.data)):
            self.myData.append([self.data[a], self.labels[a]])

        self.__repr__()

class SelfCollectedDataset(DatasetBase):
    '''
    Class to include MNIST specific details
    '''

    def __init__(self, directory="data"):
        super().__init__("smartdoc")
        self.data = []
        self.labels = []

        for image in os.listdir(directory):
            # print (image)
            if image.endswith("jpg") or image.endswith("JPG"):
                if os.path.isfile(os.path.join(directory, image + ".csv")):
                    with open(os.path.join(directory, image + ".csv"), 'r') as csvfile:
                        spamwriter = csv.reader(csvfile, delimiter=' ',
                                                quotechar='|', quoting=csv.QUOTE_MINIMAL)

                        img_path = os.path.join(directory, image)

                        gt = []
                        for row in spamwriter:
                            gt.append(row)
                        gt = np.array(gt).astype(np.float32)
                        ground_truth = utils.sort_gt(gt)
                        self.labels.append(ground_truth)
                        self.data.append(img_path)

        self.labels = np.array(self.labels)
        self.labels = np.reshape(self.labels, (-1, 8))
        self.myData = []
        for a in range(len(self.data)):
            self.myData.append([self.data[a], self.labels[a]])

        self.__repr__()

class SmartDocCorner(DatasetBase):
    '''
    data - list of strings. path to images. N is the number of files.
    labels - np array Nx2 (x, y). normalized coordinated.
    '''

    def __init__(self, directories=[]):
        super().__init__("smartdoc")
        self.data = []
        self.labels = []
        directories = [directories] if isinstance(directories, str) else directories
        for d in directories:
            self.directory = d
            self.train_transform = transforms.Compose([transforms.Resize([32, 32]),
                                                       transforms.ColorJitter(0.5, 0.5, 0.5, 0.5),
                                                       transforms.ToTensor()])

            self.test_transform = transforms.Compose([transforms.Resize([32, 32]),
                                                      transforms.ToTensor()])

            logger.info("Pass train/test data paths here")

            file_names = []
            with open(os.path.join(self.directory, "gt.csv"), 'r') as csvfile:
                spamreader = csv.reader(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
                import ast
                for row in spamreader:
                    file_names.append(row[0])
                    self.data.append(os.path.join(self.directory, row[0]))
                    test = row[1].replace("array", "")
                    self.labels.append((ast.literal_eval(test)))
        self.labels = np.array(self.labels)
        self.labels = np.reshape(self.labels, (-1, 2))

        self.myData = [self.data, self.labels]

        self.__repr__()

class Dataset():
    def __init__(self, gt) -> None:
        self.gt = gt

    @classmethod
    def from_directory(cls, directory, ignore=False):
        gt_cols = ['img_name', 'topleft_x', 'topleft_y', 'topright_x', 'topright_y', 'botright_x', 'botright_y', 'botleft_x', 'botleft_y', 'w', 'h', 'ignore', 'comment']
        gt = pd.read_csv(f'{directory}/gt.csv', names=gt_cols)
        gt['directory'] = directory
        if ignore:
            gt = gt.drop(gt[gt.ignore == 1].index).reset_index()
        return cls(gt)

    def read_sample(self, i):
        im = Image.open(self.get_im_path(i))
        tl = self.gt.loc[i, ['topleft_x', 'topleft_y']].to_list()
        tr = self.gt.loc[i, ['topright_x', 'topright_y']].to_list()
        br = self.gt.loc[i, ['botright_x', 'botright_y']].to_list()
        bl = self.gt.loc[i, ['botleft_x', 'botleft_y']].to_list()
        return im, [tl, tr, br, bl]

    def get_im_path(self, i):
        return f"{self.gt.directory[i]}/{self.gt.img_name[i]}"

    def get_name(self, i):
        name = self.gt.img_name[i]
        return name[:-4]

    def __len__(self):
        return len(self.gt)

    def __add__(self, otherDataset):
        gt = pd.concat([self.gt, otherDataset.gt], ignore_index=True)
        return Dataset(gt)

max_stars_repo_name: morhadar/Recursive-CNNs

max_issues_repo_name: morhadar/Recursive-CNNs

max_forks_repo_name: morhadar/Recursive-CNNs