text:
import sys,os
import numpy as np
import pandas as pd

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
plt.rcParams.update({'font.size': 10})

def main(argv):
  infile = argv[0]
  mode = argv[1]
  dataset = argv[2]
  outdir = argv[3]

  df = pd.read_csv(infile)
  fold_cols = [col for col in df.columns if col.startswith('Fold')]
  df['importance_mean'] = df[fold_cols].mean(axis=1)
  df['importance_std'] = df[fold_cols].std(axis=1)
  df = df.sort_values(by='importance_mean', ascending=True)

  y = np.arange(2*len(df))[::2]
  plt.Figure()
  plt.barh(y, width=df['importance_mean'], xerr=df['importance_std'], height=1.5)
  plt.xlabel('Feature importance', fontsize=14)
  plt.yticks(y, labels=df['Features'], fontsize=8)
  plt.tight_layout()
  plt.savefig(os.path.join(outdir, '-'.join(('featimp',mode,dataset))+'.jpg'))
  plt.close()

if __name__ == "__main__":
  main(sys.argv[1:])

max_stars_repo_name: wadpac/SleepStageClassification

max_issues_repo_name: wadpac/SleepStageClassification

max_forks_repo_name: wadpac/SleepStageClassification


text:
# Copyright 2015, Yahoo Inc.
# Licensed under the terms of the Apache License, Version 2.0. See the LICENSE file associated with the project for terms.
from nose.tools import assert_true, assert_equal

import pickle as pkl
import sys
import os
import numpy as np
from sklearn.model_selection import train_test_split

sys.path.insert(1, os.path.abspath('..'))
from lopq.model import LOPQModel, eigenvalue_allocation, accumulate_covariance_estimators, compute_rotations_from_accumulators
from lopq.search import LOPQSearcher, LOPQSearcherLMDB
from lopq.eval import compute_all_neighbors, get_cell_histogram, get_recall

########################################
# Helpers
########################################

relpath = lambda x: os.path.abspath(os.path.join(os.path.dirname(__file__), x))

def load_oxford_data():
    from lopq.utils import load_xvecs

    data = load_xvecs(relpath('../../data/oxford/oxford_features.fvecs'))
    return data

def make_random_model():
    m = LOPQModel(V=5, M=4, subquantizer_clusters=10)
    m.fit(np.random.RandomState(42).rand(200, 8), n_init=1)
    return m

########################################
# Tests
########################################

def test_eigenvalue_allocation():
    a = pkl.load(open(relpath('./testdata/test_eigenvalue_allocation_input.pkl')))

    vals, vecs = np.linalg.eigh(a)
    res = eigenvalue_allocation(4, vals)

    expected = np.array([
        63, 56, 52, 48, 44, 40, 36, 30, 26, 22, 18, 14, 10, 6, 3, 0,
        62, 57, 53, 51, 45, 41, 39, 33, 32, 31, 29, 25, 21, 17, 13, 9,
        61, 58, 54, 49, 47, 42, 38, 34, 28, 24, 20, 16, 12, 8, 5, 2,
        60, 59, 55, 50, 46, 43, 37, 35, 27, 23, 19, 15, 11, 7, 4, 1
    ])

    assert_true(np.equal(res, expected).all())

def test_eigenvalue_allocation_normalized_features():
    eigenvalues = np.array([
        2.02255824, 1.01940991, 0.01569471, 0.01355569, 0.01264379,
        0.01137654, 0.01108961, 0.01054673, 0.01023358, 0.00989679,
        0.00939045, 0.00900322, 0.00878857, 0.00870027, 0.00850136,
        0.00825236, 0.00813437, 0.00800231, 0.00790201, 0.00782219,
        0.00763405, 0.00752334, 0.00739174, 0.00728246, 0.00701366,
        0.00697365, 0.00677283, 0.00669658, 0.00654397, 0.00647679,
        0.00630645, 0.00621057
    ])
    indices = eigenvalue_allocation(2, eigenvalues)

    first_half = eigenvalues[indices[:16]]
    second_half = eigenvalues[indices[16:]]
    diff = np.abs(np.sum(np.log(first_half)) - np.sum(np.log(second_half)))
    assert_true(diff < .1, "eigenvalue_allocation is not working correctly")

def test_accumulate_covariance_estimators():
    data, centroids = pkl.load(open(relpath('./testdata/test_accumulate_covariance_estimators_input.pkl')))
    expected = pkl.load(open(relpath('./testdata/test_accumulate_covariance_estimators_output.pkl')))

    actual = accumulate_covariance_estimators(data, centroids)

    # Summed residual outer products
    assert_true(np.allclose(expected[0], actual[0]))

    # Summed residuals
    assert_true(np.allclose(expected[1], actual[1]))

    # Assignment count per cluster
    assert_true(np.array_equal(expected[2], actual[2]))

    # Assignments over data
    assert_true(np.array_equal(expected[3], actual[3]))

    # Residual data
    assert_true(np.allclose(expected[4], actual[4]))

def test_compute_rotations_from_accumulators():

    A, mu, count, num_buckets = pkl.load(open(relpath('./testdata/test_compute_rotations_from_accumulators_input.pkl')))
    expected = pkl.load(open(relpath('./testdata/test_compute_rotations_from_accumulators_output.pkl')))

    actual = compute_rotations_from_accumulators(A, mu, count, num_buckets)

    # Rotations
    assert_true(np.allclose(expected[0], actual[0]))

    # Mean residuals
    assert_true(np.allclose(expected[1], actual[1]))

def test_reconstruction():
    m = LOPQModel.load_proto(relpath('./testdata/random_test_model.lopq'))

    code = ((0, 1), (0, 1, 2, 3))
    r = m.reconstruct(code)
    expected = [-2.27444688, 6.47126941, 4.5042611, 4.76683476, 0.83671082, 9.36027283, 8.11780532, 6.34846377]

    assert_true(np.allclose(expected, r))

def test_oxford5k():

    random_state = 40
    data = load_oxford_data()
    train, test = train_test_split(data, test_size=0.2, random_state=random_state)

    # Compute distance-sorted neighbors in training set for each point in test set
    nns = compute_all_neighbors(test, train)

    # Fit model
    m = LOPQModel(V=16, M=8)
    m.fit(train, n_init=1, random_state=random_state)

    # Assert correct code computation
    assert_equal(m.predict(test[0]), ((3, 2), (14, 164, 83, 49, 185, 29, 196, 250)))

    # Assert low number of empty cells
    h = get_cell_histogram(train, m)
    assert_equal(np.count_nonzero(h == 0), 6)

    # Assert true NN recall on test set
    searcher = LOPQSearcher(m)
    searcher.add_data(train)
    recall, _ = get_recall(searcher, test, nns)
    assert_true(np.all(recall > [0.51, 0.92, 0.97, 0.97]))

    # Test partial fitting with just coarse quantizers
    m2 = LOPQModel(V=16, M=8, parameters=(m.Cs, None, None, None))
    m2.fit(train, n_init=1, random_state=random_state)

    searcher = LOPQSearcher(m2)
    searcher.add_data(train)
    recall, _ = get_recall(searcher, test, nns)
    assert_true(np.all(recall > [0.51, 0.92, 0.97, 0.97]))

    # Test partial fitting with coarse quantizers and rotations
    m3 = LOPQModel(V=16, M=8, parameters=(m.Cs, m.Rs, m.mus, None))
    m3.fit(train, n_init=1, random_state=random_state)

    searcher = LOPQSearcher(m3)
    searcher.add_data(train)
    recall, _ = get_recall(searcher, test, nns)
    assert_true(np.all(recall > [0.51, 0.92, 0.97, 0.97]))

def test_proto():
    import os

    filename = './temp_proto.lopq'
    m = make_random_model()
    m.export_proto(filename)
    m2 = LOPQModel.load_proto(filename)

    assert_equal(m.V, m2.V)
    assert_equal(m.M, m2.M)
    assert_equal(m.subquantizer_clusters, m2.subquantizer_clusters)

    assert_true(np.allclose(m.Cs[0], m2.Cs[0]))
    assert_true(np.allclose(m.Rs[0], m2.Rs[0]))
    assert_true(np.allclose(m.mus[0], m2.mus[0]))
    assert_true(np.allclose(m.subquantizers[0][0], m.subquantizers[0][0]))

    os.remove(filename)

def test_mat():
    import os

    filename = './temp_mat.mat'
    m = make_random_model()
    m.export_mat(filename)
    m2 = LOPQModel.load_mat(filename)

    assert_equal(m.V, m2.V)
    assert_equal(m.M, m2.M)
    assert_equal(m.subquantizer_clusters, m2.subquantizer_clusters)

    assert_true(np.allclose(m.Cs[0], m2.Cs[0]))
    assert_true(np.allclose(m.Rs[0], m2.Rs[0]))
    assert_true(np.allclose(m.mus[0], m2.mus[0]))
    assert_true(np.allclose(m.subquantizers[0][0], m.subquantizers[0][0]))

    os.remove(filename)

def searcher_instance_battery(searcher, q):
    """
    Helper to perform battery of searcher tests.
    """
    retrieved, visited = searcher.get_result_quota(q)
    assert_equal(len(retrieved), 12)
    assert_equal(visited, 3)

    retrieved, visited = searcher.search(q)
    assert_equal(len(retrieved), 10)
    assert_equal(visited, 3)

    retrieved, visited = searcher.get_result_quota(q, quota=20)
    assert_equal(len(retrieved), 28)
    assert_equal(visited, 5)

    retrieved, visited = searcher.search(q, quota=20)
    assert_equal(len(retrieved), 20)
    assert_equal(visited, 5)

    retrieved, visited = searcher.search(q, quota=20, limit=10)
    assert_equal(len(retrieved), 10)
    assert_equal(visited, 5)

def test_searcher():
    data = pkl.load(open(relpath('./testdata/test_searcher_data.pkl')))
    m = LOPQModel.load_proto(relpath('./testdata/random_test_model.lopq'))

    q = np.ones(8)

    # Test add_data
    searcher = LOPQSearcher(m)
    searcher.add_data(data)
    searcher_instance_battery(searcher, q)

    # Test add_codes
    searcher = LOPQSearcher(m)
    codes = [m.predict(x) for x in data]
    searcher.add_codes(codes)
    searcher_instance_battery(searcher, q)

def test_searcher_lmdb():
    import shutil

    data = pkl.load(open(relpath('./testdata/test_searcher_data.pkl')))
    m = LOPQModel.load_proto(relpath('./testdata/random_test_model.lopq'))

    lmbd_test_path = './test_lopq_lmbd'
    q = np.ones(8)

    # Test add_data
    searcher = LOPQSearcherLMDB(m, lmbd_test_path)
    searcher.add_data(data)
    searcher_instance_battery(searcher, q)

    # Clean up
    shutil.rmtree(lmbd_test_path)

    # Test add_codes
    searcher = LOPQSearcherLMDB(m, lmbd_test_path)
    codes = [m.predict(x) for x in data]
    searcher.add_codes(codes)
    searcher_instance_battery(searcher, q)

    # Clean up
    shutil.rmtree(lmbd_test_path)

def test_proto_partial():
    import os

    filename = './temp_proto_partial.lopq'
    c = (np.random.rand(8, 8), np.random.rand(8,8))
    m = LOPQModel(parameters=(c, None, None, None))
    m.export_proto(filename)
    m2 = LOPQModel.load_proto(filename)

    assert_equal(m.V, m2.V)
    assert_equal(m.M, m2.M)
    assert_equal(m.subquantizer_clusters, m2.subquantizer_clusters)

    assert_true(np.allclose(m.Cs[0], m2.Cs[0]))
    assert_true(m.Rs == m2.Rs)
    assert_true(m.mus == m2.mus)
    assert_true(m.subquantizers ==  m.subquantizers)

    os.remove(filename)

ext: py

max_stars_repo_name: kavorite/lopq

max_issues_repo_name: kavorite/lopq

max_forks_repo_name: kavorite/lopq

reason: import numpy


text:
import matplotlib.pyplot as plt
import math
from sklearn import metrics
from scipy.integrate import odeint
import numpy as np

""" ------ SIMULACIÓN DE SECADOR ROTATORIO ------ """
### Abbasfard H. (et. al) (2013),
# Mathematical modeling and simulation of an industrial rotary dryer: A case study of
# ammonium nitrate plant

def humedad_solidos(Xs,z,k,Xe,M,S):
    """this is the rhs of the ODE to integrate, i.e. dV/dt=f(V,t)"""
    '''
    Xs: Solid moisture []
    z: Dimensionless position []
    k:  Drying constant
    Xe: Equilibrium moisture []
    M: Total load [kg]
    S: Solid flow in [kg/s]
    '''

    return -((k*(Xs-Xe))*M)/S

def humedad_aire(Y,z,k,Xs,Xe,M,G):
    """this is the rhs of the ODE to integrate, i.e. dV/dt=f(V,t)"""
    '''
    Y: Solid moisture []
    Xs: Solid moisture []
    z: Dimensionless position []
    k:  Drying constant
    Xe: Equilibrium moisture []
    M: Total load [kg]
    G: Air flow in [kg/s]
    '''

    return ((k*(Xs-Xe))*M)/G

def temp_solido(Ts,z,Q,lamda,R,M,S,Cpd,Xs,Cpw):
    """this is the rhs of the ODE to integrate, i.e. dV/dt=f(V,t)"""
    '''
    Ts: Solid temperature [°C]
    Xs: Solid moisture []
    Q: Heat transfer
    lamda: Latent heat of vaporization [kJ/kg]
    R: Drying rate [kg water/kg dry solid/s]
    M: Total load [kg]
    S: Solid flow in [kg/s]
    Cpd: Specific heat of dry solid [kJ/kg °C]
    Cpw: Specific heat of pure water [kJ/kg °C]
    '''

    return (-Q-lamda*R*M)/(S*(Cpd+Xs*Cpw))

def temp_aire(Tg,z,Q,Cpv,R,M,Ts,Qp,G,Cpg,Y):
    """this is the rhs of the ODE to integrate, i.e. dV/dt=f(V,t)"""
    '''
    Tg: Air temperature [°C]
    Y: Air moisture []
    Q: Heat transfer
    lamda: Latent heat of vaporization [kJ/kg]
    R: Drying rate [kg water/kg dry solid/s]
    M: Total load [kg]
    G: Air flow in [kg/s]
    Cpv: Specific heat of water vapor [kJ/kg °C]
    Cpg: Specific heat of dry air [kJ/kg °C]
    '''

    return (Q+(Cpv*R*M*(Ts-Tg))-Qp)/(G*(Cpg+Y*Cpv))

def heat_transfer(Uva,V,Ts,Tg,Tamb,Up,A):

    Q = Uva*V*(Ts-Tg)
    Qp = Up*A*(Tg-Tamb)

    return Q, Qp

def volumetric_heat_transfer_coeff(G,A,S):
    Uva = (0.394*(G/A)**0.289)*((S/A)**0.541)
    # Uva = (3.535*(G/A)**0.289)*((S/A)**0.541)
    Up = 0.022*(G/A)**0.879
    # Up = 0.227*(G/A)**0.879

    return Uva,Up

def eq_moisture(RH,Tg):
    a = (2.39*10**(-6))*(0.987**Tg)*Tg**(-0.832)
    b = (-5.76*10**(-5))+(1.306*10**(-5))*np.log(Tg)
    c = 0.9715*(1.024**Tg)*Tg**(-2.31)

    Xeq = RH*(a*RH*RH+b*RH+c)

    return Xeq

def drying_constant(Tg):

    k = 0.0349*math.exp(-7.95/Tg)

    return k

def residence_time(L,slope,n,S,D,Dp,G):

    # tau = (13.8*L)/(slope*(n**0.9)*D)-((614.2)/(Dp**0.5))*((L*G)/S)
    tau = (0.35*L)/(slope*(n**0.9)*D)-((5.2)/(Dp**0.5))*((L*G)/S)

    return tau

Tamb = 35

S = 60001/3600          # Solid flow in [kg/s]
G = 60979/3600          # Air flow in [kg/s]

L = 18                  # Dryer lenght [m]
D = 3.324               # Dryer inside diameter [m]
slope = 0.025           # Dryer slope [m/m]
n = 3                   # RPM
Dp = 2*10**3            # Average solid diameter [um]
A = np.pi*(D**2)/4      # Transversal area [m2]
V = A*L                 # Dryer volume [m3]

Cpd = 1.56              # Solid heat capacity [kJ/kg°C]
Cpg = 1.009             # Air heat capacity [kJ/kg°C]
Cpv = 1.88              # Vapor heat capacity [kJ/kg°C]
Cpw = 4.18              # Water heat capacity [kJ/kg°C]

lamda = 2260            # Latent heat of vaporization [kJ/kg]

Xs0 = 0.12              # Input solid moisture []
Y0 = 0.005              # Input air moisture []
Ts0 = 84                # Input solid temperature [°C]
Tg0 = 73                # Input air temperature [°C]
RH = 0.015              # Input air relative humidity []

Uva, Up = volumetric_heat_transfer_coeff(G,A,S)
Q, Qp = heat_transfer(Uva,V,Ts0,Tg0,Tamb,Up,A)
k = drying_constant(Tg0)
Xe = eq_moisture(RH,Tg0)
tau = residence_time(L,slope,n,S,D,Dp,G)

M = S*tau

R = k*(Xs0-Xe)

args_Xs = (k,Xe,M,S)
args_Y = (k,Xs0,Xe,M,G)
args_Ts = (Q,lamda,R,M,S,Cpd,Xs0,Cpw)
args_Tg = (Q,Cpv,R,M,Ts0,Qp,G,Cpg,Y0)

sample_time = 0.01
z = np.arange(0, 1, sample_time)  # values of t for
                        # which we require
                        # the solution y(t)

Xs_temp = np.zeros(z.shape)
Y_temp = np.zeros(z.shape)
Ts_temp = np.zeros(z.shape)
Tg_temp = np.zeros(z.shape)
print("K es: ",k)
print("Xe inciial es: ",Xe)
print("Tau inciial es: ",tau)
print("Q inciial es: ",Q)
print("Uva inciial es: ",Uva)
print("Volumen secador es: ",V)

"""" ----- SIMULACIÓN ------ """

for k in range(z.size):
    # print("\n ------------------CICLO ",k)
    # print("\n k es: ",k)

    Xs_temp[k] = Xs0
    Ts_temp[k] = Ts0
    Y_temp[k] = Y0
    Tg_temp[k] = Tg0

    # print("first X0: " ,Xs0)
    # print("this is first args_Xs, ",args_Xs)
    Xs = odeint(humedad_solidos, Xs0, np.array([0.0, sample_time]), args=args_Xs)
    # print("this is Xs: ",Xs)

    # print("\n first Y0: " ,Y0)
    # print("this is first args_Y, ",args_Y)
    Y = odeint(humedad_aire, Y0, np.array([0.0, sample_time]), args=args_Y)
    # print("this is Y: ",Y[1])

    # print("\n first Ts: " ,Ts0)
    # print("this is first args_Ts, ",args_Ts)
    Ts = odeint(temp_solido, Ts0, np.array([0.0, sample_time]), args=args_Ts)
    # print("this is Ts: ",Ts[1])

    # print("\n first Ta: " ,Tg0)
    # print("this is first args_Ta, ",args_Tg)
    Tg = odeint(temp_aire, Tg0, np.array([0.0, sample_time]), args=args_Tg)
    # print("this is Ta: ",Tg[1])

    Xs0 = Xs[1]
    Y0 = Y[1]
    Ts0 = Ts[1]
    Tg0 = Tg[1]

    Q, Qp = heat_transfer(Uva,V,Ts0,Tg0,Tamb,Up,A)
    k = drying_constant(Tg0)
    Xe = eq_moisture(RH,Tg0)

    R = k*(Xs0-Xe)

    args_Xs = (k,Xe,M,S)
    args_Y = (k,Xs0,Xe,M,G)
    args_Ts = (Q,lamda,R,M,S,Cpd,Xs0,Cpw)
    args_Tg = (Q,Cpv,R,M,Ts0,Qp,G,Cpg,Y0)

    # print("\n this is Xe: ",Xe)
    # print("this is K: ",k)
    # print("this is Q: ",Q)
    # print("this is Qp: ",Qp)

delta_T = np.abs(Ts0-Tg0)

"""" ------ PLOTTING ------- """

fig, axarr = plt.subplots(2, 1)
fig.canvas.set_window_title('Rotary dryer')
fig.suptitle('Rotary dryer simulation')

axarr[0].plot(z, 100*Xs_temp, 'b', label='Solid')
axarr[0].plot(z, 100*Y_temp, 'green', label='Air')
axarr[0].legend(loc='best')
axarr[0].grid()
axarr[0].set(ylabel = 'Moisture [%]')

axarr[1].plot(z, Ts_temp, 'r', label='Solid')
axarr[1].plot(z, Tg_temp, 'y', label='Air')
axarr[1].legend(loc='best')
axarr[1].grid()
axarr[1].set(ylabel = 'Temperature [°C]')
axarr[1].set_title("Delta T is: " + str(delta_T))

plt.show()

reason: import numpy,from scipy


text:
import os
import numpy as np
from torch.utils.data import Dataset

class SpecDataset(Dataset):
    def __init__(self, root='../Data/umsun_dataset_44k/spec', mean=None, std=None, time_dim_size=431, mode=None):
        self.classes = {'alarm clock': 0, 'clock': 1, 'bee': 2, 'bird': 3, 'owl': 4,
                        'phone': 5, 'church': 6, 'cow': 7, 'duck': 8, 'dog': 9,
                        'frog': 10, 'horse': 11, 'keyboard': 12, 'pencil': 13, 'guitar': 14,
                        'piano': 15, 'train': 16, 'violin': 17, 'clarinet': 18, 'pig': 19}
        self.classes_list = ['alarm clock', 'clock', 'bee', 'bird', 'owl',
                        'phone', 'church', 'cow', 'duck', 'dog',
                        'frog', 'horse', 'keyboard', 'pencil', 'guitar',
                        'piano', 'train', 'violin', 'clarinet', 'pig']
        self.paths = self.path_finder(root, mode)
        if mean is None or std is None or time_dim_size is None:
            self.mean, self.std, self.time_dim_size, self.paths = self.initialize(self.paths)
            print(self.mean, self.std, self.time_dim_size)
        else:
            self.mean = mean
            self.std = std
            self.time_dim_size = time_dim_size

    def __getitem__(self, i):
        # Get i-th path.
        path, label = self.paths[i]
        # Load the mel-spectrogram.
        spec = np.load(path)
        if self.time_dim_size is not None:
            # Slice the temporal dimension with a fixed length so that they have
            # the same temporal dimensionality in mini-batches.
            spec = spec[:, :self.time_dim_size]
        # Perform standard normalization using pre-computed mean and std.
        spec = (spec - self.mean) / self.std

        return np.expand_dims(spec, axis=0), label

    def path_finder(self, root, mode):
        paths = []
        assert mode is not None
        cls_list = os.listdir(root)
        id = 0
        for cls in cls_list:
            cls_path = os.path.join(root, cls)
            specs = os.listdir(cls_path)
            for file in specs:
                if mode == 'val' and id%8 == 0:
                    paths.append((os.path.join(cls_path, file), self.classes[cls]))
                if mode == 'train' and id%8 != 0:
                    paths.append((os.path.join(cls_path, file), self.classes[cls]))
                id += 1
        return paths

    def initialize(self, paths, limit=800):
        specs = []
        for s, l in paths:
            sp = np.load(s)
            specs.append(sp)
        time_dims = [s.shape[1] for s in specs]
        min_time_dim_size = min(time_dims)
        specs = [s[:, :min_time_dim_size] for s in specs]
        specs = np.stack(specs)
        mean = specs.mean()
        std = specs.std()
        return mean, std, min_time_dim_size, paths

    def __len__(self):
        return len(self.paths)

max_stars_repo_name: dlrudco/CoEM

max_issues_repo_name: dlrudco/CoEM

max_forks_repo_name: dlrudco/CoEM


text:
import glob
import json
from collections import defaultdict
from tabulate import tabulate
import numpy as np
import pandas as pd
import math
from metric.smd_scorer import score_SMD

data = defaultdict(lambda: defaultdict(list))

for name in glob.glob('generations/smd-*.json'):
    parse = name.replace("generation/","").replace(".json","").split("_")
    # print(parse)
    dataset, shot, model, beam_dosample, id_prefix = parse
    # print(dataset, shot, model, id_prefix)

    data[f"{model}_{id_prefix}"][shot].append(name)

def save_file(filename, results):
    filename = filename.replace("EleutherAI/","").replace("../few-shot-lm/","")
    with open(f'generations/{filename}', 'w') as fp:
        json.dump(results, fp, indent=4)

for k, v in data.items():
    model, id_prefix = k.split("_")
    for shots, list_file in v.items():
        dict_res = {
            "score": {
                "BLEU": 0.0,
                "F1": 0.0,
                "F1 navigate": 0.0,
                "F1 weather": 0.0,
                "F1 schedule": 0.0,
            },
            "generation": []
        }

        # follow this sequence cause test test is nasty
        # schedule
        # navigate
        # weather
        temp_gen = {}
        ppl = []
        for file_name in list_file:
            res = json.load(open(file_name,"r"))
            print(file_name)
            print(res['score'])
            ppl.append(res['score']["ppl"])
            if not np.isnan(res['score']['F1 navigate']):
                dict_res['score']["F1 navigate"] = res['score']["F1 navigate"]
                temp_gen["navigate"] = res["generation"]
            if not np.isnan(res['score']['F1 weather']):
                dict_res['score']["F1 weather"] = res['score']["F1 weather"]
                temp_gen["weather"] = res["generation"]
            if not np.isnan(res['score']['F1 schedule']):
                dict_res['score']["F1 schedule"] = res['score']["F1 schedule"]
                temp_gen["schedule"] = res["generation"]
        dict_res["generation"] = temp_gen["schedule"]+temp_gen["navigate"]+temp_gen["weather"]
        res = score_SMD(dict_res["generation"], "data/smd/test.json")
        dict_res['score'] = res
        dict_res['score']['ppl'] = np.mean(ppl)
        save_file(f"smd_{shots}_{model}_{beam_dosample}_{id_prefix}.json", dict_res)
        # input()
        # print(list_file)
    print()


text:
import requests
import tarfile
import os
import sys
from pathlib import Path
from datetime import datetime
import logging
import shutil
import glob
from operator import itemgetter
import collections
import pydicom
import nrrd
import numpy as np
import nibabel as nib
#import PySimpleGUI as sg
from skimage.filters.thresholding import threshold_otsu
from skimage.transform import resize
import pandas as pd
import matplotlib.pyplot as plot
import matplotlib.cbook as cbook
import subprocess as sp
from medpy.metric.binary import hd, hd95, dc
from lung_segmentation.configuration import (
        STANDARD_CONFIG, HIGH_RES_CONFIG, HUMAN_CONFIG, CUSTOM_CONFIG)

ALLOWED_EXT = ['.xlsx', '.csv']
ILLEGAL_CHARACTERS = ['/', '(', ')', '[', ']', '{', '}', ' ', '-']

class DicomInfo(object):

    def __init__(self, dicoms):

        if type(dicoms) == list:
            self.dcms = dicoms
        elif os.path.isdir(dicoms):
            dcms = list(dicoms.glob('*.dcm'))
            if not dcms:
                dcms = list(dicoms.glob('*.IMA'))
            if not dcms:
                raise Exception('No DICOM files found in {}'.format(dicoms))
            else:
                self.dcms = dcms
        else:
            self.dcms = [dicoms]

    def get_tag(self, tag):

        tags = {}

        if type(tag) is not list:
            tag = [tag]
        for t in tag:
            values = []
            for dcm in self.dcms:
                header = pydicom.read_file(str(dcm))
                try:
                    val = header.data_element(t).value
                    if isinstance(val, collections.Iterable) and type(val) is not str:
                        val = tuple(val)
                    else:
                        val = str(val)
                    values.append(val)
                except (AttributeError, KeyError):
                    print ('{} seems to do not have the requested DICOM field ({})'.format(dcm, t))

            tags[t] = list(set(values))

        return self.dcms, tags

    def check_uniqueness(self, InstanceNums, SeriesNums):

        toRemove = []
        if (len(InstanceNums) == 2*(len(set(InstanceNums)))) and len(set(SeriesNums)) == 1:
            sortedInstanceNums = sorted(zip(self.dcms, InstanceNums), key=itemgetter(1))
            uniqueInstanceNums = [x[0] for x in sortedInstanceNums[:][0:-1:2]]
            toRemove = toRemove+uniqueInstanceNums

        return toRemove

def split_filename(fname):
    """Split a filename into parts: path, base filename and extension.
    Parameters
    ----------
    fname : str
        file or path name
    Returns
    -------
    pth : str
        base path from fname
    fname : str
        filename from fname, without extension
    ext : str
        file extension from fname
    """

    special_extensions = [".nii.gz", ".tar.gz", ".niml.dset"]

    pth = os.path.dirname(fname)
    fname = os.path.basename(fname)

    ext = None
    for special_ext in special_extensions:
        ext_len = len(special_ext)
        if (len(fname) > ext_len) and \
                (fname[-ext_len:].lower() == special_ext.lower()):
            ext = fname[-ext_len:]
            fname = fname[:-ext_len]
            break
    if not ext:
        fname, ext = os.path.splitext(fname)

    return pth, fname, ext

def get_files(url, location, file, ext='.tar.gz'):

    r = requests.get(url)
    with open(os.path.join(location, file+ext), 'wb') as f:
        f.write(r.content)
    print(r.status_code)
    print(r.headers['content-type'])
    print(r.encoding)

    return os.path.join(location, file+ext)

def untar(fname, folder_name, move_weights=False):

    untar_dir = os.path.join(os.path.split(fname)[0], folder_name)
    if fname.endswith("tar.gz"):
        tar = tarfile.open(fname)
        tar.extractall(path=untar_dir)
        tar.close()
        print("Extracted in Current Directory")
    else:
        print("Not a tar.gz file: {}".format(fname))
    if move_weights:
        weights = []
        for path, _, files in os.walk(untar_dir):
            for f in files:
                if '.h5' in f:
                    weights.append(os.path.join(path, f))
        if weights:
            for w in weights:
                if os.path.split(w)[0] != untar_dir:
                    shutil.move(w, untar_dir)

def create_log(log_dir):

    now = datetime.now()
    dt_string = now.strftime("%d-%m-%Y_%H_%M_%S")
    logger = logging.getLogger('lungs_segmentation')
    logger.setLevel(logging.DEBUG)
    # create file handler which logs even debug messages
    log_file = os.path.join(
        log_dir, 'lungs_segmentation_{}.log'.format(dt_string))
    # Path(log_file).touch()
    fh = logging.FileHandler(log_file)
    fh.setLevel(logging.DEBUG)
    # create console handler with a higher log level
    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    # create formatter and add it to the handlers
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    fh.setFormatter(formatter)
    ch.setFormatter(formatter)
    # add the handlers to the logger
    logger.addHandler(fh)
    logger.addHandler(ch)

    return logger

def batch_processing(input_data, key_col1='subjects', key_col2='masks', root=''):
    """Function to process the data in batch mode. It will take a .csv or .xlsx file with
    two columns. The first one called 'subjects' contains all the paths to the raw_data folders
    (one path per raw); the second one called 'masks' contains all the corresponding paths to
    the segmented mask folders.
    Parameters
    ----------
    input_data : str
        Excel or CSV file
    root : str
        (optional) root path to pre-pend to each subject and mask in the input_data file
    Returns
    -------
    raw_data : list
        list with all the subjects to process
    masks : list
        list with the corresponding mask to use to extract the features
    """
    if os.path.isfile(input_data):
        _, _, ext = split_filename(input_data)
        if ext not in ALLOWED_EXT:
            raise Exception('The file extension of the specified input file ({}) is not supported.'
                            ' The allowed extensions are: .xlsx or .csv')
        if ext == '.xlsx':
            files = pd.read_excel(input_data)
        elif ext == '.csv':
            files = pd.read_csv(input_data)
        files=files.dropna()
        try:
            masks = [os.path.join(root, str(x)) for x in list(files[key_col2])]
        except KeyError:
            print('No "masks" column found in the excel sheet. The cropping, if selected, will be performed without it.')
            masks = None
        raw_data = [os.path.join(root, str(x)) for x in list(files[key_col1])]

        return raw_data, masks

def dicom_check(raw_data, temp_dir, deep_check=True):
    """Function to arrange the mouse lung data into a proper structure.
    In particular, this function will look into each raw_data folder searching for
    the data with H50s in the series description field in the DICOM header. Then,
    it will copy those data into another folder and will return the path to the first
    DICOM file that will be used to run the DICOM to NRRD conversion.
    Parameters
    ----------
    raw_data : str
        path to the raw data folder
    Returns
    -------
    pth : str
        path to the first DICOM volume
    """

    dicoms = sorted(glob.glob(raw_data+'/*.IMA'))
    dcm_info = {}
    processed = False
    if not dicoms:
        dicoms = sorted(glob.glob(raw_data+'/*'))
        if not dicoms:
            raise Exception('No DICOM files found in {}! Please check.'.format(raw_data))
        else:
            ext = '.dcm'
    else:
        ext = '.IMA'

    if not os.path.isdir(temp_dir):
        os.mkdir(temp_dir)
    basename = raw_data.split('/')[-1]
    for character in ILLEGAL_CHARACTERS:
        basename = basename.replace(character, '_')

    sequence_numbers = list(set([str(pydicom.read_file(x).SeriesNumber) for x in dicoms]))
    if len(sequence_numbers) > 1 and deep_check:
        for n_seq in sequence_numbers:
            dicom_vols = [x for x in dicoms if n_seq==str(pydicom.read_file(x).SeriesNumber)]
            dcm_hd = pydicom.read_file(dicom_vols[0])
            if len(dicom_vols) > 1 and '50s' in dcm_hd.SeriesDescription and not processed:
                dcm = DicomInfo(dicom_vols)
                _, tag = dcm.get_tag(['AcquisitionDate', 'SeriesTime'])
                if len(tag['SeriesTime']) > 1:
                    dicom_vols = [x for x in dicoms if str(pydicom.read_file(x).SeriesTime)==tag['SeriesTime'][0]]
                folder_name = temp_dir+'/{0}_date_{1}_time_{2}'.format(
                    basename, tag['AcquisitionDate'][0], tag['SeriesTime'][0])
                slices = [pydicom.read_file(x).InstanceNumber for x in dicom_vols]
                if len(slices) != len(set(slices)):
                    print('Duplicate slices found in {} for H50s sequence. Please check. '
                          'This subject will be excluded from the analysis.'.format(raw_data))
                    continue
                processed = True
                break
    elif len(sequence_numbers) == 1:
        folder_name = temp_dir+'/{0}'.format(basename)
        dicom_vols = dicoms
        processed = True

    if processed:
        if not os.path.isdir(folder_name):
            os.mkdir(folder_name)
        else:
            shutil.rmtree(folder_name)
            os.mkdir(folder_name)
        for x in dicom_vols:
            try:
                shutil.copy2(x, folder_name)
            except:
                continue
        filename = sorted(glob.glob(folder_name+'/*'.format(ext)))[0]

    else:
        print('No suitable CT data with name containing "H50s" were found in {}'.format(raw_data))
        filename = None
        folder_name = None

    return filename, folder_name, dcm_info

def binarization(image):

    th = threshold_otsu(image)
    image[image>=th] = 1
    image[image!=1] = 0

    return image

def normalize(image, method='zscore'):

    image = np.asanyarray(image)
    image = image.astype('float64')
    if method == 'zscore':
        mns = image.mean()
        sstd = image.std()
        res = (image - mns)/sstd
    elif method == '0-1':
        m = image.min()
        M = image.max()
        if (M - m) > 0:
            res = (image - m)/(M - m)
        else:
            res = image
    else:
        raise NotImplementedError('Normalization method called "{}" has not been implemented yet!'.format(method))

    return res

def resize_image(image, order=0, new_spacing=(0.1, 0.1, 0.1), save2file=True):

    basepath, fname, ext = split_filename(image)
    outname = os.path.join(basepath, fname+'_resampled'+ext)
    if ext == '.nrrd':
        image, hd = nrrd.read(image)
        space_x = np.abs(hd['space directions'][0, 0])
        space_y = np.abs(hd['space directions'][1, 1])
        space_z = np.abs(hd['space directions'][2, 2])
    elif ext == '.nii.gz' or ext == '.nii':
        hd = nib.load(image).header
        affine = nib.load(image).affine
        image = nib.load(image).get_data()
        space_x, space_y, space_z = hd.get_zooms()

    resampling_factor = (new_spacing[0]/space_x, new_spacing[1]/space_y, new_spacing[2]/space_z)
    new_shape = (image.shape[0]//resampling_factor[0], image.shape[1]//resampling_factor[1],
                 image.shape[2]//resampling_factor[2])
    new_image = resize(image.astype(np.float64), new_shape, order=order, mode='edge',
                       cval=0, anti_aliasing=False)
    if save2file:
        if ext == '.nrrd':
            hd['sizes'] = np.array(new_image.shape)
            hd['space directions'][0, 0] = new_spacing[0]
            hd['space directions'][1, 1] = new_spacing[1]
            hd['space directions'][2, 2] = new_spacing[2]
            nrrd.write(outname, new_image, header=hd)
        elif ext == '.nii.gz' or ext == '.nii':
            im2save = nib.Nifti1Image(new_image, affine=affine)
            nib.save(im2save, outname)

        return new_image, tuple(map(int, new_shape)), outname, image.shape
    else:
        return new_image, np.mean(resampling_factor)

def save_prediction_2D(generated_images, dict_val, binarize=False):

    n_images = len(dict_val)
    batches = generated_images.shape[0]//n_images
    for n in range(n_images):
        im_shape = dict_val[n]['im_size']
        indexes = dict_val[n]['indexes']
        image = generated_images[n*batches:(n+1)*batches, :]
        final_image = np.zeros((im_shape[0], im_shape[1], batches), dtype=np.float16)-2
        k = 0
        for j in indexes[1]:
            for i in indexes[0]:
                final_image[i[0]:i[1], j[0]:j[1], k] = image[k, :, :, 0]
                k += 1
        final_image[final_image==-2] = np.nan
        final_image = np.nanmean(final_image, axis=2)
        final_image[np.isnan(final_image)] = 0
        final_image = final_image.astype(np.float32)
        original_dim = dict_val[n]['orig_dim']
        if tuple(final_image.shape) != original_dim:
            final_image = resize(final_image, (original_dim[0], original_dim[1]),
                                 order=3, mode='edge', cval=0,
                                 anti_aliasing=False)
        if binarize:
            final_image = binarization(final_image)

        return final_image

def dice_calculation(gt, seg):

    gt, hd_gt = nrrd.read(gt)
    seg, hd_seg = nrrd.read(seg)
    if (hd_seg['space origin'] == hd_gt['space origin']).all():
        seg = np.squeeze(seg)
        gt = gt.astype('uint16')
        seg = seg.astype('uint16')
        dice = dc(seg, gt)
    else:
        dice = None
#     vox_gt = np.sum(gt)
#     vox_seg = np.sum(seg)
#     try:
#         common = np.sum(gt & seg)
#     except:
#         print(gt)
#     dice = (2*common)/(vox_gt+vox_seg)
    return dice

def violin_box_plot(to_plot, outname):

    fig = plot.figure(1, figsize=(9, 6))
    ax = fig.add_subplot(111)
    parts=ax.violinplot(to_plot, showmeans=False, showmedians=False, showextrema=False)
    for pc in parts['bodies']:
        pc.set_facecolor('xkcd:lightblue')
        pc.set_edgecolor('xkcd:blue')
        pc.set_alpha(1)
        pc.set_linewidth(2)

    medianprops = dict(linestyle='-.', linewidth=2.5, color='firebrick')
    stats = cbook.boxplot_stats(to_plot)
    flierprops = dict(marker='o', markerfacecolor='green', markersize=12, linestyle='none')
    ax.set_axisbelow(True)
    plot.gca().yaxis.grid(True, ls='-', color='white')
    ax.bxp(stats, flierprops=flierprops, medianprops=medianprops)
    ax.set_xticks([])
    ax.set_xticklabels([])
    ax.tick_params(axis='y', length=0)
    ax.set_facecolor('lightgrey')
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.spines['bottom'].set_visible(False)
    ax.spines['left'].set_visible(False)
    plot.savefig(outname)
    plot.close()

def run_cluster_correction(image, th=0.5, min_extent=10000):

    out_nii = image.split('.nrrd')[0]+'.nii.gz'
    out_image = out_nii.split('.nii.gz')[0]+'_cc.nii.gz'
    out_text = out_nii.split('.nii.gz')[0]+'_cc.txt'
    outname = out_nii.split('.nii.gz')[0]+'_corrected.nii.gz'
    outname_nrrd = out_nii.split('.nii.gz')[0]+'_corrected.nrrd'

    im_nrrd, header_nrrd = nrrd.read(image)
    nii2save = nib.Nifti1Image(im_nrrd, affine=np.eye(4))
    nib.save(nii2save, out_nii)

    cmd = 'cluster -i "{0}" -t "{1}" -o "{2}" --minextent={3} --olmax="{4}"'.format(
        out_nii, th, out_image, min_extent, out_text)
    _ = sp.check_output(cmd, shell=True)
    with open(out_text, 'r') as f:
        res = [x.split() for x in f]
    mat = np.asarray(res[1:]).astype('float')
    try:
        clusters = list(set(mat[mat[:, 1] > 0.95][:, 0]))
    except:
        print()
    add_cmd = None
    for i, cl in enumerate(clusters):
        if len(clusters) > 2:
            print('Found {0} clusters for image {1}, please check because the'
                  ' usual number of clusters should be not greater than 2.'.format(len(clusters), image))
        out_cl = image.split('.nii.gz')[0]+'_cc_{}.nii.gz'.format(cl)
        if len(clusters) > 1:
            cmd = 'fslmaths "{0}" -uthr {1} -thr {1} -bin "{2}"'.format(out_image, cl, out_cl)
        else:
            cmd = 'fslmaths "{0}" -uthr {1} -thr {1} -bin "{2}"'.format(out_image, cl, outname)
        sp.check_output(cmd, shell=True)
        if len(clusters) > 1:
            if i == 0:
                add_cmd = 'fslmaths "{}" -add'.format(out_cl)
            elif i == len(clusters)-1:
                add_cmd = add_cmd + ' "{0}" "{1}"'.format(out_cl, outname)
            else:
                add_cmd = add_cmd + ' "{0}" -add'.format(out_cl)
    if add_cmd is not None:
        sp.check_output(add_cmd, shell=True)

    nii_im = nib.load(outname).get_data()
    nrrd.write(outname_nrrd, nii_im, header=header_nrrd)

    return outname_nrrd

def run_hd(image1, image2, mode='max'):

    image1_data, hd1 = nrrd.read(image1)
    space_x = np.abs(hd1['space directions'][0, 0])
    space_y = np.abs(hd1['space directions'][1, 1])
    space_z = np.abs(hd1['space directions'][2, 2])
    image2_data, hd2 = nrrd.read(image2)
    if (hd1['space origin'] == hd2['space origin']).all():
        if mode == 'max':
            hd_val = hd(image1_data, image2_data, (space_x, space_y, space_z))
        elif mode == '95':
            hd_val = hd95(image1_data, image2_data, (space_x, space_y, space_z))
        else:
            raise Exception('Unknown mode "{}". Possible values are "max" or "95".'
                            .format(mode))
    else:
        hd_val = None
    return hd_val

# def build_gui():
#     "Function to build the GUI for inference"
#     configured = False
#
#     while not configured:
#         disable_dw_weights = False
#         sg.ChangeLookAndFeel('GreenTan')
#
#         layout = [
#             [sg.Text('Lung Segmentation using CNN', size=(30, 1), font=("Helvetica", 25))],
#             [sg.Text('Please choose a configuration file based on your data:')],
#             [sg.Listbox(values=('Low resolution mouse', 'High resolution mouse', 'Human',
#                                 'Custom configuration'),
#                         size=(30, 3), default_values='Low resolution mouse')],
#             [sg.Submit(), sg.Quit()]
#         ]
#         window = sg.Window('SeANNA - Config', default_element_size=(40, 1)).Layout(layout)
#         config_button, values = window.Read()
#         if config_button == 'Quit':
#             sys.exit()
#         else:
#             window.close()
#         if values[0][0] == 'Low resolution mouse':
#             config = STANDARD_CONFIG
#             app_name = 'Lung segmentation inference for low resolution mouse CT images'
#         elif values[0][0] == 'High resolution mouse':
#             config = HIGH_RES_CONFIG
#             app_name = 'Lung segmentation inference for high resolution mouse CT images'
#         elif values[0][0] == 'Human':
#             config = HUMAN_CONFIG
#             app_name = 'Lung segmentation inference for human CT images'
#         else:
#             config = CUSTOM_CONFIG
#             disable_dw_weights = True
#             app_name = 'Lung segmentation inference using custom settings'
#
#         dc_check = config['dicom_check']
#         new_spacing = config['spacing']
#         min_extent = config['min_extent']
#         cluster_correction = config['cluster_correction']
#         weights_url = config['weights_url']
#
#         input_layout = [
#             [sg.Text('Input path', size=(15, 1), auto_size_text=False, justification='right'),
#              sg.InputText('', key='in_path', tooltip=(
#                  'Input can be either a folder (with one or more sub-folder(s) \n'
#                  'for each CT image) or an Excel sheet with one image path \n'
#                  'per raw. See the documentation for more information.')),
#              sg.FolderBrowse()],
#             [sg.Text('Root path', size=(15, 1), auto_size_text=False, justification='right'),
#              sg.InputText('', key='root_path', tooltip=(
#                  'If an Excel sheet was specified as input, then the root path \n'
#                  'is the path that will be prepended to each raw in the \n'
#                  'input file.')),
#              sg.FolderBrowse()],
#             [sg.Text('Working directory', size=(15, 1), auto_size_text=False,
#                      justification='right'),
#              sg.InputText('', key='work_dir', tooltip=(
#                  'Path where to save the results. If the directory does not \n'
#                  'exist, it will be created.')),
#              sg.FolderBrowse()],
#             [sg.Checkbox('Download weights', change_submits = True, enable_events=True,
#                          default=True, key='download_weights', disabled=disable_dw_weights,
#                          tooltip='Whether or not to download the weights from the \n'
#                          'repository. It does not work for custom settings.')],
#             [sg.Text('Weights directory', size=(15, 1), auto_size_text=False,
#                      justification='right'),
#              sg.InputText('', disabled = not disable_dw_weights, key='weights_dir',
#                           tooltip=('Specify here the path to the folder containing the \n'
#                                    'network weights. If you do not have them, please tick \n'
#                                    'the "Download weights" box above and the application \n'
#                                    'will try to download them from the repository.\n'
#                                    'N.B. The download does not work for custom settings.')),
#              sg.FolderBrowse(disabled = not disable_dw_weights, key='weights_dir_2')]
#             ]
#
#         colx = [[sg.Text('Spacing X')],
#                 [sg.Slider(range=(0.0, 5.0), orientation='v', resolution=0.05, size=(5, 20),
#                            default_value=new_spacing[0],
#                            tooltip='New resolution (in mm) along x direction.',
#                            key='space_x')]]
#         coly = [[sg.Text('Spacing Y')],
#                 [sg.Slider(range=(0.0, 5.0), orientation='v', resolution=0.05, size=(5, 20),
#                            default_value=new_spacing[1],
#                            tooltip='New resolution (in mm) along y direction.',
#                            key='space_y')]]
#         colz = [[sg.Text('Spacing Z')],
#                 [sg.Slider(range=(0.0, 5.0), orientation='v', resolution=0.05, size=(5, 20),
#                            default_value=new_spacing[2],
#                            tooltip='New resolution (in mm) along z direction.',
#                            key='space_z')]]
#         preproc_layout = [
#             [sg.Checkbox('DICOM check', default=dc_check,
#                          tooltip='Whether or not to carefully check the DICOM header. \n'
#                                  'This check is based on our low resolution mouse data \n'
#                                  'and might be too stringent for data coming from \n'
#                                  'different sites and/or acquired with different resolution.\n'
#                                  'If the last is the case, then turn this check off.',
#                          key='dcm_check')],
#             [sg.Text('New spacing (for resampling)',
#                      tooltip='Here you can specify the voxel size (in mm) \n'
#                              'that will be used to resample the image before\n'
#                              'running the inference. The network has been trained\n'
#                              'using images with a defined resolution. The default\n'
#                              'values for the resolution have been stored in the 3 \n'
#                              'pre-defined configuration files, however, \n'
#                              'if the network was re-trained with different\n'
#                              'resolution, you can simply change it here.')],
#             [sg.Column(colx), sg.Column(coly), sg.Column(colz)]]
#
#         post_proc_layout = [
#             [sg.Radio('Cluster correction', 'pp', default=cluster_correction,
#                       key='cluster_correction', change_submits = True, enable_events=True,
#                       tooltip=('This correction should be more accurate wrt the standard\n'
#                                'binarization and it is recommended for high res and human \n'
#                                'segmentation because it should remove all the small isolated \n'
#                                'wrongly segmented areas. However is more time consuming and \n'
#                                'relies on external software (FSL), so run it only if needed.')),
#              sg.Text('Minimum extent', size=(15, 1), auto_size_text=False, justification='right',
#                      tooltip=('If "Cluster correction" is selected, then here you can \n'
#                               'specify the minimum number of voxels that a cluster can \n'
#                               'have in order to be considered as possible lung. This value \n'
#                               'depends on the resolution and size of the CT image. Usually \n'
#                               'for low res mice 350 is enough, while for high res mice and \n'
#                               'humans can be set to 100000 and 300000, respectively.')),
#              sg.InputText(min_extent, key='min_extent', disabled = not cluster_correction,
#                           size=(15, 10))],
#             [sg.Radio('Otsu binarizazion', 'pp', default=not cluster_correction,
#                       change_submits = True, enable_events=True,
#                       tooltip=('If selected, the final segmented image will be binarized\n'
#                                'using the Otsu thresholding method.'))],
#             [sg.Checkbox('Run segmentation evaluation', default=False,
#                          tooltip=('If ground truth segmentations are available, \n'
#                                   'you can specify them in the Excel sheet and select\n'
#                                   'this in order to evaluate the CNN segmentation. Both\n'
#                                   'Dice score and Hausdorff distance will be calculated.'),
#                          key='evaluate')]]
#
#         layout = [
#             [sg.Text(app_name, size=(55, 1), font=("Helvetica", 20))],
#             [sg.Frame('Input and output specs', input_layout, font='Any 12', title_color='black')],
#             [sg.Text('_'  * 80)],
#             [sg.Frame('Pre-processing specs', preproc_layout, font='Any 12', title_color='black')],
#             [sg.Text('_'  * 80)],
#             [sg.Frame('Post-processing specs', post_proc_layout, font='Any 12',
#                       title_color='black')],
#             [sg.Submit(), sg.Quit(), sg.Cancel()]]
#         main_window = sg.Window('SeANNA - Main', default_element_size=(50, 1)).Layout(layout)
#         while True:
#             main_button, values = main_window.Read()
#             if weights_url is not None:
#                 if values['download_weights'] and weights_url is not None:
#                     main_window['weights_dir'].Update(disabled = True)
#                     main_window['weights_dir_2'].Update(disabled = True)
#                 if not values['download_weights'] and weights_url is not None:
#                     main_window['weights_dir'].Update(disabled = False)
#                     main_window['weights_dir_2'].Update(disabled = False)
#             if values['cluster_correction']:
#                 main_window['min_extent'].Update(disabled = False)
#             if not values['cluster_correction']:
#                 main_window['min_extent'].Update(disabled = True)
#             if main_button == 'Quit':
#                 sys.exit()
#             elif main_button == 'Cancel':
#                 main_window.close()
#                 break
#             elif main_button == 'Submit':
#                 configured = True
#                 break
#     values['weights_url'] = weights_url
#     return values


text:
import logging
import math
from copy import deepcopy
from typing import Any, List, Optional, Union, Callable

import numpy as np
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, Dataset
from tqdm.auto import trange
from transformers import AdamW, get_linear_schedule_with_warmup, AutoTokenizer

from ..backbone import BertSeqTagger
from ..basemodel import BaseTorchSeqModel
from ..dataset.seqdataset import BaseSeqDataset
from ..utils import construct_collate_fn_trunc_pad

logger = logging.getLogger(__name__)

collate_fn = construct_collate_fn_trunc_pad('mask')

class BERTTorchSeqDataset(Dataset):
    def __init__(self, dataset: BaseSeqDataset, tokenizer, max_seq_length, use_crf, n_data: Optional[int] = 0):
        self.id2label = deepcopy(dataset.id2label)
        self.label2id = deepcopy(dataset.label2id)
        self.n_class = len(self.id2label)

        if not use_crf:
            self.dum_label = 'X'
            self.label2id[self.dum_label] = len(self.id2label)
            self.id2label.append(self.dum_label)

        self.tokenizer = tokenizer
        self.max_seq_length = max_seq_length  # set to -1 when test
        self.use_crf = use_crf

        corpus = list(map(lambda x: x["text"], dataset.examples))
        self.seq_len = list(map(len, corpus))
        input_ids_tensor, input_mask_tensor, predict_mask_tensor = self.convert_corpus_to_tensor(corpus)

        self.input_ids_tensor = input_ids_tensor
        self.input_mask_tensor = input_mask_tensor
        self.predict_mask_tensor = predict_mask_tensor

        n_data_ = len(input_ids_tensor)
        self.n_data_ = n_data_
        if n_data > 0:
            self.n_data = math.ceil(n_data / n_data_) * n_data_
        else:
            self.n_data = n_data_

    def __len__(self):
        return self.n_data

    def convert_corpus_to_tensor(self, corpus):
        input_ids_list = []
        input_mask_list = []
        predict_mask_list = []
        max_seq_length = 0

        for words in corpus:
            predict_mask = []
            input_mask = []
            tokens = []

            for i, w in enumerate(words):
                sub_words = self.tokenizer.tokenize(w)
                if not sub_words:
                    sub_words = [self.tokenizer.unk_token]
                if self.use_crf:
                    ''' if crf is used, then the padded token will be ignored '''
                    tokens.append(sub_words[0])
                else:
                    tokens.extend(sub_words)
                for j in range(len(sub_words)):
                    if j == 0:
                        input_mask.append(1)
                        predict_mask.append(1)
                    elif not self.use_crf:  # These padding will hurt performance
                        ''' '##xxx' -> 'X' (see bert paper, for non-crf model only) '''
                        input_mask.append(1)
                        predict_mask.append(0)

            max_seq_length = max(max_seq_length, len(tokens))
            input_ids_list.append(self.tokenizer.convert_tokens_to_ids(tokens))
            input_mask_list.append(input_mask)
            predict_mask_list.append(predict_mask)

        max_seq_length = min(max_seq_length, self.max_seq_length)

        n = len(input_ids_list)
        for i in range(n):
            ni = len(input_ids_list[i])
            if ni > max_seq_length:
                logger.info(f'Example is too long, length is {ni}, truncated to {max_seq_length}!')
                input_ids_list[i] = input_ids_list[i][:max_seq_length]
                input_mask_list[i] = input_mask_list[i][:max_seq_length]
                predict_mask_list[i] = predict_mask_list[i][:max_seq_length]
            else:
                input_ids_list[i].extend([self.tokenizer.pad_token_id] * (max_seq_length - ni))
                input_mask_list[i].extend([0] * (max_seq_length - ni))
                predict_mask_list[i].extend([0] * (max_seq_length - ni))

        input_ids_tensor = torch.LongTensor(input_ids_list)
        input_mask_tensor = torch.LongTensor(input_mask_list)
        predict_mask_tensor = torch.LongTensor(predict_mask_list)

        return input_ids_tensor, input_mask_tensor, predict_mask_tensor

    def prepare_labels(self, labels):
        O_id = self.label2id['O']
        if self.use_crf:
            n, max_seq_len = self.predict_mask_tensor.shape
            prepared_labels = np.ones((n, max_seq_len), dtype=int) * O_id
            for i, labels_i in enumerate(labels):
                ni = len(labels_i)
                if ni > max_seq_len:
                    prepared_labels[i, :] = labels_i[:max_seq_len]
                else:
                    prepared_labels[i, :ni] = labels_i
        else:
            prepared_labels = []
            add_label_id = self.label2id[self.dum_label]
            for labels_i, mask in zip(labels, self.predict_mask_tensor):
                pre_labels = []
                cnt = 0
                n = len(labels_i)
                for idx, flag in enumerate(mask):
                    if flag:
                        pre_labels.append(labels_i[cnt])
                        cnt += 1
                    else:
                        if n == cnt:
                            pre_labels.append(O_id)
                        else:
                            pre_labels.append(add_label_id)
                prepared_labels.append(pre_labels)

        return torch.LongTensor(prepared_labels)

    def __getitem__(self, idx):
        idx = idx % self.n_data_
        d = {
            'ids'           : idx,
            'input_ids'     : self.input_ids_tensor[idx],
            'attention_mask': self.input_mask_tensor[idx],
            'mask'          : self.predict_mask_tensor[idx],
        }
        return d

class BERTTaggerModel(BaseTorchSeqModel):
    def __init__(self,
                 model_name: Optional[str] = 'bert-base-uncased',
                 lr: Optional[float] = 2e-5,
                 l2: Optional[float] = 1e-6,
                 max_tokens: Optional[int] = 512,
                 batch_size: Optional[int] = 32,
                 real_batch_size: Optional[int] = 32,
                 test_batch_size: Optional[int] = 128,
                 n_steps: Optional[int] = 10000,
                 use_crf: Optional[bool] = False,
                 fine_tune_layers: Optional[int] = -1,
                 lr_crf: Optional[float] = 5e-5,
                 l2_crf: Optional[float] = 1e-8,
                 ):
        super().__init__()
        self.hyperparas = {
            'fine_tune_layers': fine_tune_layers,
            'model_name'      : model_name,
            'lr'              : lr,
            'l2'              : l2,
            'max_tokens'      : max_tokens,
            'batch_size'      : batch_size,
            'real_batch_size' : real_batch_size,
            'test_batch_size' : test_batch_size,
            'n_steps'         : n_steps,
            'use_crf'         : use_crf,
            'lr_crf'          : lr_crf,
            'l2_crf'          : l2_crf,
        }
        self.model = None
        self.tokenizer = AutoTokenizer.from_pretrained(model_name)

    def _init_valid_dataloader(self, dataset_valid: BaseSeqDataset) -> DataLoader:
        torch_dataset = BERTTorchSeqDataset(dataset_valid, self.tokenizer, 512, self.hyperparas['use_crf'])
        valid_dataloader = DataLoader(torch_dataset, batch_size=self.hyperparas['test_batch_size'], shuffle=False,
                                      collate_fn=collate_fn)
        return valid_dataloader

    def fit(self,
            dataset_train: BaseSeqDataset,
            y_train: Optional[List[List]] = None,
            dataset_valid: Optional[BaseSeqDataset] = None,
            y_valid: Optional[List[List]] = None,
            evaluation_step: Optional[int] = 50,
            metric: Optional[Union[str, Callable]] = 'f1_seq',
            strict: Optional[bool] = True,
            direction: Optional[str] = 'auto',
            patience: Optional[int] = 20,
            tolerance: Optional[float] = -1.0,
            device: Optional[torch.device] = None,
            verbose: Optional[bool] = True,
            **kwargs: Any):

        if not verbose:
            logger.setLevel(logging.ERROR)

        self._update_hyperparas(**kwargs)
        hyperparas = self.hyperparas
        if hyperparas['real_batch_size'] == -1 or hyperparas['batch_size'] < hyperparas['real_batch_size']:
            hyperparas['real_batch_size'] = hyperparas['batch_size']
        accum_steps = hyperparas['batch_size'] // hyperparas['real_batch_size']

        n_steps = hyperparas['n_steps']
        torch_dataset = BERTTorchSeqDataset(dataset_train, self.tokenizer, self.hyperparas['max_tokens'],
                                            self.hyperparas['use_crf'], n_data=n_steps * hyperparas['batch_size'])
        train_dataloader = DataLoader(torch_dataset, batch_size=hyperparas['real_batch_size'], shuffle=True, collate_fn=collate_fn)

        if y_train is None:
            y_train = dataset_train.labels
        y_train = torch_dataset.prepare_labels(y_train).to(device)

        n_class = dataset_train.n_class
        model = BertSeqTagger(n_class, **hyperparas).to(device)
        self.model = model

        param_optimizer = list(model.named_parameters())
        crf_param = ['crf.transitions', ]
        optimizer_grouped_parameters = [
            {'params': [p for n, p in param_optimizer if n not in crf_param]},
            {'params'      : [p for n, p in param_optimizer if n in crf_param], 'lr': hyperparas['lr_crf'],
             'weight_decay': hyperparas['l2_crf']},
        ]

        optimizer = AdamW(optimizer_grouped_parameters, lr=hyperparas['lr'], weight_decay=hyperparas['l2'])

        # Set up the learning rate scheduler
        scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=n_steps)

        valid_flag = self._init_valid_step(dataset_valid, y_valid, metric, strict, direction, patience, tolerance)

        history = {}
        last_step_log = {}
        try:
            with trange(n_steps, desc=f"[FINETUNE] {hyperparas['model_name']} Tagger", unit="steps", disable=not verbose, ncols=150, position=0, leave=True) as pbar:
                cnt = 0
                step = 0
                model.train()
                optimizer.zero_grad()
                for batch in train_dataloader:

                    batch_idx = batch['ids'].to(device)
                    batch_label = y_train[batch_idx]
                    loss = model.calculate_loss(batch, batch_label)
                    loss.backward()
                    cnt += 1

                    if cnt % accum_steps == 0:
                        # Clip the norm of the gradients to 1.0.
                        nn.utils.clip_grad_norm_(model.parameters(), 1.0)
                        optimizer.step()
                        scheduler.step()
                        optimizer.zero_grad()
                        step += 1

                        if valid_flag and step % evaluation_step == 0:
                            metric_value, early_stop_flag, info = self._valid_step(step)
                            if early_stop_flag:
                                logger.info(info)
                                break

                            history[step] = {
                                'loss'              : loss.item(),
                                f'val_{metric}'     : metric_value,
                                f'best_val_{metric}': self.best_metric_value,
                                'best_step'         : self.best_step,
                            }
                            last_step_log.update(history[step])

                        last_step_log['loss'] = loss.item()
                        pbar.update()
                        pbar.set_postfix(ordered_dict=last_step_log)

                        if step >= n_steps:
                            break

        except KeyboardInterrupt:
            logger.info(f'KeyboardInterrupt! do not terminate the process in case need to save the best model')

        self._finalize()

        return history


text:
import random

class TerrainMaker(object):
    def __init__(self, width=100, height=100):
        self.Width = width
        self.Height = height
        self.StoredTerrainsDict = {}

    def storeTerrain(self, Terrain):
        self.StoredTerrainsDict[Terrain.Name] = Terrain

    def writeToCSV(self, Terrain, filename=""):
        if filename == "":
            filename = f"terrain_gen[{self.Width}, {self.Height}]" + ".csv"
        elif filename[-4:] != ".csv":
            filename += ".csv"

        with open(filename, "w+") as file:
            for row in range(Terrain.Height):
                for col in range(Terrain.Width):
                    file.write(str(Terrain.TerrainMatrix[row][col]) + ", ")
                file.write('\n')

    def makeRandomTerrain2D(self, floor=0, ceiling=100, terrainName=""):
        """
        Generates a 2D array of random 3-D topographic-style values
        """
        if terrainName == "":
            terrainName = f"random [{self.Width}, {self.Height}]"
        m = []

        for _ in range(self.Height):
            row = []
            for _ in range(self.Width):
                alt = random.random() * (ceiling - floor)
                row.append(alt)
            m.append(row)

        return Terrain(terrainName, m)

    def makePeakTerrain(self, floor=0, ceiling=100, terrainName=""):
        """
        Generates a 2-D array of 3-D topographic-style values with a center peak
        """
        if terrainName == "":
            terrainName = f"peak [{self.Width}, {self.Height}]"
        m = []
        halfH = (self.Height / 2)
        halfW = (self.Width / 2)
        for i in range(self.Height):
            row = []
            rowFactor = 1 - abs((halfH - i)) / halfH
            for j in range(self.Width):
                colFactor = 1 - abs((halfW - j)) / halfW
                peakFactor = colFactor * rowFactor
                alt = (ceiling - floor) * peakFactor
                row.append(alt)
            m.append(row)

        return Terrain(terrainName, m)

class Terrain(object):
    def __init__(self, name, terrainMatrix):
        self.Name = name
        self.TerrainMatrix = terrainMatrix
        self.Width = len(terrainMatrix[0])
        self.Height = len(terrainMatrix)

    def getPointForm(self):
        cols = []

        for i in range(self.Height):
            for j in range(self.Width):
                row = []
                row.append(j)
                row.append(i)
                row.append(self.TerrainMatrix[i][j])
                cols.append(row)

        return cols

    def preview(self, numRows=10, numCols=10):
        r = numRows if numRows < self.Width else self.Width
        c = numCols if numCols < self.Height else self.Height
        for i in range(r):
            print("[", end="")
            for j in range(c-1):
                print(round(self.TerrainMatrix[i][j]), end=", ")
            if i != r-1:
                print(f"{round(self.TerrainMatrix[i][c])}],\n", end="")
            else:
                print(f"{round(self.TerrainMatrix[i][c])}]")

def makeTerrainPlotHtml():
    import numpy as np
    import plotly.offline as go_offline
    import plotly.graph_objects as go

    file=open('H:\\dev\\pile\\src\\data_gen\\survey_data.csv','r')
    lines=file.readlines()
    n_line=len(lines)
    x=[]
    y=[]
    z=[]
    for i in range(1,n_line):
        split_line=lines[i].split(",")
        x.append(float(split_line[0].rstrip()))
        y.append(float(split_line[1].rstrip()))
        z.append(float(split_line[2].rstrip()))

    def distance(x1,y1,x2,y2):
        d = ((x1-x2)**2 + (y1-y2)**2)**(1/2)
        return d

    #CREATING IDW FUNCTION
    def idw_npoint(xz, yz, n_point, p):
        r=10 #block radius iteration distance
        nf=0
        while nf<=n_point: #will stop when np reaching at least n_point
            x_block=[]
            y_block=[]
            z_block=[]
            r +=10 # add 10 unit each iteration
            xr_min=xz-r
            xr_max=xz+r
            yr_min=yz-r
            yr_max=yz+r
            for i in range(len(x)):
                # condition to test if a point is within the block
                if ((x[i]>=xr_min and x[i]<=xr_max) and (y[i]>=yr_min and y[i]<=yr_max)):
                    x_block.append(x[i])
                    y_block.append(y[i])
                    z_block.append(z[i])
            nf=len(x_block) #calculate number of point in the block

        #calculate weight based on distance and p value
        w_list=[]
        for j in range(len(x_block)):
            d=distance(xz,yz,x_block[j],y_block[j])
            if d>0:
                w=1/(d**p)
                w_list.append(w)
            else:
                w_list.append(0) #if meet this condition, it means d<=0, weight is set to 0

        #check if there is 0 in weight list
        w_check=0 in w_list
        if w_check==True:
            idx=w_list.index(0) # find index for weight=0
            z_idw=z_block[idx] # set the value to the current sample value
        else:
            wt=np.transpose(w_list)
            z_idw=np.dot(z_block,wt)/sum(w_list) # idw calculation using dot product
        return z_idw

    # POPULATE INTERPOLATION POINTS
    n=100 #number of interpolation point for x and y axis
    x_min=min(x)
    x_max=max(x)
    y_min=min(y)
    y_max=max(y)
    w=x_max-x_min #width
    h=y_max-y_min #length
    wn=w/n #x interval
    hn=h/n #y interval

    #list to store interpolation point and elevation
    y_init=y_min
    x_init=x_min
    x_idw_list=[]
    y_idw_list=[]
    z_head=[]
    for i in range(n):
        xz=x_init+wn*i
        yz=y_init+hn*i
        y_idw_list.append(yz)
        x_idw_list.append(xz)
        z_idw_list=[]
        for j in range(n):
            xz=x_init+wn*j
            z_idw=idw_npoint(xz,yz,5,1.5) #min. point=5, p=1.5
            z_idw_list.append(z_idw)
        z_head.append(z_idw_list)

    # CREATING 3D TERRAIN
    fig = go.Figure()
    fig.add_trace(go.Surface(z=z_head,x=x_idw_list,y=y_idw_list))
    fig.update_layout(scene=dict(aspectratio=dict(x=2, y=2, z=0.5), xaxis=dict(range=[x_min,x_max],), yaxis=dict(range=[y_min, y_max])))
    go_offline.plot(fig, filename='H:\\dev\\pile\\src\\data_gen\\3d_terrain.html', validate=True, auto_open=False)

makeTerrainPlotHtml()

max_stars_repo_name: XDwightsBeetsX/pile

max_issues_repo_name: XDwightsBeetsX/pile

max_forks_repo_name: XDwightsBeetsX/pile


text:
# -*- coding: utf-8 -*-

# This code is part of Qiskit.
#
# (C) Copyright IBM 2020.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

"""Pulses are descriptions of waveform envelopes. They can be transmitted by control electronics
to the device.
"""
import warnings
from typing import Callable, Optional
from abc import ABC, abstractmethod

import numpy as np

from ..channels import PulseChannel
from ..exceptions import PulseError
from ..instructions.play import Play

class Pulse(ABC):
    """The abstract superclass for pulses. Pulses are complex-valued waveform envelopes. The
    modulation phase and frequency are specified separately from ``Pulse``s.
    """

    @abstractmethod
    def __init__(self, duration: int, name: Optional[str] = None):
        if not isinstance(duration, (int, np.integer)):
            raise PulseError('Pulse duration should be integer.')
        self.duration = int(duration)
        self.name = (name if name is not None
                     else '{}{}'.format(str(self.__class__.__name__).lower(),
                                        self.__hash__()))

    def __call__(self, channel: PulseChannel) -> Play:
        """Return new ``Play`` instruction that is fully instantiated with both ``pulse`` and a
        ``channel``.

        Args:
            channel: The channel that will have the pulse.

        Return:
            Complete and ready to schedule ``Play``.
        """
        warnings.warn("Calling a ``Pulse`` with a channel is deprecated. Instantiate ``Play`` "
                      "directly with a pulse and a channel.", DeprecationWarning)
        return Play(self, channel)

    @abstractmethod
    def draw(self, dt: float = 1,
             style=None,
             filename: Optional[str] = None,
             interp_method: Optional[Callable] = None,
             scale: float = 1, interactive: bool = False,
             scaling: float = None):
        """Plot the interpolated envelope of pulse.

        Args:
            dt: Time interval of samples.
            style (Optional[PulseStyle]): A style sheet to configure plot appearance
            filename: Name required to save pulse image
            interp_method: A function for interpolation
            scale: Relative visual scaling of waveform amplitudes
            interactive: When set true show the circuit in a new window
                (this depends on the matplotlib backend being used supporting this)
            scaling: Deprecated, see `scale`

        Returns:
            matplotlib.figure: A matplotlib figure object of the pulse envelope
        """
        raise NotImplementedError

    @abstractmethod
    def __eq__(self, other: 'Pulse') -> bool:
        return isinstance(other, type(self))

    @abstractmethod
    def __hash__(self) -> int:
        raise NotImplementedError

max_stars_repo_name: AdityaZade/qiskit-terra

max_issues_repo_name: ismetshn/qiskit-terra

max_forks_repo_name: ismetshn/qiskit-terra


text:
# Erick Juarez CPSC 479 SEC 1
# This is the main python file containing the K-nearest neighbor classifier

# Import libraries
import numpy as np
import pandas as pd
from mpi4py import MPI      # MPI_Init() automatically called when you import MPI from library
from math import sqrt, ceil

# Distance function returns Euclidean distance between two datapoints with same number of features
def getDistance(x, y):
    distance = 0
    for i in range(len(x) - 1):
        distance += ((x[i] - y[i]) ** 2)
    return sqrt(distance)

# Initialize multiprocess communication, each process is identified by it's rank
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()

# Initialize global vairables
training_set = []
partitions = []
testing_set = []
distances = []
k = 15

# Tasks for the master rank
if rank == 0:
    # Import training and testing data
    df = pd.read_csv("training_set.csv")
    training_set = df[['MovingTime', 'Distance', 'MaxSpeed', 'ElevationGain', 'Class']].to_numpy()
    df = pd.read_csv("testing_set.csv")
    testing_set = df[['MovingTime', 'Distance', 'MaxSpeed', 'ElevationGain', 'Class']].to_numpy()

    # Partition training data to send to worker processes
    pts_per_proc = len(training_set)//size

    for i in range(size):
        partitions.append([])
    for i in range(len(training_set)):
        if i < pts_per_proc:
            partitions[0].append(training_set[i])
        elif i < (pts_per_proc * 2):
            partitions[1].append(training_set[i])
        elif i < (pts_per_proc * 3):
            partitions[2].append(training_set[i])
        else:
            partitions[3].append(training_set[i])

# Scatter training data to worker porcesses
partitions = comm.scatter(partitions, root=0)
# Broadcast testing data to worker processes
testing_set = comm.bcast(testing_set, root=0)

# Each process calculataes distance from testing data to training data
for point in partitions:
    distances.append(getDistance(point, testing_set[0]))

# Gather results from worker processes
distances = comm.gather(distances, root=0)

# Master process does the rest of the work
if rank == 0:
    # Choose k closest points to training point
    closest = []    # array of closest indices
    k_dist = []     # distance of the k closest neigbors
    dist_vect = []  # distances from training set and testing point

    # Reshape data gathered from other processors
    for i in range(len(distances)):
        for j in range(len(distances[i])):
            dist_vect.append(distances[i][j])

    # create a copy of distance array and sort it to get closest values
    k_dist = dist_vect.copy()
    k_dist = np.sort(k_dist)

    # Find the corresponding datapoints from closest values
    for i in range(k):
        for j in range(len(dist_vect)):
            if k_dist[i] == dist_vect[j]:
                closest.append(j)

    # Classify test point
    classes = [0, 0, 0]
    for i in closest:
        if training_set[i][4] == 0:
            classes[0] += 1
        elif training_set[i][4] == 1:
            classes[1] += 1
        elif training_set[i][4] == 2:
            classes[2] += 1
    predicted = classes.index(max(classes))
    actual = int(testing_set[0][4])

    # print results
    print("---------------------------------------------")
    print("     Training points -", len(dist_vect))
    print("                   K -", k)
    if classes[0] != 0:
        print("Neighbors in class 0 -", classes[0])
    if classes[1] != 0:
        print("Neighbors in class 1 -", classes[1])
    if classes[2] != 0:
        print("Neighbors in class 2 -", classes[2])
    print("     Predicted class -", predicted)
    print("        Actual class -", actual)
    print("---------------------------------------------")


text:
# import the necessary packages
from keras.preprocessing import image as image_utils
from imagenet_utils import decode_predictions
from imagenet_utils import preprocess_input
from vgg16 import VGG16
import numpy as np
import argparse
import cv2
from keras.utils import np_utils
import matplotlib.pyplot as plt
from matplotlib import pyplot as plt

# construct the argument parse and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required=True,
                help="path to the input image")
args = vars(ap.parse_args())

# load the original image via OpenCV so we can draw on it and display
# it to our screen later
orig = cv2.imread(args["image"])
#cv2.imshow("test",orig)

# load the input image using the Keras helper utility while ensuring
# that the image is resized to 224x224 pxiels, the required input
# dimensions for the network -- then convert the PIL image to a
# NumPy array
print("[INFO] loading and preprocessing image...")
image = image_utils.load_img(args["image"], target_size=(224, 224))
image = image_utils.img_to_array(image)

# our image is now represented by a NumPy array of shape (3, 224, 224),
# but we need to expand the dimensions to be (1, 3, 224, 224) so we can
# pass it through the network -- we'll also preprocess the image by
# subtracting the mean RGB pixel intensity from the ImageNet dataset
image = np.expand_dims(image, axis=0)
image = preprocess_input(image)

# load the VGG16 network
print("[INFO] loading network...")
model = VGG16(weights="imagenet")

# classify the image
print("[INFO] classifying image...")
preds = model.predict(image)
result = decode_predictions(preds, top=1)

(inID, label, val) = decode_predictions(preds)[0][0]
print(result[0])
print(len(result))
#result1 = ([col.strip() for col in part] for part in result)
#print(result1)
#print(decode_predictions(preds)[0])
# display the predictions to our screen
print("ImageNet ID: {}, Label: {}".format(inID, label))

cv2.putText(orig, "Label: {}".format(label), (10, 30),
            cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 255, 0), 2)

plt.ioff()
plt.imshow(orig)

plt.pause(1)
plt.show()

#cv2.imshow("Classification", orig)
#cv2.waitKey(0)

P = decode_predictions(preds)
(imagenetID, label, prob) = P[0][0]
#plt.show()


text:
import tensorflow as tf
import tensorflow_hub as hub
import os
import pickle
import datetime
import itertools
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from IPython.display import display, Image
%load_ext tensorboard
IMG_SIZE = 224
BATCH_SIZE = 32

def show_25_images(images, labels):
  plt.figure(figsize=(12, 12))

  for i in range(25):
    ax = plt.subplot(5, 5, i+1)
    plt.imshow(images[i])
    plt.axis("off")
    plt.title(unique_labels[labels[i].argmax()])

def process_image(image_path):
  image = tf.io.read_file(image_path)
  image = tf.image.decode_jpeg(image, channels=3)
  image = tf.image.convert_image_dtype(image, tf.float32)
  image = tf.image.resize(image, size=[IMG_SIZE, IMG_SIZE])
  return image

def train_model():
  model = create_model()
  tensorboard = create_tensorboard_callback()
  model.fit(x=train_data,
            epochs=30,
            validation_data=val_data,
            validation_freq=1,
            callbacks=[tensorboard, early_stopping])

  return model

def create_model(input_shape, output_shape, model_url):
  model = tf.keras.Sequential([
    hub.KerasLayer(MODEL_URL),
    tf.keras.layers.Dense(units=OUTPUT_SHAPE,   activation="softmax")
  ])

  model.compile(
      loss=tf.keras.losses.CategoricalCrossentropy(),
      optimizer=tf.keras.optimizers.Adam(),
      metrics=["accuracy"]
  )

  model.build(INPUT_SHAPE)

  return model

def unbatchify(data):
    data_as_list = list(data.unbatch().as_numpy_iterator())
    images = [ ]
    true_labels = [ ]
    for image, label in data_as_list:
      images.append(image)
      true_labels.append(get_pred_label(label))
    return images, true_labels

def get_image_label(image_path, label):
  """
  Takes an image file path name and the associated label,
  processes the image and returns a tuple of (image, label).
  """
  image = process_image(image_path)
  return image, label

def get_pred_label(prediction_probabilities):
  """  Turns an array of prediction probabilities into a label. """
  return unique_labels[np.argmax(prediction_probabilities)]

def plot_pred_conf(prediction_probabilities, labels, n=1):
  pred_prob, true_label = prediction_probabilities[n], labels[n]

  pred_label = get_pred_label(pred_prob)

  top_10_pred_indexes = pred_prob.argsort()[-10:][::-1]
  top_10_pred_values = pred_prob[top_10_pred_indexes]
  top_10_pred_labels = unique_labels[top_10_pred_indexes]

  top_plot = plt.bar(np.arange(len(top_10_pred_labels)),
                     top_10_pred_values,
                     color="grey")
  plt.xticks(np.arange(len(top_10_pred_labels)),
             labels=top_10_pred_labels,
             rotation="vertical")

  if np.isin(true_label, top_10_pred_labels):
    top_plot[np.argmax(top_10_pred_labels == true_label)].set_color("green")
  else:
    pass

def create_tensorboard_callback():
  logdir = os.path.join("/content/drive/My Drive/Colab Notebooks", datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
  return tf.keras.callbacks.TensorBoard(logdir)

print("TF version is : ", tf.__version__)
print("TF Hub version is : ", hub.__version__)

def plot_predicted_truth_labels(image_list, true_labels_list, predictions, index):
    true_label = true_labels_list[index]
    predicted_label = get_pred_label(predictions[index])
    prediction_probabilty = np.max(predictions[index])
    plt.imshow(image_list[index])
    plt.xticks([])
    plt.yticks([])
    color = 'green' if true_label == predicted_label else 'red'
    plt.title("{} / {} / {}".format(true_label,
                                    predicted_label,
                                    prediction_probabilty),
                                    color=color)

def create_data_batches(x, y=None, batch_size=BATCH_SIZE, valid_data=False, test_data=False):
  if test_data:
    data = tf.data.Dataset.from_tensor_slices((tf.constant(x)))
    data = data.map(process_image)
    data_batch = data.batch(BATCH_SIZE)
    return data_batch

  elif valid_data:
    data = tf.data.Dataset.from_tensor_slices((tf.constant(x), tf.constant(y)))
    data = data.map(get_image_label)
    data_batch = data.batch(BATCH_SIZE)
    return data_batch

  else:
    data = tf.data.Dataset.from_tensor_slices((tf.constant(x), tf.constant(y)))
    data = data.shuffle(buffer_size=len(x))
    data = data.map(get_image_label)
    data_batch = data.batch(BATCH_SIZE)
  return data_batch

if __name__=='__main__':
  if tf.config.list_physical_devices("GPU"):
    print('GPU Used')

  labels_csv = pd.read_csv('/content/labels.csv')
  filenames = ['/content/train/'+file+'.jpg' for file in labels_csv.id]

  string_labels = np.array(labels_csv.breed)

  unique_labels = np.unique(string_labels)

  string_labels[0] == unique_labels

  boolean_labels = [label == np.array(unique_labels) for label in string_labels]

  X = filenames
  y = boolean_labels
  NUM_IMAGES = 1000
  NUM_IMAGES
  X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2)

  train_data = create_data_batches(X_train, y_train)
  val_data = create_data_batches(X_valid, y_valid, valid_data=True)
  train_images, train_labels = next(train_data.as_numpy_iterator())
  show_25_images(train_images, train_labels)
  INPUT_SHAPE = [None, IMG_SIZE, IMG_SIZE, 3]

  OUTPUT_SHAPE = len(unique_labels)

  MODEL_URL = "https://tfhub.dev/google/imagenet/mobilenet_v2_130_224/classification/4"
  model = create_model(INPUT_SHAPE,OUTPUT_SHAPE,MODEL_URL)
  model.summary()
  early_stopping = tf.keras.callbacks.EarlyStopping(monitor="val_accuracy",
                                                  patience=3)
  predictions  = model.predict(val_data)
  index = 0

  images, true_labels = unbatchify(val_data)
  plot_predicted_truth_labels(images, true_labels, predictions, 7)
  print(unique_labels)
  plot_pred_conf(prediction_probabilities=predictions, labels=true_labels,   n=9)
  model_path = "/content/drive/My Drive/Colab Notebooks/mobilnetV2-9000-images1.h5"
  print(f"Saving model to: {model_path}...")
  model.save(model_path)