text:
import config
import os
import cv2
import lmdb
import random
import mtcnn_pb2
import numpy as np

class BatchLoader:
    def __init__(self, param, data_cat):
        self.net = param['net']
        self.img_size = config.NET_IMG_SIZES[self.net]
        self.data_cat = data_cat
        self.db_envs = {}
        self.db_path = os.path.join(config.DB_DIR % self.data_cat, self.net)

        self.db_entries = {}

        self.label_ratio = {}
        self.db_indices = {}

        self.total_entries = 0
        self.total_labels = 0
        self.total_bboxes = 0
        self.total_landm5 = 0
        for dt in config.DATA_TYPES:
            self.db_envs[dt]= lmdb.open("%s/%s.lmdb" % (self.db_path, dt),
                                        map_size=config.DB_MAPSIZES[dt])
            self.db_entries[dt] = self.db_envs[dt].stat()['entries']
            self.total_entries += self.db_entries[dt]
            self.db_indices[dt] = 0

        for dt in config.DATA_TYPES:
            self.label_ratio[dt] = self.db_entries[dt] / self.total_entries

        self.task_dts_map = {'label': ['pos', 'neg', 'landm5'],
                             'bbox': ['pos', 'part'],
                             'land5': ['landm5']}
        self.task_dts_ratios =  config.DATA_TYPE_RATIOS[self.net]

    def is_loaded():
        return self.total_entries > 0

    def next_batch(self, batch, task):
        # return self.get_data(self.task_dts_map[task], batch)
        return self.get_data(self.task_dts_ratios.keys(), batch)

    def get_data(self, dts, batch):
        # total = sum(self.db_entries[dt] for dt in dts)
        # self.ratios = {dt:float(self.db_entries[dt]) / total for dt in dts}
        data = []
        for k, dt in enumerate(dts):
            if k == len(dts) - 1:
                sample_num = int(batch - len(data))
            else:
                sample_num = max(round(batch * self.task_dts_ratios[dt]), 1)
                # sample_num = max(round(batch * ratios[dt]), 1)
            with self.db_envs[dt].begin() as txn:
                cursor = txn.cursor()
                for i in range(0, sample_num):
                    if self.db_indices[dt] >= self.db_entries[dt]:
                        self.db_indices[dt] = 0
                    cursor.set_key(str(self.db_indices[dt]).encode())
                    datum = mtcnn_pb2.Datum()
                    datum.ParseFromString(cursor.value())
                    img = self.recover_img(datum)
                    bbox = datum.bbox
                    if random.choice([0, 1]) == 1:
                        if dt == 'neg':
                            img = cv2.flip(img, random.choice([-1, 0, 1]))
                        # elif dt == 'pos' or dt == 'part':
                        #     img = cv2.flip(img, 1)
                        #     bbox = [bbox[0], bbox[3], bbox[2], bbox[1]]
                    if len(datum.landm5) != 10:
                        print("invalid landm5 ", datum.landm5)
                        continue
                    data.append([img, datum.label, bbox, datum.landm5])
                    self.db_indices[dt] += 1
        return data

    def recover_img(self, datum):
        return np.frombuffer(datum.img).reshape((3, self.img_size, self.img_size))


text:
"""implementation of a basic classifier class which allows to use different algorithms in the same framework."""
# classifiers
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier

# preprocessing
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_val_score

# Feature/model selection
from sklearn.decomposition import PCA
from sklearn.model_selection import GridSearchCV
from scipy.cluster import hierarchy
#from sklearn.feature_selection import RFECV

# visualization
from sklearn.metrics import classification_report
import matplotlib.pyplot as plt
import seaborn as sns

# metrics
from sklearn.metrics import f1_score
from sklearn.metrics import r2_score
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import average_precision_score
from sklearn.calibration import calibration_curve

# third party libararies
import numpy as np
import pathlib
import pandas as pd
import pickle
from collections import defaultdict

# own libraries
from . import utils
from . import plotting

classifier_dict = {'lr': LogisticRegression, 'rf': RandomForestClassifier}

gridcv_parameters_dict = {'lr':
                          {'cls__solver': 'liblinear',
                           'cls__C': np.arange(0, 1, 0.1),
                           'cls__penalty': ["l1", "l2"]},
                          'rf':
                          {'cls__max_depth': [50, 100, None],
                              'cls__max_features': ['auto', 'log2'],
                              'cls__min_samples_leaf': [1, 3, 5],
                              'cls__min_samples_split': [2, 4, 6, 10],
                              'cls__n_estimators': [300, 500, 1000],
                              'cls__oob_score': [True]}}

tuned_parameters_dict = {'lr':
                         {'cls__solver': 'liblinear',
                          'cls__C': 0.1,
                          'cls__penalty': "l1"},
                         'rf':
                         {'cls__max_depth': 50,
                             'cls__max_features': 'auto',
                             'cls__min_samples_leaf': 5,
                             'cls__min_samples_split': 4,
                             'cls__n_estimators': 500,
                             'cls__oob_score': True}}

class Classifier():
    def __init__(self, classifier='lr', name='', **kwargs):
        """Initialization of a new Classifier object. This requires the name of classifier."""
        if name:
            self.name = name
        else:
            self.name = classifier
        self.classes = ['non-pathogenic', 'pathogenic']
        self.pipeline = Pipeline(steps=([('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')), (
            'scaler', MinMaxScaler()), ('cls', classifier_dict[classifier](**kwargs))]))
        self.trained = False

    def feature_selection(self, train_set, feature_type, save=False, output_dir='./'):
        """Visualize the correlation in between the features."""
        labels = train_set[0].columns

        # scale and impute the training data
        scaled_train_df = self.pipeline[0:2].fit_transform(train_set[0])

        output_path = pathlib.Path(output_dir)

        # compute Pearson correlation between features
        # TODO: update phi_coeff for numpy array
        # if 'binary' in feature_type:
        #     cor = scaled_train_df.corr(utils.phi_coeff)
        # else:
        cor = np.corrcoef(scaled_train_df, rowvar=False)

        fig = plotting.plot_correlation_heatmap(cor, labels)
        plt.savefig(output_path / f'{self.name}_feature_correlations.png')

        # compute the partial correlation for all features
        P_corr = utils.partial_corr(scaled_train_df, feature_type)

        # plot partial correlation heatmap
        fig = plotting.plot_correlation_heatmap(P_corr, labels)
        plt.savefig(
            output_path / f'{self.name}_feature_partial_correlations.png')

        # plot partial correlation as node graph
        P_corr = pd.DataFrame(P_corr, columns=labels)
        P_corr.rename(
            {idx: label for idx, label in enumerate(labels)}, inplace=True)
        fig = plotting.plot_correlation_node_graph(P_corr)
        fig.savefig(
            output_path / f'{self.name}_correlation_node_graph.png', bbox_inches='tight')

        # plot correlation dendrogram based on hierachical clustering
        plt.figure(figsize=(10, 12))
        cor_linkage = hierarchy.ward(cor)
        dendro = hierarchy.dendrogram(cor_linkage, labels=train_set[0].columns,
                                      leaf_rotation=90)
        plt.tight_layout()
        plt.savefig(output_path / f'{self.name}_correlation_dendogram.png')

        # PCA
        # plt.figure(figsize=(12,10))
        # pca = PCA(n_components=2)
        # projected = pca.fit_transform(scaled_train_df)
        # colors = ['red' if label else 'green' for label in merged_labels]
        # plt.scatter(projected[:, 0], projected[:, 1],
        #             c=colors, edgecolor='none', alpha=0.05)
        # plt.xlabel('PC 1')
        # plt.ylabel('PC 2')
        # plt.yticks(np.arange(train_set[0].shape[1])+0.5, va="center")
        # plt.tight_layout()
        # plt.savefig(pathlib.Path(output_dir) / f'{self.name}_PCA.png')

    def train(self, train_set, output_dir='./', permut_importance=True, gridcv=False):
        skf = StratifiedKFold(n_splits=5)

        # extract feature names
        names = train_set[0].columns

        # transform panda dataframes into np.arrays
        if any([type(set) == pd.DataFrame for set in train_set]):
            train_set = [set.values for set in train_set]

        if gridcv:
            # Initialize GridSearch object
            gscv = GridSearchCV(
                self.pipeline, gridcv_parameters_dict[self.name], n_jobs=3, cv=skf, verbose=1, scoring="roc_auc")
            # Fit gscv
            print(f"Tuning {self.name}...")
            gscv.fit(train_set[0], train_set[1])

            # Get best parameters and score
            print(f'Best parameters: {gscv.best_params_}')
            print(f'Best ROC-AUC score {gscv.best_score_}')

            # train pipeline on all data with the best parameters
            self.pipeline.set_params(**gscv.best_params_)
            self.pipeline.fit(train_set[0], train_set[1])
        else:
            # set classifier params
            self.pipeline.set_params(**tuned_parameters_dict[self.name])
            # compute cross validation score
            roc_auc_scores = cross_val_score(
                self.pipeline, train_set[0], train_set[1], cv=skf, scoring='roc_auc')
            print(f'5-fold CV ROC-AUC Mean: {np.mean(roc_auc_scores)}')
            self.pipeline.fit(train_set[0], train_set[1])

        self.trained = True

        # save model to pickle file
        with open(pathlib.Path(output_dir) / f'{self.name}_model.p', 'wb') as model_output:
            pickle.dump(self, model_output)

        # print feature importance by decreased accuracy
        if permut_importance:
            # scale and impute training data
            scaled_train_df = self.pipeline[0:2].transform(train_set[0])

            # compute pearson correlation and cluster feature with a distance <= 1
            cor = np.corrcoef(scaled_train_df, rowvar=False)
            corr_linkage = hierarchy.ward(cor)
            cluster_ids = hierarchy.fcluster(
                corr_linkage, 1, criterion='distance')

            # save clustered feature indices
            cluster_id_to_feature_ids = defaultdict(list)
            for idx, cluster_id in enumerate(cluster_ids):
                cluster_id_to_feature_ids[cluster_id].append(idx)

            # iterate through clusters, permutating the corresponding features
            acc = utils.oob_classifier_accuracy(
                self.pipeline, train_set[0], train_set[1])
            scores = defaultdict(list)
            for i in np.random.choice(100, 30, replace=False):
                np.random.seed(i)
                for cluster, feature_idx in cluster_id_to_feature_ids.items():
                    X_t = train_set[0].copy()
                    X_t[:, feature_idx] = np.random.permutation(
                        X_t[:, feature_idx])
                    shuff_acc = utils.oob_classifier_accuracy(
                        self.pipeline, X_t, train_set[1])
                    scores[cluster].append(acc - shuff_acc)
                mean_scores = {cluster:[np.mean(score),np.std(score)] for cluster, score in scores.items()}

            # sort the scores and plot them in decending order
            sorted_scores, sorted_clusters, sorted_std = list(zip(
                *sorted([(round(score[0], 4), cluster, round(score[1], 4)) for cluster, score in mean_scores.items()], reverse=False)))
            feature_idx = [cluster_id_to_feature_ids[cluster_id]
                           for cluster_id in sorted_clusters]
            feature_names = [[names[idx] for idx in feature_list]
                             for feature_list in feature_idx]
            feature_ticks = ['\n'.join(features) for features in feature_names]
            heights = [len(features) for features in feature_names]
            coordinates = np.arange(len(feature_ticks))
            fig = plt.figure(figsize=(12, 10))
            plt.barh(coordinates, sorted_scores, xerr=sorted_std, color='#007878')
            plt.yticks(coordinates, feature_ticks)
            plt.xlabel('Mean-Decrease in Accuracy')
            plt.gca().spines['top'].set_visible(False)
            plt.gca().spines['right'].set_visible(False)
            plt.tight_layout()
            plt.savefig(pathlib.Path(output_dir) /
                        'feature_importance_acc_decrease.png')

        # # plot roc auc as a function of added features by importance
        # features = []
        # roc_auc_scores = []
        # for scores, feat in sorted_scores:
        #     features.append(np.argwhere(names == feat).flat[0])
        #     X_t = train_set[0][:,features]
        #     roc_auc_scores.append(cross_val_score(self.pipeline, X_t, train_set[1], cv=skf, scoring='roc_auc'))
        # plt.figure(figsize=(12,10))
        # plt.plot(coordinates,[np.mean(roc_auc_score) for roc_auc_score in roc_auc_scores],color='#007878')
        # plt.xticks(coordinates,[feat for scores, feat in sorted_scores],rotation=90)
        # plt.tight_layout()
        # plt.savefig(pathlib.Path(output_dir) / 'roc_auc_over_features.png')

    def test(self, test_set, save=False, output_dir='./', plot=False):
        print('testing...')
        if self.trained:
            # predict class labels and probabilities
            y_pred = self.pipeline.predict(test_set[0])
            y_pred_scores = self.pipeline.predict_proba(test_set[0])[:, 1]

            if save:
                with open(pathlib.Path(output_dir) /'ROC_scores.txt','w') as test_output:
                    roc_scores = [f"{test_set[1].values[idx]}\t{pred}" for idx, pred in enumerate(y_pred_scores)]
                    test_output.write("\n".join(roc_scores))

            # report classification metrics
            print(
                f'ROC AUC on test-set: {roc_auc_score(test_set[1],y_pred_scores)}')
            print(classification_report(
                test_set[1], y_pred, target_names=self.classes))

            # calculate roc curve
            fpr, tpr, thresholds = roc_curve(test_set[1], y_pred_scores)
            auc = roc_auc_score(test_set[1], y_pred_scores)

            if plot:
                # plot confusion matrix
                plotting.plot_confusion_matrix(test_set[1], y_pred, classes=[
                                               'Non-pathogenic', 'Pathogenic'], cmap=plt.cm.BuGn)
                if save:
                    plt.tight_layout()
                    plt.savefig(pathlib.Path(output_dir) /
                                f'{self.name}_Confusion_Matrix.png')
                # plot feature importance
                if self.name == 'rf':
                    coef = self.pipeline['cls'].feature_importances_
                    ylabel = 'Feature Importance'
                else:
                    coef = self.pipeline['cls'].coef_[0]
                    ylabel = 'Model Coefficients'
                plt.figure(figsize=(12, 10))
                feature_index = np.arange(len(coef))
                plt.bar(feature_index, coef, color='#007878')
                plt.ylabel(ylabel, fontsize=25)
                plt.tick_params(labelsize=25)
                if len(coef) < 5:
                    plt.xticks(ticks=feature_index, labels=test_set[0].columns)
                else:
                    plt.xticks(ticks=feature_index,
                               labels=test_set[0].columns, rotation='vertical')
                if save:
                    plt.tight_layout()
                    plt.savefig(pathlib.Path(output_dir) /
                                f'{self.name}_Coefficients.png')

                # plot the roc curve
                plt.figure(figsize=(12, 10))
                plt.plot(fpr, tpr, color='#007878', label='Random Forest')
                plt.plot([0, 1], [0, 1], color='grey', lw=2, linestyle='--')
                # axis labels
                plt.xlabel('False Positive Rate')
                plt.ylabel('True Positive Rate')
                # show the legend
                plt.legend()
                if save:
                    plt.tight_layout()
                    plt.savefig(pathlib.Path(output_dir) /
                                f'{self.name}_ROC.png')

                # plot calibration curve
                plt.rcdefaults()
                plt.figure(figsize=(12, 10))
                ax1 = plt.subplot2grid((3, 1), (0, 0), rowspan=2)
                ax2 = plt.subplot2grid((3, 1), (2, 0))
                ax1.plot([0, 1], [0, 1], "k:", label="Perfectly calibrated")
                fraction_of_positives, mean_predicted_value = calibration_curve(test_set[1], y_pred_scores, n_bins=10)

                ax1.plot(mean_predicted_value, fraction_of_positives, "s-",label=f'{self.name}')
                ax2.hist(y_pred_scores, range=(0, 1), bins=10, label=f'{self.name}',histtype="step", lw=2)

                ax1.set_ylabel("Fraction of positives")
                ax1.set_ylim([-0.05, 1.05])
                ax1.legend(loc="lower right")
                ax1.set_title('Calibration plots  (reliability curve)')

                ax2.set_xlabel("Mean predicted value")
                ax2.set_ylabel("Count")
                ax2.legend(loc="upper center", ncol=2)

                plt.tight_layout()
                plt.savefig(pathlib.Path(output_dir) / f'{self.name}_calibration.png')

ext: py

max_stars_repo_name: jakob-he/TADA

max_issues_repo_name: jakob-he/TADA

max_forks_repo_name: jakob-he/TADA

reason: import numpy,from scipy


text:
import argparse
import logging
import pickle
import neat
import gym
import numpy as np
import os
import csv
from ppaquette_gym_super_mario.wrappers import *
from testbed.logging import visualize
from testbed.training import neat as neat_

# -----------------------------------------------------------------------------
#  GLOBALS
# -----------------------------------------------------------------------------

GYM_NAME = 'ppaquette/SavingSuperMarioBros-1-1-Tiles-v0'
STATE_DIR = None
SESSION_DIR = None
CHECKPOINTS_DIR = None
SAVE_INTERVAL = 5
ENV = None

# -----------------------------------------------------------------------------
#  HELPERS
# -----------------------------------------------------------------------------

def get_env():
    env = gym.make(GYM_NAME)
    save_wrapper = EnableStateSavingAndLoading(STATE_DIR)
    env = save_wrapper(env)
    return env

def mkdir_p(directory):
    if not os.path.exists(directory):
        os.makedirs(directory)

# -----------------------------------------------------------------------------
#  NEAT TRAINING
# -----------------------------------------------------------------------------

def play_best(genome, config):
    net = neat.nn.FeedForwardNetwork.create(genome, config)
    stuck = 0
    stuck_max = 600
    done = False
    observation = ENV.reset()

    while not done:
        outputs = neat_.clean_outputs(net.activate(observation.flatten()))

        observation, reward, done, info = ENV.step(outputs)

        # Check if Mario is progressing in level
        stuck += 1 if reward <= 0 else 0

        if stuck > stuck_max:
            break

    ENV.close()

    return info['distance']

def evaluate(genome, config):
    net = neat.nn.FeedForwardNetwork.create(genome, config)
    fitnesses = []

    train_length = 300

    for stuck_point in [530, 1320, 1600, 2204, 2800]:
        done = False
        stuck = 0
        stuck_max = 600
        ENV.loadSaveStateFile(stuck_point)
        observation = ENV.reset()
        info = {}

        while not done:
            # Get move from NN
            outputs = neat_.clean_outputs(net.activate(observation.flatten()))

            # Make move
            observation, reward, done, info = ENV.step(outputs)

            # Check if Mario is progressing in level
            stuck += 1 if reward <= 0 else 0

            if stuck > stuck_max or info['distance'] > stuck_point + train_length:
                break

        # print("STUCKPOINT : " + str(stuck_point) + " Fitness : " + str(info['distance'] - stuck_point))
        fitnesses.append(max(0, info['distance'] - stuck_point))

        ENV.close()

    return np.array(fitnesses).mean()

def load_checkpoint(config):
    try:
        checkpoint = max([x.split("-")[-1] for x in os.listdir(CHECKPOINTS_DIR) if x.startswith("neat-checkpoint-")])
        print("Found checkpoint at gen : " + str(checkpoint) + "... Loading...")
        return neat.Checkpointer.restore_checkpoint(CHECKPOINTS_DIR + "neat-checkpoint-" + checkpoint)
    except Exception as e:
        print("No saved session found, creating new population")
        return neat.Population(config)

def evolve(config, num_cores):
    pop = load_checkpoint(config)
    pop.add_reporter(neat.Checkpointer(1, 600, CHECKPOINTS_DIR + "neat-checkpoint-"))
    pop.add_reporter(neat.StdOutReporter(True))
    stats = neat.StatisticsReporter()
    pop.add_reporter(stats)

    pe = neat.ParallelEvaluator(num_cores, evaluate)

    for gen in range(500):
        winner = pop.run(pe.evaluate, 1)

        winner_distance = play_best(winner, config)

        with open(SESSION_DIR + 'stats.csv', mode='a') as stats_file:
            stats_writer = csv.writer(stats_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
            stats_writer.writerow([gen, winner_distance])

        with open(SESSION_DIR + 'Best-{}.pkl'.format(len(stats.most_fit_genomes)), 'wb') as output:
            pickle.dump(winner, output, 1)

def main():
    parser = argparse.ArgumentParser(description='Mario NEAT Agent Trainer')
    parser.add_argument('--config-path', type=str, default="/opt/train/NEAT/config-feedforward",
                        help="The path to the NEAT parameter config file to use")
    parser.add_argument('--num-cores', type=int, default=1,
                        help="The number of cores on your computer for parallel execution")
    parser.add_argument('--state-path', type=str, default="/opt/train/Experiment_2/AllYouCanEat-SavePoints/",
                        help="The directory to pull and store states from")
    parser.add_argument('--session-path', type=str, default="/opt/train/Experiment_2/session/",
                        help="The directory to store output files within")

    args = parser.parse_args()

    # Setup logger
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)

    # Load Config
    config = neat_.load_config_with_defaults(args.config_path)

    # Setup globals
    global STATE_DIR
    STATE_DIR = args.state_path
    mkdir_p(STATE_DIR)

    global SESSION_DIR
    SESSION_DIR = args.session_path
    mkdir_p(SESSION_DIR)

    global CHECKPOINTS_DIR
    CHECKPOINTS_DIR = SESSION_DIR + "checkpoints/"
    mkdir_p(CHECKPOINTS_DIR)

    global ENV
    ENV = get_env()

    # Evolve!
    evolve(config=config, num_cores=args.num_cores)

if __name__ == '__main__':
    main()

max_stars_repo_name: NES-NN/OpenAI-Testbed

max_issues_repo_name: NES-NN/OpenAI-Testbed

max_forks_repo_name: NES-NN/OpenAI-Testbed

reason: import numpy


text:
"""
A simple regression analysis on the Boston housing data
========================================================

Here we perform a simple regression analysis on the Boston housing
data, exploring two types of regressors.

"""

from sklearn.datasets import load_boston
from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingRegressor
import matplotlib.pyplot as plt
import numpy as np
import pickle

# Load data
data = load_boston()

# Data split
X_train, X_test, y_train, y_test = train_test_split(data.data, data.target)

# Fit train set
clf = GradientBoostingRegressor()
clf.fit(X_train, y_train)

# Predict test set
predicted = clf.predict(X_test)
expected = y_test

# Check metrics
print("RMS: %r " % np.sqrt(np.mean((predicted - expected) ** 2)))

# Dump model ina pickle file
with open("predict_housing_price.pkl", "wb") as f:
    pickle.dump(clf,f)

max_stars_repo_name: Abimbola-ai/House-price-prediction

max_issues_repo_name: Abimbola-ai/House-price-prediction

max_forks_repo_name: Abimbola-ai/House-price-prediction


text:
from __future__ import print_function
from __future__ import division
import torch
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
import numpy as np
import torchvision
from torchvision import datasets, transforms
import matplotlib.pyplot as plt
import time
import os
import copy
from models import initialize_model
from tools import check_dir
from datasets import DatasetImgTxtPair
from losses import AdaptiveWingLoss, WingLoss, AWing
from tqdm import tqdm
from torch import Tensor

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
torch.backends.cudnn.benchmark = True
#data_dir = "/home/admin/data/xfeng/COMDATA/face_pts_train_data/face_pts_train_data_torch_comb_6parts"
data_dir = "../comdata/insurance_angle_train_data"
save_model_dir = '../commodel/tmp_class2/'

model_name = "efficientnet"  # 从[my_resnet,resnet, alexnet, vgg, squeezenet, densenet, inception,efficientnet]中选择模型
pretrain_model = None#"/home/admin/data/xfeng/COMMODEL/face_pts_20th_with_init_final_no_open_small2/best_acc.pt"
layers = [1,1,1,1,1]
input_size = 224
num_classes = 4
batch_size = 32
num_epochs = 500
lr = 0.01
lr_sch = [num_epochs//2,int(num_epochs*7/8)]
gamma = 0.1
feature_extract = False

criterion = nn.CrossEntropyLoss()
#criterion = AdaptiveWingLoss()
#criterion = nn.MSELoss()
#criterion = WingLoss()
#criterion = AWing()

data_transforms = {
    'train': transforms.Compose([
        #transforms.Grayscale(num_output_channels=1),
        transforms.Resize((input_size,input_size)),
        #transforms.RandomApply([transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.2)],p=0.5),
        #transforms.RandomResizedCrop(input_size),
        #transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        #transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        transforms.Normalize([0.5,0.5,0.5], [0.5,0.5,0.5])
    ]),
    'val': transforms.Compose([
        #transforms.Grayscale(num_output_channels=1),
        transforms.Resize((input_size,input_size)),
        #transforms.CenterCrop(input_size),
        transforms.ToTensor(),
        #transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        transforms.Normalize([0.5,0.5,0.5], [0.5,0.5,0.5])
    ]),
}

check_dir(save_model_dir)
def train_model(model, dataloaders, criterion, optimizer, scheduler, num_epochs=25, is_inception=False):
    since = time.time()

    val_acc_history = []

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0
    best_loss = 1000000
    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # 每个epoch都有一个训练和验证阶段
        for phase in ['train', 'val']:
            start_time = time.time()
            if phase == 'train':
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # 迭代数据
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device)
                labels = labels.to(device)
                optimizer.zero_grad()
                with torch.set_grad_enabled(phase == 'train'):
                    if is_inception and phase == 'train':
                        outputs, aux_outputs = model(inputs)
                        loss1 = criterion(outputs, labels)
                        loss2 = criterion(aux_outputs, labels)
                        loss = loss1 + 0.4*loss2
                    else:
                        outputs = model(inputs)
                        loss = criterion(outputs, labels)
                    _, preds = torch.max(outputs, 1)
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                # 统计
                scheduler.step()
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)
            #print(len(dataloaders[phase].dataset))
            epoch_loss = running_loss / len(dataloaders[phase].dataset)
            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)
            train_time = time.time()-start_time
            print('{} Loss: {:.4f} Acc: {:.4f} Time cost:{:.2f}s'.format(phase, epoch_loss, epoch_acc, train_time))

            # deep copy the model

            #if phase == 'val':
               #torch.save(model, save_model_dir+'model_epoch_%d'%epoch+'.pth')
            if (phase == 'val' and epoch_loss < best_loss):#epoch_acc > best_acc
                best_loss = epoch_loss
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())
                torch.save(model.state_dict(), os.path.join(save_model_dir,'best_acc.pt'))
                #torch.save(model, os.path.join(save_model_dir,'best_acc.pt'))
                print("better model is saved in %s"%(os.path.join(save_model_dir,'best_acc.pt')))
            if phase == 'val':
                val_acc_history.append(epoch_acc)

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model, val_acc_history

if __name__ == '__main__':
    # 在这步中初始化模型
    model_ft = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True, pretrain_model=pretrain_model)

    # 打印我们刚刚实例化的模型
    print(model_ft)

    print("Initializing Datasets and Dataloaders...")

    # 创建训练和验证数据集
    #image_datasets = {x: DatasetImgTxtPair(data_dir, train_val=x, transform=data_transforms[x]) for x in ['train', 'val']}
    image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x]) for x in ['train', 'val']}
    print(image_datasets["train"].class_to_idx)
    #print(image_datasets["train"].imgs)
    # 创建训练和验证数据加载器
    dataloaders_dict = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size, shuffle=True, num_workers=4) for x in ['train', 'val']}

    # 将模型发送到GPU
    model_ft = model_ft.to(device)

    # 在此运行中收集要优化/更新的参数。
    # 如果我们正在进行微调，我们将更新所有参数。
    # 但如果我们正在进行特征提取方法，我们只会更新刚刚初始化的参数，即`requires_grad`的参数为True。
    params_to_update = model_ft.parameters()
    print("Params to learn:")
    if feature_extract:
        params_to_update = []
        for name,param in model_ft.named_parameters():
            if param.requires_grad == True:
                params_to_update.append(param)
                print("\t",name)
    else:
        for name,param in model_ft.named_parameters():
            if param.requires_grad == True:
                print("\t",name)
    optimizer_ft = optim.Adam(params_to_update, lr=lr)
    scheduler = lr_scheduler.MultiStepLR(optimizer_ft, milestones=lr_sch, gamma=gamma)

    model_ft, hist = train_model(model_ft, dataloaders_dict, criterion, optimizer_ft, scheduler, num_epochs=num_epochs, is_inception=(model_name=="inception"))


text:
# -*- coding: utf-8 -*-
"""
Created on Fri Jun  1 22:22:31 2018

@author: lijie
"""
from numpy import *
from completed_SMO import *

'''load data'''
def loadDataSet(fileName):
    dataMat = []
    labelMat = []
    fr = open(fileName)
    for line in fr.readlines():
        lineArr = line.strip().split('\t')
        dataMat.append([float(lineArr[0]), float(lineArr[1])])#alpha1,alpha2
        labelMat.append(float(lineArr[2]))#label
    return dataMat,labelMat

'''package all the functions'''
def testRbf(k1):    #k1 : Velocity parameter of rbf
    #train
    dataArr,labelArr = loadDataSet('testSetRBF.txt')
    b,alphas = smoP(dataArr, labelArr, 200, 0.0001, 10000, ('rbf', k1)) #C=200
    datMat=mat(dataArr); labelMat = mat(labelArr).transpose()
    svInd=nonzero(alphas.A>0)[0] #get matrix of only support vectors
    sVs=datMat[svInd]
    labelSV = labelMat[svInd];
    print( "there are %d Support Vectors" % shape(sVs)[0])
    m,n = shape(datMat)
    errorCount = 0
    # key part to do classification & calculate the error rate
    for i in range(m):
        kernelEval = kernelTrans(sVs,datMat[i,:],('rbf', k1))
        predict=kernelEval.T * multiply(labelSV,alphas[svInd]) + b
        if sign(predict)!=sign(labelArr[i]): errorCount += 1
    print( "the training error rate is: %f" % (float(errorCount)/m))
    #test
    dataArr,labelArr = loadDataSet('testSetRBF2.txt')
    errorCount = 0
    datMat=mat(dataArr); labelMat = mat(labelArr).transpose()
    m,n = shape(datMat)
    for i in range(m):
        kernelEval = kernelTrans(sVs,datMat[i,:],('rbf', k1))
        predict=kernelEval.T * multiply(labelSV,alphas[svInd]) + b
        if sign(predict)!=sign(labelArr[i]): errorCount += 1
    print ("the test error rate is: %f" % (float(errorCount)/m) )

if __name__=='__main__' :
#test
    testRbf(k1=1.3)
'''
    #the result:
        iteration number: 6
        there are 26 Support Vectors
        the training error rate is: 0.090000
        the test error rate is: 0.180000
'''

reason: from numpy


text:
"""
Script for downloading and exploring data available on the NHL API
"""
import streamlit as st
import numpy as np
import autodraft.api as api
# import plotly.express as px

def main():
    st.title('NHL API Dataset Assembly')
    st.write('This report will cover assembling a player-level'
             'game-by-game dataset for 4 NHL seasons.')
    st.write('For the most complete dataset, we will go through'
             'each game on each day so as to get all stats,'
             'but only for those players that played in the game.')

    st.write('Here is the full dataframe of NHL seasons with available stats:')
    seasons_df = api.get_seasons(streamlit=True)
    st.dataframe(seasons_df)

    st.write("Let's grab the 4 most recent, COMPLETE, seasons.")
    seasons_df = seasons_df.iloc[-5:-1, :]
    st.dataframe(seasons_df)
    st.write('Now on to getting our team rosters.')

    st.dataframe(api.get_teams(streamlit=True))
    st.write('From here, we will get the rosters for each team in the seasons we want.')

    # roster_team = st.text_input('Enter a team ID from the table above:', '22')
    roster_team = '22'
    # roster_year = st.text_input('Enter a season ID (eg. 20182019 for the 2018-2019 season)', '20182019')
    roster_year = '20182019'

    st.write("Here is that team's roster for that year:")
    st.dataframe(api.get_roster(team_id=int(roster_team), season_id=int(roster_year), streamlit=True))

    st.write("Now we'll get a roster that covers all players that were active"
             "in a range of season. NOTE: this uses the team entered earlier.")

    merged_roster = api.merge_team_rosters(team_id=roster_team)
    st.dataframe(merged_roster)

    st.write("We'll start collecting time-series sets now for exploration.")
    # roster_player = st.text_input('Enter a player ID from the table above:', '8477934')
    roster_player = '8477934'

    player_df = api.get_player_season_game_stats(player_id=roster_player, streamlit=True)
    player_df.sort_index(inplace=True)
    st.write('{0} ({1}) stat dataframe for {2} season:'.format(merged_roster.loc[np.int(roster_player), 'name'],
                                                               merged_roster.loc[np.int(roster_player), 'position'],
                                                               roster_year))
    st.dataframe(player_df)
    st.write('Season (sorted) point history:')
    st.dataframe(player_df.loc[:, 'statPoints'])

    combined_player_df = api.get_combined_player_season_game_stats(player_id=roster_player)
    combined_player_df.sort_index(inplace=True)

    augmented_df = api.augment_player_dataframe(player_df=combined_player_df)

    st.write('{0} ({1}) combined and augmented stat dataframe for {2} seasons:' \
             .format(merged_roster.loc[np.int(roster_player), 'name'],
                     merged_roster.loc[np.int(roster_player), 'position'],
                     3))
    st.dataframe(augmented_df)

    # line_fig = px.line(augmented_df, x='gameNumber', y='cumStatpoints')
    # line_fig.update_layout(title_text='test', title_x=0.5)
    # line_fig.update_yaxes(title_text='Cumulative Points')
    # st.plotly_chart(line_fig)

    # multi_season_df = api.augment_player_dataframe(combined_player_df)
    # line_fig = px.line(multi_season_df, x='gameNumber', y='cumStatpoints')
    # line_fig.update_layout(title_text='test', title_x=0.5)
    # line_fig.update_yaxes(title_text='Cumulative Points')
    # st.plotly_chart(line_fig)

    # multiplayer_df = api.assemble_multiplayer_stat_dataframe(player_id_list=list(merge_team_rosters(team_id=roster_team).index), stat_list=[])
    # st.dataframe(multiplayer_df)
    # line_fig = px.line(multiplayer_df, x='date', y='cumStatpoints', color='name')
    # line_fig.update_layout(title_text='test', title_x=0.5, xaxis_rangeslider_visible=True, showlegend=True, legend=dict(x=1, y=-2))
    # line_fig.update_yaxes(title_text='Cumulative Points')
    # st.plotly_chart(line_fig)

    all_rosters = api.get_all_rosters(streamlit=True)
    st.dataframe(all_rosters)
    st.write(all_rosters.shape)

    full_df = api.assemble_multiplayer_stat_dataframe(player_id_list=list(all_rosters.index), stat_list=[])
    st.write(full_df.shape)

    # full_df.to_csv('./data/full_dataset_4_seasons.csv')

main()


text:
# crop image based on the annotation information and save the image and label information
import json
from dataset  import JsonDataset
from common_lib.tools.coding_related import decode_distribution
#from common_lib import decode_labelme_shape
from coding_related import decode_labelme_shape
from common_lib.tools.cnn_json_related import ClassNameManager
import os
import cv2
import glob
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt

def convert_NG_label(distribution,class_dict):
      distribution_list=decode_distribution(distribution)
      class_name_manager=ClassNameManager(class_dict)
      NG_type=class_name_manager.get_key_from_distribution(distribution_list)
      for class_num in class_dict:
            if NG_type[-1]==class_num["class_name"]:
                  return class_num["class_id"]-4
#image_data_root = '/workspace/dataSet/raw/unsupervised-learning/kangqiang/'
image_data_root="/git/dataSet/raw/unsupervised-learning/kangqiang/"
json_path='assets_new_new/data/2021-03-05/json_for_classification'

save_image_root="/git/PaDiM-master/kangqiang_result/croped_images"
save_folder="json_for_classification"
json_file_list=["train.json","val.json","test.json"]
save_file_list=["train.txt","val.txt","test.txt"]
data_list=[]
for json_file in json_file_list:
    with open(os.path.join(json_path,json_file)) as fid:
        dataset_info=json.load(fid)
        data_list.append(dataset_info['record'])

for data in data_list:
    anomaly_image_path_label_list=[]
    for i, rec in enumerate(data):
        image_name=rec['info']['image_path']
        image_dir,image_name_part=os.path.split(image_name)
        print(i)
        img = cv2.imread(os.path.join(image_data_root,image_name))[...,::-1]
        assert img is not None, rec
        for p,inst in enumerate(rec['instances']):
            shapes = np.array(decode_labelme_shape(inst['points']))
            x1,y1,x2,y2 = int(shapes[:,0].min()), int(shapes[:,1].min()), int(shapes[:,0].max()), int(shapes[:,1].max())
            crop_img=img[y1:y2,x1:x2]
            plt.imshow(crop_img)
            save_dir=os.path.join(save_image_root, image_dir)
            if not os.path.exists(save_dir):
                os.makedirs(save_dir)
            save_new_image_name=os.path.join(save_dir, os.path.splitext(image_name_part)[0]+"_"+str(p)+".jpg")
            plt.savefig(save_new_image_name)
            anomaly_image_path_label_list.append([save_new_image_name,dataset_info["class_dict"]])
    with open(os.path.join(json_file,save_folder,save_file_list[i]),"w+") as fid:
        fid.writelines([k[0]+" "+str(k[1])+"\n" for k in anomaly_image_path_label_list])
        fid.close()


text:
#!/usr/bin/env python3

import numpy as np
import unittest
import warnings

from scipy.stats import rankdata

from context import lir

from lir.transformers import InstancePairing, PercentileRankTransformer

warnings.simplefilter("error")

class TestPercentileRankTransformer(unittest.TestCase):
    def test_fit_transform(self):
        """When X itself is transformed, it should return its own ranks"""
        X = np.array([[0.1, 0.4, 0.5],
                      [0.2, 0.5, 0.55],
                      [0.15, 0.51, 0.55],
                      [0.18, 0.45, 0.56]])
        rank_transformer = PercentileRankTransformer()
        rank_transformer.fit(X)
        ranks = rank_transformer.transform(X)
        self.assertSequenceEqual(ranks.tolist(),
                                 (rankdata(X, method='max', axis=0)/len(X)).tolist(),
                                 'Ranking X and PercentileRankTransformer.transform(X)'
                                 ' should give the same results')

    def test_extrapolation(self):
        """Values smaller than the lowest value should map to 0,
        values larger than the highest value should map to 1"""
        X = np.array([[0.1, 0.2, 0.3],
                      [0.2, 0.2, 0.4],
                      [0.3, 0.2, 0.5]])
        Z = np.array([[0.0, 0.1, 0.2],
                      [1.0, 1.0, 1.0]])
        rank_transformer = PercentileRankTransformer()
        rank_transformer.fit(X)
        ranks = rank_transformer.transform(Z)
        self.assertSequenceEqual(ranks.tolist(), [[0, 0, 0], [1, 1, 1]],
                                 'Elements smaller than the lowest value should'
                                 'map to 0, larger than the highest value to 1')

    def test_interpolation(self):
        """Values inbetween values of X result in interpolated ranking,
        with linear interpolation."""
        X = np.array([[0, 0, 0],
                      [1, 1, 1]])
        Z = np.array([[0.1, 0.3, 0.5]])
        rank_transformer = PercentileRankTransformer()
        rank_transformer.fit(X)
        ranks = rank_transformer.transform(Z)
        # Ranks are interpolated between 0.5 (rank of 0) and 1 (rank of 1).
        # We expect a linear interpolation.
        expected_ranks = 0.5 + np.array([[0.1, 0.3, 0.5]])*0.5
        self.assertSequenceEqual(ranks.tolist(), expected_ranks.tolist(),
                                 'Interpolation failed.')

    def test_ties(self):
        """Ties are given the maximum value (the maximum of the ranks that would
        have been assigned to all the tied values is assigned to each value)."""
        X = np.array([[0.1], [0.1], [0.1], [0.1], [0.3]])
        Z = np.array([[0.1]])
        rank_transformer = PercentileRankTransformer()
        rank_transformer.fit(X)
        ranks = rank_transformer.transform(Z)
        self.assertEqual(ranks, 0.8, "Ties should be given the maximum value")

    def test_constant_feature(self):
        """If a feature is a constant value c, the rank should be 0 for x < c
        and 1 for x >= c."""
        X = np.array([[1], [1], [1], [1]])
        Z = np.array([[0], [1], [2]])
        rank_transformer = PercentileRankTransformer()
        rank_transformer.fit(X)
        ranks = rank_transformer.transform(Z)
        self.assertEqual(ranks.tolist(), [[0], [1], [1]],
                         "If a feature is a constant value, "
                         "interpolation should still work")

class TestPairing(unittest.TestCase):
    X = np.arange(30).reshape(10, 3)
    y = np.concatenate([np.arange(5), np.arange(5)])

    def test_pairing(self):
        pairing = InstancePairing()
        X_pairs, y_pairs = pairing.transform(self.X, self.y)

        self.assertEqual(np.sum(y_pairs == 1), 5, 'number of same source pairs')
        self.assertEqual(np.sum(y_pairs == 0), 2*(8+6+4+2), 'number of different source pairs')

        pairing = InstancePairing(ratio_limit=1)
        X_pairs, y_pairs = pairing.transform(self.X, self.y)

        self.assertEqual(np.sum(y_pairs == 1), 5, 'number of same source pairs')
        self.assertEqual(np.sum(y_pairs == 0), 5, 'number of different source pairs')

        self.assertTrue(np.all(pairing.pairing[:, 0] != pairing.pairing[:, 1]), 'identity in pairs')

    def test_pairing_ratio(self):
        # test ratio
        ratio_limit = 7
        pairing_ratio = InstancePairing(ratio_limit=ratio_limit)
        X_pairs, y_pairs = pairing_ratio.transform(self.X, self.y)
        ratio = np.sum(y_pairs == 0) / np.sum(y_pairs == 1)
        self.assertEqual(ratio, ratio_limit, 'ratio ss ds pairs not correct')

        # if ratio_limit exceeds highest possible ratio, all ds pairs are selected
        ratio_limit = 1_000
        pairing_ratio_max = InstancePairing(ratio_limit=ratio_limit)
        X_pairs, y_pairs = pairing_ratio_max.transform(self.X, self.y)
        ratio = np.sum(y_pairs == 0) / np.sum(y_pairs == 1)
        self.assertLess(ratio, ratio_limit,
                        'realised ratio should be less or equal than ratio_limit')
        self.assertEqual(np.sum(y_pairs == 0), 2*(8+6+4+2),
                         'all different source pairs should be selected')

        # test ratio with same_source_limit
        ratio_limit = 5
        same_source_limit = 3
        pairing_ratio_ss_lim = InstancePairing(ratio_limit=ratio_limit,
                                            same_source_limit=same_source_limit)
        X_pairs, y_pairs = pairing_ratio_ss_lim.transform(self.X, self.y)
        ratio = np.sum(y_pairs == 0) / np.sum(y_pairs == 1)
        self.assertEqual(ratio, ratio_limit, 'ratio ss ds pairs not correct')
        self.assertEqual(same_source_limit, np.sum(y_pairs == 1), 'ss pairs limit')

        # test ratio with different_source_limit
        ratio_limit = 5
        different_source_limit = 20
        pairing_ratio_ds_lim = InstancePairing(ratio_limit=ratio_limit,
                                               different_source_limit=different_source_limit)
        X_pairs, y_pairs = pairing_ratio_ds_lim.transform(self.X, self.y)
        ratio = np.sum(y_pairs == 0) / np.sum(y_pairs == 1)
        self.assertLessEqual(ratio, ratio_limit, 'ratio ss ds pairs not correct')
        self.assertLessEqual(np.sum(y_pairs == 0), different_source_limit,
                         'ds pairs limit')

        # test ratio with same_source_limit and different_source_limit
        max_ratio = 5
        same_source_limit = 9
        different_source_limit = 30
        pairing_ratio_ds_lim = InstancePairing(ratio_limit=ratio_limit,
                                               same_source_limit=same_source_limit,
                                               different_source_limit=different_source_limit)
        X_pairs, y_pairs = pairing_ratio_ds_lim.transform(self.X, self.y)
        ratio = np.sum(y_pairs == 0) / np.sum(y_pairs == 1)
        self.assertLessEqual(ratio, ratio_limit, 'ratio_limit ss and ds pairs exceeded')
        self.assertLessEqual(np.sum(y_pairs == 1), same_source_limit,
                             'ss pairs limit')
        self.assertLessEqual(np.sum(y_pairs == 0), different_source_limit,
                             'ds pairs limit')

    def test_pairing_seed(self):
        pairing_seed_1 = InstancePairing(ratio_limit=1, seed=123)
        X_pairs_1, y_pairs_1 = pairing_seed_1.transform(self.X, self.y)

        pairing_seed_2 = InstancePairing(ratio_limit=1, seed=123)
        X_pairs_2, y_pairs_2 = pairing_seed_2.transform(self.X, self.y)

        pairing_seed_3 = InstancePairing(ratio_limit=1, seed=456)
        X_pairs_3, y_pairs_3 = pairing_seed_3.transform(self.X, self.y)

        self.assertTrue(np.all(X_pairs_1 == X_pairs_2), 'same seed, same X pairs')
        self.assertTrue(np.any(X_pairs_1 != X_pairs_3), 'different seed, different X pairs')

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: HolmesNL/lir

max_issues_repo_name: HolmesNL/lir

max_forks_repo_name: HolmesNL/lir


text:
# Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
from __future__ import absolute_import

import gzip
import os
import pickle
import sys
import time

import pytest

from sagemaker import KMeans, KMeansModel
from sagemaker.utils import unique_name_from_base
from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES
from tests.integ.timeout import timeout, timeout_and_delete_endpoint_by_name

@pytest.mark.continuous_testing
def test_kmeans(sagemaker_session):
    with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
        data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
        pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}

        # Load the data into memory as numpy arrays
        with gzip.open(data_path, 'rb') as f:
            train_set, _, _ = pickle.load(f, **pickle_args)

        kmeans = KMeans(role='SageMakerRole', train_instance_count=1,
                        train_instance_type='ml.c4.xlarge',
                        k=10, sagemaker_session=sagemaker_session, base_job_name='test-kmeans')

        kmeans.init_method = 'random'
        kmeans.max_iterations = 1
        kmeans.tol = 1
        kmeans.num_trials = 1
        kmeans.local_init_method = 'kmeans++'
        kmeans.half_life_time_size = 1
        kmeans.epochs = 1
        kmeans.center_factor = 1

        assert kmeans.hyperparameters() == dict(
            init_method=kmeans.init_method,
            local_lloyd_max_iter=str(kmeans.max_iterations),
            local_lloyd_tol=str(kmeans.tol),
            local_lloyd_num_trials=str(kmeans.num_trials),
            local_lloyd_init_method=kmeans.local_init_method,
            half_life_time_size=str(kmeans.half_life_time_size),
            epochs=str(kmeans.epochs),
            extra_center_factor=str(kmeans.center_factor),
            k=str(kmeans.k),
            force_dense='True',
        )

        kmeans.fit(kmeans.record_set(train_set[0][:100]))

    endpoint_name = unique_name_from_base('kmeans')
    with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
        model = KMeansModel(kmeans.model_data, role='SageMakerRole', sagemaker_session=sagemaker_session)
        predictor = model.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name)
        result = predictor.predict(train_set[0][:10])

        assert len(result) == 10
        for record in result:
            assert record.label["closest_cluster"] is not None
            assert record.label["distance_to_cluster"] is not None

    predictor.delete_model()
    with pytest.raises(Exception) as exception:
        sagemaker_session.sagemaker_client.describe_model(ModelName=model.name)
        assert 'Could not find model' in str(exception.value)

def test_async_kmeans(sagemaker_session):
    training_job_name = ""
    endpoint_name = unique_name_from_base('kmeans')

    with timeout(minutes=5):
        data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
        pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}

        # Load the data into memory as numpy arrays
        with gzip.open(data_path, 'rb') as f:
            train_set, _, _ = pickle.load(f, **pickle_args)

        kmeans = KMeans(role='SageMakerRole', train_instance_count=1,
                        train_instance_type='ml.c4.xlarge',
                        k=10, sagemaker_session=sagemaker_session, base_job_name='test-kmeans')

        kmeans.init_method = 'random'
        kmeans.max_iterations = 1
        kmeans.tol = 1
        kmeans.num_trials = 1
        kmeans.local_init_method = 'kmeans++'
        kmeans.half_life_time_size = 1
        kmeans.epochs = 1
        kmeans.center_factor = 1

        assert kmeans.hyperparameters() == dict(
            init_method=kmeans.init_method,
            local_lloyd_max_iter=str(kmeans.max_iterations),
            local_lloyd_tol=str(kmeans.tol),
            local_lloyd_num_trials=str(kmeans.num_trials),
            local_lloyd_init_method=kmeans.local_init_method,
            half_life_time_size=str(kmeans.half_life_time_size),
            epochs=str(kmeans.epochs),
            extra_center_factor=str(kmeans.center_factor),
            k=str(kmeans.k),
            force_dense='True',
        )

        kmeans.fit(kmeans.record_set(train_set[0][:100]), wait=False)
        training_job_name = kmeans.latest_training_job.name

        print("Detached from training job. Will re-attach in 20 seconds")
        time.sleep(20)
        print("attaching now...")

    with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
        estimator = KMeans.attach(training_job_name=training_job_name, sagemaker_session=sagemaker_session)
        model = KMeansModel(estimator.model_data, role='SageMakerRole', sagemaker_session=sagemaker_session)
        predictor = model.deploy(1, 'ml.c4.xlarge', endpoint_name=endpoint_name)
        result = predictor.predict(train_set[0][:10])

        assert len(result) == 10
        for record in result:
            assert record.label["closest_cluster"] is not None
            assert record.label["distance_to_cluster"] is not None

max_stars_repo_name: wweic/sagemaker-python-sdk

max_issues_repo_name: wweic/sagemaker-python-sdk

max_forks_repo_name: wweic/sagemaker-python-sdk

reason: from sage


text:
import numpy as np
import random

from pyrado.environments.pysim.base import SimEnv
from pyrado.utils.data_types import RenderMode

class MockEnv(SimEnv):
    """
    A mock version of SimEnv, used in env wrapper tests.
    Observation, action, and init spaces as well as the task are passed to the constructor.
    The step() method saves the last action into the last_act attribute. The action value is converted to a list to
    ease assertions. step() and reset() return next_obs as observation. If it is None, a random vector is returned.
    """

    def __init__(self, obs_space=None, act_space=None, init_space=None, task=None):
        # Set spaces
        self._obs_space = obs_space
        self._act_space = act_space
        self._init_space = init_space
        self._task = task

        # Set empty domain param
        self._domain_param = {}

        # Init check attributes
        self.next_obs = None
        self.next_reward = None
        self.next_step_done = False
        self.last_act = None

    @property
    def obs_space(self):
        if self._obs_space is None:
            raise NotImplementedError
        return self._obs_space

    @property
    def state_space(self):
        # Just use observation space here for now.
        if self._obs_space is None:
            raise NotImplementedError
        return self._obs_space

    @property
    def init_space(self):
        if self._init_space is None:
            raise NotImplementedError
        return self._init_space

    @property
    def act_space(self):
        if self._act_space is None:
            raise NotImplementedError
        return self._act_space

    @property
    def task(self):
        raise self._task

    def _create_task(self, task_args: dict = None):
        pass  # unused

    @property
    def domain_param(self):
        return self._domain_param.copy()

    @domain_param.setter
    def domain_param(self, param):
        self._domain_param.clear()
        self._domain_param.update(param)

    def get_nominal_domain_param(self):
        return {}

    def _get_obs(self):
        # Return None if no obs space
        if self._obs_space is None:
            return None

        # Return random if no next_obs set
        if self.next_obs is None:
            return self._obs_space.sample_uniform()

        return np.array(self.next_obs)

    def reset(self, init_state=None, domain_param=None):
        # Init state is not needed for now.

        # Set domain params
        if domain_param is not None:
            self.domain_param = domain_param

        # Return next observation
        return self._get_obs()

    def step(self, act):
        # Store as last action as list, to simplify asserts
        if self._act_space is not None:
            self.last_act = list(act)

        # Return next observation
        obs = self._get_obs()

        # And next reward
        rew = self.next_reward
        if rew is None:
            rew = random.random()

        return obs, rew, self.next_step_done, dict()

    def render(self, mode=RenderMode(), render_step=1):
        # No visualization
        pass

max_stars_repo_name: jacarvalho/SimuRLacra

max_issues_repo_name: jacarvalho/SimuRLacra

max_forks_repo_name: jacarvalho/SimuRLacra


text:
# Copyright 2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
Testing GriffinLim op in DE
"""
import numpy as np
import pytest

import mindspore.dataset as ds
import mindspore.dataset.audio.transforms as c_audio
from mindspore import log as logger

DATA_DIR = "../data/dataset/audiorecord/"

def count_unequal_element(data_expected, data_me, rtol, atol):
    assert data_expected.shape == data_me.shape
    total_count = len(data_expected.flatten())
    error = np.abs(data_expected - data_me)
    greater = np.greater(error, atol + np.abs(data_expected) * rtol)
    loss_count = np.count_nonzero(greater)
    assert (loss_count / total_count) < rtol, \
        "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \
        format(data_expected[greater], data_me[greater], error[greater])

def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True):
    if np.any(np.isnan(data_expected)):
        assert np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan)
    elif not np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan):
        count_unequal_element(data_expected, data_me, rtol, atol)

def test_griffin_lim_pipeline():
    """
    Feature: GriffinLim
    Description: test GriffinLim cpp op in pipeline
    Expectation: equal results from Mindspore and benchmark
    """
    # <101, 6>
    in_data = np.load(DATA_DIR + "griffinlim_101x6.npy")[np.newaxis, :]
    out_expect = np.load(DATA_DIR + "griffinlim_101x6_out.npy")
    dataset = ds.NumpySlicesDataset(in_data, column_names=["multi_dimensional_data"], shuffle=False)
    transforms = [c_audio.GriffinLim(n_fft=200, rand_init=False)]
    dataset = dataset.map(operations=transforms, input_columns=["multi_dimensional_data"])
    for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True):
        out_put = item["multi_dimensional_data"]
        allclose_nparray(out_put, out_expect, 0.001, 0.001)

    # <151, 8>
    in_data = np.load(DATA_DIR + "griffinlim_151x8.npy")[np.newaxis, :]
    out_expect = np.load(DATA_DIR + "griffinlim_151x8_out.npy")
    dataset = ds.NumpySlicesDataset(in_data, column_names=["multi_dimensional_data"], shuffle=False)
    transforms = [c_audio.GriffinLim(n_fft=300, n_iter=20, win_length=240, hop_length=120, rand_init=False, power=1.2)]
    dataset = dataset.map(operations=transforms, input_columns=["multi_dimensional_data"])
    for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True):
        out_put = item["multi_dimensional_data"]
        allclose_nparray(out_put, out_expect, 0.001, 0.001)

    # <2, 301, 4> hop_length greater than half of win_length
    in_data = np.load(DATA_DIR + "griffinlim_2x301x4.npy")[np.newaxis, :]
    out_expect = np.load(DATA_DIR + "griffinlim_2x301x4_out.npy")
    dataset = ds.NumpySlicesDataset(in_data, column_names=["multi_dimensional_data"], shuffle=False)
    transforms = [c_audio.GriffinLim(n_fft=600, n_iter=10, win_length=240, hop_length=130, rand_init=False)]
    dataset = dataset.map(operations=transforms, input_columns=["multi_dimensional_data"])
    for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True):
        out_put = item["multi_dimensional_data"]
        allclose_nparray(out_put, out_expect, 0.001, 0.001)

def test_griffin_lim_pipeline_invalid_param_range():
    """
    Feature: GriffinLim
    Description: test GriffinLim with invalid input parameters
    Expectation: throw ValueError
    """
    logger.info("test GriffinLim op with default values")
    in_data = np.load(DATA_DIR + "griffinlim_151x8.npy")[np.newaxis, :]
    data1 = ds.NumpySlicesDataset(in_data, column_names=["multi_dimensional_data"], shuffle=False)

    with pytest.raises(ValueError, match=r"Input n_fft is not within the required interval of \[1, 2147483647\]."):
        transforms = [c_audio.GriffinLim(n_fft=-10)]
        data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
        for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
            _ = item["multi_dimensional_data"]

    with pytest.raises(ValueError, match=r"Input n_iter is not within the required interval of \[1, 2147483647\]."):
        transforms = [c_audio.GriffinLim(n_fft=300, n_iter=-10)]
        data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
        for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
            _ = item["multi_dimensional_data"]

    with pytest.raises(ValueError, match=r"Input win_length is not within the required interval of \[0, 2147483647\]."):
        transforms = [c_audio.GriffinLim(n_fft=300, n_iter=10, win_length=-10)]
        data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
        for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
            _ = item["multi_dimensional_data"]

    with pytest.raises(ValueError,
                       match=r"Input win_length should be no more than n_fft, but got win_length: 400 " +
                       r"and n_fft: 300."):
        transforms = [c_audio.GriffinLim(n_fft=300, n_iter=10, win_length=400)]
        data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
        for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
            _ = item["multi_dimensional_data"]

    with pytest.raises(ValueError, match=r"Input hop_length is not within the required interval of \[0, 2147483647\]."):
        transforms = [c_audio.GriffinLim(n_fft=300, n_iter=10, win_length=0, hop_length=-10)]
        data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
        for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
            _ = item["multi_dimensional_data"]

    with pytest.raises(ValueError, match=r"Input power is not within the required interval of \(0, 16777216\]."):
        transforms = [c_audio.GriffinLim(n_fft=300, n_iter=10, win_length=0, hop_length=0, power=-3)]
        data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
        for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
            _ = item["multi_dimensional_data"]

    with pytest.raises(ValueError, match=r"Input momentum is not within the required interval of \[0, 16777216\]."):
        transforms = [c_audio.GriffinLim(n_fft=300, n_iter=10, win_length=0, hop_length=0, power=2, momentum=-10)]
        data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
        for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
            _ = item["multi_dimensional_data"]

    with pytest.raises(ValueError, match=r"Input length is not within the required interval of \[0, 2147483647\]."):
        transforms = [
            c_audio.GriffinLim(n_fft=300, n_iter=10, win_length=0, hop_length=0, power=2, momentum=0.9, length=-2)
        ]
        data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
        for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
            _ = item["multi_dimensional_data"]

def test_griffin_lim_pipeline_invalid_param_constraint():
    """
    Feature: GriffinLim
    Description: test GriffinLim with invalid input parameters
    Expectation: throw RuntimeError
    """
    logger.info("test GriffinLim op with default values")
    in_data = np.load(DATA_DIR + "griffinlim_151x8.npy")[np.newaxis, :]
    data1 = ds.NumpySlicesDataset(in_data, column_names=["multi_dimensional_data"], shuffle=False)

    with pytest.raises(RuntimeError,
                       match=r"Unexpected error. map operation: \[GriffinLim\] failed. " +
                       r"GriffinLim: the frequency of the input should equal to n_fft / 2 \+ 1"):
        transforms = [c_audio.GriffinLim(n_fft=100)]
        data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
        for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
            _ = item["multi_dimensional_data"]

    with pytest.raises(RuntimeError,
                       match=r"Unexpected error. map operation: \[GriffinLim\] failed. " +
                       r"GriffinLim: the frequency of the input should equal to n_fft / 2 \+ 1"):
        transforms = [c_audio.GriffinLim(n_fft=300, n_iter=10, win_length=0, hop_length=120)]
        data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
        for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
            _ = item["multi_dimensional_data"]

    with pytest.raises(RuntimeError,
                       match=r"Syntax error. GriffinLim: momentum equal to or greater than 1 can be unstable, " +
                       "but got: 1.000000"):
        transforms = [c_audio.GriffinLim(n_fft=300, n_iter=10, win_length=0, hop_length=0, power=2, momentum=1)]
        data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
        for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
            _ = item["multi_dimensional_data"]

def test_griffin_lim_pipeline_invalid_param_type():
    """
    Feature: GriffinLim
    Description: test GriffinLim with invalid input parameters
    Expectation: throw TypeError
    """
    logger.info("test GriffinLim op with default values")
    in_data = np.load(DATA_DIR + "griffinlim_151x8.npy")[np.newaxis, :]
    data1 = ds.NumpySlicesDataset(in_data, column_names=["multi_dimensional_data"], shuffle=False)

    with pytest.raises(TypeError,
                       match=r"Argument window_type with value type is not of type " +
                       r"\[<enum \'WindowType\'>\], but got <class \'str\'>."):
        transforms = [c_audio.GriffinLim(n_fft=300, n_iter=10, win_length=0, hop_length=0, window_type="type")]
        data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
        for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
            _ = item["multi_dimensional_data"]

    with pytest.raises(TypeError,
                       match=r"Argument rand_init with value true is not of type \[<class \'bool\'>\], " +
                       r"but got <class \'str\'>."):
        transforms = [
            c_audio.GriffinLim(n_fft=300,
                               n_iter=10,
                               win_length=0,
                               hop_length=0,
                               power=2,
                               momentum=0.9,
                               length=0,
                               rand_init='true')
        ]
        data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
        for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
            _ = item["multi_dimensional_data"]

def test_griffin_lim_eager():
    """
    Feature: GriffinLim
    Description: test GriffinLim cpp op with eager mode
    Expectation: equal results from Mindspore and benchmark
    """
    # <freq, time>
    spectrogram = np.load(DATA_DIR + "griffinlim_101x6.npy").astype(np.float64)
    out_expect = np.load(DATA_DIR + "griffinlim_101x6_out.npy").astype(np.float64)
    out_ms = c_audio.GriffinLim(n_fft=200, rand_init=False)(spectrogram)
    allclose_nparray(out_ms, out_expect, 0.001, 0.001)
    # <1, freq, time>
    spectrogram = np.load(DATA_DIR + "griffinlim_1x201x6.npy").astype(np.float64)
    out_expect = np.load(DATA_DIR + "griffinlim_1x201x6_out.npy").astype(np.float64)
    out_ms = c_audio.GriffinLim(rand_init=False)(spectrogram)
    allclose_nparray(out_ms, out_expect, 0.001, 0.001)
    # <2, freq, time>
    spectrogram = np.load(DATA_DIR + "griffinlim_2x301x6.npy").astype(np.float64)
    out_expect = np.load(DATA_DIR + "griffinlim_2x301x6_out.npy").astype(np.float64)
    out_ms = c_audio.GriffinLim(n_fft=600, rand_init=False)(spectrogram)
    allclose_nparray(out_ms, out_expect, 0.001, 0.001)

if __name__ == "__main__":
    test_griffin_lim_pipeline()
    test_griffin_lim_pipeline_invalid_param_range()
    test_griffin_lim_pipeline_invalid_param_constraint()
    test_griffin_lim_pipeline_invalid_param_type()
    test_griffin_lim_eager()

max_stars_repo_name: httpsgithu/mindspore


text:
#!/usr/bin/python3

import sys

import pandas as pd
import statsmodels.formula.api as smf
from scipy.stats import stats

import matplotlib.pyplot as plt
from matplotlib.offsetbox import AnchoredText
import seaborn as sns

import itertools as it
import numpy as np
from slugify import slugify
from keyname import keyname as kn
from iterpop import iterpop as ip

try:
    __, synchronous = sys.argv
except:
    raise ValueError('specify "synchronous" as argument')

try:
    synchronous = int(synchronous)
except:
    raise ValueError('"synchronous" should be an integer')

assert synchronous in (0, 1), '"synchronous" should be 0 or 1'

def estimate_latency(group):
  '''
  Use a linear regression to estimate time per work from a Pandas GroupBy.
  '''
  model = smf.glm(
    formula='Time ~ Work',
    data=group,
  ).fit()

  # pack up information about parameter estimates
  # so they can be programatically unpacked later
  decoder = {
    'Intercept' : 'Overhead',
    'Work' : 'Latency',
  }

  for spec, val in (
        (
            {
              'statistic' : statistic,
              'name' : decoder[parameter],
            },
            getter(parameter)
        ) for parameter in model.params.keys()
        for statistic, getter in (
          ('Estimated', lambda param: model.params[parameter]),
          ('Lower Bound', lambda param: model.conf_int()[0][param]),
          ('Upper Bound', lambda param: model.conf_int()[1][param]),
         )
    ):
    group['{statistic} {name}'.format(**spec)] = val

  return group

def _do_plot(df, x, y, ax=None):
  '''
  Wrap a call to Seaborn's regplot.
  '''
  res = sns.regplot(
    data=df,
    x=x,
    y=y,
    truncate=False,
    ax=ax
  )
  res.set_xlim(0, )
  res.set_ylim(0, )

  return res

def plot_regression(df, x, y, extra_names={}):
  '''Plot a regression with annotated statistics.'''
  # ugly hack to include origin in plot bounds
  plt.clf()
  ax = _do_plot(df, x, y)
  xlim, ylim = ax.get_xlim(), ax.get_ylim()
  ax.cla()
  ax.set_xlim(*xlim)
  ax.set_ylim(*ylim)
  _do_plot(df, x, y, ax=ax)

  # calculate some regression statistics...
  info = [
    ("{} = " + ("{}" if isinstance(v, int) else "{:.2f}")).format(k, v)
    for k, v in it.chain(zip(
      [
        'Slope',
        'Intercept',
        '$R^2$',
        '$p$',
        'Standard Error',
      ],
      stats.linregress(df[x],df[y]),
    ), [
      ('$n$', len(df))
    ])
  ]

  # ... and annotate regression statistics onto upper left
  at = AnchoredText(
    '\n'.join(info),
    frameon=True,
    loc='upper left',
  )
  ax.add_artist(at)

  # save to file
  # and assert df['Load'] is homogeneous
  plt.savefig(
    kn.pack({**{
      'x' : slugify(x),
      'y' : slugify(y),
      'synchronous' : str(synchronous),
      'ext': '.png',
    },**extra_names}),
    transparent=True,
    dpi=300,
  )

def deduce_properties(**kwargs):
  '''
  Deduce scaling properties we're trying to estimate from params computed
  via linear regression between Inverse Threads and Estimated Latency.
  '''
  res = {}

  res['Execution Seconds per Unit Work'] =  (
    kwargs['Intercept'] + kwargs['Q("Inverse Threads")']
  )
  res['Serial Fraction'] = (
    kwargs['Intercept'] / res['Execution Seconds per Unit Work']
  )
  res['Parallel Fraction'] = 1.0 - res['Serial Fraction']
  res['Threading Efficiency'] = (
    res['Serial Fraction'] / kwargs['Q("Inverse Threads")']
  )

  return res

def estimate_properties(model):
  '''
  Calculate estimates and bounds on scaling properties of interest
  from estimates and confidence intervals of params computed
  via linear regression between Inverse Threads and Estimated Latency
  '''
  # from DataFrame...
  #             0     1
  # Parameter1  val1  val2
  # Parameter2  val3  val4
  # ... to dict of lists
  # {
  #   Parameter1 : [val1, val2]
  #   Parameter2 : [val3, val4]
  # }
  conf_ints = {
      col : list(bounds)
      for col, bounds in model.conf_int().iterrows()
  }

  # take the product of dicts
  # use .items() to preserve ordering
  # adapted from http://stephantul.github.io/python/2019/07/20/product-dict/
  #
  # from to dict of lists...
  # {
  #   Parameter1 : [val1, val2]
  #   Parameter2 : [val3, val4]
  # }
  # ... to list of dicts (product of value lists)
  # [
  #   {
  #     Parameter1 : val1,
  #     Parameter2 : val3,
  #   },
  #   {
  #     Parameter1 : val1,
  #     Parameter2 : val4,
  #   },
  #   {
  #     Parameter1 : val2,
  #     Parameter2 : val3,
  #   },
  #   {
  #     Parameter1 : val2,
  #     Parameter2 : val4,
  #   },
  # ]
  keys, values = zip(*conf_ints.items())
  corners = [
    dict(zip(keys, bundle))
    for bundle in it.product(*values)
  ]
  assert(
    len(corners) == np.prod([len(val) for val in conf_ints.values()])
  )

  # from to dict of lists of raw parameter values...
  # [
  #   {
  #     Parameter1 : val1,
  #     Parameter2 : val3,
  #   },
  #   {
  #     Parameter1 : val1,
  #     Parameter2 : val4,
  #   },
  #   etc.
  # ]
  # ... to DataFrame of deduced parameters
  #    Parameter3   Parameter4   Parameter5
  # 0  val          val          val
  # 1  val          val          val
  # etc.
  extremes = pd.DataFrame([
    deduce_properties(**corner)
    for corner in corners
  ])

  # from DataFrame of deduced parameters...
  #    Parameter3   Parameter4   Parameter5
  # 0  val5         val6         val7
  # 1  val8         val9         val10
  # etc.
  # ... to DataFrame of deduced parameter bounds and estimates
  #    Parameter    Lower Bound   Upper Bound   Estimate
  # 0  Parameter3   val           val           val
  # 1  Parameter4   val           val           val
  # 2  Parameter5   val           val
  res = pd.DataFrame([
    {
      'Parameter' : param,
      'Lower Bound' : extremes[param].min(),
      'Upper Bound' : extremes[param].max(),
      'Estimate' : deduce_properties(
        **model.params
      )[param],
    } for param in extremes
  ])

  return res

# load and set up DataFrame
df = pd.read_csv(kn.pack({
  "synchronous" : str(synchronous),
  "ext" : ".csv",
}))
df['Inverse Threads'] = 1 / df['Threads']

# independently for each replicate, estimate overhead and latency
df = df.groupby([
  'Threads',
  'Replicate',
]).apply(
  estimate_latency,
)

# model relationship between latency (inverse throughput) and thread count
model = smf.glm(
  formula='Q("Estimated Latency") ~ Q("Inverse Threads")',
  data=df.drop_duplicates([
    'Threads',
    'Replicate',
  ]),
).fit()

# start a dataframe to collect estimates of scaling properties
res = estimate_properties(model)

# visualize regression
plot_regression(
  df.drop_duplicates([
    'Threads',
    'Replicate',
  ]),
  'Inverse Threads',
  'Estimated Latency',
)

# now, for posterity...
# estimate overhead and latency using /all/ replicates
df = df.groupby(
  'Threads',
).apply(
  estimate_latency,
)

# plot regressions used to estimate overhead and latency
for threads, chunk in df.groupby(
  'Threads',
):
  plot_regression(
    chunk,
    'Work',
    'Time',
    extra_names={'threads' : threads},
  )

# extract estimates of overhead and latency
res = res.append(pd.DataFrame([
    {
      'Parameter' : '{} @ {} Threads'.format(name, threads),
      'Lower Bound' : ip.pophomogeneous(chunk['Lower Bound ' + name]),
      'Upper Bound' : ip.pophomogeneous(chunk['Upper Bound ' + name]),
      'Estimate' : ip.pophomogeneous(chunk['Estimated ' + name]),
    } for threads, chunk in df.groupby('Threads')
    for name in ('Overhead', 'Latency')
]))

# consolidate and save computed estimates and bounds
res.sort_values([
  'Parameter',
]).to_csv(
  kn.pack({
    'title' : 'parameter_estimates',
    'synchronous' : str(synchronous),
    'ext' : '.csv',
  }),
  index=False,
)

reason: import numpy,from scipy,import statsmodels


text:
import glob
import json
import pdb
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import csv

with open('k80_only_JCT.json', 'r') as fp:
    k80_only = json.load(fp)
with open('oracle_JCT.json', 'r') as fp:
    oracle_only = json.load(fp)
with open('v100_only_JCT.json', 'r') as fp:
    v100_only = json.load(fp)
with open('oracle_K80_time.json', 'r') as fp:
    oracle_K80_only = json.load(fp)
with open('oracle_V100_time.json', 'r') as fp:
    oracle_V100_only = json.load(fp)
with open('oracle_overhead.json', 'r') as fp:
    oracle_overhead_only = json.load(fp)
with open('oracle_epoch_waste.json', 'r') as fp:
    oracle_epoch_waste_only = json.load(fp)
with open('oracle_num_mig.json', 'r') as fp:
    oracle_num_mig_only = json.load(fp)
with open('total_time.json', 'r') as fp:
    total_time_only = json.load(fp)
with open('load_time.json', 'r') as fp:
    load_time_only = json.load(fp)
with open('save_time.json', 'r') as fp:
    save_time_only = json.load(fp)
with open('speedup.json', 'r') as fp:
    speedup_only = json.load(fp)

job_list = []
oracle = []
k80 = []
v100 = []
oracle_K80 = []
oracle_V100 = []
oracle_overhead = []
oracle_epoch_waste = []

oracle_num_mig = []
total_time = []
load_time = []
save_time = []
speedup = []

for i in range(50):
    job = str(i+1)
    job_list.append('job'+job)
    oracle.append(oracle_only[job])
    k80.append(k80_only[job])
    v100.append(v100_only[job])
    oracle_K80.append(oracle_K80_only[job])
    oracle_V100.append(oracle_V100_only[job])
    oracle_overhead.append(oracle_overhead_only[job])
    oracle_epoch_waste.append(oracle_epoch_waste_only['job'+job])
    oracle_num_mig.append(oracle_num_mig_only[job])
    total_time.append(total_time_only[job])
    load_time.append(load_time_only[job])
    save_time.append(save_time_only[job])
    speedup.append(round(speedup_only[job],2))

job_list = np.asarray(job_list)
oracle = np.asarray(oracle)
k80 = np.asarray(k80)
v100 = np.asarray(v100)
oracle_K80 = np.asarray(oracle_K80)
oracle_V100 = np.asarray(oracle_V100)
oracle_overhead = np.asarray(oracle_overhead)
oracle_epoch_waste = np.asarray(oracle_epoch_waste)

oracle_num_mig = np.asarray(oracle_num_mig)
total_time = np.asarray(total_time)
load_time = np.asarray(load_time)
save_time = np.asarray(save_time)
speedup = np.asarray(speedup)

rows = zip(job_list, k80, v100, oracle, oracle_K80, oracle_V100, oracle_overhead, oracle_epoch_waste, oracle_num_mig,
total_time, save_time, load_time, speedup)
with open('comparison.csv', 'w') as f:
    writer = csv.writer(f)
    for row in rows:
        writer.writerow(row)

#np.savetxt('comparison.csv', (job_list, k80, v100, oracle, oracle_K80, oracle_V100, oracle_overhead, oracle_num_mig,
#total_time, save_time, load_time), fmt='%s')


text:
#!/usr/bin/env python3
# This file is covered by the LICENSE file in the root of this project.

import torch
import torch.nn as nn
import torch.optim as optim
import torch.backends.cudnn as cudnn
import torchvision.transforms as transforms
import imp
import yaml
import time
from PIL import Image
import numpy as np
import __init__ as booger
import collections
import copy
import os

from backbones.config import *
from tasks.segmentation.modules.head import *
from tasks.segmentation.modules.segmentator import *
import onnx

class TraceSaver():
  def __init__(self, path, new_path, force_img_prop=(None, None)):
    # parameters
    self.path = path
    self.new_path = new_path

    # config from path
    try:
      yaml_path = self.path + "/cfg.yaml"
      print("Opening config file %s" % yaml_path)
      self.CFG = yaml.safe_load(open(yaml_path, 'r'))
    except Exception as e:
      print(e)
      print("Error opening cfg.yaml file from trained model.")
      quit()

    # if force img prop is a tuple with 2 elements, force image props
    if force_img_prop[0] is not None or force_img_prop[1] is not None:
      if force_img_prop[0] is not None:
        self.CFG["dataset"]["img_prop"]["height"] = force_img_prop[0]
      if force_img_prop[1] is not None:
        self.CFG["dataset"]["img_prop"]["width"] = force_img_prop[1]
      print("WARNING: FORCING IMAGE PROPERTIES TO")
      print(self.CFG["dataset"]["img_prop"])

    # get the data
    parserModule = imp.load_source("parserModule",
                                   booger.TRAIN_PATH + '/tasks/segmentation/dataset/' +
                                   self.CFG["dataset"]["name"] + '/parser.py')
    self.parser = parserModule.Parser(img_prop=self.CFG["dataset"]["img_prop"],
                                      img_means=self.CFG["dataset"]["img_means"],
                                      img_stds=self.CFG["dataset"]["img_stds"],
                                      classes=self.CFG["dataset"]["labels"],
                                      train=False)
    self.data_h, self.data_w, self.data_d = self.parser.get_img_size()

    # get architecture and build backbone (with pretrained weights)
    self.bbone_cfg = BackboneConfig(name=self.CFG["backbone"]["name"],
                                    os=self.CFG["backbone"]["OS"],
                                    h=self.data_h,
                                    w=self.data_w,
                                    d=self.data_d,
                                    dropout=self.CFG["backbone"]["dropout"],
                                    bn_d=self.CFG["backbone"]["bn_d"],
                                    extra=self.CFG["backbone"]["extra"])

    self.decoder_cfg = DecoderConfig(name=self.CFG["decoder"]["name"],
                                     dropout=self.CFG["decoder"]["dropout"],
                                     bn_d=self.CFG["decoder"]["bn_d"],
                                     extra=self.CFG["decoder"]["extra"])

    self.head_cfg = HeadConfig(n_class=self.parser.get_n_classes(),
                               dropout=self.CFG["head"]["dropout"])

    # concatenate the encoder and the head
    with torch.no_grad():
      self.model = Segmentator(self.bbone_cfg,
                               self.decoder_cfg,
                               self.head_cfg,
                               self.path,
                               strict=True)

    # CUDA speedup?
    if torch.cuda.is_available():
      cudnn.fastest = True
      cudnn.benchmark = True
      self.model = self.model.cuda()

    # don't train
    self.model.eval()
    for w in self.model.parameters():
      w.requires_grad = False

    # print number of parameters and the ones requiring gradients
    weights_total = sum(p.numel()
                        for p in self.model.parameters())
    weights_grad = sum(p.numel()
                       for p in self.model.parameters() if p.requires_grad)
    print("Total number of parameters: ", weights_total)
    print("Total number of parameters requires_grad: ", weights_grad)

    # profiler based saver, so create a dummy input to infer
    print("Creating dummy input to profile")
    self.dummy_input = torch.randn(1, self.CFG["dataset"]["img_prop"]["depth"],
                                   self.CFG["dataset"]["img_prop"]["height"],
                                   self.CFG["dataset"]["img_prop"]["width"])
    # gpu?
    if torch.cuda.is_available():
      self.dummy_input = self.dummy_input.cuda()

  def export_config(self):
    # save the config file in the log folder
    try:
      new_yaml_path = self.new_path + "/cfg.yaml"
      print("Saving config file %s" % new_yaml_path)
      with open(new_yaml_path, 'w') as outfile:
        yaml.dump(self.CFG, outfile, default_flow_style=False)
    except Exception as e:
      print(e)
      print("Error saving cfg.yaml in new model dir ", new_yaml_path)
      quit()

  def export_ONNX(self):
    # convert to ONNX traced model

    # create profile
    onnx_path = os.path.join(self.new_path, "model.onnx")
    with torch.no_grad():
      print("Profiling model")
      print("saving model in ", onnx_path)
      torch.onnx.export(self.model, self.dummy_input, onnx_path)

    # check that it worked
    print("Checking that it all worked out")
    model_onnx = onnx.load(onnx_path)
    onnx.checker.check_model(model_onnx)

    # Print a human readable representation of the graph
    # print(onnx.helper.printable_graph(model_onnx.graph))

  def export_pytorch(self):
    # convert to Pytorch traced model

    # create profile
    pytorch_path = os.path.join(self.new_path, "model.pytorch")
    with torch.no_grad():
      print("Profiling model")
      pytorch_model = torch.jit.trace(self.model, self.dummy_input)
      print("saving model in ", pytorch_path)
      pytorch_model.save(pytorch_path)

  def export(self):
    """Export config file, ONNX model, and Pytorch traced model to log directory
    """
    self.export_config()
    self.export_ONNX()
    self.export_pytorch()

max_stars_repo_name: hamzaMahdi/bonnetal

max_issues_repo_name: hamzaMahdi/bonnetal

max_forks_repo_name: hamzaMahdi/bonnetal


text:
"""
Microsimulation base class - common functionality
"""

import pandas as pd
import numpy as np

import ukcensusapi.Nomisweb as Api_ew
import ukcensusapi.NRScotland as Api_sc
import humanleague as hl
import microsimulation.utils as utils

class Base(object):
  """
  Microsimulation base class - common functionality
  """

  def __init__(self, region, resolution, cache_dir):
    self.region = region
    self.resolution = resolution
    self.data_api_en = Api_ew.Nomisweb(cache_dir)
    self.data_api_sc = Api_sc.NRScotland(cache_dir)

  def get_census_data(self):
    if self.region[0] == "S":
      return self.__get_census_data_sc()
    elif self.region[0] == "N":
      raise NotImplementedError("NI support not yet implemented")
    else:
      return self.__get_census_data_ew()

  def __get_census_data_sc(self):

    print("Synthesising Scottish DC1117/DC2101 tables from LAD-level seeds and univariate data")

    # age only, no gender
    qs103sc = self.data_api_sc.get_data("QS103SC", self.region, self.resolution, category_filters={"QS103SC_0_CODE": range(1,102)})
    qs103sc = utils.cap_value(qs103sc, "QS103SC_0_CODE", 86, "OBS_VALUE")
    # sex only
    qs104sc = self.data_api_sc.get_data("QS104SC", self.region, self.resolution, category_filters={"QS104SC_0_CODE": [1,2]})

    ngeogs = len(qs103sc.GEOGRAPHY_CODE.unique())
    nages = len(qs103sc.QS103SC_0_CODE.unique())
    nsexes = 2

    # Get a LAD-level seed population by age and gender
    dc1117lad = self.data_api_sc.get_data("DC1117SC", self.region, "LAD", category_filters={"DC1117SC_0_CODE": [1,2], "DC1117SC_1_CODE": range(1,102)})
    dc1117lad = utils.cap_value(dc1117lad, "DC1117SC_1_CODE", 86, "OBS_VALUE")
    dc1117seed = utils.unlistify(dc1117lad, ["DC1117SC_0_CODE", "DC1117SC_1_CODE"], [2, 86], "OBS_VALUE").astype(float)
    # expand to all geogs within LAD
    dc1117seed = np.dstack([dc1117seed] * ngeogs).T

    ga = utils.unlistify(qs103sc, ["GEOGRAPHY_CODE", "QS103SC_0_CODE"], [ngeogs, nages], "OBS_VALUE")
    gs = utils.unlistify(qs104sc, ["GEOGRAPHY_CODE", "QS104SC_0_CODE"], [ngeogs, nsexes], "OBS_VALUE")
    msynth = hl.qisi(dc1117seed, [np.array([0,1]), np.array([0,2])], [ga,gs])
    #msynth = hl.qis([np.array([0,1]), np.array([0,2])], [ga,gs])
    utils.check_result(msynth)
    # TODO pending humanleague seed consistency check
    assert dc1117seed.shape == msynth["result"].shape

    dc1117sc = utils.listify(msynth["result"], "OBS_VALUE", ["GEOGRAPHY_CODE", "C_AGE", "C_SEX"])
    dc1117sc.GEOGRAPHY_CODE = utils.remap(dc1117sc.GEOGRAPHY_CODE, qs103sc.GEOGRAPHY_CODE.unique())
    dc1117sc.C_AGE = utils.remap(dc1117sc.C_AGE, qs103sc.QS103SC_0_CODE.unique())
    dc1117sc.C_SEX = utils.remap(dc1117sc.C_SEX, [1, 2])
    #print(dc1117sc.head())

    # These ETH codes are slightly different to E&W codes...
    # ETH Totals = [1,8,9,15,18,22]
    #eths = [2,3,4,5,6,7,8,10,11,12,13,14,16,17,19,20,21,23,24]
    eths = [1,8,9,15,18,22]
    ks201sc = self.data_api_sc.get_data("KS201SC", self.region, self.resolution, category_filters={"KS201SC_0_CODE": eths})
    neths = len(ks201sc.KS201SC_0_CODE.unique())

    # Get a LAD-level seed population by age and gender
    dc2101lad = self.data_api_sc.get_data("DC2101SC", self.region, "LAD", category_filters={"DC2101SC_0_CODE": eths, "DC2101SC_1_CODE": [1,2], "DC2101SC_2_CODE": 0})
    dc2101seed = utils.unlistify(dc2101lad, ["DC2101SC_1_CODE", "DC2101SC_0_CODE"], [2, neths], "OBS_VALUE").astype(float)
    # expand to all geogs within LAD
    dc2101seed = np.dstack([dc2101seed] * ngeogs).T

    #print(ks201sc.head())
    ge = utils.unlistify(ks201sc, ["GEOGRAPHY_CODE", "KS201SC_0_CODE"], [ngeogs, neths], "OBS_VALUE")
    # TODO use a LAD-level seed population
    msynth = hl.qisi(dc2101seed, [np.array([0,1]), np.array([0,2])], [ge,gs])
    utils.check_result(msynth)
    assert dc2101seed.shape == msynth["result"].shape

    dc2101sc = utils.listify(msynth["result"], "OBS_VALUE", ["GEOGRAPHY_CODE", "C_ETHPUK11", "C_SEX"])
    dc2101sc.GEOGRAPHY_CODE = utils.remap(dc2101sc.GEOGRAPHY_CODE, qs103sc.GEOGRAPHY_CODE.unique())
    dc2101sc.C_ETHPUK11 = utils.remap(dc2101sc.C_ETHPUK11, ks201sc.KS201SC_0_CODE.unique())
    dc2101sc.C_SEX = utils.remap(dc2101sc.C_SEX, [1, 2])
    #print(dc2101sc.head())

    assert dc1117sc.OBS_VALUE.sum() == dc2101sc.OBS_VALUE.sum()

    #print(self.data_api_sc.get_metadata("DC6206SC", "LAD"))
    # TODO Aberdeen has 174869 in this table
    # dc6206sc = self.data_api_sc.get_data("DC6206SC", self.region, "LAD", category_filters={"DC6206SC_1_CODE": 0,
    #                                                                                "DC6206SC_0_CODE": [1,2,3,4,5,6],
    #                                                                                "DC6206SC_2_CODE": [1,2,3,4,5,6,7,8,9]})
    #print(dc6206sc.OBS_VALUE.sum())
    #print(dc6206sc.DC6206SC_2_CODE.unique())
    # # dc6206sc = self.data_api_sc.get_data("DC6206SC", "MSOA11", self.region)
    # #raise NotImplementedError("Problem with MSOA-level detailed characteristics in Scottish census data")

    return (dc1117sc, dc2101sc, None)

  def __get_census_data_ew(self):
    """
    Download/cache census data
    """
    # convert input string to enum
    resolution = self.data_api_en.GeoCodeLookup[self.resolution]

    region_codes = self.data_api_en.get_lad_codes(self.region)

    if not region_codes:
      raise ValueError("no regions match the input: \"" + self.region + "\"")

    area_codes = self.data_api_en.get_geo_codes(region_codes, resolution)

    # Census: sex by age by MSOA
    table = "DC1117EW"
    query_params = {"MEASURES": "20100",
                    "date": "latest",
                    "C_AGE": "1...86",
                    "select": "GEOGRAPHY_CODE,C_SEX,C_AGE,OBS_VALUE",
                    "C_SEX": "1,2",
                    "geography": area_codes}

    # problem - data only available at MSOA and above
    dc1117ew = self.data_api_en.get_data(table, query_params)

    # Census: sex by ethnicity by MSOA
    table = "DC2101EW"
    query_params = {"MEASURES": "20100",
                    "date": "latest",
                    "C_AGE": "0",
                    "C_ETHPUK11": "2,3,4,5,7,8,9,10,12,13,14,15,16,18,19,20,22,23",
                    "select": "GEOGRAPHY_CODE,C_SEX,C_ETHPUK11,OBS_VALUE",
                    "C_SEX": "1,2",
                    "geography": area_codes}
    # problem - data only available at MSOA and above
    dc2101ew = self.data_api_en.get_data(table, query_params)

    # This table contains only 16+ persons (under-16s do not have NS-SeC)
    table = "DC6206EW"
    query_params = {"date": "latest",
                    "MEASURES": "20100",
                    "C_SEX": "1,2",
                    "C_ETHPUK11": "2,3,4,5,7,8,9,10,12,13,14,15,16,18,19,20,22,23",
                    "C_NSSEC": "2...9,11,12,14,15,16",
                    "C_AGE": "1,2,3,4",
                    "select": "GEOGRAPHY_CODE,C_SEX,C_ETHPUK11,C_NSSEC,C_AGE,OBS_VALUE",
                    "geography": area_codes}
    dc6206ew = self.data_api_en.get_data(table, query_params)

    return (dc1117ew, dc2101ew, dc6206ew)

  def append_children(self, full_table, adults_table):
    """
      Append adult-only (16+) table (e.g DC6206) with children from full population table
    """
    # remember "17" census value means age 16
    children_table = full_table[full_table.C_AGE < 17].copy()
    children_table["C_NSSEC"] = 100 #

    # rename DC6206 C_AGE (band) so as not to conflict with DC1117 C_AGE (single year)
    adults_table = adults_table.rename(columns={"C_AGE": "C_AGEBAND"})

    x = pd.concat([adults_table, children_table], axis=0, ignore_index=True)
    assert full_table.OBS_VALUE.sum() == x.OBS_VALUE.sum()
    return x

max_stars_repo_name: niklomax/daedalus

max_issues_repo_name: niklomax/daedalus

max_forks_repo_name: Leeds-MRG/Minos


text:
from eqsig import sdof
import numpy as np
from tests import conftest
from tests.conftest import TEST_DATA_DIR

def test_nigam_and_jennings_vs_duhamels():
    record_path = TEST_DATA_DIR
    record_filename = 'test_motion_dt0p01.txt'
    dt = 0.01
    rec = np.loadtxt(record_path + record_filename, skiprows=2)
    periods = np.linspace(0.1, 2.5, 2)
    xi = 0.05
    resp_u, resp_v, sdof_acc = sdof.response_series(rec, dt, periods, xi)
    i = 0
    duhamel_u = sdof.single_elastic_response(rec, dt, periods[i], xi)
    diff_disp = np.sum(abs(resp_u[i] - duhamel_u))
    assert diff_disp < 2.0e-2, diff_disp
    i = 1
    duhamel_u = sdof.single_elastic_response(rec, dt, periods[i], xi)
    diff_disp = np.sum(abs(resp_u[i] - duhamel_u))
    assert diff_disp < 3.0e-2, diff_disp
    # import matplotlib.pyplot as plt
    # bf, sps = plt.subplots(figsize=(9, 5))
    # plt.plot(resp_u[i], lw=0.7)
    # plt.plot(duhamel_u,lw=0.7)
    # plt.show()

def show_nigam_and_jennings_vs_duhamels():
    record_path = TEST_DATA_DIR
    record_filename = 'test_motion_dt0p01.txt'
    dt = 0.01
    rec = np.loadtxt(record_path + record_filename, skiprows=2)
    periods = np.array([0.0, 0.05])
    xi = 0.05
    resp_u, resp_v, sdof_acc = sdof.response_series(rec, dt, periods, xi)

    import matplotlib.pyplot as plt
    bf, sps = plt.subplots(figsize=(9, 5))
    plt.plot(sdof_acc[0], lw=0.7)
    plt.plot(sdof_acc[1], lw=0.7)
    plt.show()

def test_pseudo_response_spectra():
    asig = conftest.t_asig()
    periods = [0, 2, 4]
    xi = 0.05
    sd, sv, sa = sdof.pseudo_response_spectra(asig.values, asig.dt, periods, xi)
    assert sa[0] == asig.pga

if __name__ == '__main__':
    show_nigam_and_jennings_vs_duhamels()

max_stars_repo_name: geosharma/eqsig

max_issues_repo_name: geosharma/eqsig

max_forks_repo_name: geosharma/eqsig


text:
import pylab
import tables
import math
import numpy

def exactSol(a, X):
    c0 = a/12.0 - 1.0/2.0
    c1 = 0.0
    return X**2/2 - a*X**4/12 + c0*X + c1

fh = tables.openFile("s79-poisson-1d_phi.h5")
q = fh.root.StructGridField
nx, nc = q.shape
Xf = pylab.linspace(0, 1, nx)
dx = Xf[1]-Xf[0]

Xhr = pylab.linspace(0, 1, 101)
fhr = exactSol(2, Xhr)

# make plot comparing exact to numerical solution
pylab.plot(Xhr, fhr, '-r', Xf, q[:,0], '-k')

# compute error
fex = exactSol(2, Xf)
error = numpy.abs(fex-q[:,0]).sum()/nx;

print "%g %g" % (dx, error)

pylab.show()

max_stars_repo_name: ammarhakim/ammar-simjournal

max_stars_repo_licenses: MIT Unlicense

max_issues_repo_name: ammarhakim/ammar-simjournal

max_issues_repo_licenses: MIT Unlicense

max_forks_repo_name: ammarhakim/ammar-simjournal

max_forks_repo_licenses: MIT Unlicense