text:
import numpy as np
import re
import os
import pickle
import itertools
from collections import Counter

cop = re.compile("[^\u4e00-\u9fa5^.^a-z^A-Z^0-9]")
company_x ="../data/formated_data/company/company_x_v1.pkl"
text_y = "../data/formated_data/company/company_texty_v1.pkl"
sen3_y = "../data/formated_data/company/company_sen3_v1.pkl"
sen5_y = "../data/formated_data/company/company_sen5_v1.pkl"

def clean_str(string):
    """
    Tokenization/string cleaning for all datasets except for SST.
    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    # string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    string = cop.sub("", string)
    return string.strip().lower()

def load_data_and_labels(dataset_name):
    """
    formated data: x.pkl & y.pkl
    """
    formated_xfile = "../data/formated_data/"+dataset_name+"/x.pkl"
    formated_yfile = "../data/formated_data/"+dataset_name+"/y.pkl"

    if os.path.exists(formated_xfile) and os.path.exists(formated_yfile):
        with open(formated_xfile, "rb") as file:
            x = pickle.load(file)
        with open(formated_yfile, "rb") as file:
            y = pickle.load(file)
    else:
        if dataset_name == "mr":
            if os.path.isdir("../data/formated_data/"+dataset_name+"/") == False:
                os.mkdir("../data/formated_data/"+dataset_name+"/")
            x, y = preprocess_mr("../data/rt-polaritydata/rt-polarity.pos.utf8","../data/rt-polaritydata/rt-polarity.neg.utf8")
            with open(formated_xfile, "wb") as file:
                pickle.dump(x, file)
            with open(formated_yfile, "wb") as file:
                pickle.dump(y, file)
        else:
            print("UNKNOWN dataset")
            exit(2)
    return x, y

def load_train_dev_test(dataset_name):
    formated_train_xfile = "../data/formated_data/"+dataset_name+"/train_x.pkl"
    formated_train_yfile = "../data/formated_data/"+dataset_name+"/train_y.pkl"
    formated_dev_xfile = "../data/formated_data/"+dataset_name+"/dev_x.pkl"
    formated_dev_yfile = "../data/formated_data/"+dataset_name+"/dev_y.pkl"
    formated_test_xfile = "../data/formated_data/"+dataset_name+"/test_x.pkl"
    formated_test_yfile = "../data/formated_data/"+dataset_name+"/test_y.pkl"
    with open(formated_train_xfile, "rb") as file:
        train_x = pickle.load(file)
    with open(formated_train_yfile, "rb") as file:
        train_y = pickle.load(file)
    with open(formated_dev_xfile, "rb") as file:
        dev_x = pickle.load(file)
    with open(formated_dev_yfile, "rb") as file:
        dev_y = pickle.load(file)
    with open(formated_test_xfile, "rb") as file:
        test_x = pickle.load(file)
    with open(formated_test_yfile, "rb") as file:
        test_y = pickle.load(file)
    return train_x, train_y, dev_x, dev_y, test_x, test_y

def get_train_dev_test(x, y, dataset_name):
    formated_train_xfile = "../data/formated_data/"+dataset_name+"/train_x.pkl"
    formated_train_yfile = "../data/formated_data/"+dataset_name+"/train_y.pkl"
    formated_dev_xfile = "../data/formated_data/"+dataset_name+"/dev_x.pkl"
    formated_dev_yfile = "../data/formated_data/"+dataset_name+"/dev_y.pkl"
    formated_test_xfile = "../data/formated_data/"+dataset_name+"/test_x.pkl"
    formated_test_yfile = "../data/formated_data/"+dataset_name+"/test_y.pkl"
    np.random.seed(10)
    shuffle_indices = np.random.permutation(np.arange(len(y)))
    x_shuffled = x[shuffle_indices]
    y_shuffled = y[shuffle_indices]
    dev_sample_index = -1 * int(0.1 * float(len(y)))
    x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[dev_sample_index:]
    y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[dev_sample_index:]
    dev_test_len = int(len(x_dev)/2)
    x_test, y_test = x_dev[:dev_test_len], y_dev[:dev_test_len]
    x_dev, y_dev = x_dev[dev_test_len:], y_dev[dev_test_len:]

    with open(formated_train_xfile, "wb") as file:
        pickle.dump(x_train, file)
    with open(formated_train_yfile, "wb") as file:
        pickle.dump(y_train, file)
    with open(formated_dev_xfile, "wb") as file:
        pickle.dump(x_dev, file)
    with open(formated_dev_yfile, "wb") as file:
        pickle.dump(y_dev, file)
    with open(formated_test_xfile, "wb") as file:
        pickle.dump(x_test, file)
    with open(formated_test_yfile, "wb") as file:
        pickle.dump(y_test, file)
    return x_train, y_train, x_dev, y_dev, x_test, y_test

def batch_iter(data, batch_size, num_epochs, shuffle=True):
    """
    Generates a batch iterator for a dataset.
    """
    data = np.array(data)
    data_size = len(data)
    num_batches_per_epoch = int((len(data)-1)/batch_size) + 1
    for epoch in range(num_epochs):
        # Shuffle the data at each epoch
        if shuffle:
            shuffle_indices = np.random.permutation(np.arange(data_size))
            shuffled_data = data[shuffle_indices]
        else:
            shuffled_data = data
        for batch_num in range(num_batches_per_epoch):
            start_index = batch_num * batch_size
            end_index = min((batch_num + 1) * batch_size, data_size)
            yield shuffled_data[start_index:end_index]

def preprocess_mr(pos_data_file, neg_data_file):
    """
    Loads MR polarity data from files, splits the data into words and generates labels.
    Returns split sentences and labels.
    """
    # Load data from files
    positive_examples = list(open(pos_data_file, "r").readlines())
    positive_examples = [s.strip() for s in positive_examples]
    negative_examples = list(open(neg_data_file, "r").readlines())
    negative_examples = [s.strip() for s in negative_examples]
    # Split by words
    x_text = positive_examples + negative_examples
    x_text = [clean_str(sent) for sent in x_text]
    # Generate labels
    positive_labels = [[0, 1] for _ in positive_examples]
    negative_labels = [[1, 0] for _ in negative_examples]
    y = np.concatenate([positive_labels, negative_labels], 0)
    return [x_text, y]

def float_to_int(float_str):
    float_str = float(float_str)
    if float_str <= 0.2:
        return [1,0,0,0,0]
    elif float_str <= 0.4:
        return [0,1,0,0,0]
    elif float_str <= 0.6:
        return [0,0,1,0,0]
    elif float_str <= 0.8:
        return [0,0,0,1,0]
    else:
        return [0,0,0,0,1]

def preprocess_sst(file_name):
    x_text = list(open(file_name, "r").readlines())
    y = [float_to_int(x.strip().split("\t")[1]) for x in x_text]
    x_text = [x.strip().split("\t")[0] for x in x_text]
    x_text = [clean_str(sent) for sent in x_text]
    return x_text, y

def preprocess_sst1(dataset_name):
    if os.path.isdir("../data/formated_data/"+dataset_name+"/") == False:
        os.mkdir("../data/formated_data/"+dataset_name+"/")
    x, y = preprocess_sst("../data/sst1/train.txt")
    with open("../data/formated_data/"+dataset_name+"/train_x.pkl", "wb") as file:
        pickle.dump(x, file)
    with open("../data/formated_data/"+dataset_name+"/train_y.pkl", "wb") as file:
        pickle.dump(y, file)
    x, y = preprocess_sst("../data/sst1/dev.txt")
    with open("../data/formated_data/"+dataset_name+"/dev_x.pkl", "wb") as file:
        pickle.dump(x, file)
    with open("../data/formated_data/"+dataset_name+"/dev_y.pkl", "wb") as file:
        pickle.dump(y, file)
    x, y = preprocess_sst("../data/sst1/test.txt")
    with open("../data/formated_data/"+dataset_name+"/test_x.pkl", "wb") as file:
        pickle.dump(x, file)
    with open("../data/formated_data/"+dataset_name+"/test_y.pkl", "wb") as file:
        pickle.dump(y, file)

def shuffle_data(x, y):
    np.random.seed(10)
    shuffle_indices = np.random.permutation(np.arange(len(y)))
    # print(y)
    x_shuffled = x[shuffle_indices]
    y_shuffled = y[shuffle_indices]
    return x_shuffled, y_shuffled

def load_company_data_x(filename):
    with open(filename, "rb") as file:
        train_x = pickle.load(file)
        train_x = [" ".join(list(x)) for x in train_x]
        # print(train_x)
        return train_x

def load_company_data_y(filename):
    with open(filename, "rb") as file:
        y = pickle.load(file)
        return y

def split_data(flag):
    x = load_company_data_x(company_x)
    if flag == "text":
        y = load_company_data_y(text_y)
    elif flag == "sen3":
        y = load_company_data_y(sen3_y)
    elif flag == "sen5":
        y = load_company_data_y(sen5_y)
    np.random.seed(10)
    # shuffle_indices = np.random.permutation(np.arange(len(y)))
    # x_shuffled = x[shuffle_indices]
    # y_shuffled = y[shuffle_indices]
    dev_sample_index = -1 * int(0.2 * float(len(y)))
    x_train, x_dev = x[:dev_sample_index], x[dev_sample_index:]
    y_train, y_dev = y[:dev_sample_index], y[dev_sample_index:]
    dev_test_len = int(len(x_dev)/2)
    x_test, y_test = x_dev[:dev_test_len], y_dev[:dev_test_len]
    x_dev, y_dev = x_dev[dev_test_len:], y_dev[dev_test_len:]
    with open("../data/formated_data/company/"+flag+"/train_x.pkl", "wb") as file:
        pickle.dump(x_train, file)
    with open("../data/formated_data/company/"+flag+"/train_y.pkl", "wb") as file:
        pickle.dump(y_train, file)
    with open("../data/formated_data/company/"+flag+"/dev_x.pkl", "wb") as file:
        pickle.dump(x_dev, file)
    with open("../data/formated_data/company/"+flag+"/dev_y.pkl", "wb") as file:
        pickle.dump(y_dev, file)
    with open("../data/formated_data/company/"+flag+"/test_x.pkl", "wb") as file:
        pickle.dump(x_test, file)
    with open("../data/formated_data/company/"+flag+"/test_y.pkl", "wb") as file:
        pickle.dump(y_test, file)

def load_company_data(flag):
    formated_train_xfile = "../data/formated_data/company/"+flag+"/train_x.pkl"
    formated_train_yfile = "../data/formated_data/company/"+flag+"/train_y.pkl"
    formated_dev_xfile = "../data/formated_data/company/"+flag+"/dev_x.pkl"
    formated_dev_yfile = "../data/formated_data/company/"+flag+"/dev_y.pkl"
    formated_test_xfile = "../data/formated_data/company/"+flag+"/test_x.pkl"
    formated_test_yfile = "../data/formated_data/company/"+flag+"/test_y.pkl"
    with open(formated_train_xfile, "rb") as file:
        train_x = pickle.load(file)
    with open(formated_train_yfile, "rb") as file:
        train_y = pickle.load(file)
    with open(formated_dev_xfile, "rb") as file:
        dev_x = pickle.load(file)
    with open(formated_dev_yfile, "rb") as file:
        dev_y = pickle.load(file)
    with open(formated_test_xfile, "rb") as file:
        test_x = pickle.load(file)
    with open(formated_test_yfile, "rb") as file:
        test_y = pickle.load(file)
    return train_x, train_y, dev_x, dev_y, test_x, test_y


text:
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as colors
from matplotlib.colors import LinearSegmentedColormap
from matplotlib.ticker import AutoMinorLocator
import conf

# set rcParams to configure plots
plt.rcParams.update({'figure.autolayout': True})
plt.rcParams.update({'figure.autolayout': True})
plt.rcParams.update({'font.size': 18})
plt.rcParams.update({'axes.titlesize': 18})
plt.rcParams.update({'axes.labelsize': 18})
plt.rcParams.update({'xtick.labelsize': 18})
plt.rcParams.update({'ytick.labelsize': 18})
plt.rcParams.update({'axes.linewidth': 1.5})
plt.rcParams.update({'xtick.major.size': 6})
plt.rcParams.update({'xtick.major.width': 1.5})
plt.rcParams.update({'ytick.major.size': 6})
plt.rcParams.update({'ytick.major.width': 1.5})
plt.rcParams.update({'xtick.minor.size': 3})
plt.rcParams.update({'xtick.minor.width': 1.0})
plt.rcParams.update({'ytick.minor.size': 3})
plt.rcParams.update({'ytick.minor.width': 1.0})
plt.rcParams.update({'ytick.minor.visible': False})
plt.rcParams.update({'xtick.minor.visible': False})
plt.rcParams.update({'xtick.top': False})
plt.rcParams.update({'ytick.right': False})
plt.rcParams.update({'xtick.direction': 'out'})
plt.rcParams.update({'ytick.direction': 'out'})
plt.rcParams.update({'mathtext.fontset': 'cm'})
plt.rcParams["font.family"] = "Times New Roman"

# midpoint normalization which is applied to color scales and color bars, such that a diverging scale from a fixed midpoint value is obtained

class MidpointNormalize(colors.Normalize):

    def __init__(self, vmin=None, vmax=None, midpoint=None, clip=False):
        self.midpoint = midpoint
        colors.Normalize.__init__(self, vmin, vmax, clip)

    def __call__(self, value, clip=None):
        x, y = [self.vmin, self.midpoint, self.vmax], [0, 0.5, 1]
        return np.ma.masked_array(np.interp(value, x, y), np.isnan(value))

# plot indicator along selected linescan of constant Nf

def plot_linescan(U_range, Y_true_U, Y_pred_U, indicator_array, dim):

    # construct color map from selected colors
    basic_cols = ['black', 'darkred', 'indianred', 'red']
    my_cmap = LinearSegmentedColormap.from_list('mycmap', basic_cols)

    # construct normalization for color bar and color scale
    elev_min = 0
    elev_max = np.max(indicator_array)
    mid_val = np.mean(indicator_array)
    norm = MidpointNormalize(midpoint=mid_val, vmin=elev_min, vmax=elev_max)

    # plot indicator along selected linescan
    fig, ax = plt.subplots()
    ax.scatter(Y_true_U[1:-1], indicator_array, c=indicator_array, cmap=my_cmap,
               norm=norm, label='divergence')

    # set ticks and axis labels
    ax.set_xlabel('$U$')
    ax.set_ylabel('$\partial \delta U/\partial U$')
    ax.tick_params(axis='y', which='both', labelcolor='k')

    # plot predictions \hat{U} along selected linescan
    ax2 = ax.twinx()
    ax2.plot(U_range, Y_pred_U, color='b', linestyle='dashed')

    # set ticks and axis labels
    ax2.tick_params(axis='y', which='both', colors='b')
    ax2.set_ylabel('$\hat{U}$', color='b', ha='right', labelpad=10)

    ax.xaxis.set_minor_locator(AutoMinorLocator(2))
    ax.yaxis.set_minor_locator(AutoMinorLocator(2))
    ax2.yaxis.set_minor_locator(AutoMinorLocator(2))

    # save and close plot
    plt.savefig('./results/result_linear_models_{}_{}_nf_{:.0f}.pdf'.format(
        conf.case, conf.input_type, conf.nf))
    plt.close(fig)

ext: py

reason: import numpy


text:
#!/usr/bin/env python3

# Copyright 2018 Nagoya University (Tomoki Hayashi)
#  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)

"""Text-to-speech model training script."""

import logging
import multiprocessing as mp
import os
import random
import subprocess
import sys

import configargparse
import numpy as np

from espnet.nets.tts_interface import TTSInterface
from espnet.utils.cli_utils import strtobool
from espnet.utils.training.batchfy import BATCH_COUNT_CHOICES

# NOTE: you need this func to generate our sphinx doc
def get_parser():
    """Get parser of training arguments."""
    parser = configargparse.ArgumentParser(
        description='Train a new text-to-speech (TTS) model on one CPU, one or multiple GPUs',
        config_file_parser_class=configargparse.YAMLConfigFileParser,
        formatter_class=configargparse.ArgumentDefaultsHelpFormatter)

    # general configuration
    parser.add('--config', is_config_file=True,
               help='config file path')
    parser.add('--config2', is_config_file=True,
               help='second config file path that overwrites the settings in `--config`.')
    parser.add('--config3', is_config_file=True,
               help='third config file path that overwrites the settings in `--config` and `--config2`.')

    parser.add_argument('--ngpu', default=None, type=int,
                        help='Number of GPUs. If not given, use all visible devices')
    parser.add_argument('--backend', default='pytorch', type=str,
                        choices=['chainer', 'pytorch'],
                        help='Backend library')
    parser.add_argument('--outdir', type=str, required=True,
                        help='Output directory')
    parser.add_argument('--debugmode', default=1, type=int,
                        help='Debugmode')
    parser.add_argument('--seed', default=1, type=int,
                        help='Random seed')
    parser.add_argument('--resume', '-r', default='', type=str, nargs='?',
                        help='Resume the training from snapshot')
    parser.add_argument('--minibatches', '-N', type=int, default='-1',
                        help='Process only N minibatches (for debug)')
    parser.add_argument('--verbose', '-V', default=0, type=int,
                        help='Verbose option')
    parser.add_argument('--tensorboard-dir', default=None, type=str, nargs='?',
                        help="Tensorboard log directory path")
    parser.add_argument('--eval-interval-epochs', default=1, type=int,
                        help="Evaluation interval epochs")
    parser.add_argument('--save-interval-epochs', default=1, type=int,
                        help="Save interval epochs")
    parser.add_argument('--report-interval-iters', default=100, type=int,
                        help="Report interval iterations")
    # task related
    parser.add_argument('--train-json', type=str, required=True,
                        help='Filename of training json')
    parser.add_argument('--valid-json', type=str, required=True,
                        help='Filename of validation json')
    # network architecture
    parser.add_argument('--model-module', type=str, default="espnet.nets.pytorch_backend.e2e_tts_tacotron2:Tacotron2",
                        help='model defined module')
    # minibatch related
    parser.add_argument('--sortagrad', default=0, type=int, nargs='?',
                        help="How many epochs to use sortagrad for. 0 = deactivated, -1 = all epochs")
    parser.add_argument('--batch-sort-key', default='shuffle', type=str,
                        choices=['shuffle', 'output', 'input'], nargs='?',
                        help='Batch sorting key. "shuffle" only work with --batch-count "seq".')
    parser.add_argument('--batch-count', default='auto', choices=BATCH_COUNT_CHOICES,
                        help='How to count batch_size. The default (auto) will find how to count by args.')
    parser.add_argument('--batch-size', '--batch-seqs', '-b', default=0, type=int,
                        help='Maximum seqs in a minibatch (0 to disable)')
    parser.add_argument('--batch-bins', default=0, type=int,
                        help='Maximum bins in a minibatch (0 to disable)')
    parser.add_argument('--batch-frames-in', default=0, type=int,
                        help='Maximum input frames in a minibatch (0 to disable)')
    parser.add_argument('--batch-frames-out', default=0, type=int,
                        help='Maximum output frames in a minibatch (0 to disable)')
    parser.add_argument('--batch-frames-inout', default=0, type=int,
                        help='Maximum input+output frames in a minibatch (0 to disable)')
    parser.add_argument('--maxlen-in', '--batch-seq-maxlen-in', default=100, type=int, metavar='ML',
                        help='When --batch-count=seq, batch size is reduced if the input sequence length > ML.')
    parser.add_argument('--maxlen-out', '--batch-seq-maxlen-out', default=200, type=int, metavar='ML',
                        help='When --batch-count=seq, batch size is reduced if the output sequence length > ML')
    parser.add_argument('--num-iter-processes', default=0, type=int,
                        help='Number of processes of iterator')
    parser.add_argument('--preprocess-conf', type=str, default=None,
                        help='The configuration file for the pre-processing')
    parser.add_argument('--use-speaker-embedding', default=False, type=strtobool,
                        help='Whether to use speaker embedding')
    parser.add_argument('--use-second-target', default=False, type=strtobool,
                        help='Whether to use second target')
    # optimization related
    parser.add_argument('--opt', default='adam', type=str,
                        choices=['adam', 'noam'],
                        help='Optimizer')
    parser.add_argument('--accum-grad', default=1, type=int,
                        help='Number of gradient accumuration')
    parser.add_argument('--lr', default=1e-3, type=float,
                        help='Learning rate for optimizer')
    parser.add_argument('--eps', default=1e-6, type=float,
                        help='Epsilon for optimizer')
    parser.add_argument('--weight-decay', default=1e-6, type=float,
                        help='Weight decay coefficient for optimizer')
    parser.add_argument('--epochs', '-e', default=30, type=int,
                        help='Number of maximum epochs')
    parser.add_argument('--early-stop-criterion', default='validation/main/loss', type=str, nargs='?',
                        help="Value to monitor to trigger an early stopping of the training")
    parser.add_argument('--patience', default=3, type=int, nargs='?',
                        help="Number of epochs to wait without improvement before stopping the training")
    parser.add_argument('--grad-clip', default=1, type=float,
                        help='Gradient norm threshold to clip')
    parser.add_argument('--num-save-attention', default=5, type=int,
                        help='Number of samples of attention to be saved')
    parser.add_argument('--keep-all-data-on-mem', default=False, type=strtobool,
                        help='Whether to keep all data on memory')

    return parser

def main(cmd_args):
    """Run training."""
    parser = get_parser()
    args, _ = parser.parse_known_args(cmd_args)

    from espnet.utils.dynamic_import import dynamic_import
    model_class = dynamic_import(args.model_module)
    assert issubclass(model_class, TTSInterface)
    model_class.add_arguments(parser)
    args = parser.parse_args(cmd_args)

    # logging info
    if args.verbose > 0:
        logging.basicConfig(
            level=logging.INFO, format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s')
    else:
        logging.basicConfig(
            level=logging.WARN, format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s')
        logging.warning('Skip DEBUG/INFO messages')

    # If --ngpu is not given,
    #   1. if CUDA_VISIBLE_DEVICES is set, all visible devices
    #   2. if nvidia-smi exists, use all devices
    #   3. else ngpu=0
    if args.ngpu is None:
        cvd = os.environ.get("CUDA_VISIBLE_DEVICES")
        if cvd is not None:
            ngpu = len(cvd.split(','))
        else:
            logging.warning("CUDA_VISIBLE_DEVICES is not set.")
            try:
                p = subprocess.run(['nvidia-smi', '-L'],
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)
            except (subprocess.CalledProcessError, FileNotFoundError):
                ngpu = 0
            else:
                ngpu = len(p.stderr.decode().split('\n')) - 1
    else:
        ngpu = args.ngpu
    logging.info(f"ngpu: {ngpu}")

    # set random seed
    logging.info('random seed = %d' % args.seed)
    random.seed(args.seed)
    np.random.seed(args.seed)

    if args.backend == "pytorch":
        from espnet.tts.pytorch_backend.tts import train
        train(args)
    else:
        raise NotImplementedError("Only pytorch is supported.")

if __name__ == "__main__":
    # NOTE(kan-bayashi): setting multiple times causes RuntimeError
    #   See also https://github.com/pytorch/pytorch/issues/3492
    try:
        mp.set_start_method('spawn')
    except RuntimeError:
        pass
    main(sys.argv[1:])

max_stars_repo_name: smolendawid/espnet

max_issues_repo_name: Jo-ms/espnet

max_forks_repo_name: Jo-ms/espnet


text:
import numpy as np
from rk.utils import LinkNode, find_mother
import pytest

@pytest.fixture
def biped_ulink():
    UX = np.array([[1, 0, 0]]).T
    UY = np.array([[0, 1, 0]]).T
    UZ = np.array([[0, 0, 1]]).T

    ro_biped_ulink = {}
    ro_biped_ulink[0] = LinkNode(id=0,  name='NULL')
    ro_biped_ulink[1] = LinkNode(id=1,  name='BODY'   , children=[2, 8], b=np.array([[0, 0, 0.7]]).T,  a=UZ, q=0)
    ro_biped_ulink[2] = LinkNode(id=2,  name='RLEG_J0', children=[3],    b=np.array([[0, -0.1, 0]]).T, a=UZ, q=0)
    ro_biped_ulink[3] = LinkNode(id=3,  name='RLEG_J1', children=[4],    b=np.array([[0, 0, 0]]).T,    a=UX, q=0)
    ro_biped_ulink[4] = LinkNode(id=4,  name='RLEG_J2', children=[5],    b=np.array([[0, 0, 0]]).T,    a=UY, q=0)
    ro_biped_ulink[5] = LinkNode(id=5,  name='RLEG_J3', children=[6],    b=np.array([[0, 0, -0.3]]).T, a=UY, q=0)
    ro_biped_ulink[6] = LinkNode(id=6,  name='RLEG_J4', children=[7],    b=np.array([[0, 0, -0.3]]).T, a=UY, q=0)
    ro_biped_ulink[7] = LinkNode(id=7,  name='RLEG_J5', children=[0],    b=np.array([[0, 0, 0]]).T,    a=UX, q=0)
    ro_biped_ulink[8] = LinkNode(id=8,  name='LLEG_J0', children=[9],    b=np.array([[0, 0.1, 0]]).T,  a=UZ, q=0)
    ro_biped_ulink[9] = LinkNode(id=9,  name='LLEG_J1', children=[10],   b=np.array([[0, 0, 0]]).T,    a=UX, q=0)
    ro_biped_ulink[10]= LinkNode(id=10, name='LLEG_J2', children=[11],   b=np.array([[0, 0, 0]]).T,    a=UY, q=0)
    ro_biped_ulink[11]= LinkNode(id=11, name='LLEG_J3', children=[12],   b=np.array([[0, 0, -0.3]]).T, a=UY, q=0)
    ro_biped_ulink[12]= LinkNode(id=12, name='LLEG_J4', children=[13],   b=np.array([[0, 0, -0.3]]).T, a=UY, q=0)
    ro_biped_ulink[13]= LinkNode(id=13, name='LLEG_J5', children=[0],    b=np.array([[0, 0, 0]]).T,    a=UX, q=0)
    return ro_biped_ulink

def test_find_mother(biped_ulink):
    mother_found_ulink = find_mother(biped_ulink, 1)
    assert mother_found_ulink[1].mother == 0
    assert mother_found_ulink[2].mother == 1
    assert mother_found_ulink[3].mother == 2
    assert mother_found_ulink[4].mother == 3
    assert mother_found_ulink[5].mother == 4
    assert mother_found_ulink[6].mother == 5
    assert mother_found_ulink[7].mother == 6
    assert mother_found_ulink[8].mother == 1
    assert mother_found_ulink[9].mother == 8
    assert mother_found_ulink[10].mother == 9
    assert mother_found_ulink[11].mother == 10
    assert mother_found_ulink[12].mother == 11
    assert mother_found_ulink[13].mother == 12

max_stars_repo_name: jihoonerd/Robot-Kinematics


text:
##############################################
# Author: -- (--)
# Description:
#   Analyze thresholds for ontology mapping
#   Include: Perform a greedy search to find the optimum threshold for type and properties
##############################################

import collections
import functools
import operator
import json
from request_handler import RequestHandler, Configuration
import numpy as np

config = Configuration()
config.set_url('http://45.113.235.161:1313/resolve')
handler = RequestHandler(config)

def mapping(label, thg, thl, key='type'):
    return handler.post_minimal({'key': key, 'value': label, 'thg': thg, 'thl': thl})

def measure(real, matched):
    intersection = len(list(set(real).intersection(matched)))
    if intersection == 0:
        precision = 0
        recall = 0
    else:
        precision = intersection/len(matched)
        recall = intersection/len(real)
    if precision == 0 and recall == 0:
        fscore = 0
    else:
        fscore = 2 * precision * recall / (precision + recall)
    return {'fscore': fscore, 'precision': precision, 'recall': recall}

def average(measures):
    add = dict(functools.reduce(operator.add, map(collections.Counter, measures)))
    return {k: v / len(measures) for k, v in add.items()}

if __name__ == "__main__":
    print('start fixing thresholds')
    with open('evaluation/types-manually-labelled.json', encoding='utf-8') as fp:
        types = json.load(fp)
    with open('evaluation/properties-manually-labelled.json', encoding='utf-8') as fp:
        props = json.load(fp)
    thgs = np.arange(0.5, 1, 0.05).tolist()
    thls = np.arange(0.5, 1, 0.05).tolist()
    measures = []
    result = {}
    is_type = False
    for thg in thgs:
        result[thg] = {}
        for thl in thls:
            if is_type:
                for key, real in types.items():
                    measures.append(measure(real=real, matched=mapping(label=key, thg=thg, thl=thl)))
            else:
                for key, real in props.items():
                    measures.append(measure(real=real, matched=mapping(label=key, thg=thg, thl=thl, key='property')))
            result[thg][thl] = average(measures)
            print('average measure gloss threshold: {0} label threshold:{1}: {2}'.format(thg, thl, result[thg][thl]))
            measures = []
    print('process finished... ')
    if is_type:
        with open('evaluation/type-threshold-results.json', 'w', encoding='utf-8') as fp:
            json.dump(obj=result, fp=fp)
    else:
        with open('evaluation/property-threshold-results.json', 'w', encoding='utf-8') as fp:
            json.dump(obj=result, fp=fp)

max_stars_repo_name: hamzeiehsan/Questions-To-GeoSPARQL

max_issues_repo_name: hamzeiehsan/Questions-To-GeoSPARQL

max_forks_repo_name: hamzeiehsan/Questions-To-GeoSPARQL


text:
"""Distribution of real-time messages using Apache Arrow Plasma store (shared memory).

This module demonstrates how to efficiently distribute real-time messages via shared memory using
the Apache Arrow Plasma store.

The messages are read and processed (concurrently) by a number of consumer processes. The producer
process is not affected by consumer(s) performance. Both the producer and the consumer processes
utilize the same logic for generating object ids based on batch numbers.
"""

import logging
from multiprocessing import Pool
import random
import subprocess
import time

import numpy as np
import pandas as pd
import pyarrow as pa
import pyarrow.plasma as plasma
from sklearn.linear_model import SGDRegressor

PLASMA_STORE_LOCATION = "/tmp/store"
PLASMA_STORE_SIZE_BYTES = 1 * 10 ** 9  # 1 GB
PLASMA_OBJID_SIZE_BYTES = 20

# Producer related values
NUM_ROWS = 10000
RANDOM_SEED = 1234
BATCH_MAX_ROWS = 40

# Consumer related values
NUM_CONSUMERS = 3
CONSUMER_TIMEOUT_MS = 1000

def get_object_id(batch_num):
    """Generates an object id used by the plasma store for a given batch number.

    Args:
        batch_num (int): The batch number.

    Returns:
        The plasma object id.
    """
    return plasma.ObjectID(batch_num.to_bytes(PLASMA_OBJID_SIZE_BYTES, byteorder="big"))

def put_df(client, object_num, df):
    """Saves a data frame into the plasma store.

    Code adjusted from:
    https://github.com/apache/arrow/blob/master/python/examples/plasma/sorting/sort_df.py

    Args:
        client: Plasma store connection.
        object_num (int): Object number to be used to generate a corresponding object id.
        df (pandas): Data frame to be saved.
    """
    record_batch = pa.RecordBatch.from_pandas(df)

    # Get size of record batch and schema
    mock_sink = pa.MockOutputStream()
    stream_writer = pa.RecordBatchStreamWriter(mock_sink, record_batch.schema)
    stream_writer.write_batch(record_batch)
    data_size = mock_sink.size()

    # Generate an ID and allocate a buffer in the object store for the
    # serialized DataFrame
    object_id = get_object_id(object_num)
    buf = client.create(object_id, data_size)

    # Write the serialized DataFrame to the object store
    sink = pa.FixedSizeBufferWriter(buf)
    stream_writer = pa.RecordBatchStreamWriter(sink, record_batch.schema)
    stream_writer.write_batch(record_batch)

    # Seal the object
    client.seal(object_id)

def get_data_checksum(checksum, data):
    """Calculates the data checksum and adds it to the running checksum.

    Args:
        checksum (int): The running checksum.
        data (numpy): The data for which checksum is calculated.

    Returns:
        int: The checksum value.
    """
    return checksum + np.sum(np.sum(data))

def get_random_data(num_points, seed):
    """Generate linear-looking data.

    Args:
        num_points (int): Number of points to be generated.
        seed (int): Seed for the random generator.

    Returns:
        numpy: Generated random data.
    """
    np.random.seed(seed)

    a_0 = 4
    a_1 = 3
    logging.info(
        "Simulating linear-looking data these coefficients (plus some error): a = [%f, %f]",
        a_0, a_1)

    x = 10 * np.random.rand(num_points)
    y = a_0 + a_1 * x + np.random.randn(num_points)
    df = pd.DataFrame(data={'x': x, 'y': y})
    return df

def producer(data, batch_max_rows=BATCH_MAX_ROWS):
    """Saves the input data as a set of batches (of random size) into the plasma store.

    Args:
        data (numpy): The input data saved into the plasma store.
        batch_max_rows (int): The maximum size allowed for each batch.

    Returns:
        float: The checksum value for all the input data.
    """
    logging.info("Producer: connecting to the plasma store")
    client = plasma.connect(PLASMA_STORE_LOCATION)

    logging.info("Producer: starting to load data (%i rows) onto the plasma store", len(data))
    row_num = 0
    batch_num = 0
    checksum = 0.0
    while row_num < len(data):
        k = random.randint(1, batch_max_rows)
        rows = data[row_num: row_num + k]
        checksum = get_data_checksum(checksum, rows)

        logging.debug("Producer: storing batch number: %i", batch_num)
        put_df(client, batch_num, rows)

        row_num += k
        batch_num += 1

    client.disconnect()
    logging.info("Producer: total %i rows distributed (checksum: %f)", len(data), checksum)
    return checksum

def consumer(cid, timeout=CONSUMER_TIMEOUT_MS):
    """Reads and processes data batches from the plasma store (shared memory). This function blocks
     (with a specified timeout) if a batch number has not yet been saved into the plasma store.

    Args:
        cid (int): Unique consumer id.
        timeout (int): How long we should wait for new messages to be saved into the plasma store.

    Returns:
        float: The checksum value calculated for all the read and processed data.
    """
    logging.info("Consumer %i: connecting to the plasma store", cid)
    client = plasma.connect(PLASMA_STORE_LOCATION)

    model = SGDRegressor()

    batch_num = 0
    row_cnt = 0
    checksum = 0
    while True:
        object_id = get_object_id(batch_num)
        logging.debug("Consumer %i: retrieving object id '%s'", cid, object_id)

        [data] = client.get_buffers([object_id], timeout)
        if not data:
            # This means we hit the timeout condition. Normally we would check if we should
            # continue to poll the plasma store for more data, but in our case we will just return.
            client.disconnect()
            logging.info("Consumer %i: total %i rows processed (checksum: %f)",
                         cid, row_cnt, checksum)
            logging.info("Consumer %i: calculated linear coefficients: a = [%f, %f]",
                         cid, model.intercept_, model.coef_)
            break

        buffer = pa.BufferReader(data)
        reader = pa.RecordBatchStreamReader(buffer)
        record_batch = reader.read_next_batch()
        df = record_batch.to_pandas()

        logging.debug("Consumer %i: processing data:\n%s", cid, df)

        # Incremental model training
        x = df['x'].to_numpy().reshape(-1, 1)
        y = df['y'].to_numpy()
        model.partial_fit(x, y)

        checksum = get_data_checksum(checksum, df)
        row_cnt += len(df)

        batch_num += 1

    return checksum

def main():
    """Starts up a plasma store instance and runs one producer and multiple consumer processes."""
    logging.basicConfig(level=logging.INFO)

    # Start the plasma store.
    plasma_store = subprocess.Popen(["plasma_store",
                                     "-s", PLASMA_STORE_LOCATION,
                                     "-m", str(PLASMA_STORE_SIZE_BYTES)])

    # Wait for the plasma store to come up and make sure it is running.
    time.sleep(1)
    assert plasma_store.poll() is None

    num_processes = NUM_CONSUMERS + 1
    pool = Pool(processes=num_processes)

    data = get_random_data(NUM_ROWS, RANDOM_SEED)
    producer_result = pool.map_async(producer, [data])
    consumer_result = pool.map_async(consumer, range(NUM_CONSUMERS))

    producer_check = producer_result.get()
    consumer_check = consumer_result.get()

    # Make sure data is identical across all consumers and the producer process.
    assert len(np.unique(consumer_check)) == 1 and consumer_check[0] == producer_check
    plasma_store.kill()

if __name__ == "__main__":
    main()

max_stars_repo_name: awasielak/plasma-rt-example

max_issues_repo_name: awasielak/plasma-rt-example

max_forks_repo_name: awasielak/plasma-rt-example


text:
# crop image based on the annotation information and save the image and label information
from common_lib.tools.coding_related import decode_distribution
#from common_lib import decode_labelme_shape
from coding_related import decode_labelme_shape
from common_lib.tools.cnn_json_related import ClassNameManager
import os
import cv2
import glob
import json
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt

def convert_NG_label(distribution,class_dict):
      distribution_list=decode_distribution(distribution)
      class_name_manager=ClassNameManager(class_dict)
      NG_type=class_name_manager.get_key_from_distribution(distribution_list)
      for class_num in class_dict:
            if NG_type[-1]==class_num["class_name"]:
                  return class_num["class_id"]-4
#image_data_root = '/workspace/dataSet/raw/unsupervised-learning/kangqiang/'

def image_crop(data):
    anomaly_image_path_label_list=[]
    for i, rec in enumerate(data):
        image_name=rec['info']['image_path']
        image_dir,image_name_part=os.path.split(image_name)
        img = cv2.imread(os.path.join(image_data_root,image_name))[...,::-1]
        assert img is not None, rec
        for p,inst in enumerate(rec['instances']):
            save_dir=os.path.join(save_image_root, image_dir)
            if not os.path.exists(save_dir):
                os.makedirs(save_dir,exist_ok=True)
            save_new_image_name=os.path.join(save_dir, os.path.splitext(image_name_part)[0]+"_"+str(p)+".jpg")
            if not os.path.exists(save_new_image_name):
                shapes = np.array(decode_labelme_shape(inst['points']))
                x1,y1,x2,y2 = int(shapes[:,0].min()), int(shapes[:,1].min()), int(shapes[:,0].max()), int(shapes[:,1].max())
                crop_img=img[y1:y2,x1:x2]
                plt.imshow(crop_img)
                plt.axis("off")
                # save new images
                plt.imsave(save_new_image_name,crop_img)
            # delete the corrupted file
            img1 = cv2.imread(save_new_image_name)
            if img1 is None:
                print(save_new_image_name)
                os.remove(save_new_image_name)
                continue
            if [save_new_image_name,convert_NG_label(rec['instances'][0]['distribution'],dataset_info["class_dict"])] in anomaly_image_path_label_list:
                pass
            else:
                anomaly_image_path_label_list.append([save_new_image_name,convert_NG_label(rec['instances'][0]['distribution'],dataset_info["class_dict"])])
    return anomaly_image_path_label_list

def func(listTemp, n):
    for i in range(0, len(listTemp), n):
        yield listTemp[i:i + n]

if __name__ == '__main__':
    from multiprocessing import Pool
    image_data_root="/git/dataSet/raw/unsupervised-learning/kangqiang/"
    save_image_root="/git/PaDiM-master/kangqiang_result/croped_images_with_original_annotation"
    json_path='assets_new_new/data/2021-03-05/json_for_classification'
    #save_folder="json_for_classification"
    json_file_list=["train.json","val.json","test.json"]
    save_file_list=["train.txt","val.txt","test.txt"]
    data_list=[]
    for json_file in json_file_list:
        with open(os.path.join(json_path,json_file)) as fid:
            dataset_info=json.load(fid)
            data_list.append(dataset_info['record'])

    cpu_num=24
    for k, data in enumerate(data_list):
        print("the current process is {} dataset".format(json_file_list[k]))
        with Pool(cpu_num) as p:
            anomaly_image_path_label_list=p.map(image_crop, list(func(data, cpu_num)))
        print(len(anomaly_image_path_label_list[0]))
        print(os.path.join(json_path,save_file_list[k]))
        with open(os.path.join(json_path,save_file_list[k]),"w+") as fid:
            num=0
            for i in  range(len(anomaly_image_path_label_list)):
                for j in range(len(anomaly_image_path_label_list[i])):
                    num+=1
                    fid.writelines(anomaly_image_path_label_list[i][j][0]+" "+str(anomaly_image_path_label_list[i][j][1])+"\n")


text:
# some snippets of code to show how one might
# get initial trace and wavecal files for APO instruments

import copy
import numpy as np
import os
import pickle
import pdb
import yaml
import matplotlib.pyplot as plt
from pyvista import imred
from pyvista import image
from pyvista import spectra
from pyvista import tv
from astropy.io import fits
from astropy import units
from astropy.nddata import CCDData, StdDevUncertainty
from tools import plots
import scipy.signal

ROOT = os.path.dirname(os.path.abspath(__file__)) + '/../../'

import readmultispec
import os

def kosmos(blue=True):

    if blue :
      kred=imred.Reducer('KOSMOS',dir='/home/holtz/raw/apo/oct21/Q4NM01/UT211031/kosmos')
    else :
      kred=imred.Reducer('KOSMOS',dir='/home/holtz/raw/apo/oct21/Q4NM01/UT211030/kosmos')

    if blue :
        wref=[6402,266]
        disp=(6153-6402)/(614-266)
        disp=(4319-6402)/(3093-266)
        lamp='henearkr.dat'
    else :
        wref=[7245,1701]
        disp=-1.026
        lamp='henear.dat'

    def model(x) : return(np.zeros(len(x))+1024)
    traces=spectra.Trace('KOSMOS',rad=5,model=[model],sc0=2048)
    pdb.set_trace()
    if blue : arc=kred.sum([14,16,1001])
    else : arc=kred.sum([15,16,17])
    #ne=kred.reduce(16)
    arc=kred.imtranspose(arc)
    spec=traces.extract(arc)
    spec2d=traces.extract2d(arc)
    spec.data-=scipy.signal.medfilt(spec.data,kernel_size=101)
    kwav=spectra.WaveCal(degree=2)
    fig=plt.figure()
    kwav.identify(spec,file=lamp,wref=wref,disp=disp,plot=fig,rad=5,thresh=100)
    pdb.set_trace()
    kwav.fit()

    return kwav,spec

def dis(lowres=True) :
    pdb.set_trace()
    if lowres : dred=imred.Reducer(inst='DIS',dir='UT191019/DIS/')
    else : dred=imred.Reducer(inst='DIS',dir='UT191027/DIS/')
    arc=dred.sum([1,2,3])

    # define a flat trace model
    def model(x) : return(np.zeros(len(x))+500)
    traces=spectra.Trace(rad=10,model=[model],sc0=1024)

    fig,ax=plots.multi(1,2)
    for chip in range(2) :
        spec=traces.extract(arc[chip])
        spec.data-=scipy.signal.medfilt(spec.data,kernel_size=101)
        rad=10
        if lowres :
            degree=2
            suffix='lowres'
            disp=spec.header['DISPDW']
            wref=[5015,1367]
            disp=(5015-3888)/(1367-749)
            thresh=40
            rad=10
            if chip==1 :
              #wref=[7635,974]
              #disp=(7635-6402)/(974-1508)
              wref=[8377,653]
              disp=(8377-6143)/(653-1622)
              thresh=200
              rad=3
              degree=4
        else :
            suffix='hires'
            wref=[5015,1481]
            degree=2
            disp=spec.header['DISPDW']
            thresh=None
            if chip==1 :
                disp=-1*disp
                disp=-0.58
                degree=4
                wref=[6402,933]
        wcal=spectra.WaveCal(type='chebyshev',degree=degree)
        wcal.identify(spec,file='henear.dat',rad=rad,plot=fig,disp=disp,wref=wref,thresh=thresh)
        wcal.fit()
        wcal.wave(image=spec.shape[1])
        if chip==0 : wcal.save(suffix+'blue.pkl')
        elif chip==1 : wcal.save(suffix+'red.pkl')

    ##wcal.set_spectrum(spec)
    ##f=open(os.environ['PYVISTA_DIR']+'/data/dis/dis_blue_lowres.pkl','rb')
    #f=open(os.environ['PYVISTA_DIR']+'/data/DIS/DIS_hires_6400_waves.pkl','rb')
    #wcal0=pickle.load(f)
    #wcal0[0][0].orders=[1]
    #wav=wcal0[0][0].wave(image=spec.shape)
#
#    wcal.identify(spec,wav=wav,file=os.environ['PYVISTA_DIR']+'/data/lamps/henear.dat',rad=3)
#    wcal.fit()
#    w=wcal.wave(image=np.array(spec.data.shape))

    spec2d=traces.extract2d(arc[0],rows=[200,900])
    pdb.set_trace()

    star=dred.reduce(26)
    return spec,w,wcal

def arces() :
    ered=imred.Reducer(inst='ARCES',dir='UT191020/ARCES/')

    flat=ered.sum([11,15])
    thar=ered.sum([19,20])

    t=tv.TV()

    apertures=np.loadtxt('newap')[:,1]
    #traces=trace(flat,apertures/4)
    traces=spectra.Trace(inst='ARCES')
    traces.trace(flat,apertures/4,sc0=1024,thresh=40,plot=t)
    ec=traces.extract(thar,scat=True)

    wcal=spectra.WaveCal(type='chebyshev2D',orders=54+np.arange(107),ydegree=3)
    wcal.set_spectrum(ec)
    wav=readmultispec.readmultispec('w131102.0004.ec.fits')['wavelen']
    new=ec.data*0.
    new[:,204:1855]=wav
    fig=plt.figure()
    wcal.identify(ec,wav=new,file='thar_arces',xmin=201,xmax=1849,thresh=10,rad=3,display=t,plot=fig)
    wcal.fit()

    wcal.identify(ec,file='thar_arces',xmin=150,xmax=1900,thresh=10,rad=3,display=t,plot=fig)
    wcal.fit()

    w=wcal.wave(image=np.array(ec.data.shape))

    hr7950=ered.reduce(1,flat=flat)
    hr7950ec=traces.extract(hr7950,scat=True)

    return flat,thar,traces,ec,wcal,w

def tspec() :

    tspec=imred.Reducer(inst='TSPEC',dir='UT191026/TSPEC',nfowler=8)
    a=tspec.reduce(21)
    acr=tspec.reduce(21,crbox=[11,1])
    b=tspec.reduce(22)

    t=tv.TV()
    fig,ax=plots.multi(1,2)

    rows=[[135,235],[295,395],[435,535],[560,660],[735,830]]
    apers=[155,316,454,591,761]
    traces=[]
    waves=[]

    # wavelength arrays from TSpexTool reduction, note that flips wavelengths, and has K band first, not last
    wav=fits.open('tspec_wave.fits')[0].data
    #w0=[23147.049,17388.174,13927.875,11621.16,9972.335]
    #w0.reverse()
    #disp=[-2.8741264,-2.156124, -1.7261958,-1.4418244,-1.2400593]
    #disp.reverse()
    order=7
    for aper,row in zip(apers,rows) :
        trace=spectra.Trace(order=3,rows=row,lags=range(-75,75))
        trace.trace(a.subtract(b),aper,sc0=350,thresh=100,plot=t)
        traces.append(trace)
        trace.pix0 +=30
        out=trace.extract(acr,rad=20,plot=t)
        out.data=out.data - scipy.signal.medfilt(out.data,kernel_size=[1,201])
        wcal=spectra.WaveCal(type='chebyshev',orders=[order],degree=3)
        w=np.atleast_2d(wav[order-3,0,:][::-1])*1.e4
        bd=np.where(~np.isfinite(w))
        w[bd[0],bd[1]]=9000.
        #wcal.identify(out,file=os.environ['PYVISTA_DIR']+'/data/sky/OHll.dat',rad=10,wref=[wr,2048-1500],disp=d,plot=fig)
        wcal.identify(out,file=os.environ['PYVISTA_DIR']+'/data/sky/OHll.dat',rad=3,wav=w,plot=fig)
        wcal.fit()
        delattr(wcal,'ax')
        waves.append(wcal)
        order-=1

    #spec2d=traces.extract2d(a.subtract(b),rows=range(200,900))
    pickle.dump([traces],open('TSPEC_traces.pkl','wb'))
    pickle.dump([waves],open('TSPEC_waves.pkl','wb'))

    return traces,waves

def test():
    a=np.loadtxt('ecnewarc.ec')
    x=a[:,2]
    y=a[:,1]
    w=a[:,4]
    fitter=fitting.LinearLSQFitter()
    mod=models.Chebyshev2D(x_degree=3,y_degree=3,x_domain=[1,2000],y_domain=[55,160])
    fit=fitter(mod,x,y,w*y)
    plt.figure()
    plt.plot(w,w-fit(x,y)/y,'ro')

def testfit() :
    a=np.loadtxt('ecnewarc.ec')
    a[:,2]
    x=a[:,2]
    y=a[:,1]
    w=a[:,4]
    fitter=fitting.LinearLSQFitter()
    mod=models.Chebyshev2D(x_degree=3,y_degree=3,x_domain=[1,2000],y_domain=[55,160])
    fit=fitter(mod,x,y,w*y)
    plt.figure()
    plt.plot(w,w-fit(x,y)/y)
    plt.clf()
    plt.plot(w,w-fit(x,y)/y,'ro')

def echfit() :
    mod=models.custom_model(echelle_model)

def ripple(w,ord,amp=1,alpha=1,wc=5500) :
#def echelle_model(ind,amp,alpha,wc) :
#    w=ind[0,:]
#    ord=ind[1,:]
    x = ord * (1 - wc/w)
    out=amp*(np.sin(np.pi*alpha*x)/(np.pi*alpha*x))**2
    bd=np.where(np.pi*alpha*x == 0)[0]
    out[bd] = amp[bd]
    return out

def ripple2d(w,ord,amp0=1,amp1=0,amp2=0,alpha0=1,alpha1=0,alpha2=0,wc0=5500,wc1=0,wc2=0) :
#def echelle_model(ind,amp,alpha,wc) :
#    w=ind[0,:]
#    ord=ind[1,:]
    wc = wc0 + wc1*(ord-109) + wc2*(ord-109)**2
    amp = amp0 + amp1*(ord-109) + amp2*(ord-109)**2
    alpha = alpha0 + alpha1*(ord-109) + alpha2*(ord-109)**2
    #print(ord,amp,alpha,wc)
    x = ord * (1 - wc/w)
    out=amp*(np.sin(np.pi*alpha*x)/(np.pi*alpha*x))**2
    bd=np.where(np.pi*alpha*x == 0)[0]
    out[bd] = amp[bd]
    return out

def echelle_model_deriv(w,ord,amp=1,alpha=1,wc=5500) :
    x = ord * (1 - wc/w)
    f=amp*(np.sin(np.pi*alpha*x)/(np.pi*alpha*x))**2
    deriv=[]
    deriv.append(f/amp)
    deriv.append(2*amp*np.sin(np.pi*alpha*x)/(np.pi*alpha*x)**2*np.cos(np.pi*alpha*x)*np.pi*x
                 - 2*f/alpha )
    deriv.append(2*amp*np.sin(np.pi*alpha*x)/(np.pi*alpha*x)**2*np.cos(np.pi*alpha*x)*np.pi*alpha*-1*ord/w
                 + 2*f/x*ord/w)

    return np.array(deriv)

def mktrace(group,display=None) :

    # trace from scratch
    red=imred.Reducer(inst=group['inst'],dir=group['rawdir'])
    try :
        traces=group['traces']
        print('create trace')
        pdb.set_trace()
        if group['traces']['frame'] == 'sflat' :
            a=sflat
        else :
            a= red.reduce(group['traces']['frame']) #,bias=sbias,dark=sdark,flat=sflat,scat=red.scat)
        try :
            b= red.reduce(group['traces']['dark']) #,bias=sbias,dark=sdark,flat=sflat,scat=red.scat)
            a=a.subtract(b)
        except: pass
        if group['inst'] == 'TSPEC' :
            apers=[155,316,454,591,761]
            apers.reverse()
            traces=spectra.Trace(inst=group['inst'])
            traces.trace(a,apers,sc0=350,plot=display)
            traces=[traces]
        elif group['inst'] == 'ARCES' :
            apers=np.loadtxt('newap')[:,1]/4
            traces=spectra.Trace(inst=group['inst'])
            traces.trace(a,apers,sc0=1024,plot=display)
            traces=[[traces]]
        elif group['inst'] == 'DIS' :
            apers=[662,562]
            traces=[]
            for i,(im,ap) in enumerate(zip(a,apers)) :
                trace=spectra.Trace(inst=group['inst'],channel=i)
                trace.trace(im,ap,sc0=1024,plot=display)
                traces.append(trace)
            traces=[[traces[0]],[traces[1]]]
        elif group['inst'] == 'KOSMOS' :
            apers=[940]
            traces.trace(a,apers,sc0=2048,plot=display)
            traces=[[traces]]

        pickle.dump(traces,open(group['inst']+'_traces.pkl','wb'))
    except:
        print('no trace frames given')

    # existing trace template
    trace=pickle.load(open(group['inst']+'_traces.pkl','rb'))

max_stars_repo_name: holtzmanjon/pyvista

max_issues_repo_name: holtzmanjon/pyvista

max_forks_repo_name: holtzmanjon/pyvista

reason: import numpy,import scipy,from astropy


text:
import airsim
import cv2
import numpy as np
import os
import setup_path
import time

# connect to the AirSim simulator
client = airsim.CarClient()
client.confirmConnection()
client.enableApiControl(True)

if not os.path.exists('c:/temp/'):
    os.makedirs('c:/temp/')

filename = 'c:/temp/' + "none"
png_image = client.simGetImage(str(0), airsim.ImageType.Scene)
airsim.write_file(os.path.normpath(filename+".png"),png_image)

w = client.simGetCameraInfo("0").pose.orientation.w_val
x = client.simGetCameraInfo("0").pose.orientation.x_val
y = client.simGetCameraInfo("0").pose.orientation.y_val
z = client.simGetCameraInfo("0").pose.orientation.z_val

q = airsim.Quaternionr(x,y,z,w).conjugate()
client.simSetCameraOrientation("0",q)
filename = 'c:/temp/' + "conjugate"
png_image = client.simGetImage(str(0), airsim.ImageType.Scene)
airsim.write_file(os.path.normpath(filename+".png"),png_image)

q = airsim.Quaternionr(x,y,z,w).inverse()
client.simSetCameraOrientation("0",q)
filename = 'c:/temp/' + "inverse"
png_image = client.simGetImage(str(0), airsim.ImageType.Scene)
airsim.write_file(os.path.normpath(filename+".png"),png_image)

q = airsim.Quaternionr(x,y,z,w).conjugate().inverse()
client.simSetCameraOrientation("0",q)
filename = 'c:/temp/' + "conjugate_inverse"
png_image = client.simGetImage(str(0), airsim.ImageType.Scene)
airsim.write_file(os.path.normpath(filename+".png"),png_image)

q = airsim.Quaternionr(x,y,z,w).inverse().conjugate()
client.simSetCameraOrientation("0",q)
filename = 'c:/temp/' + "inverse_conjugate"
png_image = client.simGetImage(str(0), airsim.ImageType.Scene)
airsim.write_file(os.path.normpath(filename+".png"),png_image)

q = airsim.Quaternionr(x,y,z,w).conjugate().conjugate()
client.simSetCameraOrientation("0",q)
filename = 'c:/temp/' + "conjugate_conjugate"
png_image = client.simGetImage(str(0), airsim.ImageType.Scene)
airsim.write_file(os.path.normpath(filename+".png"),png_image)

q = airsim.Quaternionr(x,y,z,w).inverse().inverse()
client.simSetCameraOrientation("0",q)
filename = 'c:/temp/' + "inverse_inverse"
png_image = client.simGetImage(str(0), airsim.ImageType.Scene)
airsim.write_file(os.path.normpath(filename+".png"),png_image)

q = airsim.Quaternionr(x,y,z,w).conjugate().conjugate().conjugate()
client.simSetCameraOrientation("0",q)
filename = 'c:/temp/' + "conjugate_conjugate_conjugate"
png_image = client.simGetImage(str(0), airsim.ImageType.Scene)
airsim.write_file(os.path.normpath(filename+".png"),png_image)

q = airsim.Quaternionr(x,y,z,w).inverse().inverse().inverse()
client.simSetCameraOrientation("0",q)
filename = 'c:/temp/' + "inverse_inverse_inverse"
png_image = client.simGetImage(str(0), airsim.ImageType.Scene)
airsim.write_file(os.path.normpath(filename+".png"),png_image)

q = airsim.Quaternionr(x,y,z,w).conjugate().conjugate().conjugate().conjugate()
client.simSetCameraOrientation("0",q)
filename = 'c:/temp/' + "conjugate_conjugate_conjugate_conjugate"
png_image = client.simGetImage(str(0), airsim.ImageType.Scene)
airsim.write_file(os.path.normpath(filename+".png"),png_image)

q = airsim.Quaternionr(x,y,z,w).inverse().inverse().inverse().inverse()
client.simSetCameraOrientation("0",q)
filename = 'c:/temp/' + "inverse_inverse_inverse_inverse"
png_image = client.simGetImage(str(0), airsim.ImageType.Scene)
airsim.write_file(os.path.normpath(filename+".png"),png_image)

# w = client.simGetCameraInfo("0").pose.orientation.w_val
# x = client.simGetCameraInfo("0").pose.orientation.x_val
# y = client.simGetCameraInfo("0").pose.orientation.y_val
# z = client.simGetCameraInfo("0").pose.orientation.z_val
# q = airsim.Quaternionr(x,y,z,w).conjugate().conjugate().inverse()
# client.simSetCameraOrientation("0",q)
# filename = 'c:/temp/py' + str(3)
# png_image = client.simGetImage(str(0), airsim.ImageType.Scene)
# airsim.write_file(os.path.normpath(filename+".png"),png_image)

# w = client.simGetCameraInfo("0").pose.orientation.w_val
# x = client.simGetCameraInfo("0").pose.orientation.x_val
# y = client.simGetCameraInfo("0").pose.orientation.y_val
# z = client.simGetCameraInfo("0").pose.orientation.z_val
# q = airsim.Quaternionr(x,y,z,w).conjugate().conjugate().conjugate()
# client.simSetCameraOrientation("0",q)
# filename = 'c:/temp/py' + str(4)
# png_image = client.simGetImage(str(0), airsim.ImageType.Scene)
# airsim.write_file(os.path.normpath(filename+".png"),png_image)

# filename = 'c:/temp/py' + str(5)
# png_image = client.simGetImage(str(4), airsim.ImageType.Scene)
# airsim.write_file(os.path.normpath(filename+".png"),png_image)

#restore to original state
client.reset()
client.enableApiControl(False)

max_stars_repo_name: chocolatedisco/airsim-python

max_issues_repo_name: chocolatedisco/airsim-python

max_forks_repo_name: chocolatedisco/airsim-python


text:
# MIT License

# Copyright (c) 2019 Runway AI, Inc

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import random
from PIL import Image
import dlib
import face_recognition
import numpy as np

def get_bounding_box(face, h, w):
    top, right, bottom, left = face
    z = [float(left)/w, float(top)/h, float(right)/w, float(bottom)/h]
    print(z)
    return z

class FaceTracker():

    def __init__(self, options):
        self.known_faces = {}
        self.index = 0

    # Generate an image based on some text.
    def process(self, input):

        img = np.array(input)
        h, w = img.shape[0:2]
        print("HW;,", h,w)
        face_locations = face_recognition.face_locations(img)
        face_encodings = face_recognition.face_encodings(img, face_locations)

        faces = []
        for face_encoding, face_location in zip(face_encodings, face_locations):
            matches = face_recognition.compare_faces([self.known_faces[f]["encoding"] for f in self.known_faces], face_encoding)
            if True not in matches:
                self.known_faces[self.index] = {"index": self.index, "encoding": face_encoding}
                match_index = self.index
                self.index += 1
            else:
                match_index = matches.index(True)

            print("look for", face_location)
            faces.append({"index": match_index, "location": get_bounding_box(face_location, h, w)})

        return faces


text:
import torchvision.transforms as transforms
# from torchvision.transforms.functional import InterpolationMode
from PIL import Image, ImageFilter
import random
import torch
import numpy as np
import logging
from enum import Enum

# 0: InterpolationMode.NEAREST,
# 2: InterpolationMode.BILINEAR,
# 3: InterpolationMode.BICUBIC,
# 4: InterpolationMode.BOX,
# 5: InterpolationMode.HAMMING,
# 1: InterpolationMode.LANCZOS,
class InterpolationMode():
    NEAREST = 0
    BILINEAR = 2
    BICUBIC = 3
    BOX = 4
    HAMMING = 5
    LANCZOS = 1

logger = logging.getLogger(__name__)

class ResizePad(object):

    def __init__(self, imgH=64, imgW=3072, keep_ratio_with_pad=True):
        self.imgH = imgH
        self.imgW = imgW
        assert keep_ratio_with_pad == True
        self.keep_ratio_with_pad = keep_ratio_with_pad

    def __call__(self, im):

        old_size = im.size  # old_size[0] is in (width, height) format

        ratio = float(self.imgH)/old_size[1]
        new_size = tuple([int(x*ratio) for x in old_size])
        im = im.resize(new_size, Image.BICUBIC)

        new_im = Image.new("RGB", (self.imgW, self.imgH))
        new_im.paste(im, (0, 0))

        return new_im

class WeightedRandomChoice:

    def __init__(self, trans, weights=None):
        self.trans = trans
        if not weights:
            self.weights = [1] * len(trans)
        else:
            assert len(trans) == len(weights)
            self.weights = weights

    def __call__(self, img):
        t = random.choices(self.trans, weights=self.weights, k=1)[0]
        try:
            tfm_img = t(img)
        except Exception as e:
            logger.warning('Error during data_aug: '+str(e))
            return img

        return tfm_img

    def __repr__(self):
        format_string = self.__class__.__name__ + '('
        for t in self.transforms:
            format_string += '\n'
            format_string += '    {0}'.format(t)
        format_string += '\n)'
        return format_string

class Dilation(torch.nn.Module):

    def __init__(self, kernel=3):
        super().__init__()
        self.kernel=kernel

    def forward(self, img):
        return img.filter(ImageFilter.MaxFilter(self.kernel))

    def __repr__(self):
        return self.__class__.__name__ + '(kernel={})'.format(self.kernel)

class Erosion(torch.nn.Module):

    def __init__(self, kernel=3):
        super().__init__()
        self.kernel=kernel

    def forward(self, img):
        return img.filter(ImageFilter.MinFilter(self.kernel))

    def __repr__(self):
        return self.__class__.__name__ + '(kernel={})'.format(self.kernel)

class Underline(torch.nn.Module):

    def __init__(self):
        super().__init__()

    def forward(self, img):
        img_np = np.array(img.convert('L'))
        black_pixels = np.where(img_np < 50)
        try:
            y1 = max(black_pixels[0])
            x0 = min(black_pixels[1])
            x1 = max(black_pixels[1])
        except:
            return img
        for x in range(x0, x1):
            for y in range(y1, y1-3, -1):
                try:
                    img.putpixel((x, y), (0, 0, 0))
                except:
                    continue
        return img

class KeepOriginal(torch.nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, img):
        return img

def build_data_aug(size, mode, resnet=False, resizepad=False):
    if resnet:
        norm_tfm = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
    else:
        norm_tfm = transforms.Normalize(0.5, 0.5)
    if resizepad:
        resize_tfm = ResizePad(imgH=size[0], imgW=size[1])
    else:
        resize_tfm = transforms.Resize(size, interpolation=InterpolationMode.BICUBIC)
    if mode == 'train':
        return transforms.Compose([
            WeightedRandomChoice([
                # transforms.RandomHorizontalFlip(p=1),
                transforms.RandomRotation(degrees=(-10, 10), expand=True, fill=255),
                transforms.GaussianBlur(3),
                Dilation(3),
                Erosion(3),
                transforms.Resize((size[0] // 3, size[1] // 3), interpolation=InterpolationMode.NEAREST),
                Underline(),
                KeepOriginal(),
            ]),
            resize_tfm,
            transforms.ToTensor(),
            norm_tfm
        ])
    else:
        return transforms.Compose([
            resize_tfm,
            transforms.ToTensor(),
            norm_tfm
        ])

if __name__ == '__main__':
    tfm = ResizePad()
    img = Image.open('temp.jpg')
    tfm(img).save('temp2.jpg')

max_stars_repo_name: aimore-globality/unilm

max_issues_repo_name: aimore-globality/unilm

max_forks_repo_name: aimore-globality/unilm


text:
import os
from collections import defaultdict
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
from PIL import Image
from IPython.display import Markdown, display

def printmd(string):
    display(Markdown(string))

def batch_predict_results(dat_list):
    dat = defaultdict(list)
    for d in dat_list:
        for k, v in d.items():
            if v is not None:
                dat[k] += v

    for k in dat.keys():
        if isinstance(dat[k][0], torch.Tensor):
            dat[k] = torch.cat([d.unsqueeze(0) for d in dat[k]])
            dat[k] = dat[k].squeeze().cpu()
    return dat

def remove_spines(ax):
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.spines['bottom'].set_visible(False)
    ax.spines['left'].set_visible(False)

def plot_explanation(raster, ax):
    length = raster.shape[-1]
    ax.imshow(raster)
    # grid
    ax.set_xticks(np.arange(.5, length - 0.5, 1), minor=True)
    ax.set_yticks(np.arange(.5, 1, 1), minor=True)
    ax.grid(which='minor', color='w', linestyle='-', linewidth=2)
    # Remove y-axis
    ax.get_yaxis().set_visible(False)
    ax.set_xticks(range(length))
    remove_spines(ax)

def unnorm_cub(img):
    sd = np.array([0.229, 0.224, 0.225])
    mu = np.array([0.485, 0.456, 0.406])
    img = img.transpose(0, 2).transpose(0, 1)
    return img * sd + mu

def plot_cub_gt(sample):
    """
        plot_cub(data_module.cub_test[2])
    """
    img, expl, spatial_expl, label = sample

    im = Image.fromarray(np.uint8(unnorm_cub(img)*255)).convert("RGBA")

    n_patch = int(np.sqrt(spatial_expl.shape[0]))
    patch_idx = ~torch.isnan(spatial_expl[:,0])
    patches = np.zeros(n_patch**2) + 0.3
    patches[patch_idx] = 1.0
    patches = patches.reshape(n_patch, n_patch)

    im_p = Image.fromarray(np.uint8(patches * 255)).convert("L")
    im_p = im_p.resize(im.size, Image.ANTIALIAS)

    im.putalpha(im_p)

    plt.imshow(im)
    plt.axis('off')
    plt.show()

def plot_cub_expl(results, ind, data_module):
    idx = results['idx'][ind].item()

    img = data_module.cub_test[idx][0]
    im = Image.fromarray(np.uint8(unnorm_cub(img)*255)).convert("RGBA")

    # Prediction
    pred = results['preds'][ind].item()
    prediction = data_module.cub_test.class_names[pred].split('/')[0][4:]

    # Spatial attention
    attn = results['spatial_concept_attn'][ind]
    n_patch = int(np.sqrt(attn.shape[0]))

    # Get most active patches
    patch_idx = attn.max(axis=1)[0] > 0.6

    patches = np.zeros(n_patch**2) + 0.4
    patches[patch_idx] = 1.0
    patches = patches.reshape(n_patch, n_patch)

    # Get corresponding most active attributes
    attr_idx = attn[patch_idx,:].max(axis=0)[0] > 0.3
    attr_ind = np.nonzero(attr_idx)

    attr_list = load_attributes()
    attributes = attr_list[np.array(data_module.cub_test.spatial_attributes_pos)[attr_ind] - 1]

    # Nonspatial explanation
    expl = results['concept_attn'][ind]
    expl_idx = expl > 0.2
    nonspatial_attributes = attr_list[np.array(data_module.cub_test.non_spatial_attributes_pos)[expl_idx] - 1]

    # Plot
    im_p = Image.fromarray(np.uint8(patches * 255)).convert("L")
    im_p = im_p.resize(im.size, Image.ANTIALIAS)

    im.putalpha(im_p)

    plt.imshow(im)
    plt.axis('off')
    plt.show()

    correct = results['correct'][ind].item()
    correct_wrong = ['*wrong*', '*correct*'][correct]
    if not correct:
        gt = data_module.cub_test[idx][3]
        gt = data_module.cub_test.class_names[gt].split('/')[0][4:]
        correct_wrong += f', gt is {gt}'

    printmd(f'**Prediction**: {prediction} ({correct_wrong})')

    print(' Spatial explanations:')
    if isinstance(attributes, str):
        attributes = [[attributes]]
    for t in attributes:
        print(f'   - {t[0]}')

    print(' Global explanations:')
    for t in nonspatial_attributes:
        print(f'   - {t}')

def load_attributes(root='/data/Datasets/'):
    # Load list of attributes
    attr_list = pd.read_csv(os.path.join(root, 'CUB_200_2011', 'attributes.txt'),
                            sep=' ', names=['attr_id', 'def'])
    attr_list = np.array(attr_list['def'].to_list())
    return attr_list


text:
# -*- coding: utf-8 -*-
import os
import math

import scipy.io as scio
import numpy as np
from sklearn.linear_model import LogisticRegression #加载模块
from sklearn.linear_model import LogisticRegressionCV

def LoadData(file_path):
    '''
    加载mat文件
    :param file_path: 文件路径
    :return:
    '''
    data = scio.loadmat(file_path)
    return data

def StndTrainData(x, axis):
    '''
    标准化训练数据，每列使它们满足零均值和单位方差
    :param x: numpy.ndarray类型
    :param axis:
    :return: 处理后的训练数据，均值，均方差
    '''
    #不要修改原数据
    x = np.array(x)   #拷贝
    xr = np.rollaxis(x, axis=axis)
    mean = np.mean(x, axis=axis)
    std = np.std(x, axis=axis)
    xr -= mean #减去均值
    xr /= std #除以均方差
    return xr, mean, std

def StndTestData(test, mean, std):
    '''
    标准化测试数据
    :param test: 测试数据
    :param mean: 训练数据的均值
    :param std:  训练数据的均方差
    :return: 处理后的测试数据
    '''
    #不要修改原数据
    testr = np.array(test)
    testr -= mean #减去均值
    testr /= std #除以均方差
    return testr

def Log(x):
    '''
    使用log(x[i][j] + 0.1)变换特征数据
    :param x: 原数据
    :return: 处理后的数据
    '''
    #不要修改原数据
    xr = np.array(x) #copy
    xr = np.log(xr + 0.1)
    return xr

def Binary(x):
    '''
    二值化数据 >0为1 否则为0
    :param x: 原数据
    :return: 处理后的数据
    '''
    #不要修改原数据
    xr = np.array(x) #copy
    xr = np.where(xr>0, 1, 0)
    return xr

def Sigmod(z):
    '''
    for i in xrange(np.size(z)):
        try:
            a = math.exp(-z[i])
        except:
            pass
    '''
    return 1.0 / (1.0 + np.exp(-z))

def ChangeWeight(w1, w2):
    '''
    计算两次权重改变了多少
    :param w1:
    :param w2:
    :return:
    '''
    return math.sqrt(np.sum(np.square(w1 - w2)) / np.shape(w1)[0])

def GradAscent(x, y, l2_param=0.05, round=2000, step=0.001):
    '''
    梯度上升算法，采用L2正则化方法
    :param x: 数据集
    :param y: 结果集
    :return: 权重
    '''
    x = np.asmatrix(x)
    y = np.asmatrix(y)
    m, n = np.shape(x)
    weights = np.mat(np.ones((n, 1)))
    #weights = np.matrix(np.random.rand(n) - 0.5).T  # 初始化系数
    #delta = np.mat(np.full((n, 1), 0))
    change = 1.0
    i = 0
    #print(l2_param, step, round)
    while change > 0.0001 and i < round:
    #while i < 500:
        w = np.mat(weights)
        z = Sigmod(x * weights)
        error = y - z # + delta
        weights = weights + step * x.T * error - step * l2_param * weights
        #delta = np.mat(np.full((n, 1), math.sqrt(np.sum(np.square(weights))/m)))
        change = ChangeWeight(weights, w)
        '''if change > 0.005:
            alpha = 0.0003
        else:
            alpha = 0.0001'''
        i += 1
    #print i, change
    return weights

def Classify(data, weights):
    '''
    对数据进行分类
    :param data:
    :param weights:
    :return:
    '''
    data = np.asmatrix(data)
    z = Sigmod(data * weights)
    for i in range(len(z)):
        if z[i][0] > 0.5:
            z[i][0] = 1.0
        else:
            z[i][0] = 0.0
    return z

def CaclErrorRate(test_ret, real_ret):
    '''
    计算错误率
    :param test_ret: 测试结果
    :param real_ret: 真实结果
    :return:
    '''
    count = len(test_ret)
    count_error = 0
    for i in range(count):
        if test_ret[i][0] != real_ret[i][0]:
            count_error += 1
    return count_error / float(count)

def CrossValidator(data):
    '''
    十折交叉验证
    :param data:
    :return:
    '''
    params = []
    err_set = []
    l2_param = [0.0, 0.0, 0.0]
    for k in xrange(20):
        params.append(1.0 + 1.0 * k)
        err_set.append([0.0, 0.0, 0.0])
    splits = np.array_split(data['Xtrain'], 5)
    splits_y = np.array_split(data['ytrain'], 5)
    train_set = np.ndarray([])
    validate_set = np.ndarray([])
    train_set_y = np.ndarray([])
    validate_set_y = np.ndarray([])
    is_train = False
    for i in xrange(5): #第i个切块是验证集
        print i
        for j in xrange(5):
            if j == i:
                validate_set = splits[i]
                validate_set_y = splits_y[i]
            else:
                if False == is_train:
                    train_set = splits[i]
                    train_set_y = splits_y[i]
                    is_train = True
                else:
                    train_set = np.concatenate((train_set, splits[j]))
                    train_set_y = np.concatenate((train_set_y, splits_y[j]))
        # 1、使用均值和单位方差进行标准化
        train_set_stnd, mean, std = StndTrainData(train_set, 0)
        validate_set_stnd = StndTestData(validate_set, mean, std)
        for k in xrange(20):
            weights = GradAscent(train_set_stnd, train_set_y, params[k], 2000)
            ret = Classify(train_set_stnd, weights)
            print(CaclErrorRate(ret, train_set_y))
            # 对验证数据分类，计算错误率
            ret = Classify(validate_set_stnd, weights)
            err_rate = CaclErrorRate(ret, validate_set_y)
            err_set[k][0] += err_rate
            print(params[k], err_rate, err_set[k][0])

        # 2、Log
        train_set_log = Log(train_set)
        validate_set_log = Log(validate_set)
        for k in xrange(20):
            weights = GradAscent(train_set_log, train_set_y, params[k], 2000)
            ret = Classify(train_set_log, weights)
            print(CaclErrorRate(ret, train_set_y))
            # 对验证数据分类，计算错误率
            ret = Classify(validate_set_log, weights)
            err_rate = CaclErrorRate(ret, validate_set_y)
            err_set[k][1] += err_rate
            print(params[k], err_rate, err_set[k][1])

        # 3、Binary
        train_set_bin = Binary(train_set)
        validate_set_bin = Binary(validate_set)
        for k in xrange(20):
            weights = GradAscent(train_set_bin, train_set_y, params[k], 2000)
            ret = Classify(train_set_bin, weights)
            print(CaclErrorRate(ret, train_set_y))
            # 对验证数据分类，计算错误率
            ret = Classify(validate_set_bin, weights)
            err_rate = CaclErrorRate(ret, validate_set_y)
            err_set[k][2] += err_rate
            print(params[k], err_rate, err_set[k][2])

    for i in xrange(3):
        min_err_index = 0
        for k in xrange(1, 20):
            if err_set[min_err_index][i] > err_set[k][i]:
                min_err_index = k
        l2_param[i] = params[min_err_index]

    return  l2_param

def main():
    #加载数据
    data = LoadData('spamData.mat')

    #交叉验证速度很慢，注释掉了，返回结果为[20.0, 10.0, 1.0]
    #l2_param = CrossValidator(data)
    l2_param = [20.0, 10.0, 1.0]

    print("%-8s\t%-8s\t%-8s") % ("method", "train", "test")

    #1、使用均值和单位方差进行标准化
    print("%-8s\t") % ("Stnd"),
    train_stnd, mean, std = StndTrainData(data['Xtrain'], 0)
    # 梯度上升法拟合回归参数
    weights = GradAscent(train_stnd, data['ytrain'], l2_param[0])
    #print(weights)
    # 对训练数据分类，计算错误率
    ret = Classify(train_stnd, weights)
    print("%8.6f\t") % CaclErrorRate(ret, data["ytrain"]),
    # 对测试数据分类，计算错误率
    test_stnd = StndTestData(data['Xtest'], mean, std)
    ret = Classify(test_stnd, weights)
    print("%8.6f\t") % (CaclErrorRate(ret, data["ytest"]))

    #2、Log
    print("%-8s\t") % ("Log"),
    train_log = Log(data['Xtrain'])
    # 梯度上升法拟合回归参数
    weights = GradAscent(train_log, data['ytrain'], l2_param[1])
    #print(weights)
    # 对训练数据分类，计算错误率
    ret = Classify(train_log, weights)
    print("%8.6f\t") % (CaclErrorRate(ret, data["ytrain"])),
    # 对测试数据分类，计算错误率
    test_log = Log(data['Xtest'])
    ret = Classify(test_log, weights)
    print("%8.6f\t") % (CaclErrorRate(ret, data["ytest"]))

    #3、Binary
    print("%-8s\t") % ("Binary"),
    train_bin = Binary(data['Xtrain'])
    # 梯度上升法拟合回归参数
    weights = GradAscent(train_bin, data['ytrain'], l2_param[2])
    #print(weights)
    # 对训练数据分类，计算错误率
    ret = Classify(train_bin, weights)
    print("%8.6f\t") % (CaclErrorRate(ret, data["ytrain"])),
    # 对测试数据分类，计算错误率
    test_bin = Binary(data['Xtest'])
    ret = Classify(test_bin, weights)
    print("%8.6f\t") % (CaclErrorRate(ret, data["ytest"]))

    '''
    #调用sklearn库测试 结果为0.07145187601957581  0.06901041666666663
    model = LogisticRegression(penalty='l2', dual=False, C=1.0, n_jobs=1, random_state=20, fit_intercept=True)
    #model = LogisticRegressionCV(cv=10, penalty='l2', dual=False, C=1.0, n_jobs=1, random_state=20, fit_intercept=True)
    model.fit(data['Xtrain'], data['ytrain'])  # 对模型进行训练
    acc = model.score(data['Xtrain'], data['ytrain'])  # 根据给定数据与标签返回正确率的均值
    print 1-acc #0.07145187601957581
    acc = model.score(data['Xtest'], data['ytest'])  # 根据给定数据与标签返回正确率的均值
    print 1-acc #0.06901041666666663
    '''

if __name__ == "__main__":
    main()

max_stars_repo_name: yanxdd/ML

max_issues_repo_name: yanxxd/ML

max_forks_repo_name: yanxxd/ML

reason: import numpy,import scipy


text:
from __future__ import division
import numpy as np
# import matplotlib.pyplot as plt
# from mpl_toolkits.mplot3d import Axes3D
import bpy

### Plot the red and blue circular pulses.

##### PARAMETERS
c = 299792458

def Efield(times=np.linspace(-10e-15, 10e-15, 5000), pdur=7e-15, E0=1, lambda1=7.90e-7, ellip=1):

    def gauss(x, A=E0, x0=0, width=1):
        return A*np.exp(-(x-x0)**2/(2*(width/2.35482)**2))

    w1 = c/lambda1 * 2 * np.pi

    x = np.sin(w1 * times)  * gauss(times, width=pdur)
    y = ellip * np.cos(w1 * times)  * gauss(times, width=pdur)

    return times, x, y

def make_field(x, y, z, name=''):

    print(name)
    print(name + '_BevObj')

    # Create the data block for the curve.
    curveD = bpy.data.curves.new(name, type='CURVE')
    curveD.dimensions = '3D'
    curveD.resolution_u = 2

    # Create the bevel object for the curve.
    bevel_name = name + '_BevObj'
    bpy.ops.curve.primitive_nurbs_circle_add(radius=0.118, location=(0.0, 0.0, 10.0))
    bevelobj = bpy.context.object
    bevelobj.name = bevel_name

    coords = np.vstack((x,y,z)).transpose()

    print(coords.shape)

    # Map the coords to the figure.
    polyline = curveD.splines.new('POLY')
    polyline.points.add(len(coords-1))

    for i, coord in enumerate(coords):
        x,y,z = coord
        polyline.points[i].co = (x, y, z, 1)

    # Create the object.
    curveOBJ = bpy.data.objects.new(name, curveD)

    # Attach to scene and validate context.
    scn = bpy.context.scene
    scn.objects.link(curveOBJ)
    scn.objects.active = bpy.data.objects[name]

    bpy.context.object.data.bevel_object = bevelobj

#tt, tx, ty = Efield()
#fig = plt.figure()
#ax = fig.add_subplot(111, projection='3d')
#ax.plot(tt, tx, ty, lw=2)
#
#plt.show()

# Create a few different E-fields based on the following intensity ratios.
Irats = np.array([0.1, 0.5, 1.0, 2.0, 4.0])

for rat in Irats:

    tr, xr, yr =  Efield(lambda1=7.90e-7, ellip=1)
    tb, xb, yb =  Efield(lambda1=3.95e-7, ellip=-1)

    # Make the combined field.
    xc = (xr + xb) / np.max((xr + xb))
    yc = (yr + yb) / np.max((yr + yb))

    # Now, get the APT with the correct intensity ratio.
    fn = '/Users/keV_Dawg/Desktop/Blender-rifiric Shiz/CLEO 2017 - CPHHG/APT Data/_%1.1f.txt'%rat
    tp, Expt, Eypt = np.genfromtxt('/Users/keV_Dawg/Desktop/Blender-rifiric Shiz/CLEO 2017 - CPHHG/APT Data/APT_0.1.txt', skip_header=1, unpack=True)

    # Create the Blender objects.
    make_field(tr*1e15, xr, yr, name='RedField_%1.1f'%rat)
    make_field(tr*1e15, xb, yb, name='BlueField_%1.1f'%rat)
    make_field(tr*1e15, xc, yc, name='BiCircField_%1.1f'%rat)
    make_field(tp, Expt, Eypt, name='APTField_%1.1f'%rat)

max_stars_repo_path: files/Blender Gallery/Bicircular HHG/BHHG General Scheme - CLEO 2017/Make Bicircular - v2.py


text:
# coding=utf-8
# Copyright 2019 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Timeseries data generator tests."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

from tensor2tensor.data_generators import timeseries_data_generator

import tensorflow as tf

class TimeseriesDataGeneratorTest(tf.test.TestCase):

  def testGenerateData(self):
    timeseries_params = [{
        "m": 0.006,
        "b": 300.0,
        "A": 50.0,
        "freqcoeff": 1500.0,
        "rndA": 15.0,
        "fn": np.sin
    }, {
        "m": 0.000,
        "b": 500.0,
        "A": 35.0,
        "freqcoeff": 3500.0,
        "rndA": 25.0,
        "fn": np.cos
    }, {
        "m": -0.003,
        "b": 800.0,
        "A": 65.0,
        "freqcoeff": 2500.0,
        "rndA": 5.0,
        "fn": np.sin
    }, {
        "m": 0.009,
        "b": 600.0,
        "A": 20.0,
        "freqcoeff": 1000.0,
        "rndA": 1.0,
        "fn": np.cos
    }, {
        "m": 0.002,
        "b": 700.0,
        "A": 40.0,
        "freqcoeff": 2000.0,
        "rndA": 35.0,
        "fn": np.sin
    }, {
        "m": -0.008,
        "b": 1000.0,
        "A": 70.0,
        "freqcoeff": 3000.0,
        "rndA": 25.0,
        "fn": np.cos
    }, {
        "m": 0.000,
        "b": 100.0,
        "A": 25.0,
        "freqcoeff": 1500.0,
        "rndA": 10.0,
        "fn": np.sin
    }, {
        "m": 0.004,
        "b": 1500.0,
        "A": 54.0,
        "freqcoeff": 900.0,
        "rndA": 55.0,
        "fn": np.cos
    }, {
        "m": 0.005,
        "b": 2000.0,
        "A": 32.0,
        "freqcoeff": 1100.0,
        "rndA": 43.0,
        "fn": np.sin
    }, {
        "m": 0.010,
        "b": 2500.0,
        "A": 43.0,
        "freqcoeff": 1900.0,
        "rndA": 53.0,
        "fn": np.cos
    }]
    multi_timeseries = timeseries_data_generator.generate_data(
        20, timeseries_params)

    self.assertEqual(10, len(multi_timeseries))
    self.assertEqual(20, len(multi_timeseries[0]))

if __name__ == "__main__":
  tf.test.main()