text:
// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "exec/olap_scan_node.h"

#include <algorithm>
#include <boost/foreach.hpp>
#include <sstream>
#include <iostream>
#include <utility>
#include <string>

#include "codegen/llvm_codegen.h"
#include "common/logging.h"
#include "exprs/expr.h"
#include "exprs/binary_predicate.h"
#include "exprs/in_predicate.h"
#include "gen_cpp/PlanNodes_types.h"
#include "runtime/exec_env.h"
#include "runtime/runtime_state.h"
#include "runtime/row_batch.h"
#include "runtime/string_value.h"
#include "runtime/tuple_row.h"
#include "util/runtime_profile.h"
#include "util/thread_pool.hpp"
#include "util/debug_util.h"
#include "agent/cgroups_mgr.h"
#include "common/resource_tls.h"
#include "olap/olap_reader.h"
#include <boost/variant.hpp>

using llvm::Function;

namespace palo {

#define DS_SUCCESS(x) ((x) >= 0)

OlapScanNode::OlapScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs):
        ScanNode(pool, tnode, descs),
        _tuple_id(tnode.olap_scan_node.tuple_id),
        _olap_scan_node(tnode.olap_scan_node),
        _tuple_desc(NULL),
        _tuple_idx(0),
        _eos(false),
        _scanner_pool(new ObjectPool()),
        _max_materialized_row_batches(config::palo_scanner_queue_size),
        _start(false),
        _scanner_done(false),
        _transfer_done(false),
        _wait_duration(0, 0, 1, 0),
        _status(Status::OK),
        _resource_info(nullptr),
        _buffered_bytes(0),
        _running_thread(0),
        _eval_conjuncts_fn(nullptr) {
}

OlapScanNode::~OlapScanNode() {
}

Status OlapScanNode::init(const TPlanNode& tnode, RuntimeState* state) {
    RETURN_IF_ERROR(ExecNode::init(tnode, state));
    _direct_conjunct_size = _conjunct_ctxs.size();

    if (tnode.olap_scan_node.__isset.sort_column) {
        _is_result_order = true;
    } else {
        _is_result_order = false;
    }

    // Before, we support scan data ordered, but is not used in production
    // Now, we drop this functional
    DCHECK(!_is_result_order) << "ordered result don't support any more";

    return Status::OK;
}

void OlapScanNode::_init_counter(RuntimeState* state) {
#if 0
    ADD_TIMER(profile, "GetTabletTime");
    ADD_TIMER(profile, "InitReaderTime");
    ADD_TIMER(profile, "ShowHintsTime");
    ADD_TIMER(profile, "BlockLoadTime");
    ADD_TIMER(profile, "IndexLoadTime");
    ADD_TIMER(profile, "VectorPredicateEvalTime");
    ADD_TIMER(profile, "ScannerTimer");
    ADD_TIMER(profile, "IOTimer");
    ADD_TIMER(profile, "DecompressorTimer");
    ADD_TIMER(profile, "RLETimer");

    ADD_COUNTER(profile, "RawRowsRead", TUnit::UNIT);
    ADD_COUNTER(profile, "IndexStreamCacheMiss", TUnit::UNIT);
    ADD_COUNTER(profile, "IndexStreamCacheHit", TUnit::UNIT);
    ADD_COUNTER(profile, "BlockLoadCount", TUnit::UNIT);
#endif
    ADD_TIMER(_runtime_profile, "ShowHintsTime");

    _read_compressed_counter =
        ADD_COUNTER(_runtime_profile, "CompressedBytesRead", TUnit::BYTES);
    _read_uncompressed_counter =
        ADD_COUNTER(_runtime_profile, "UncompressedBytesRead", TUnit::BYTES);
    _block_load_timer = ADD_TIMER(_runtime_profile, "BlockLoadTime");
    _block_load_counter =
        ADD_COUNTER(_runtime_profile, "BlocksLoad", TUnit::UNIT);
    _block_fetch_timer =
        ADD_TIMER(_runtime_profile, "BlockFetchTime");
    _raw_rows_counter =
        ADD_COUNTER(_runtime_profile, "RawRowsRead", TUnit::UNIT);

    _rows_vec_cond_counter =
        ADD_COUNTER(_runtime_profile, "RowsVectorPredFiltered", TUnit::UNIT);
    _vec_cond_timer =
        ADD_TIMER(_runtime_profile, "VectorPredEvalTime");

    _stats_filtered_counter =
        ADD_COUNTER(_runtime_profile, "RowsStatsFiltered", TUnit::UNIT);
    _del_filtered_counter =
        ADD_COUNTER(_runtime_profile, "RowsDelFiltered", TUnit::UNIT);

    _io_timer = ADD_TIMER(_runtime_profile, "IOTimer");
    _decompressor_timer = ADD_TIMER(_runtime_profile, "DecompressorTimer");
    _index_load_timer = ADD_TIMER(_runtime_profile, "IndexLoadTime");

    _scan_timer = ADD_TIMER(_runtime_profile, "ScanTime");
}

Status OlapScanNode::prepare(RuntimeState* state) {
    RETURN_IF_ERROR(ScanNode::prepare(state));
    // create scanner profile
    // create timer
    _tablet_counter =
        ADD_COUNTER(runtime_profile(), "TabletCount ", TUnit::UNIT);
    _rows_pushed_cond_filtered_counter =
        ADD_COUNTER(_runtime_profile, "RowsPushedCondFiltered", TUnit::UNIT);
    _init_counter(state);

    _tuple_desc = state->desc_tbl().get_tuple_descriptor(_tuple_id);
    if (_tuple_desc == NULL) {
        // TODO: make sure we print all available diagnostic output to our error log
        return Status("Failed to get tuple descriptor.");
    }

    const std::vector<SlotDescriptor*>& slots = _tuple_desc->slots();

    for (int i = 0; i < slots.size(); ++i) {
        if (!slots[i]->is_materialized()) {
            continue;
        }

        if (!slots[i]->type().is_string_type()) {
            continue;
        }

        _string_slots.push_back(slots[i]);
    }

    if (state->codegen_level() > 0) {
        LlvmCodeGen* codegen = NULL;
        RETURN_IF_ERROR(state->get_codegen(&codegen));
        Function* codegen_eval_conjuncts_fn = codegen_eval_conjuncts(state, _conjunct_ctxs);
        if (codegen_eval_conjuncts_fn != NULL) {
            codegen->add_function_to_jit(codegen_eval_conjuncts_fn,
                                         reinterpret_cast<void**>(&_eval_conjuncts_fn));
            // AddRuntimeExecOption("Probe Side Codegen Enabled");
        }
    }

    _runtime_state = state;
    return Status::OK;
}

Status OlapScanNode::open(RuntimeState* state) {
    VLOG(1) << "OlapScanNode::Open";
    SCOPED_TIMER(_runtime_profile->total_time_counter());
    RETURN_IF_CANCELLED(state);
    RETURN_IF_ERROR(ExecNode::open(state));

    for (int conj_idx = 0; conj_idx < _conjunct_ctxs.size(); ++conj_idx) {
        // if conjunct is constant, compute direct and set eos = true
        if (_conjunct_ctxs[conj_idx]->root()->is_constant()) {
            void* value = _conjunct_ctxs[conj_idx]->get_value(NULL);
            if (value == NULL || *reinterpret_cast<bool*>(value) == false) {
                _eos = true;
            }
        }
    }

    _resource_info = ResourceTls::get_resource_tls();

    return Status::OK;
}

Status OlapScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) {
    RETURN_IF_ERROR(exec_debug_action(TExecNodePhase::GETNEXT));
    SCOPED_TIMER(_runtime_profile->total_time_counter());

    // check if Canceled.
    if (state->is_cancelled()) {
        boost::unique_lock<boost::mutex> l(_row_batches_lock);
        _transfer_done = true;
        boost::lock_guard<boost::mutex> guard(_status_mutex);
        if (LIKELY(_status.ok())) {
            _status = Status::CANCELLED;
        }
        return _status;
    }

    if (_eos) {
        *eos = true;
        return Status::OK;
    }

    // check if started.
    if (!_start) {
        Status status = start_scan(state);

        if (!status.ok()) {
            LOG(ERROR) << "StartScan Failed cause " << status.get_error_msg();
            *eos = true;
            return status;
        }

        _start = true;
    }

    // wait for batch from queue
    RowBatch* materialized_batch = NULL;
    {
        boost::unique_lock<boost::mutex> l(_row_batches_lock);

        while (_materialized_row_batches.empty() && !_transfer_done) {
            if (state->is_cancelled()) {
                _transfer_done = true;
            }

            _row_batch_added_cv.timed_wait(l, _wait_duration);
        }

        if (!_materialized_row_batches.empty()) {
            materialized_batch = dynamic_cast<RowBatch*>(_materialized_row_batches.front());
            DCHECK(materialized_batch != NULL);
            _materialized_row_batches.pop_front();
        }
    }

    // return batch
    if (NULL != materialized_batch) {
        // notify scanner
        _row_batch_consumed_cv.notify_one();
        // get scanner's batch memory
        row_batch->acquire_state(materialized_batch);
        _num_rows_returned += row_batch->num_rows();
        COUNTER_SET(_rows_returned_counter, _num_rows_returned);

        // reach scan node limit
        if (reached_limit()) {
            int num_rows_over = _num_rows_returned - _limit;
            row_batch->set_num_rows(row_batch->num_rows() - num_rows_over);
            _num_rows_returned -= num_rows_over;
            COUNTER_SET(_rows_returned_counter, _num_rows_returned);

            {
                boost::unique_lock<boost::mutex> l(_row_batches_lock);
                _transfer_done = true;
            }

            _row_batch_consumed_cv.notify_all();
            *eos = true;
            LOG(INFO) << "OlapScanNode ReachedLimit.";
        } else {
            *eos = false;
        }

        if (VLOG_ROW_IS_ON) {
            for (int i = 0; i < row_batch->num_rows(); ++i) {
                TupleRow* row = row_batch->get_row(i);
                VLOG_ROW << "OlapScanNode output row: "
                    << print_tuple(row->get_tuple(0), *_tuple_desc);
            }
        }
        __sync_fetch_and_sub(&_buffered_bytes,
                             row_batch->tuple_data_pool()->total_reserved_bytes());

        delete materialized_batch;
        return Status::OK;
    }

    // all scanner done, change *eos to true
    *eos = true;
    boost::lock_guard<boost::mutex> guard(_status_mutex);
    return _status;
}

Status OlapScanNode::close(RuntimeState* state) {
    if (is_closed()) {
        return Status::OK;
    }
    RETURN_IF_ERROR(exec_debug_action(TExecNodePhase::CLOSE));

    // change done status
    {
        boost::unique_lock<boost::mutex> l(_row_batches_lock);
        _transfer_done = true;
    }
    // notify all scanner thread
    _row_batch_consumed_cv.notify_all();
    _row_batch_added_cv.notify_all();
    _scan_batch_added_cv.notify_all();

    // join transfer thread
    _transfer_thread.join_all();

    // clear some row batch in queue
    for (auto row_batch : _materialized_row_batches) {
        delete row_batch;
    }

    _materialized_row_batches.clear();

    for (auto row_batch : _scan_row_batches) {
        delete row_batch;
    }

    _scan_row_batches.clear();

    // OlapScanNode terminate by exception
    // so that initiative close the Scanner
    for (auto scanner : _all_olap_scanners) {
        scanner->close(state);
    }

    VLOG(1) << "OlapScanNode::close()";
    return ScanNode::close(state);
}

Status OlapScanNode::set_scan_ranges(const std::vector<TScanRangeParams>& scan_ranges) {
    for (auto& scan_range : scan_ranges) {
        DCHECK(scan_range.scan_range.__isset.palo_scan_range);
        boost::shared_ptr<PaloScanRange> palo_scan_range(
            new PaloScanRange(scan_range.scan_range.palo_scan_range));
        RETURN_IF_ERROR(palo_scan_range->init());
        VLOG(1) << "palo_scan_range table=" << scan_range.scan_range.palo_scan_range.table_name <<
                " version" << scan_range.scan_range.palo_scan_range.version;
        _palo_scan_ranges.push_back(palo_scan_range);
        COUNTER_UPDATE(_tablet_counter, 1);
    }

    return Status::OK;
}

Status OlapScanNode::start_scan(RuntimeState* state) {
    RETURN_IF_CANCELLED(state);

    VLOG(1) << "NormalizeConjuncts";
    // 1. Convert conjuncts to ColumnValueRange in each column
    RETURN_IF_ERROR(normalize_conjuncts());

    VLOG(1) << "BuildOlapFilters";
    // 2. Using ColumnValueRange to Build OlapEngine filters
    RETURN_IF_ERROR(build_olap_filters());

    VLOG(1) << "SelectScanRanges";
    // 3. Using `Partition Column`'s ColumnValueRange to select ScanRange
    RETURN_IF_ERROR(select_scan_ranges());

    VLOG(1) << "BuildScanKey";
    // 4. Using `Key Column`'s ColumnValueRange to split ScanRange to serval `Sub ScanRange`
    RETURN_IF_ERROR(build_scan_key());

    VLOG(1) << "SplitScanRange";
    // 5. Query OlapEngine to split `Sub ScanRange` to serval `Sub Sub ScanRange`
    RETURN_IF_ERROR(split_scan_range());

    VLOG(1) << "StartScanThread";
    // 6. Start multi thread to read serval `Sub Sub ScanRange`
    RETURN_IF_ERROR(start_scan_thread(state));

    return Status::OK;
}

Status OlapScanNode::normalize_conjuncts() {
    std::vector<SlotDescriptor*> slots = _tuple_desc->slots();

    for (int slot_idx = 0; slot_idx < slots.size(); ++slot_idx) {
        switch (slots[slot_idx]->type().type) {
            // TYPE_TINYINT use int32_t to present
            // because it's easy to convert to string for build Olap fetch Query
        case TYPE_TINYINT: {
            ColumnValueRange<int32_t> range(slots[slot_idx]->col_name(),
                                            slots[slot_idx]->type().type,
                                            std::numeric_limits<int8_t>::min(),
                                            std::numeric_limits<int8_t>::max());
            normalize_predicate(range, slots[slot_idx]);
            break;
        }

        case TYPE_SMALLINT: {
            ColumnValueRange<int16_t> range(slots[slot_idx]->col_name(),
                                            slots[slot_idx]->type().type,
                                            std::numeric_limits<int16_t>::min(),
                                            std::numeric_limits<int16_t>::max());
            normalize_predicate(range, slots[slot_idx]);
            break;
        }

        case TYPE_INT: {
            ColumnValueRange<int32_t> range(slots[slot_idx]->col_name(),
                                            slots[slot_idx]->type().type,
                                            std::numeric_limits<int32_t>::min(),
                                            std::numeric_limits<int32_t>::max());
            normalize_predicate(range, slots[slot_idx]);
            break;
        }

        case TYPE_BIGINT: {
            ColumnValueRange<int64_t> range(slots[slot_idx]->col_name(),
                                            slots[slot_idx]->type().type,
                                            std::numeric_limits<int64_t>::min(),
                                            std::numeric_limits<int64_t>::max());
            normalize_predicate(range, slots[slot_idx]);
            break;
        }

        case TYPE_LARGEINT: {
            __int128 min = MIN_INT128;
            __int128 max = MAX_INT128;
            ColumnValueRange<__int128> range(slots[slot_idx]->col_name(),
                                             slots[slot_idx]->type().type,
                                             min,
                                             max);
            normalize_predicate(range, slots[slot_idx]);
            break;
        }

        case TYPE_CHAR:
        case TYPE_VARCHAR:
        case TYPE_HLL: {
            static char min_char = 0x00;
            static char max_char = 0xff;
            ColumnValueRange<StringValue> range(slots[slot_idx]->col_name(),
                                                slots[slot_idx]->type().type,
                                                StringValue(&min_char, 0),
                                                StringValue(&max_char, 1));
            normalize_predicate(range, slots[slot_idx]);
            break;
        }

        case TYPE_DATE:
        case TYPE_DATETIME: {
            DateTimeValue max_value = DateTimeValue::datetime_max_value();
            DateTimeValue min_value = DateTimeValue::datetime_min_value();
            ColumnValueRange<DateTimeValue> range(slots[slot_idx]->col_name(),
                                                  slots[slot_idx]->type().type,
                                                  min_value,
                                                  max_value);
            normalize_predicate(range, slots[slot_idx]);
            break;
        }

        case TYPE_DECIMAL: {
            DecimalValue min = DecimalValue::get_min_decimal();
            DecimalValue max = DecimalValue::get_max_decimal();
            ColumnValueRange<DecimalValue> range(slots[slot_idx]->col_name(),
                                                 slots[slot_idx]->type().type,
                                                 min,
                                                 max);
            normalize_predicate(range, slots[slot_idx]);
            break;
        }

        default: {
            VLOG(2) << "Unsupport Normalize Slot [ColName="
                    << slots[slot_idx]->col_name() << "]";
            break;
        }
        }
    }

    return Status::OK;
}

Status OlapScanNode::build_olap_filters() {
    _olap_filter.clear();

    for (auto iter : _column_value_ranges) {
        ToOlapFilterVisitor visitor;
        boost::variant<std::list<TCondition>> filters;
        boost::apply_visitor(visitor, iter.second, filters);

        std::list<TCondition> new_filters = boost::get<std::list<TCondition>>(filters);
        if (new_filters.empty()) {
            continue;
        }

        for (auto filter : new_filters) {
           _olap_filter.push_back(filter);
        }
    }

    return Status::OK;
}

Status OlapScanNode::select_scan_ranges() {

    std::list<boost::shared_ptr<PaloScanRange> >::iterator scan_range_iter
        = _palo_scan_ranges.begin();

    while (scan_range_iter != _palo_scan_ranges.end()) {
        if (!select_scan_range(*scan_range_iter)) {
            if ((*scan_range_iter)->scan_range().partition_column_ranges.size() != 0) {
                VLOG(1) << "Remove ScanRange: ["
                        << (*scan_range_iter)->scan_range().partition_column_ranges[0].begin_key << ", "
                        << (*scan_range_iter)->scan_range().partition_column_ranges[0].end_key << ")";
            } else {
                VLOG(1) << "Remove ScanRange";
            }

            _palo_scan_ranges.erase(scan_range_iter++);
        } else {
            scan_range_iter++;
        }
    }

    return Status::OK;
}

Status OlapScanNode::build_scan_key() {
    const std::vector<std::string>& column_names = _olap_scan_node.key_column_name;
    const std::vector<TPrimitiveType::type>& column_types = _olap_scan_node.key_column_type;
    DCHECK(column_types.size() == column_names.size());

    // 1. construct scan key except last olap engine short key
    int column_index = 0;
    _scan_keys.set_is_convertible(limit() == -1);

    for (; column_index < column_names.size() && !_scan_keys.has_range_value(); ++column_index) {
        std::map<std::string, ColumnValueRangeType>::iterator column_range_iter
            = _column_value_ranges.find(column_names[column_index]);

        if (_column_value_ranges.end() == column_range_iter) {
            break;
        }

        ExtendScanKeyVisitor visitor(_scan_keys);
        RETURN_IF_ERROR(boost::apply_visitor(visitor, column_range_iter->second));
    }

    _scan_keys.debug();

    return Status::OK;
}

Status OlapScanNode::split_scan_range() {
    std::vector<OlapScanRange> sub_ranges;
    VLOG(1) << "_palo_scan_ranges.size()=" << _palo_scan_ranges.size();

    for (auto scan_range : _palo_scan_ranges) {
        sub_ranges.clear();
        RETURN_IF_ERROR(get_sub_scan_range(scan_range, &sub_ranges));

        for (auto sub_range : sub_ranges) {
            VLOG(1) << "SubScanKey=" << (sub_range.begin_include ? "[" : "(")
                    << OlapScanKeys::to_print_key(sub_range.begin_scan_range)
                    << " : " << OlapScanKeys::to_print_key(sub_range.end_scan_range) <<
                    (sub_range.end_include ? "]" : ")");
            _query_key_ranges.push_back(sub_range);
            _query_scan_ranges.push_back(scan_range);
        }
    }

    DCHECK(_query_key_ranges.size() == _query_scan_ranges.size());

    return Status::OK;
}

Status OlapScanNode::start_scan_thread(RuntimeState* state) {
    VLOG(1) << "Query ScanRange Num: " << _query_scan_ranges.size();

    // thread num
    if (0 == _query_scan_ranges.size()) {
        _transfer_done = true;
        return Status::OK;
    }

    int key_range_size = _query_key_ranges.size();
    int key_range_num_per_scanner = key_range_size / 64;

    if (0 == key_range_num_per_scanner) {
        key_range_num_per_scanner = 1;
    }

    // scan range per therad
    for (int i = 0; i < key_range_size;) {
        boost::shared_ptr<PaloScanRange> scan_range = _query_scan_ranges[i];
        std::vector<OlapScanRange> key_ranges;
        key_ranges.push_back(_query_key_ranges[i]);
        ++i;

        for (int j = 1;
             j < key_range_num_per_scanner
             && i < key_range_size
             && _query_scan_ranges[i] == _query_scan_ranges[i - 1]
             && _query_key_ranges[i].end_include == _query_key_ranges[i - 1].end_include;
             j++, i++) {
            key_ranges.push_back(_query_key_ranges[i]);
        }

        OlapScanner* scanner = new OlapScanner(
            state, this, _olap_scan_node.is_preaggregation,
            scan_range.get(), key_ranges);

        _scanner_pool->add(scanner);
        _olap_scanners.push_back(scanner);
        _all_olap_scanners = _olap_scanners;
    }

    // init progress
    std::stringstream ss;
    ss << "ScanThread complete (node=" << id() << "):";
    _progress = ProgressUpdater(ss.str(), _olap_scanners.size(), 1);
    _progress.set_logging_level(1);

    _transfer_thread.add_thread(
        new boost::thread(
            &OlapScanNode::transfer_thread, this, state));

    return Status::OK;
}

template<class T>
Status OlapScanNode::normalize_predicate(ColumnValueRange<T>& range, SlotDescriptor* slot) {
    // 1. Normalize InPredicate, add to ColumnValueRange
    RETURN_IF_ERROR(normalize_in_predicate(slot, &range));

    // 2. Normalize BinaryPredicate , add to ColumnValueRange
    RETURN_IF_ERROR(normalize_binary_predicate(slot, &range));

    // 3. Add range to Column->ColumnValueRange map
    _column_value_ranges[slot->col_name()] = range;

    return Status::OK;
}

static bool ignore_cast(SlotDescriptor* slot, Expr* expr) {
    if (slot->type().is_date_type() && expr->type().is_date_type()) {
        return true;
    }
    if (slot->type().is_string_type() && expr->type().is_string_type()) {
        return true;
    }
    return false;
}

template<class T>
Status OlapScanNode::normalize_in_predicate(SlotDescriptor* slot, ColumnValueRange<T>* range) {
    for (int conj_idx = 0; conj_idx < _conjunct_ctxs.size(); ++conj_idx) {
        // 1. Normalize in conjuncts like 'where col in (v1, v2, v3)'
        if (TExprOpcode::FILTER_IN == _conjunct_ctxs[conj_idx]->root()->op()) {
            InPredicate* pred = dynamic_cast<InPredicate*>(_conjunct_ctxs[conj_idx]->root());
            if (pred->is_not_in()) {
                continue;
            }

            if (Expr::type_without_cast(pred->get_child(0)) != TExprNodeType::SLOT_REF) {
                continue;
            }

            if (pred->get_child(0)->type() != slot->type()) {
                if (!ignore_cast(slot, pred->get_child(0))) {
                    continue;
                }
            }

            if (pred->is_not_in()) {
                continue;
            }
            std::vector<SlotId> slot_ids;

            if (1 == pred->get_child(0)->get_slot_ids(&slot_ids)) {
                // 1.1 Skip if slot not match conjunct
                if (slot_ids[0] != slot->id()) {
                    continue;
                }

                VLOG(1) << slot->col_name() << " fixed_values add num: "
                        << pred->hybird_set()->size();

                // 1.2 Skip if InPredicate value size larger then max_scan_key_num
                if (pred->hybird_set()->size() > config::palo_max_scan_key_num) {
                    LOG(WARNING) << "Predicate value num " << pred->hybird_set()->size()
                                 << " excede limit " << config::palo_max_scan_key_num;
                    continue;
                }

                // 1.3 Push InPredicate value into ColumnValueRange
                HybirdSetBase::IteratorBase* iter = pred->hybird_set()->begin();
                while (iter->has_next()) {
                    // column in (NULL,...) counldn't push down to OlapEngine
                    // so that discard whole ColumnValueRange
                    if (NULL == iter->get_value()) {
                        range->clear();
                        break;
                    }

                    switch (slot->type().type) {
                    case TYPE_TINYINT: {
                        int32_t v = *reinterpret_cast<int8_t*>(const_cast<void*>(iter->get_value()));
                        range->add_fixed_value(*reinterpret_cast<T*>(&v));
                        break;
                    }
                    case TYPE_DATE: {
                        DateTimeValue date_value =
                            *reinterpret_cast<const DateTimeValue*>(iter->get_value());
                        date_value.cast_to_date();
                        range->add_fixed_value(*reinterpret_cast<T*>(&date_value));
                        break;
                    }
                    case TYPE_DECIMAL:
                    case TYPE_LARGEINT:
                    case TYPE_CHAR:
                    case TYPE_VARCHAR:
	                case TYPE_HLL:
                    case TYPE_SMALLINT:
                    case TYPE_INT:
                    case TYPE_BIGINT:
                    case TYPE_DATETIME: {
                        range->add_fixed_value(*reinterpret_cast<T*>(const_cast<void*>(iter->get_value())));
                        break;
                    }
                    default: {
                        break;
                    }
                    }
                    iter->next();
                }
            }
        }

        // 2. Normalize eq conjuncts like 'where col = value'
        if (TExprNodeType::BINARY_PRED == _conjunct_ctxs[conj_idx]->root()->node_type()
                && FILTER_IN == to_olap_filter_type(_conjunct_ctxs[conj_idx]->root()->op(), false)) {
            Expr* pred = _conjunct_ctxs[conj_idx]->root();
            DCHECK(pred->get_num_children() == 2);

            for (int child_idx = 0; child_idx < 2; ++child_idx) {
                if (Expr::type_without_cast(pred->get_child(child_idx))
                        != TExprNodeType::SLOT_REF) {
                    continue;
                }
                if (pred->get_child(child_idx)->type() != slot->type()) {
                    if (!ignore_cast(slot, pred->get_child(child_idx))) {
                        continue;
                    }
                }

                std::vector<SlotId> slot_ids;
                if (1 == pred->get_child(child_idx)->get_slot_ids(&slot_ids)) {
                    if (slot_ids[0] != slot->id()) {
                        continue;
                    }

                    Expr* expr = pred->get_child(1 - child_idx);
                    if (!expr->is_constant()) {
                        continue;
                    }

                    void* value = _conjunct_ctxs[conj_idx]->get_value(expr, NULL);
                    // for case: where col = null
                    if (value == NULL) {
                        continue;
                    }

                    switch (slot->type().type) {
                    case TYPE_TINYINT: {
                        int32_t v = *reinterpret_cast<int8_t*>(value);
                        range->add_fixed_value(*reinterpret_cast<T*>(&v));
                        break;
                    }
                    case TYPE_DATE: {
                        DateTimeValue date_value =
                            *reinterpret_cast<DateTimeValue*>(value);
                        date_value.cast_to_date();
                        range->add_fixed_value(*reinterpret_cast<T*>(&date_value));
                        break;
                    }
                    case TYPE_DECIMAL:
                    case TYPE_CHAR:
                    case TYPE_VARCHAR:
                    case TYPE_HLL:
                    case TYPE_DATETIME:
                    case TYPE_SMALLINT:
                    case TYPE_INT:
                    case TYPE_BIGINT:
                    case TYPE_LARGEINT: {
                        range->add_fixed_value(*reinterpret_cast<T*>(value));
                        break;
                    }
                    default: {
                        LOG(WARNING) << "Normalize filter fail, Unsupport Primitive type. [type="
                                     << expr->type() << "]";
                        return Status("Normalize filter fail, Unsupport Primitive type");
                    }
                    }
                }
            }
        }
    }

    return Status::OK;
}

void OlapScanNode::construct_is_null_pred_in_where_pred(Expr* expr, SlotDescriptor* slot, std::string is_null_str) {
    if (Expr::type_without_cast(expr) != TExprNodeType::SLOT_REF) {
        return;
    }

    std::vector<SlotId> slot_ids;
    if (1 != expr->get_slot_ids(&slot_ids)) {
        return;
    }

    if (slot_ids[0] != slot->id()) {
        return;
    }
    TCondition is_null;
    is_null.column_name = slot->col_name();
    is_null.condition_op = "is";
    is_null.condition_values.push_back(is_null_str);
    _is_null_vector.push_back(is_null);
    return;
}

template<class T>
Status OlapScanNode::normalize_binary_predicate(SlotDescriptor* slot, ColumnValueRange<T>* range) {
    for (int conj_idx = 0; conj_idx < _conjunct_ctxs.size(); ++conj_idx) {
        Expr *root_expr =  _conjunct_ctxs[conj_idx]->root();
        if (TExprNodeType::BINARY_PRED != root_expr->node_type()
                || FILTER_IN == to_olap_filter_type(root_expr->op(), false)
                || FILTER_NOT_IN == to_olap_filter_type(root_expr->op(), false)) {
            if (TExprNodeType::FUNCTION_CALL == root_expr->node_type()) {
                std::string is_null_str;
                if (root_expr->is_null_scalar_function(is_null_str)) {
                    construct_is_null_pred_in_where_pred(root_expr->get_child(0),
                        slot, is_null_str);
                }
            }
            continue;
        }

        Expr* pred = _conjunct_ctxs[conj_idx]->root();
        DCHECK(pred->get_num_children() == 2);

        for (int child_idx = 0; child_idx < 2; ++child_idx) {
            if (Expr::type_without_cast(pred->get_child(child_idx)) != TExprNodeType::SLOT_REF) {
                continue;
            }
            if (pred->get_child(child_idx)->type() != slot->type()) {
                if (!ignore_cast(slot, pred->get_child(child_idx))) {
                    continue;
                }
            }

            std::vector<SlotId> slot_ids;

            if (1 == pred->get_child(child_idx)->get_slot_ids(&slot_ids)) {
                if (slot_ids[0] != slot->id()) {
                    continue;
                }

                Expr* expr = pred->get_child(1 - child_idx);

                // for case: where col_a > col_b
                if (!expr->is_constant()) {
                    continue;
                }

                void* value = _conjunct_ctxs[conj_idx]->get_value(expr, NULL);
                // for case: where col > null
                if (value == NULL) {
                    continue;
                }

                switch (slot->type().type) {
                case TYPE_TINYINT: {
                    int32_t v = *reinterpret_cast<int8_t*>(value);
                    range->add_range(to_olap_filter_type(pred->op(), child_idx),
                                    *reinterpret_cast<T*>(&v));
                    break;
                }

                case TYPE_DATE: {
                    DateTimeValue date_value = *reinterpret_cast<DateTimeValue*>(value);
                    date_value.cast_to_date();
                    range->add_range(to_olap_filter_type(pred->op(), child_idx),
                                     *reinterpret_cast<T*>(&date_value));
                    break;
                }
                case TYPE_DECIMAL:
                case TYPE_CHAR:
                case TYPE_VARCHAR:
			    case TYPE_HLL:
                case TYPE_DATETIME:
                case TYPE_SMALLINT:
                case TYPE_INT:
                case TYPE_BIGINT:
                case TYPE_LARGEINT: {
                    range->add_range(to_olap_filter_type(pred->op(), child_idx),
                                    *reinterpret_cast<T*>(value));
                    break;
                }

                default: {
                    LOG(WARNING) << "Normalize filter fail, Unsupport Primitive type. [type="
                                 << expr->type() << "]";
                    return Status("Normalize filter fail, Unsupport Primitive type");
                }
                }

                VLOG(1) << slot->col_name() << " op: "
                        << static_cast<int>(to_olap_filter_type(pred->op(), child_idx))
                        << " value: " << *reinterpret_cast<T*>(value);
            }
        }
    }

    return Status::OK;
}

bool OlapScanNode::select_scan_range(boost::shared_ptr<PaloScanRange> scan_range) {
    std::map<std::string, ColumnValueRangeType>::iterator iter
        = _column_value_ranges.begin();

    while (iter != _column_value_ranges.end()) {
        // return false if it's partition column range has no intersection with
        // column range deduce by conjunct
        if (0 == scan_range->has_intersection(iter->first, iter->second)) {
            return false;
        }

        ++iter;
    }

    return true;
}

Status OlapScanNode::get_sub_scan_range(
    boost::shared_ptr<PaloScanRange> scan_range,
    std::vector<OlapScanRange>* sub_range) {
    std::vector<OlapScanRange> scan_key_range;
    RETURN_IF_ERROR(_scan_keys.get_key_range(&scan_key_range));

    if (limit() != -1 ||
        scan_key_range.size() > 64) {
        if (scan_key_range.size() != 0) {
            *sub_range = scan_key_range;
        } else { // [-oo, +oo]
            sub_range->resize(1);
        }
    } else {
        EngineMetaReader olap_meta_reader(scan_range);
        RETURN_IF_ERROR(olap_meta_reader.open());

        if (!olap_meta_reader.get_hints(
                    config::palo_scan_range_row_count,
                    _scan_keys.begin_include(),
                    _scan_keys.end_include(),
                    scan_key_range,
                    sub_range,
                    _runtime_profile.get()).ok()) {
            if (scan_key_range.size() != 0) {
                *sub_range = scan_key_range;
            } else { // [-oo, +oo]
                sub_range->resize(1);
            }
        }

        RETURN_IF_ERROR(olap_meta_reader.close());
    }

    return Status::OK;
}

void OlapScanNode::transfer_thread(RuntimeState* state) {
    // scanner open pushdown to scanThread
    Status status = Status::OK;
    for (auto scanner : _olap_scanners) {
        status = Expr::clone_if_not_exists(_conjunct_ctxs, state, scanner->conjunct_ctxs());
        if (!status.ok()) {
            boost::lock_guard<boost::mutex> guard(_status_mutex);
            _status = status;
            break;
        }
    }

    /*********************************
     * 优先级调度基本策略:
     * 1. 通过查询拆分的Range个数来确定初始nice值
     *    Range个数越多，越倾向于认定为大查询，nice值越小
     * 2. 通过查询累计读取的数据量来调整nice值
     *    读取的数据越多，越倾向于认定为大查询，nice值越小
     * 3. 通过nice值来判断查询的优先级
     *    nice值越大的，越优先获得的查询资源
     * 4. 定期提高队列内残留任务的优先级，避免大查询完全饿死
     *********************************/
    PriorityThreadPool* thread_pool = state->exec_env()->thread_pool();
    _total_assign_num = 0;
    _nice = 18 + std::max(0, 2 - (int)_olap_scanners.size() / 5);
    std::list<OlapScanner*> olap_scanners;

    int64_t mem_limit = 512 * 1024 * 1024;
    // TODO(zc): use memory limit
    int64_t mem_consume = __sync_fetch_and_add(&_buffered_bytes, 0);
    if (state->fragment_mem_tracker() != nullptr) {
        mem_limit = state->fragment_mem_tracker()->limit();
        mem_consume = state->fragment_mem_tracker()->consumption();
    }
    int max_thread = _max_materialized_row_batches;
    if (config::palo_scanner_row_num > state->batch_size()) {
        max_thread /= config::palo_scanner_row_num / state->batch_size();
    }
    // read from scanner
    while (LIKELY(status.ok())) {
        int assigned_thread_num = 0;
        // copy to local
        {
            boost::unique_lock<boost::mutex> l(_scan_batches_lock);
            assigned_thread_num = _running_thread;
            // int64_t buf_bytes = __sync_fetch_and_add(&_buffered_bytes, 0);
            // How many thread can apply to this query
            size_t thread_slot_num = 0;
            mem_consume = __sync_fetch_and_add(&_buffered_bytes, 0);
            if (state->fragment_mem_tracker() != nullptr) {
                mem_consume = state->fragment_mem_tracker()->consumption();
            }
            if (mem_consume < (mem_limit * 6) / 10) {
                thread_slot_num = max_thread - assigned_thread_num;
            } else {
                // Memory already exceed
                if (_scan_row_batches.empty()) {
                    // NOTE(zc): here need to lock row_batches_lock_
                    //  be worried about dead lock, so don't check here
                    // if (materialized_row_batches_.empty()) {
                    //     LOG(FATAL) << "Scan_row_batches_ and materialized_row_batches_"
                    //         " are empty when memory exceed";
                    // }
                    // Just for notify if scan_row_batches_ is empty and no running thread
                    if (assigned_thread_num == 0) {
                        thread_slot_num = 1;
                        // NOTE: if olap_scanners_ is empty, scanner_done_ should be true
                    }
                }
            }
            thread_slot_num = std::min(thread_slot_num, _olap_scanners.size());
            for (int i = 0; i < thread_slot_num; ++i) {
                olap_scanners.push_back(_olap_scanners.front());
                _olap_scanners.pop_front();
                _running_thread++;
                assigned_thread_num++;
            }
        }

        auto iter = olap_scanners.begin();
        while (iter != olap_scanners.end()) {
            PriorityThreadPool::Task task;
            task.work_function = boost::bind(&OlapScanNode::scanner_thread, this, *iter);
            task.priority = _nice;
            if (thread_pool->offer(task)) {
                olap_scanners.erase(iter++);
            } else {
                LOG(FATAL) << "Failed to assign scanner task to thread pool!";
            }
            ++_total_assign_num;
        }

        RowBatchInterface* scan_batch = NULL;
        {
            // 1 scanner idle task not empty, assign new sanner task
            boost::unique_lock<boost::mutex> l(_scan_batches_lock);

            // scanner_row_num = 16k
            // 16k * 10 * 12 * 8 = 15M(>2s)  --> nice=10
            // 16k * 20 * 22 * 8 = 55M(>6s)  --> nice=0
            while (_nice > 0
                       && _total_assign_num > (22 - _nice) * (20 - _nice) * 6) {
                --_nice;
            }

            // 2 wait when all scanner are running & no result in queue
            while (UNLIKELY(_running_thread == assigned_thread_num
                            && _scan_row_batches.empty()
                            && !_scanner_done)) {
                _scan_batch_added_cv.wait(l);
            }

            // 3 transfer result row batch when queue is not empty
            if (LIKELY(!_scan_row_batches.empty())) {
                scan_batch = _scan_row_batches.front();
                _scan_row_batches.pop_front();

                // delete scan_batch if transfer thread should be stoped
                // because scan_batch wouldn't be useful anymore
                if (UNLIKELY(_transfer_done)) {
                    delete scan_batch;
                    scan_batch = NULL;
                }
            } else {
                if (_scanner_done) {
                    break;
                }
            }
        }

        if (NULL != scan_batch) {
            add_one_batch(scan_batch);
        }
    }

    VLOG(1) << "TransferThread finish.";
    boost::unique_lock<boost::mutex> l(_row_batches_lock);
    _transfer_done = true;
    _row_batch_added_cv.notify_all();
}

void OlapScanNode::scanner_thread(OlapScanner* scanner) {
    Status status = Status::OK;
    bool eos = false;
    RuntimeState* state = scanner->runtime_state();
    DCHECK(NULL != state);
    if (!scanner->is_open()) {
        status = scanner->open();
        if (!status.ok()) {
            boost::lock_guard<boost::mutex> guard(_status_mutex);
            _status = status;
            eos = true;
        }
        scanner->set_opened();
    }

    // apply to cgroup
    if (_resource_info != nullptr) {
        CgroupsMgr::apply_cgroup(_resource_info->user, _resource_info->group);
    }

    std::vector<RowBatch*> row_batchs;

    // Because we use thread pool to scan data from storage. One scanner can't
    // use this thread too long, this can starve other query's scanner. So, we
    // need yield this thread when we do enough work. However, OlapStorage read
    // data in pre-aggregate mode, then we can't use storage returned data to
    // judge if we need to yield. So we record all raw data read in this round
    // scan, if this exceed threshold, we yield this thread.
    int64_t raw_rows_read = scanner->raw_rows_read();
    int64_t raw_rows_threshold = raw_rows_read + config::palo_scanner_row_num;
    while (!eos && raw_rows_read < raw_rows_threshold) {
        if (UNLIKELY(_transfer_done)) {
            eos = true;
            status = Status::CANCELLED;
            LOG(INFO) << "Scan thread cancelled, cause query done, maybe reach limit.";
            break;
        }
        RowBatch *row_batch = new RowBatch(
                this->row_desc(), state->batch_size(), _runtime_state->fragment_mem_tracker());
        row_batch->set_scanner_id(scanner->id());
        status = scanner->get_batch(_runtime_state, row_batch, &eos);
        if (!status.ok()) {
            LOG(WARNING) << "Scan thread read OlapScanner failed!";
            eos = true;
            break;
        }
        // 4. if status not ok, change status_.
        if (UNLIKELY(row_batch->num_rows() == 0)) {
            // may be failed, push already, scan node delete this batch.
            delete row_batch;
            row_batch = NULL;
        } else {
            row_batchs.push_back(row_batch);
            __sync_fetch_and_add(&_buffered_bytes,
                                 row_batch->tuple_data_pool()->total_reserved_bytes());
        }
        raw_rows_read = scanner->raw_rows_read();
    }

    boost::unique_lock<boost::mutex> l(_scan_batches_lock);
    // if we failed, check status.
    if (UNLIKELY(!status.ok())) {
        _transfer_done = true;
        boost::lock_guard<boost::mutex> guard(_status_mutex);
        if (LIKELY(_status.ok())) {
            _status = status;
        }
    }

    bool global_status_ok = false;
    {
        boost::lock_guard<boost::mutex> guard(_status_mutex);
        global_status_ok = _status.ok();
    }
    if (UNLIKELY(!global_status_ok)) {
        eos = true;
        for (auto rb : row_batchs) {
            delete rb;
        }
    } else {
        for (auto rb : row_batchs) {
            _scan_row_batches.push_back(rb);
        }
    }
    // Scanner thread completed. Take a look and update the status
    if (UNLIKELY(eos)) {
        _progress.update(1);
        if (_progress.done()) {
            // this is the right out
            _scanner_done = true;
        }
        scanner->close(_runtime_state);
    } else {
        _olap_scanners.push_front(scanner);
    }
    _running_thread--;
    _scan_batch_added_cv.notify_one();
}

Status OlapScanNode::add_one_batch(RowBatchInterface* row_batch) {
    {
        boost::unique_lock<boost::mutex> l(_row_batches_lock);

        while (UNLIKELY(_materialized_row_batches.size()
                        >= _max_materialized_row_batches
                        && !_transfer_done)) {
            _row_batch_consumed_cv.wait(l);
        }

        VLOG(2) << "Push row_batch to materialized_row_batches";
        _materialized_row_batches.push_back(row_batch);
    }
    // remove one batch, notify main thread
    _row_batch_added_cv.notify_one();
    return Status::OK;
}

void OlapScanNode::debug_string(
    int /* indentation_level */,
    std::stringstream* /* out */) const {
}

} // namespace palo

max_stars_repo_name: cyongli/palo

max_issues_repo_name: aiplat/palo

max_forks_repo_name: aiplat/palo


text:
#include <boost/test/unit_test.hpp>

#include <scorum/protocol/betting/invariants_validation.hpp>
#include <scorum/protocol/betting/market.hpp>

#include <defines.hpp>

namespace {
using namespace scorum;
using namespace scorum::protocol;

BOOST_AUTO_TEST_SUITE(invariants_validation_tests)

SCORUM_TEST_CASE(validate_wincases_test_positive)
{
    {
        auto wincase = total::over{ 500 };
        BOOST_REQUIRE_NO_THROW(validate_wincase(wincase));
    }
    {
        auto wincase = total::under{ 1000 };
        BOOST_REQUIRE_NO_THROW(validate_wincase(wincase));
    }
    {
        auto wincase = handicap::over{ -500 };
        BOOST_REQUIRE_NO_THROW(validate_wincase(wincase));
    }
    {
        auto wincase = handicap::over{ 0 };
        BOOST_REQUIRE_NO_THROW(validate_wincase(wincase));
    }
    {
        auto wincase = handicap::under{ 1000 };
        BOOST_REQUIRE_NO_THROW(validate_wincase(wincase));
    }
    {
        auto wincase = result_home::yes{};
        BOOST_REQUIRE_NO_THROW(validate_wincase(wincase));
    }
}

SCORUM_TEST_CASE(validate_wincases_test_negative)
{
    {
        auto wincase = total::over{ 400 }; // must be divisible by 500
        BOOST_REQUIRE_THROW(validate_wincase(wincase), fc::assert_exception);
    }
    {
        auto wincase = total::under{ -1000 }; // must be positive
        BOOST_REQUIRE_THROW(validate_wincase(wincase), fc::assert_exception);
    }
    {
        auto wincase = total::under{ 0 }; // must be positive
        BOOST_REQUIRE_THROW(validate_wincase(wincase), fc::assert_exception);
    }
    {
        auto wincase = handicap::over{ -400 }; // must be divisible by 500
        BOOST_REQUIRE_THROW(validate_wincase(wincase), fc::assert_exception);
    }
    {
        auto wincase = handicap::over{ 499 }; // must be divisible by 500
        BOOST_REQUIRE_THROW(validate_wincase(wincase), fc::assert_exception);
    }
}

SCORUM_TEST_CASE(validate_game_full_market_list_test_positive)
{
    soccer_game game;
    // clang-format off
    fc::flat_set<market_type> markets = { result_home{},
                                          result_draw{},
                                          result_away{},
                                          round_home{},
                                          handicap{ 500 },
                                          correct_score_home{},
                                          correct_score_draw{},
                                          correct_score_away{},
                                          correct_score{1, 2},
                                          goal_home{},
                                          goal_both{},
                                          goal_away{},
                                          total{ 500 },
                                          total_goals_home{},
                                          total_goals_away{}};
    // clang-format on

    BOOST_REQUIRE_NO_THROW(validate_game(game, markets));
}

SCORUM_TEST_CASE(validate_game_invalid_market_should_negative)
{
    hockey_game game; // there are some alien markets (handicap_market, total_goals_market)
    // clang-format off
    fc::flat_set<market_type> markets = { result_home{},
                                          goal_home{},
                                          handicap{ 500 },
                                          total_goals_home{} };
    // clang-format on

    BOOST_REQUIRE_THROW(validate_game(game, markets), fc::assert_exception);
}

SCORUM_TEST_CASE(validate_if_wincase_in_game_positive)
{
    soccer_game game;

    BOOST_REQUIRE_NO_THROW(validate_if_wincase_in_game(game, correct_score::yes{ 2, 1 }));
}

SCORUM_TEST_CASE(validate_if_wincase_in_game_negative)
{
    hockey_game game;

    BOOST_REQUIRE_NO_THROW(validate_if_wincase_in_game(game, goal_home::yes{}));

    BOOST_REQUIRE_THROW(validate_if_wincase_in_game(game, correct_score::yes{ 2, 1 }), fc::assert_exception);
}

BOOST_AUTO_TEST_SUITE_END()
}

ext: cpp

max_stars_repo_name: scorum/scorum

max_issues_repo_name: Scorum/Scorum

max_forks_repo_name: Scorum/Scorum


text:
/**********************************************************************
*  Copyright (c) 2008-2014, Alliance for Sustainable Energy.
*  All rights reserved.
*
*  This library is free software; you can redistribute it and/or
*  modify it under the terms of the GNU Lesser General Public
*  License as published by the Free Software Foundation; either
*  version 2.1 of the License, or (at your option) any later version.
*
*  This library is distributed in the hope that it will be useful,
*  but WITHOUT ANY WARRANTY; without even the implied warranty of
*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
*  Lesser General Public License for more details.
*
*  You should have received a copy of the GNU Lesser General Public
*  License along with this library; if not, write to the Free Software
*  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
**********************************************************************/

#ifndef UTILITIES_DOCUMENT_DOCUMENTELEMENT_IMPL_HPP
#define UTILITIES_DOCUMENT_DOCUMENTELEMENT_IMPL_HPP

#include <utilities/UtilitiesAPI.hpp>
#include <utilities/core/Logger.hpp>
#include <utilities/core/Serialization.hpp>

#include <utilities/document/DocumentEnums.hpp>

#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <boost/optional.hpp>

#include <string>
#include <vector>

namespace openstudio {

namespace detail {

  class UTILITIES_API DocumentElement_Impl : public boost::enable_shared_from_this<DocumentElement_Impl> {
   public:

    /** @name Constructors */
    //@{

    virtual ~DocumentElement_Impl() {}

    virtual DocumentElement clone() const=0;

    //@}
    /** @name Printing */
    //@{

    virtual std::string print(DocumentFormat fmt) const=0;

    virtual std::ostream& printToStream(std::ostream& os, DocumentFormat fmt) const=0;

    //@}
    /** @name Type Casting */
    //@{

    /** Get an object that wraps this impl. */
    template<typename T>
    T getObject() const {
      T result(boost::dynamic_pointer_cast<typename T::ImplType>(
                 boost::const_pointer_cast<DocumentElement_Impl>(shared_from_this())));
      return result;
    }

    //@}
   private:
    REGISTER_LOGGER("openstudio.DocumentElement");

    // serialization support
    friend class boost::serialization::access;
    template<class Archive>
    void serialize(Archive& ar, const unsigned int version) {}
  };

  BOOST_SERIALIZATION_ASSUME_ABSTRACT(DocumentElement_Impl);

} // detail

} // openstudio

#endif // UTILITIES_DOCUMENT_DOCUMENTELEMENT_IMPL_HPP

ext: hpp

max_stars_repo_name: ORNL-BTRIC/OpenStudio

max_issues_repo_name: ORNL-BTRIC/OpenStudio

max_forks_repo_name: ORNL-BTRIC/OpenStudio


text:
#include "utility.h"
#include <iostream>
#include <fstream>
#include <ros/ros.h>
#include <nav_msgs/Odometry.h>
#include <Eigen/Eigen>
#include <string>

using namespace std;

static int frame_count1 = 0;
static int frame_count2 = 0;
// ofstream destFile1;  //("/home/belight/Dataset/test20191231/predict_odom_test3.txt", ios::out);
// ofstream destFile2;
ofstream destFile;

ros::Time time_local_odometry_;
ros::Time time_mapped_odometry_;
bool new_local_odometry_;
bool new_mapped_odometry_;

Eigen::Vector3d t_odom_curr;
Eigen::Vector3d t_map_curr;
Eigen::Vector3d euler_odom;
Eigen::Vector3d euler_map;

// Z-Y-X Euler Angles
Eigen::Vector3d Quat2Euler(Eigen::Quaterniond q){
  Eigen::Vector3d euler;

  Eigen::Matrix3d rot_mat = q.toRotationMatrix().transpose();  //这里必须转置一下，不转置的旋转矩阵为body到world的转换矩阵
  double R11 = q.w()*q.w() + q.x()*q.x() - q.y()*q.y() - q.z()*q.z();
  double R12 = 2 * (q.x()*q.y() + q.w()*q.z());
  double R13 = 2 * (q.x()*q.z() - q.w()*q.y());
  double R23 = 2 * (q.y()*q.z() + q.w()*q.x());
  double R33 = q.w()*q.w() - q.x()*q.x() - q.y()*q.y() + q.z()*q.z();

  double cy = sqrt(R11*R11 + R12*R12);  // cy = |cos(pitch)|

  // euler.x() = atan2(R23,R33);
  // euler.y() = asin(-R13);
  // euler.z() = atan2(R12,R11);

  euler.x() = atan2(rot_mat(1,2),rot_mat(2,2));  // roll
  // euler.y() = asin(-rot_mat(0,2));
  euler.y() = atan2(-rot_mat(0,2),sqrt(rot_mat(0,0)*rot_mat(0,0) + rot_mat(0,1)*rot_mat(0,1)));  // pitch
  euler.z() = atan2(rot_mat(0,1),rot_mat(0,0));  // yaw

  // euler2.x() = atan2(2 * (q.y()*q.z() + q.w()*q.x()), q.w()*q.w() - q.x()*q.x() - q.y()*q.y() + q.z()*q.z());  // roll
  // euler2.y() = asin(-2 * (q.x()*q.z() - q.w()*q.y()));  // pitch
  // euler2.z() = atan2(2 * (q.x()*q.y() + q.w()*q.z()), q.w()*q.w() + q.x()*q.x() - q.y()*q.y() - q.z()*q.z());  // yaw

  return euler;
}

void LaserOdomHandler(const nav_msgs::Odometry::ConstPtr &laser_odom_msg) {

  time_local_odometry_ = laser_odom_msg->header.stamp;

  Eigen::Quaterniond q_odom_curr;
//   Eigen::Vector3d t_odom_curr;
  q_odom_curr.x() = laser_odom_msg->pose.pose.orientation.x;
  q_odom_curr.y() = laser_odom_msg->pose.pose.orientation.y;
  q_odom_curr.z() = laser_odom_msg->pose.pose.orientation.z;
  q_odom_curr.w() = laser_odom_msg->pose.pose.orientation.w;
  t_odom_curr.x() = laser_odom_msg->pose.pose.position.x;
  t_odom_curr.y() = laser_odom_msg->pose.pose.position.y;
  t_odom_curr.z() = laser_odom_msg->pose.pose.position.z;
  double odom_time = laser_odom_msg->header.stamp.toSec();
//   cout << setprecision(10) << odom_time << endl;

  // Eigen::Vector3d euler_odom = q_odom_curr.matrix().eulerAngles(2,1,0)*57.3;
  euler_odom = Quat2Euler(q_odom_curr)*180/M_PI;
//   cout << "predict position: " << t_odom_curr.transpose() << endl;

//   destFile1.fill('0');
//   destFile1 <<setw(10)<< frame_count1 <<setw(15)<< odom_time <<setw(15)<< t_odom_curr.x() <<setw(15)<< t_odom_curr.y() <<setw(15)<< t_odom_curr.z() <<setw(15)\
//            << euler_odom.x() <<setw(15)<< euler_odom.y() <<setw(15)<< euler_odom.z() << endl;

  frame_count1++;
  new_local_odometry_ = true;
}

void MapOdomHandler(const nav_msgs::Odometry::ConstPtr &mapped_odom_msg) {

  time_mapped_odometry_ = mapped_odom_msg->header.stamp;

  Eigen::Quaterniond q_map_curr;
//   Eigen::Vector3d t_map_curr;
  q_map_curr.x() = mapped_odom_msg->pose.pose.orientation.x;
  q_map_curr.y() = mapped_odom_msg->pose.pose.orientation.y;
  q_map_curr.z() = mapped_odom_msg->pose.pose.orientation.z;
  q_map_curr.w() = mapped_odom_msg->pose.pose.orientation.w;
  t_map_curr.x() = mapped_odom_msg->pose.pose.position.x;
  t_map_curr.y() = mapped_odom_msg->pose.pose.position.y;
  t_map_curr.z() = mapped_odom_msg->pose.pose.position.z;
  double map_time = mapped_odom_msg->header.stamp.toSec();

  euler_map = Quat2Euler(q_map_curr)*180/M_PI;

//   destFile2 <<setw(10)<< frame_count2 <<setw(15)<< map_time <<setw(15)<< t_map_curr.x() <<setw(15)<< t_map_curr.y() <<setw(15)<< t_map_curr.z() <<setw(15)\
//            << euler_map.x() <<setw(15)<< euler_map.y() <<setw(15)<< euler_map.z() << endl;

  frame_count2++;
  new_mapped_odometry_ = true;
}

bool HasNewData(){
    return new_local_odometry_ && new_mapped_odometry_ && (frame_count1 == frame_count2);
}

void ProcessSaving(){

    if(!HasNewData()){
        return;
    }

    new_local_odometry_ = false;
    new_mapped_odometry_ = false;

  destFile <<setw(10)<< frame_count1 <<setw(15)<< time_local_odometry_.toSec() <<setw(15)<< t_odom_curr.x() <<setw(15)<< t_odom_curr.y() <<setw(15)<< t_odom_curr.z() <<setw(15)\
           << euler_odom.x() <<setw(15)<< euler_odom.y() <<setw(15)<< euler_odom.z() <<setw(15)<< t_map_curr.x() <<setw(15)<< t_map_curr.y() <<setw(15)<< t_map_curr.z() <<setw(15)\
           << euler_map.x() <<setw(15)<< euler_map.y() <<setw(15)<< euler_map.z() << endl;

}

int main(int argc, char** argv)
{
    // destFile1.setf(ios::left);  //设置对齐方式（左对齐）
    // // destFile1.fill('0');
    // destFile1.setf(ios::fixed, ios::floatfield);  //设定为fixed模式，以小数点表示浮点数
    // destFile1.precision(4);  //设置小数点位数（精度）

    // destFile2.setf(ios::left);
    // destFile2.setf(ios::fixed, ios::floatfield);
    // destFile2.precision(4);

    destFile.setf(ios::left);
    destFile.setf(ios::fixed, ios::floatfield);
    destFile.precision(4);

    ros::init(argc, argv, "save_local_mapped_odom");
    ros::NodeHandle nh("~");

    string output_file;
    // nh.getParam("output_local_odom",output_file1);
    // nh.getParam("output_map_to_init",output_file2);
    // destFile1.open(output_file1,ios::out);
    // destFile2.open(output_file2,ios::out);
    nh.getParam("output_local_mapped_odom",output_file);
    destFile.open(output_file,ios::out);

    ros::Subscriber sub_laser_odom_ = nh.subscribe<nav_msgs::Odometry>("/laser_odom_to_init", 100, LaserOdomHandler);
    ros::Subscriber sub_mapped_odom_ = nh.subscribe<nav_msgs::Odometry>("/aft_mapped_to_init", 100, MapOdomHandler);

    ros::Rate r(100);
    while(ros::ok()) {
      ProcessSaving();
      ros::spinOnce();
      r.sleep();
    }
    // ros::spin();

    // destFile1.close();
    // destFile2.close();
    destFile.close();

    return 0;
}


text:
// Author: Michael Kaufmann <kau@zurich.ibm.com>
//
// Copyright (c) 2018, IBM Corporation, All rights reserved.
//
// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.

#include "cluster/cluster.h"
#include "common.h"
#include "event/engine.h"
#include "resourcemanager/resourcemanager.h"
#include "server/server.h"

#include <chrono>
#include <iostream>
#include <string>
#include <thread>
#include <vector>

#include <boost/program_options.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <boost/property_tree/ptree.hpp>

namespace pt = boost::property_tree;
namespace po = boost::program_options;

Config        cfg; // global configuration
Config *const config = &cfg;

Engine *         engine          = NULL;
ResourceManager *resourcemanager = NULL;

int main(int argc, const char **argv)
{
    try {
        std::string cls_filename;
        std::string perfdb_filename;

 	po::options_description global_opts("Global options");
	global_opts.add_options()
	    ("cluster",            po::value<std::string>(), "Path to cluster model")
	    ("perfdb",             po::value<std::string>(), "Path to app/task performance database directory")
	    ("seed",               po::value<uint64_t>(),    "Seed for random number generator(s)")
	    ("log",                po::value<std::string>(), "Log level: trace, debug, info, warning, error, fatal")
	    ("io_task_weight",     po::value<double>(),      "")
	    ("io_task_io_weight",  po::value<double>(),      "")
	    ("cmp_task_weight",    po::value<double>(),      "")
	    ("cmp_task_io_weight", po::value<double>(),      "")
	    ("node_load_weight",   po::value<double>(),      "")
	    ("interference_mode",  po::value<int>(),         "0-6")
	    ("rsf",                po::value<double>(),      "Resource scaling factor (0.0 - 1.0), default = 1.0")
	    ("hostname",           po::value<std::string>(), "Which hostname to listen on")
	    ("port",               po::value<uint>(),        "Which port to listen on")
	    ("lweight",            po::value<int>(),         "Weight of the stage level (distance to final stage).")
	    ("train",                                        "Enable training mode")
	    ("help",                                         "Print this message");

	po::variables_map vm;
        po::parsed_options
            parsed = po::command_line_parser(argc, argv).options(global_opts).allow_unregistered().run();
        po::store(parsed, vm);

        if (vm.count("cluster")) {
            cls_filename = vm["cluster"].as<std::string>();
        } else {
            throw std::invalid_argument("Cluster definition file (--cluster) missing");
        }

        if (vm.count("perfdb")) {
            cfg.perfdbdir(vm["perfdb"].as<std::string>());
        } else {
            throw std::invalid_argument("Performance database directory (--perfdb) missing");
        }

        if (vm.count("seed")) {
            cfg.seed(vm["seed"].as<uint64_t>());
	} else {
            cfg.seed(std::chrono::system_clock::now().time_since_epoch().count());
	}

        if (vm.count("log")) {
            if (vm["log"].as<std::string>() == "trace") {
                std::cout << "Setting output level to trace" << std::endl;
                boost::log::core::get()->set_filter(boost::log::trivial::severity >=
                                                    boost::log::trivial::trace);
            } else if (vm["log"].as<std::string>() == "debug") {
                std::cout << "Setting output level to debug" << std::endl;
                boost::log::core::get()->set_filter(boost::log::trivial::severity >=
                                                    boost::log::trivial::debug);
            } else if (vm["log"].as<std::string>() == "info") {
                std::cout << "Setting output level to info" << std::endl;
                boost::log::core::get()->set_filter(boost::log::trivial::severity >=
						    boost::log::trivial::info);
            } else if (vm["log"].as<std::string>() == "warning") {
                std::cout << "Setting output level to warning" << std::endl;
                boost::log::core::get()->set_filter(boost::log::trivial::severity >=
                                                    boost::log::trivial::warning);
            } else if (vm["log"].as<std::string>() == "error") {
                std::cout << "Setting output level to error" << std::endl;
                boost::log::core::get()->set_filter(boost::log::trivial::severity >=
                                                    boost::log::trivial::error);
            } else if (vm["log"].as<std::string>() == "fatal") {
                std::cout << "Setting output level to fatal" << std::endl;
                boost::log::core::get()->set_filter(boost::log::trivial::severity >=
                                                    boost::log::trivial::fatal);
            }
        }

        if (vm.count("train")) {
            config->setFlag(ConfigSchedulerFlags::training_mode);
        }

        if (vm.count("io_task_weight")) {
            config->set(ConfigVariable::io_task_weight, vm["io_task_weight"].as<double>());
        }

        if (vm.count("io_task_io_weight")) {
            config->set(ConfigVariable::io_task_io_weight, vm["io_task_io_weight"].as<double>());
        }

        if (vm.count("cmp_task_weight")) {
            config->set(ConfigVariable::cmp_task_weight, vm["cmp_task_weight"].as<double>());
        }

        if (vm.count("cmp_task_io_weight")) {
            config->set(ConfigVariable::cmp_task_io_weight, vm["cmp_task_io_weight"].as<double>());
        }

        if (vm.count("node_load_weight")) {
            config->set(ConfigVariable::node_load_weight, vm["node_load_weight"].as<double>());
        }

        if (vm.count("interference_mode")) {
            config->set(ConfigVariable::interference_mode, vm["interference_mode"].as<int>());
        }

        if (vm.count("rsf")) {
            config->set(ConfigVariable::resource_scaling_factor, vm["rsf"].as<double>());
        }

	if (vm.count("hostname")) {
            config->listenHost(vm["hostname"].as<std::string>());
        }
	if (vm.count("port")) {
            config->listenPort(vm["port"].as<uint>());
        }

	if (vm.count("lweight")) {
            config->lweight(vm["lweight"].as<int>());
        }

        pt::ptree  clusterData;

	try {
	    std::ifstream file;
	    file.open(cls_filename);
	    pt::read_json(file, clusterData);
	    file.close();
	} catch (std::exception const &e) {
	    throw(std::invalid_argument("Cannot read read cluster definition file " + cls_filename + " : " + e.what()));
	}

        config->setFlag(ConfigSchedulerFlags::consider_io_time);
        config->setFlag(ConfigSchedulerFlags::consider_io_size);
        //    config->setFlag(ConfigSchedulerFlags::signal_executor_idle);
        //    config->setFlag(ConfigSchedulerFlags::compact_schedule);
        //    config->setFlag(ConfigSchedulerFlags::best_fit);
        config->print();

        Cluster cluster(clusterData);

        Engine  myengine(EngineMode::Real);
        engine  = &myengine; // global object
        ResourceManager myresourcemanager(&cluster);
        resourcemanager = &myresourcemanager; // global object

        Server myserver(&myengine, &cluster, &myresourcemanager);

        std::thread engineThread(static_cast<void (*)(Engine *)>(Engine::run), &myengine);
        std::thread resourcemanagerThread(static_cast<void (*)(ResourceManager *)>(ResourceManager::run), &myresourcemanager);
        std::thread serverThread(static_cast<void (*)(Server *)>(Server::run), &myserver);

        serverThread.join();
        resourcemanagerThread.join();
        engineThread.join();

        fflush(stdout);
        fflush(stderr);
        fclose(stdout);
        fclose(stderr);

        return 0;
    } catch (std::exception const &e) {
        logError << "Error: " << e.what();
        return -1;
    }
}

ext: cc

max_stars_repo_name: zrlkau/hcs

max_issues_repo_name: zrlkau/hcs

max_forks_repo_name: zrlkau/hcs


text:
// MIT License
//
// shrinkler-gba: Port of the Shrinkler Amiga executable cruncher for the GBA
// Copyright (c) 2021 Thomas Mathys
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

#include <algorithm>
#include <boost/numeric/conversion/cast.hpp>
#include <fstream>
#include <stdexcept>
#include <system_error>
#include "fmt/core.h"
#include "shrinklergba/elfio_wrapper.hpp"
#include "shrinklergba/elf_strings.hpp"
#include "shrinklergba/input_file.hpp"
#include "shrinklergba/table_printer.hpp"

namespace shrinklergba
{

using boost::numeric_cast;
using ELFIO::elfio;
using ELFIO::Elf64_Addr;
using ELFIO::Elf_Half;
using ELFIO::segment;
using fmt::format;
using std::runtime_error;

static table_printer create_table_printer()
{
    table_printer p;
    p.table_indent(2);
    return p;
}

void input_file::load(const std::filesystem::path& path)
{
    try
    {
        CONSOLE_VERBOSE(m_console) << "Loading: " << path.string() << std::endl;
        std::ifstream stream(path, std::ios::binary);

        if (!stream)
        {
            auto e = errno;
            throw std::system_error(e, std::generic_category());
        }

        load(stream);
    }
    catch (const std::exception& e)
    {
        throw std::runtime_error(path.string() + ": " + e.what());
    }
}

void input_file::load(std::istream& stream)
{
    load_elf(stream);

    CONSOLE_VERBOSE(m_console) << format("Entry: {:#x}", m_entry) << std::endl;
    CONSOLE_VERBOSE(m_console) << format("Load address: {:#x}", m_load_address) << std::endl;
    CONSOLE_VERBOSE(m_console) << format("Total size of loaded data: {0:#x} ({0})", m_data.size()) << std::endl;
}

void input_file::load_elf(std::istream& stream)
{
    reset();

    elfio reader;
    open_elf(reader, stream);
    check_header(reader);
    read_entry(reader);
    log_program_headers(reader);
    log_section_headers(reader);
    convert_to_binary(reader);

    // TODO: final checks:
    //       * what do we do if the resulting raw binary is 0 bytes big (or too small in general?)
    //         * Well in principle it depends on our packers what they consider 'too small'. If they can cope with zero bytes, that's fine by me.
    //       * what do we do if the resulting raw binary is too big?
    //         * If it goes into IWRAM it can be at most 32K, and even that is too big due to stack and BIOS reserved area
    //         * If it goes into EWRAM it can be at most 256K
    //         * Moreover, maximum size does not only depend on where the binary is loaded to, but also how large the binary itself is,
    //           and how much memory is used by other things, including the depacker in the case of IWRAM.
    //       * Do we limit the entry points we are going to accept? Well perhaps, but not in this class.
    //         * Well basically entries must be in IWRAM or EWRAM. Or, more specifically, inside the
    //           memory area occupied by the binary. But then, that's probably too much to worry about
    //       * Thing is, all of this is mostly target specific and does not belong into input_file
}

void input_file::reset()
{
    m_entry = 0;
    m_load_address = 0;
    std::vector<unsigned char>().swap(m_data);
}

void input_file::read_entry(elfio& reader)
{
    m_entry = numeric_cast<uint32_t>(reader.get_entry());
}

void input_file::log_program_headers(elfio& reader) const
{
    if (!m_console.is_verbose_enabled())
    {
        return;
    }

    Elf_Half nheaders = reader.segments.size();
    if (nheaders == 0)
    {
        CONSOLE_VERBOSE(m_console) << "File has no program headers" << std::endl;
        return;
    }

    auto printer = create_table_printer();
    printer.add_row({ "Nr", "Type", "Offset", "VirtAddr", "PhysAddr", "FileSiz", "MemSiz", "Align", "Flg" });
    for (Elf_Half i = 0; i < nheaders; ++i)
    {
        const auto& s = *reader.segments[i];
        printer.add_row({
            std::to_string(i),
            elf_strings::get_segment_type(s.get_type()),
            elf_strings::to_hex(s.get_offset(), 6),
            elf_strings::to_hex(s.get_virtual_address(), 8),
            elf_strings::to_hex(s.get_physical_address(), 8),
            elf_strings::to_hex(s.get_file_size(), 5),
            elf_strings::to_hex(s.get_memory_size(), 5),
            elf_strings::to_hex(s.get_align(), 5),
            elf_strings::get_segment_flags(s.get_flags())});
    }

    CONSOLE_VERBOSE(m_console) << "Program headers" << std::endl;
    printer.print(*m_console.verbose());
}

void input_file::log_section_headers(ELFIO::elfio& reader) const
{
    if (!m_console.is_verbose_enabled())
    {
        return;
    }

    Elf_Half nheaders = reader.sections.size();
    if (nheaders == 0)
    {
        CONSOLE_VERBOSE(m_console) << "File has no section headers" << std::endl;
        return;
    }

    auto printer = create_table_printer();
    printer.add_row({ "Nr", "Name", "Type", "Addr", "Off", "Size", "ES", "Flg", "Lk", "Inf", "Al", "Inc" });
    for (Elf_Half i = 0; i < nheaders; ++i)
    {
        const ELFIO::section* s = reader.sections[i];
        printer.add_row({
            std::to_string(i),
            s->get_name(),
            elf_strings::get_section_type(s->get_type()),
            elf_strings::to_hex(s->get_address(), 8),
            elf_strings::to_hex(s->get_offset(), 6),
            elf_strings::to_hex(s->get_size(), 6),
            elf_strings::to_hex(s->get_entry_size(), 2),
            elf_strings::get_section_flags(s->get_flags()),
            elf_strings::to_hex(s->get_link(), 2),
            elf_strings::to_hex(s->get_info(), 3),
            elf_strings::to_hex(s->get_addr_align(), 2),
            is_section_included(s) ? "Y" : "N"
            });
    }

    CONSOLE_VERBOSE(m_console) << "Section headers" << std::endl;
    printer.print(*m_console.verbose());
}

void input_file::convert_to_binary(ELFIO::elfio& reader)
{
    std::vector<const ELFIO::section*> included_sections = get_included_sections(reader);
    sort_sections_by_address(included_sections);

    const ELFIO::section* previous_section = nullptr;
    Elf64_Addr output_address = 0;

    for (const ELFIO::section* s : included_sections)
    {
        if (m_data.size())
        {
            if (s->get_address() < output_address)
            {
                throw std::runtime_error(format("section {} overlaps with previous section {}", s->get_name(), previous_section->get_name()));
            }

            const auto npadding_bytes = s->get_address() - output_address;
            if (npadding_bytes > 0)
            {
                // There is a hole between the current and the last section.
                // Pad it with zeros. Zeros are required by ELF.
                m_data.insert(m_data.end(), numeric_cast<size_t>(npadding_bytes), 0);
                output_address += npadding_bytes;
            }
        }
        else
        {
            // No bytes written to output yet. Record initial output address and load address.
            output_address = s->get_address();
            m_load_address = numeric_cast<uint32_t>(output_address);
        }

        // Copy section data to output.
        m_data.insert(m_data.end(), s->get_data(), s->get_data() + s->get_size());
        output_address += s->get_size();
        previous_section = s;
    }
}

void input_file::open_elf(elfio& reader, std::istream& stream)
{
    if (!reader.load(stream))
    {
        throw runtime_error("file is not a valid ELF file");
    }
}

void input_file::check_header(elfio& reader)
{
    check_executable_type(reader);
    check_elf_version(reader);

    // Not sure these matter. Checking them to be on the safe side.
    check_os_abi(reader);
    check_abi_version(reader);
    check_object_file_version(reader);
}

void input_file::check_executable_type(elfio& reader)
{
    if ((reader.get_class() != ELFCLASS32) ||
        (reader.get_encoding() != ELFDATA2LSB) ||
        (reader.get_type() != ET_EXEC) ||
        (reader.get_machine() != EM_ARM))
    {
        throw runtime_error("file is not a 32-bit little endian ARM executable ELF file");
    }
}

void input_file::check_elf_version(elfio& reader)
{
    const auto expected_elf_version = 1;

    auto ei_version = reader.get_elf_version();
    if (ei_version != expected_elf_version)
    {
        throw runtime_error(format("unknown ELF format version {}. Expected {}", ei_version, expected_elf_version));
    }
}

void input_file::check_os_abi(elfio& reader)
{
    const auto expected_abi = ELFOSABI_NONE;

    auto ei_osabi = reader.get_os_abi();
    if (ei_osabi != expected_abi)
    {
        throw runtime_error(format("unknown ELF OS ABI {}. Expected none ({})", ei_osabi, expected_abi));
    }
}

void input_file::check_abi_version(elfio& reader)
{
    const auto expected_abi_version = 0;

    auto ei_abiversion = reader.get_abi_version();
    if (ei_abiversion != expected_abi_version)
    {
        throw runtime_error(format("unknown ABI version {}. Expected {}", ei_abiversion, expected_abi_version));
    }
}

void input_file::check_object_file_version(elfio& reader)
{
    const auto expected_object_file_version = 1;

    auto e_version = reader.get_version();
    if (e_version != expected_object_file_version)
    {
        throw runtime_error(format("unknown object file version {}. Expected {}", e_version, expected_object_file_version));
    }
}

bool input_file::is_section_included(const ELFIO::section* s)
{
    if ((s->get_type() == SHT_NULL) || (s->get_type() == SHT_NOBITS))
    {
        return false;
    }

    if (s->get_address() == 0)
    {
        return false;
    }

    if (s->get_size() == 0)
    {
        return false;
    }

    if (!(s->get_flags() & SHF_ALLOC))
    {
        return false;
    }

    return true;
}

std::vector<const ELFIO::section*> input_file::get_included_sections(ELFIO::elfio& reader)
{
    std::vector<const ELFIO::section*> included_sections;
    std::copy_if(
        reader.sections.begin(),
        reader.sections.end(),
        std::back_inserter(included_sections),
        is_section_included);
    return included_sections;
}

void input_file::sort_sections_by_address(std::vector<const ELFIO::section*>& sections)
{
    std::sort(
        sections.begin(),
        sections.end(),
        [](const ELFIO::section* lhs, const ELFIO::section* rhs) { return lhs->get_address() < rhs->get_address(); });
}

}


text:
/*****************************************************************************
 *
 * numinquire.cpp     Blitz++ <numinquire.h> example, illustrating how to
 *                    get at properties of numeric types.
 * $Id$
 *****************************************************************************/

#include <blitz/blitz.h>
#include <blitz/numinquire.h>

BZ_USING_NAMESPACE(blitz)

int main()
{
    double z = 1.0;

    cout << "Some inquiries into the nature of double:" << endl
         << "digits(z) = " << digits(z) << endl
         << "epsilon(z) = " << epsilon(z) << endl
         << "huge(z) = " << huge(z) << endl
         << "tiny(z) = " << tiny(z) << endl
         << "max_exponent(z) = " << max_exponent(z) << endl
         << "min_exponent(z) = " << min_exponent(z) << endl
         << "max_exponent10(z) = " << max_exponent10(z) << endl
         << "min_exponent10(z) = " << min_exponent10(z) << endl
         << "precision(z) = " << precision(z) << endl
         << "radix(z) = " << radix(z) << endl;

    Range r = range(z);
    cout << "range(z) = [ " << r.first() << ", " << r.last() << " ]"
         << endl;

    cout << endl << "More obscure properties:" << endl
         << "is_signed(z) = " << is_signed(z) << endl
         << "is_integer(z) = " << is_integer(z) << endl
         << "is_exact(z) = " << is_exact(z) << endl
         << "round_error(z) = " << round_error(z) << endl
         << "has_infinity(z) = " << has_infinity(z) << endl
         << "has_quiet_NaN(z) = " << has_quiet_NaN(z) << endl
         << "has_signaling_NaN(z) = " << has_signaling_NaN(z) << endl
         << "has_denorm(z) = " << has_denorm(z) << endl
         << "has_denorm_loss(z) = " << has_denorm_loss(z) << endl
         << "infinity(z) = " << infinity(z) << endl
         << "quiet_NaN(z) = " << quiet_NaN(z) << endl
         << "signaling_NaN(z) = " << signaling_NaN(z) << endl
         << "denorm_min(z) = " << denorm_min(z) << endl
         << "is_iec559(z) = " << is_iec559(z) << endl
         << "is_bounded(z) = " << is_bounded(z) << endl
         << "is_modulo(z) = " << is_modulo(z) << endl
         << "traps(z) = " << traps(z) << endl
         << "tinyness_before(z) = " << tinyness_before(z) << endl;

    return 0;
}

max_stars_repo_name: fraguela/depspawn

max_issues_repo_name: MSV-Project/IBAMR

max_forks_repo_name: MSV-Project/IBAMR


text:
//////////////////////////////////////////////////////////////////////////////
///   Copyright 2003 and onward LASMEA UMR 6602 CNRS/U.B.P Clermont-Ferrand
///   Copyright 2009 and onward LRI    UMR 8623 CNRS/Univ Paris Sud XI
///
///          Distributed under the Boost Software License, Version 1.0
///                 See accompanying file LICENSE.txt or copy at
///                     http://www.boost.org/LICENSE_1_0.txt
//////////////////////////////////////////////////////////////////////////////
#ifndef NT2_TOOLBOX_IEEE_FUNCTION_SIMD_COMMON_FAST_FREXP_HPP_INCLUDED
#define NT2_TOOLBOX_IEEE_FUNCTION_SIMD_COMMON_FAST_FREXP_HPP_INCLUDED
#include <nt2/sdk/meta/adapted_traits.hpp>
#include <nt2/sdk/constant/properties.hpp>
#include <nt2/sdk/meta/as_integer.hpp>
#include <boost/fusion/tuple.hpp>
#include <nt2/sdk/meta/strip.hpp>
#include <nt2/include/functions/shri.hpp>
#include <nt2/include/functions/bitwise_andnot.hpp>
#include <boost/fusion/include/vector.hpp>

/////////////////////////////////////////////////////////////////////////////
// Implementation when type  is arithmetic_
/////////////////////////////////////////////////////////////////////////////
NT2_REGISTER_DISPATCH(tag::fast_frexp_, tag::cpu_,
                             (A0)(X),
                             ((simd_<arithmetic_<A0>,X>))
                            );

namespace nt2 { namespace ext
{
  template<class X, class Dummy>
  struct call<tag::fast_frexp_(tag::simd_<tag::arithmetic_, X> ),
              tag::cpu_, Dummy> : callable
  {
    template<class Sig> struct result;
    template<class This,class A0>
    struct result<This(A0)>
     {
       typedef typename meta::strip<A0>::type                     A00;
       typedef typename meta::as_integer<A00, signed>::type  exponent;
       typedef boost::fusion::vector<A00,exponent>                type;
     };

    NT2_FUNCTOR_CALL(1)
    {
      typename NT2_RETURN_TYPE(1)::type res;
      eval( a0
          , boost::fusion::at_c<0>(res)
          , boost::fusion::at_c<1>(res)
          );
      return res;
    }
  private:
    template<class A0,class R0,class R1> inline void
    eval(A0 const& a0,R0& r0, R1& r1)const
    {
      typedef typename meta::as_integer<A0, signed>::type      int_type;
      typedef typename meta::scalar_of<int_type>::type        sint_type;
      typedef typename meta::scalar_of<A0>::type                 s_type;
      static const sint_type me = Maxexponent<s_type>()-1;
      static const sint_type nmb= Nbmantissabits<s_type>();
      static const int_type vn1 = nt2::splat<int_type>((2*me+3)<<nmb);
      static const sint_type n2 = me<<nmb;
      r1 = b_and(vn1, a0);                                 //extract exponent
      A0 x = b_andnot(a0, vn1);                            //clear exponent in a0
      r1 = shri(r1,nmb) - splat<int_type>(me);             //compute exponent
      r0 = b_or(x,splat<int_type>(n2));                    //insert expon+1 in x
    }
  };
} }

#endif
// modified by jt the 04/01/2011


text:
#include "parseTitle.hpp"
#include <boost/core/demangle.hpp>
#include <unordered_map>

std::string_view aima::util::detail::parseTitle( std::type_index typeIndex ) {
    static std::unordered_map<std::type_index, std::string> titles;

    auto& title = titles[ typeIndex ];

    if ( title.empty()) {
        auto raw  = boost::core::demangle( typeIndex.name());
        auto name = raw.substr( raw.find_last_of( "::" ) + 1 );

        // add spaces before each capital letter except the first character
        for ( size_t i = 1; i < name.size(); ++i )
            if ( isupper( name[ i ] )) {
                name.insert( i, 1, ' ' );
                ++i;
            }

        title = name;
    }

    return title;
}