text:
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import paddle
import os
import ast
import argparse
import warnings
import numpy as np
from functools import partial
from data import read, load_dict, convert_example_to_feature
from model import JointModel
from utils import set_seed
from metric import SeqEntityScore, MultiLabelClassificationScore
from evaluate import evaluate

import paddle
import paddle.nn.functional as F
from paddlenlp.datasets import load_dataset
from paddlenlp.transformers import ErnieTokenizer, ErnieModel, LinearDecayWithWarmup
from paddlenlp.data import Stack, Pad, Tuple

warnings.filterwarnings("ignore")

# yapf: disable
parser = argparse.ArgumentParser(__doc__)
parser.add_argument("--num_epoch", type=int, default=3, help="Number of epoches for fine-tuning.")
parser.add_argument("--learning_rate", type=float, default=5e-5, help="Learning rate used to train with warmup.")
parser.add_argument("--slot_dict_path", type=str, default=None, help="slot dict path")
parser.add_argument("--intent_dict_path", type=str, default=None, help="intent dict path")
parser.add_argument("--train_path", type=str, default=None, help="train data")
parser.add_argument("--dev_path", type=str, default=None, help="dev data")
parser.add_argument("--weight_decay", type=float, default=0.01, help="Weight decay rate for L2 regularizer.")
parser.add_argument("--warmup_proportion", type=float, default=0.1, help="Warmup proportion params for warmup strategy")
parser.add_argument("--max_seq_len", type=int, default=512, help="Number of words of the longest seqence.")
parser.add_argument("--max_grad_norm", type=float, default=1.0, help="max grad norm to clip gradient.")
parser.add_argument("--eval_step", type=int, default=500, help="evaluation step")
parser.add_argument("--log_step", type=int, default=50, help="log step")
parser.add_argument("--batch_size", type=int, default=32, help="Total examples' number in batch for training.")
parser.add_argument("--checkpoint", type=str, default=None, help="Directory to model checkpoint")
parser.add_argument("--use_history", type=bool, default=False, help="Use history in dataset or not")
parser.add_argument("--intent_weight", type=bool, default=True, help="Use intent weight strategy")
parser.add_argument("--seed", type=int, default=1000, help="random seed for initialization")
parser.add_argument('--device', choices=['cpu', 'gpu'], default="gpu", help="Select which device to train model, defaults to gpu.")
args = parser.parse_args()
# yapf: enable

class JointLoss(paddle.nn.Layer):
    def __init__(self, intent_weight=None):
        super(JointLoss, self).__init__()
        self.intent_criterion = paddle.nn.BCEWithLogitsLoss(weight=intent_weight)
        self.slot_criterion = paddle.nn.CrossEntropyLoss()

    def forward(self, intent_logits, slot_logits, intent_labels, slot_labels):
        intent_loss = self.intent_criterion(intent_logits, intent_labels)
        slot_loss = self.slot_criterion(slot_logits, slot_labels)
        loss = intent_loss + slot_loss

        return loss

def train():
    # set running envir
    paddle.set_device(args.device)
    set_seed(args.seed)

    if not os.path.exists(args.checkpoint):
        os.mkdir(args.checkpoint)

    model_name = "ernie-1.0"

    # load and process data
    intent2id, id2intent = load_dict(args.intent_dict_path)
    slot2id, id2slot = load_dict(args.slot_dict_path)

    train_ds = load_dataset(read, data_path=args.train_path, lazy=False)
    dev_ds = load_dataset(read, data_path=args.dev_path, lazy=False)

    # compute intent weight
    if args.intent_weight:
        intent_weight = [1] * len(intent2id)
        for example in train_ds:
            for intent in example["intent_labels"]:
                intent_weight[intent2id[intent]] += 1
        for intent, intent_id in intent2id.items():
            neg_pos = (len(train_ds) - intent_weight[intent_id]) / intent_weight[intent_id]
            intent_weight[intent_id] = np.log10(neg_pos)
        intent_weight = paddle.to_tensor(intent_weight)

    tokenizer = ErnieTokenizer.from_pretrained(model_name)
    trans_func = partial(convert_example_to_feature, tokenizer=tokenizer, slot2id=slot2id, intent2id=intent2id, use_history=args.use_history, pad_default_tag="O", max_seq_len=args.max_seq_len)

    train_ds = train_ds.map(trans_func, lazy=False)
    dev_ds = dev_ds.map(trans_func, lazy=False)

    batchify_fn = lambda samples, fn=Tuple(
        Pad(axis=0, pad_val=tokenizer.pad_token_id),
        Pad(axis=0, pad_val=tokenizer.pad_token_type_id),
        Stack(dtype="float32"),
        Pad(axis=0, pad_val=slot2id["O"], dtype="int64"),
        Pad(axis=0, pad_val=tokenizer.pad_token_id)
    ):fn(samples)

    train_batch_sampler = paddle.io.BatchSampler(train_ds, batch_size=args.batch_size, shuffle=True)
    dev_batch_sampler = paddle.io.BatchSampler(dev_ds, batch_size=args.batch_size, shuffle=False)
    train_loader = paddle.io.DataLoader(dataset=train_ds, batch_sampler=train_batch_sampler, collate_fn=batchify_fn, return_list=True)
    dev_loader = paddle.io.DataLoader(dataset=dev_ds, batch_sampler=dev_batch_sampler, collate_fn=batchify_fn, return_list=True)

    # configure model training
    ernie = ErnieModel.from_pretrained(model_name)
    joint_model = JointModel(ernie, len(slot2id), len(intent2id), use_history=args.use_history, dropout=0.1)

    num_training_steps = len(train_loader) * args.num_epoch
    lr_scheduler = LinearDecayWithWarmup(args.learning_rate, num_training_steps, args.warmup_proportion)
    decay_params = [p.name for n, p in joint_model.named_parameters() if not any(nd in n for nd in ["bias", "norm"])]
    grad_clip = paddle.nn.ClipGradByGlobalNorm(args.max_grad_norm)
    optimizer = paddle.optimizer.AdamW(learning_rate=lr_scheduler, parameters=joint_model.parameters(), weight_decay=args.weight_decay, apply_decay_param_fun=lambda x: x in decay_params, grad_clip=grad_clip)

    if args.intent_weight:
        joint_loss = JointLoss(intent_weight)
    else:
        joint_loss = JointLoss()

    intent_metric = MultiLabelClassificationScore(id2intent)
    slot_metric = SeqEntityScore(id2slot)
    # start to train joint_model
    global_step, intent_best_f1, slot_best_f1 = 0, 0., 0.
    joint_model.train()
    for epoch in range(1, args.num_epoch+1):
        for batch_data in train_loader:
            input_ids, token_type_ids, intent_labels, tag_ids, history_ids = batch_data
            intent_logits, slot_logits = joint_model(input_ids, token_type_ids=token_type_ids, history_ids=history_ids)

            loss = joint_loss(intent_logits, slot_logits, intent_labels, tag_ids)

            loss.backward()
            lr_scheduler.step()
            optimizer.step()
            optimizer.clear_grad()

            if global_step > 0 and global_step % args.log_step == 0:
                print(f"epoch: {epoch} - global_step: {global_step}/{num_training_steps} - loss:{loss.numpy().item():.6f}")
            if global_step > 0 and global_step % args.eval_step == 0:
                intent_results, slot_results = evaluate(joint_model, dev_loader, intent_metric, slot_metric)
                intent_result, slot_result = intent_results["Total"], slot_results["Total"]
                joint_model.train()
                intent_f1, slot_f1 = intent_result["F1"], slot_result["F1"]
                if intent_f1 > intent_best_f1 or slot_f1 > slot_best_f1:
                    paddle.save(joint_model.state_dict(), f"{args.checkpoint}/best.pdparams")
                if intent_f1 > intent_best_f1:
                    print(f"intent best F1 performence has been updated: {intent_best_f1:.5f} --> {intent_f1:.5f}")
                    intent_best_f1 = intent_f1
                if slot_f1 > slot_best_f1:
                    print(f"slot best F1 performence has been updated: {slot_best_f1:.5f} --> {slot_f1:.5f}")
                    slot_best_f1 = slot_f1
                print(f'intent evalution result: precision: {intent_result["Precision"]:.5f}, recall: {intent_result["Recall"]:.5f},  F1: {intent_result["F1"]:.5f}, current best {intent_best_f1:.5f}')
                print(f'slot evalution result: precision: {slot_result["Precision"]:.5f}, recall: {slot_result["Recall"]:.5f},  F1: {slot_result["F1"]:.5f}, current best {slot_best_f1:.5f}\n')

            global_step += 1

if __name__=="__main__":
    train()


text:
#!/usr/bin/env python
# coding: utf-8
#
# Author:   Kazuto Nakashima
# URL:      http://kazuto1011.github.io
# Created:  2017-11-15

import re
from collections import OrderedDict

import click
import numpy as np
import torch

from libs import caffe_pb2
from libs.models import DeepLabV2_ResNet101_MSC

def parse_caffemodel(model_path):
    caffemodel = caffe_pb2.NetParameter()
    with open(model_path, 'rb') as f:
        caffemodel.MergeFromString(f.read())

    # Check trainable layers
    print(*set([(layer.type, len(layer.blobs)) for layer in caffemodel.layer]), sep='\n')

    params = OrderedDict()
    previous_layer_type = None
    for layer in caffemodel.layer:
        print('{} ({}): {}'.format(layer.name, layer.type, len(layer.blobs)))

        # Skip the shared branch
        if 'res075' in layer.name or 'res05' in layer.name:
            continue

        # Convolution or Dilated Convolution
        if 'Convolution' in layer.type:
            params[layer.name] = {}
            params[layer.name]['kernel_size'] = layer.convolution_param.kernel_size[0]
            params[layer.name]['weight'] = list(layer.blobs[0].data)
            if len(layer.blobs) == 2:
                params[layer.name]['bias'] = list(layer.blobs[1].data)
            if len(layer.convolution_param.stride) == 1:  # or []
                params[layer.name]['stride'] = layer.convolution_param.stride[0]
            else:
                params[layer.name]['stride'] = 1
            if len(layer.convolution_param.pad) == 1:  # or []
                params[layer.name]['padding'] = layer.convolution_param.pad[0]
            else:
                params[layer.name]['padding'] = 0
            if isinstance(layer.convolution_param.dilation, int):
                params[layer.name]['dilation'] = layer.convolution_param.dilation
            elif len(layer.convolution_param.dilation) == 1:
                params[layer.name]['dilation'] = layer.convolution_param.dilation[0]
            else:
                params[layer.name]['dilation'] = 1
        # Batch Normalization
        elif 'BatchNorm' in layer.type:
            params[layer.name] = {}
            params[layer.name]['running_mean'] = np.array(layer.blobs[0].data) / layer.blobs[2].data[0]  # NOQA
            params[layer.name]['running_var'] = np.array(layer.blobs[1].data) / layer.blobs[2].data[0]  # NOQA
            params[layer.name]['eps'] = layer.batch_norm_param.eps
            params[layer.name]['momentum'] = layer.batch_norm_param.moving_average_fraction
            batch_norm_layer = layer.name
        # Scale
        elif 'Scale' in layer.type:
            assert (previous_layer_type == 'BatchNorm')
            params[batch_norm_layer]['weight'] = list(layer.blobs[0].data)
            params[batch_norm_layer]['bias'] = list(layer.blobs[1].data)

        previous_layer_type = layer.type

    return params

# Hard coded translater
def translate_layer_name(source):
    def layer_block_branch(source, target):
        target += '.layer{}'.format(source[0][0])
        if len(source[0][1:]) == 1:
            block = {'a': 1, 'b': 2, 'c': 3}.get(source[0][1:])
        else:
            block = int(source[0][2:]) + 1
        target += '.block{}'.format(block)
        branch = source[1][6:]
        if branch == '1':
            target += '.proj'
        elif branch == '2a':
            target += '.reduce'
        elif branch == '2b':
            target += '.conv3x3'
        elif branch == '2c':
            target += '.increase'
        return target

    source = source.split('_')
    target = 'scale'

    if 'conv1' in source[0]:
        target += '.layer1.conv1.conv'
    elif 'conv1' in source[1]:
        target += '.layer1.conv1.bn'
    elif 'res' in source[0]:
        source[0] = source[0].replace('res', '')
        target = layer_block_branch(source, target)
        target += '.conv'
    elif 'bn' in source[0]:
        source[0] = source[0].replace('bn', '')
        target = layer_block_branch(source, target)
        target += '.bn'
    elif 'fc' in source[0]:
        # Skip if coco_init
        if len(source) == 3:
            stage = source[2]
            target += '.aspp.stages.{}'.format(stage)

    return target

@click.command()
@click.option('--dataset', required=True, type=click.Choice(['voc12', 'coco_init', 'init']))
def main(dataset):
    WHITELIST = ['kernel_size', 'stride', 'padding', 'dilation', 'eps', 'momentum']  # NOQA
    CONFIG = {
        'voc12': {
            'path_caffe_model': 'data/models/deeplab_resnet101/voc12/train2_iter_20000.caffemodel',
            'path_pytorch_model': 'data/models/deeplab_resnet101/voc12/deeplabv2_resnet101_VOC2012.pth',
            'n_classes': 21,
        },
        'coco_init': {
            'path_caffe_model': 'data/models/deeplab_resnet101/coco_init/init.caffemodel',
            'path_pytorch_model': 'data/models/deeplab_resnet101/coco_init/deeplabv2_resnet101_COCO_init.pth',
            'n_classes': 91,
        },
        'init': {
            # The same as the coco_init parameters
            'path_caffe_model': 'data/models/deeplab_resnet101/init/deeplabv2_resnet101_init.caffemodel',
            'path_pytorch_model': 'data/models/deeplab_resnet101/init/deeplabv2_resnet101_init.pth',
            'n_classes': 91,
        },
    }.get(dataset)

    params = parse_caffemodel(CONFIG['path_caffe_model'])

    model = DeepLabV2_ResNet101_MSC(n_classes=CONFIG['n_classes'])
    model.eval()
    own_state = model.state_dict()

    state_dict = OrderedDict()
    for layer_name, layer_dict in list(params.items()):
        for param_name, values in list(layer_dict.items()):
            if param_name in WHITELIST and dataset != 'coco_init' and dataset != 'init':
                attribute = translate_layer_name(layer_name)
                attribute = eval('model.' + attribute + '.' + param_name)
                if isinstance(attribute, tuple):
                    if attribute[0] != values:
                        raise ValueError
                else:
                    if abs(attribute - values) > 1e-4:
                        raise ValueError
                print(layer_name.ljust(20), '->', param_name, attribute, values, ': Checked!')
                continue
            param_name = translate_layer_name(layer_name) + '.' + param_name
            if param_name in own_state:
                values = torch.FloatTensor(values)
                values = values.view_as(own_state[param_name])
                state_dict[param_name] = values
                print(layer_name.ljust(20), '->', param_name, ': Copied!')

    torch.save(state_dict, CONFIG['path_pytorch_model'])

if __name__ == '__main__':
    main()

ext: py

reason: import numpy


text:
import re
from datetime import datetime
from unittest import mock

import pytest

from cognite.client.experimental import CogniteClient

COGNITE_CLIENT = CogniteClient()

@pytest.fixture(scope="session")
def test_time_series():
    time_series = {}
    for ts in COGNITE_CLIENT.time_series.list(limit=1000):
        if ts.name in ["test__constant_{}_with_noise".format(i) for i in range(0, 10)]:
            value = int(re.match(r"test__constant_(\d+)_with_noise", ts.name).group(1))
            time_series[value] = ts
    yield time_series

@pytest.fixture
def post_spy():
    with mock.patch.object(
        COGNITE_CLIENT.datapoints.synthetic, "_post", wraps=COGNITE_CLIENT.datapoints.synthetic._post
    ) as _:
        yield

class TestSyntheticDatapointsAPI:
    def test_retrieve(self, test_time_series, post_spy):
        query = "ts{id:%d} + ts{id:%d}" % (test_time_series[0].id, test_time_series[1].id)
        dps = COGNITE_CLIENT.datapoints.synthetic.retrieve(
            expression=query, start=datetime(2017, 1, 1), end="now", limit=23456
        )
        assert 23456 == len(dps)
        assert 3 == COGNITE_CLIENT.datapoints.synthetic._post.call_count

    def test_retrieve_with_errors(self, test_time_series, post_spy):
        dps = COGNITE_CLIENT.datapoints.synthetic.retrieve(
            expression="A / (B - B)",
            start=datetime(2017, 1, 1),
            end="now",
            limit=100,
            variables={"A": test_time_series[0], "B": test_time_series[1]},
        )
        assert 100 == len(dps)
        assert 100 == len(dps.error)
        assert all(x is not None for x in dps.error)
        assert all(x is None for x in dps.value)
        assert (100, 1) == dps.to_pandas().shape
        assert (100, 2) == dps.to_pandas(include_errors=True).shape

    @pytest.mark.dsl
    def test_expression_builder_time_series_vs_string(self, test_time_series):
        from sympy import symbols

        dps1 = COGNITE_CLIENT.datapoints.synthetic.retrieve(
            expression=symbols("a"),
            start=datetime(2017, 1, 1),
            end="now",
            limit=100,
            variables={"a": test_time_series[0].external_id},
        )
        dps2 = COGNITE_CLIENT.datapoints.synthetic.retrieve(
            expression=symbols("a"),
            start=datetime(2017, 1, 1),
            end="now",
            limit=100,
            variables={"a": test_time_series[0]},
        )
        assert 100 == len(dps1)
        assert 100 == len(dps2)
        assert dps1 == dps2

    @pytest.mark.dsl
    def test_expression_builder_complex(self, test_time_series):
        from sympy import symbols, cos, sin, pi, log, sqrt

        abc = list("abcdefghij")
        syms = symbols(abc)
        expression = syms[0]
        for s in syms:
            expression = expression + s
        expression = (
            (expression * expression)
            + sqrt(sin(pi * 0.1 ** syms[1]))
            + log(23 + syms[5] ** 1.234)
            + cos(syms[3] ** (1 + 0.1 ** syms[4]))
            + sqrt(log(abs(syms[8]) + 1))
        )
        dps1 = COGNITE_CLIENT.datapoints.synthetic.retrieve(
            expression=expression,
            start=datetime(2017, 1, 1),
            end="now",
            limit=100,
            variables={v: test_time_series[tsi] for v, tsi in zip(abc, range(10))},
        )
        assert 100 == len(dps1)

max_stars_repo_name: CsabaAlmasi/cognite-sdk-python

max_issues_repo_name: CsabaAlmasi/cognite-sdk-python

max_forks_repo_name: CsabaAlmasi/cognite-sdk-python

reason: from sympy


text:
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
import logging
import numpy as np
from abc import ABC, abstractmethod
from typing import Dict, Optional, List, Tuple

from syne_tune.config_space import Domain, is_log_space, Categorical
from syne_tune.optimizer.schedulers.searchers.bayesopt.utils.debug_log \
    import DebugLogPrinter
from syne_tune.optimizer.schedulers.searchers.bayesopt.datatypes.hp_ranges_factory \
    import make_hyperparameter_ranges
from syne_tune.optimizer.schedulers.searchers.bayesopt.tuning_algorithms.common \
    import ExclusionList

__all__ = ['BaseSearcher',
           'SearcherWithRandomSeed',
           'RandomSearcher',
           'impute_points_to_evaluate',
           'extract_random_seed']

logger = logging.getLogger(__name__)

def _impute_default_config(default_config, config_space):
    new_config = dict()
    for name, hp_range in config_space.items():
        if isinstance(hp_range, Domain):
            if name not in default_config:
                if isinstance(hp_range, Categorical):
                    # For categorical: Pick first entry
                    new_config[name] = hp_range.categories[0]
                else:
                    lower, upper = float(hp_range.lower), float(hp_range.upper)
                    if not is_log_space(hp_range):
                        midpoint = 0.5 * (upper + lower)
                    else:
                        midpoint = np.exp(0.5 * (
                                np.log(upper) + np.log(lower)))
                    # Casting may involve rounding to nearest value in
                    # a finite range
                    midpoint = hp_range.cast(midpoint)
                    midpoint = np.clip(
                        midpoint, hp_range.lower, hp_range.upper)
                    new_config[name] = midpoint
            else:
                # Check validity
                # Note: For `FiniteRange`, the value is mapped to
                # the closest one in the range
                val = hp_range.cast(default_config[name])
                if isinstance(hp_range, Categorical):
                    assert val in hp_range.categories, \
                        f"default_config[{name}] = {val} is not in " +\
                        f"categories = {hp_range.categories}"
                else:
                    assert hp_range.lower <= val <= hp_range.upper, \
                        f"default_config[{name}] = {val} is not in " +\
                        f"[{hp_range.lower}, {hp_range.upper}]"
                new_config[name] = val
    return new_config

def _to_tuple(config: Dict, keys: List) -> Tuple:
    return tuple(config[k] for k in keys)

def _sorted_keys(config_space: Dict) -> List[str]:
    return sorted(k for k, v in config_space.items() if isinstance(v, Domain))

def impute_points_to_evaluate(
        points_to_evaluate: Optional[List[Dict]],
        config_space: Dict) -> List[Dict]:
    """
    Transforms `points_to_evaluate` argument to `BaseSearcher`. Each config in
    the list can be partially specified, or even be an empty dict. For each
    hyperparameter not specified, the default value is determined using a
    midpoint heuristic. Also, duplicate entries are filtered out.
    If None (default), this is mapped to [dict()], a single default config
    determined by the midpoint heuristic. If [] (empty list), no initial
    configurations are specified.

    :param points_to_evaluate:
    :param config_space:
    :return: List of fully specified initial configs
    """
    if points_to_evaluate is None:
        points_to_evaluate = [dict()]
    # Impute and filter out duplicates
    result = []
    excl_set = set()
    keys = _sorted_keys(config_space)
    for point in points_to_evaluate:
        config = _impute_default_config(point, config_space)
        config_tpl = _to_tuple(config, keys)
        if config_tpl not in excl_set:
            result.append(config)
            excl_set.add(config_tpl)
    return result

class BaseSearcher(ABC):
    """Base Searcher (virtual class to inherit from if you are creating a custom Searcher).

    Parameters
    ----------
    config_space : Dict
        The configuration space to sample from. It contains the full
        specification of the Hyperparameters with their priors
    metric : str
        Name of metric passed to update.
    points_to_evaluate : List[Dict] or None
        List of configurations to be evaluated initially (in that order).
        Each config in the list can be partially specified, or even be an
        empty dict. For each hyperparameter not specified, the default value
        is determined using a midpoint heuristic.
        If None (default), this is mapped to [dict()], a single default config
        determined by the midpoint heuristic. If [] (empty list), no initial
        configurations are specified.
    """
    def __init__(
            self, config_space, metric, points_to_evaluate=None):
        self.config_space = config_space
        assert metric is not None, "Argument 'metric' is required"
        self._metric = metric
        self._points_to_evaluate = impute_points_to_evaluate(
            points_to_evaluate, config_space)

    def configure_scheduler(self, scheduler):
        """
        Some searchers need to obtain information from the scheduler they are
        used with, in order to configure themselves.
        This method has to be called before the searcher can be used.

        The implementation here sets _metric for schedulers which specify it.

        Args:
            scheduler: TaskScheduler
                Scheduler the searcher is used with.

        """
        from syne_tune.optimizer.schedulers.fifo import FIFOScheduler

        if isinstance(scheduler, FIFOScheduler):
            self._metric = scheduler.metric

    def _next_initial_config(self) -> Optional[Dict]:
        if self._points_to_evaluate:
            return self._points_to_evaluate.pop(0)
        else:
            return None  # No more initial configs

    @abstractmethod
    def get_config(self, **kwargs):
        """Function to sample a new configuration

        This function is called inside TaskScheduler to query a new
        configuration.

        Note: Query `_next_initial_config` for initial configs to return first.

        Args:
        kwargs:
            Extra information may be passed from scheduler to searcher
        returns: New configuration. The searcher may return None if a new
            configuration cannot be suggested. In this case, the tuning will
            stop. This happens if searchers never suggest the same config more
            than once, and all configs in the (finite) search space are
            exhausted.
        """
        pass

    def on_trial_result(
            self, trial_id: str, config: Dict, result: Dict, update: bool):
        """Inform searcher about result

        The scheduler passes every result. If `update` is True, the searcher
        should update its surrogate model (if any), otherwise `result` is an
        intermediate result not modelled.

        The default implementation calls self._update if `update` is True. It
        can be overwritten by searchers which also react to intermediate
        results.

        :param trial_id:
        :param config:
        :param result:
        :param update: Should surrogate model be updated?
        """
        if update:
            self._update(trial_id, config, result)

    @abstractmethod
    def _update(self, trial_id: str, config: Dict, result: Dict):
        """Update surrogate model with result

        :param trial_id:
        :param config:
        :param result:
        """
        pass

    def register_pending(
            self, trial_id: str, config: Optional[Dict] = None,
            milestone=None):
        """
        Signals to searcher that evaluation for trial has started, but not
        yet finished, which allows model-based searchers to register this
        evaluation as pending.
        For multi-fidelity schedulers, milestone is the next milestone the
        evaluation will attend, so that model registers (config, milestone)
        as pending.
        The configuration for the trial has to be passed in `config` for a
        new trial, which the searcher has not seen before. If the trial is
        already registered with th searcher, `config` is ignored.
        """
        pass

    def remove_case(self, trial_id: str, **kwargs):
        """Remove data case previously appended by update

        For searchers which maintain the dataset of all cases (reports) passed
        to update, this method allows to remove one case from the dataset.
        """
        pass

    def evaluation_failed(self, trial_id: str):
        """
        Called by scheduler if an evaluation job for a trial failed. The
        searcher should react appropriately (e.g., remove pending evaluations
        for this trial, not suggest the configuration again).
        """
        pass

    def cleanup_pending(self, trial_id: str):
        """
        Removes all pending candidates whose configuration is equal to
        `config`.
        This should be called after an evaluation terminates. For various
        reasons (e.g., termination due to convergence), pending candidates
        for this evaluation may still be present.

        """
        pass

    def dataset_size(self):
        """
        :return: Size of dataset a model is fitted to, or 0 if no model is
            fitted to data
        """
        return 0

    def model_parameters(self):
        """
        :return: Dictionary with current model (hyper)parameter values if
            this is supported; otherwise empty
        """
        return dict()

    def get_state(self) -> dict:
        """
        Together with clone_from_state, this is needed in order to store and
        re-create the mutable state of the searcher.

        The state returned here must be pickle-able.

        :return: Pickle-able mutable state of searcher
        """
        return {'points_to_evaluate': self._points_to_evaluate}

    @abstractmethod
    def clone_from_state(self, state: dict):
        """
        Together with get_state, this is needed in order to store and
        re-create the mutable state of the searcher.

        Given state as returned by get_state, this method combines the
        non-pickle-able part of the immutable state from self with state
        and returns the corresponding searcher clone. Afterwards, self is
        not used anymore.

        :param state: See above
        :return: New searcher object
        """
        pass

    def _restore_from_state(self, state: dict):
        self._points_to_evaluate = state['points_to_evaluate'].copy()

    @property
    def debug_log(self):
        """
        Some BaseSearcher subclasses support writing a debug log, using
        DebugLogPrinter. See RandomSearcher for an example.

        :return: DebugLogPrinter; or None (not supported)
        """
        return None

def extract_random_seed(kwargs: dict) -> (int, dict):
    key = 'random_seed_generator'
    if kwargs.get(key) is not None:
        random_seed = kwargs[key]()
    else:
        key = 'random_seed'
        if kwargs.get(key) is not None:
            random_seed = kwargs[key]
        else:
            random_seed = 31415927
            key = None
    _kwargs = {k: v for k, v in kwargs.items() if k != key}
    return random_seed, _kwargs

class SearcherWithRandomSeed(BaseSearcher):
    """
    Base class of searchers which use random decisions. Creates the
    `random_state` member, which must be used for all random draws.

    Making proper use of this interface allows us to run experiments
    with control of random seeds, e.g. for paired comparisons or
    integration testing.

    Extra parameters
    ----------------
    random_seed_generator : RandomSeedGenerator (optional)
        If given, the random_seed for `random_state` is obtained from there,
        otherwise `random_seed` is used
    random_seed : int (optional)
        This is used if `random_seed_generator` is not given.

    """
    def __init__(
            self, config_space, metric, points_to_evaluate=None, **kwargs):
        super().__init__(
            config_space, metric=metric, points_to_evaluate=points_to_evaluate)
        random_seed, _ = extract_random_seed(kwargs)
        self.random_state = np.random.RandomState(random_seed)

    def get_state(self) -> dict:
        state = dict(
            super().get_state(),
            random_state=self.random_state.get_state())
        return state

    def _restore_from_state(self, state: dict):
        super()._restore_from_state(state)
        self.random_state.set_state(state['random_state'])

class RandomSearcher(SearcherWithRandomSeed):
    """Searcher which randomly samples configurations to try next.

    Extra parameters
    ----------------
    debug_log : bool
        If True (default), debug log printing is activated. Logs which
        configs are chosen when, and which metric values are obtained.

    """
    MAX_RETRIES = 100

    def __init__(self, config_space, metric, points_to_evaluate=None, **kwargs):
        super().__init__(
            config_space, metric, points_to_evaluate, **kwargs)
        self._hp_ranges = make_hyperparameter_ranges(config_space)
        self._resource_attr = kwargs.get('resource_attr')
        self._excl_list = ExclusionList.empty_list(self._hp_ranges)
        # Debug log printing (switched on by default)
        debug_log = kwargs.get('debug_log', True)
        if isinstance(debug_log, bool):
            if debug_log:
                self._debug_log = DebugLogPrinter()
            else:
                self._debug_log = None
        else:
            assert isinstance(debug_log, DebugLogPrinter)
            self._debug_log = debug_log

    def configure_scheduler(self, scheduler):
        """
        Some searchers need to obtain information from the scheduler they are
        used with, in order to configure themselves.
        This method has to be called before the searcher can be used.

        The implementation here sets _metric for schedulers which
        specify it.

        Args:
            scheduler: TaskScheduler
                Scheduler the searcher is used with.

        """
        from syne_tune.optimizer.schedulers.hyperband import \
            HyperbandScheduler

        super().configure_scheduler(scheduler)
        # If the scheduler is Hyperband, we want to know the resource
        # attribute, this is used for debug_log
        if isinstance(scheduler, HyperbandScheduler):
            self._resource_attr = scheduler._resource_attr

    def get_config(self, **kwargs):
        """Sample a new configuration at random
        This is done without replacement, so previously returned configs are
        not suggested again.

        Returns
        -------
        A new configuration that is valid, or None if no new config can be
        suggested.
        """
        trial_id = kwargs.get('trial_id')
        if self._debug_log is not None:
            self._debug_log.start_get_config('random', trial_id=trial_id)
        new_config = self._next_initial_config()
        if new_config is None:
            if not self._excl_list.config_space_exhausted():
                for _ in range(self.MAX_RETRIES):
                    _config = self._hp_ranges.random_config(self.random_state)
                    if not self._excl_list.contains(_config):
                        new_config = _config
                        break

        if new_config is not None:
            self._excl_list.add(new_config)  # Should not be suggested again
            if self._debug_log is not None:
                self._debug_log.set_final_config(new_config)
                # All get_config debug log info is only written here
                self._debug_log.write_block()
        else:
            msg = "Failed to sample a configuration not already chosen " + \
                  f"before. Exclusion list has size {len(self._excl_list)}."
            cs_size = self._excl_list.configspace_size
            if cs_size is not None:
                msg += f" Configuration space has size {cs_size}."
            logger.warning(msg)
        return new_config

    def _update(self, trial_id: str, config: Dict, result: Dict):
        if self._debug_log is not None:
            metric_val = result[self._metric]
            if self._resource_attr is not None:
                # For HyperbandScheduler, also add the resource attribute
                resource = int(result[self._resource_attr])
                trial_id = trial_id + ':{}'.format(resource)
            msg = f"Update for trial_id {trial_id}: metric = {metric_val:.3f}"
            logger.info(msg)

    def clone_from_state(self, state: dict):
        new_searcher = RandomSearcher(
            self.config_space, metric=self._metric, debug_log=self._debug_log)
        new_searcher._resource_attr = self._resource_attr
        new_searcher._restore_from_state(state)
        return new_searcher

    @property
    def debug_log(self):
        return self._debug_log

max_stars_repo_name: hfurkanbozkurt/syne-tune

max_issues_repo_name: hfurkanbozkurt/syne-tune

max_forks_repo_name: hfurkanbozkurt/syne-tune


text:
import os
from scipy.sparse import coo_matrix
import torch
import numpy as np
import tables as tb
from src.preprocessing.utils import DATASET_DIR, HDF5_DATASET, DATASET_NAME

from typing import Union, List, Tuple

class UserDataset(torch.utils.data.Dataset):
    """Implements User Dataloader"""
    PATH = os.path.join(DATASET_DIR, HDF5_DATASET)
    VAL_PREFIX = 'validation_'
    TRAIN_PREFIX = 'training_'

    @staticmethod
    def hdfarray_to_numpy(hdfarray: tb.CArray):
        array = np.empty(shape=hdfarray.shape, dtype=hdfarray.dtype)
        array[:] = hdfarray[:]
        return array

    @staticmethod
    def hdfarray_to_tensor(hdfarray: tb.CArray):
        return torch.from_numpy(hdfarray[:])

    @staticmethod
    def vec_to_sparse(i: torch.tensor, j: torch.tensor, v: torch.tensor,
                      size=None, load_full=False, style='tensor') -> Union[torch.Tensor, coo_matrix, np.ndarray]:
        if style == 'tensor':
            idx = torch.stack((i, j), axis=0)
            sparsed_matrix = torch.sparse_coo_tensor(idx, v, size=size)
            if load_full:
                sparsed_matrix = sparsed_matrix.to_dense()
        elif style == 'numpy':
            if len(i.shape) + len(j.shape) + len(v.shape) != 3:
                raise ValueError("scipy.coo_matrix: row, column, and data arrays must be 1-D")
            i, j = i.numpy().astype(np.uint), j.numpy().astype(np.uint)
            v = v.numpy()
            sparsed_matrix = coo_matrix( (v, (i, j)), shape=size )
            if load_full:
                sparsed_matrix = sparsed_matrix.toarray()
        else:
            raise NameError("style must be 'tensor' or 'numpy'!")

        return sparsed_matrix

    def __init__(self, data_name: str, path=PATH, masked_uid=None, masked_iid=None, masked_vid=None, mode='full'):
        self.path = path
        self.mode = mode
        self.data_name = data_name
        self.h5f = tb.open_file(path, 'r')
        cur_group = self.h5f.root[data_name]

        mask_prefix = None
        if self.mode == 'val':
            mask_prefix = UserDataset.VAL_PREFIX
        elif self.mode == 'train':
            mask_prefix = UserDataset.TRAIN_PREFIX
        elif self.mode != 'full':
            raise ValueError(f'Supported mode: {self.mode}')

        if mask_prefix is not None:
            self.userIdx = self.hdfarray_to_tensor(cur_group[mask_prefix+'masked_uid'])
            self.itemIdx = self.hdfarray_to_tensor(cur_group[mask_prefix+'masked_iid'])
            masked_vid = self.hdfarray_to_tensor(cur_group[mask_prefix+'masked_vid'])
        else:
            if masked_uid is not None:
                self.userIdx = masked_uid
            else:
                self.userIdx = self.hdfarray_to_tensor(cur_group['userIdx'])

            if masked_iid is not None:
                self.itemIdx = masked_iid
            else:
                self.itemIdx = self.hdfarray_to_tensor(cur_group['itemIdx'])

        if masked_vid is not None:
            self.masked_vid = masked_vid
            self.rating    = self.hdfarray_to_tensor(cur_group['rating'])[masked_vid]
            self.embedding = self.hdfarray_to_tensor(cur_group['embedding'])[masked_vid]
        else:
            self.masked_vid = None
            self.rating    = self.hdfarray_to_tensor(cur_group['rating'])
            self.embedding = self.hdfarray_to_tensor(cur_group['embedding'])

        self.idx = torch.stack((self.userIdx, self.itemIdx), axis=0)
        self.numIDs, self.numItems = int(torch.max(self.userIdx))+1, int(torch.max(self.itemIdx))+1
        self.interactions = self.vec_to_sparse(
            self.userIdx, self.itemIdx, self.rating,
            size=(self.numIDs, self.numItems),
            load_full=False,
            style='tensor'
        )
        self.review_embeddings = self.vec_to_sparse(
            self.userIdx, self.itemIdx, self.embedding,
            size=(self.numIDs, self.numItems, self.embedding.shape[1]),
            load_full=False,
            style='tensor'
        )

        self.h5f.close()

    def get_indices(self, style='tensor') -> Union[torch.Tensor, np.ndarray]:
        if style == 'tensor':
            return self.idx
        elif style == 'numpy':
            return np.stack((self.userIdx, self.itemIdx), axis=0)
        else:
            raise NameError("style must be 'tensor' or 'numpy'!")

    def get_reviewEmbeddings(self, load_full=False) -> Union[torch.Tensor, coo_matrix, np.ndarray]:
        if load_full:
            self.review_embeddings.to_dense()
        return self.review_embeddings

    def get_interactions(self, load_full=False, style='tensor') -> Union[torch.Tensor, coo_matrix, np.ndarray]:
        if style == 'tensor':
            if load_full:
                return self.interactions.to_dense()
            return self.interactions

        return self.vec_to_sparse(self.userIdx, self.itemIdx, self.rating,
                                  size=(self.numIDs, self.numItems),
                                  load_full=load_full,
                                  style=style)

    def get_mask(self, drop_ratio: float, masked_uid=None, masked_iid=None) -> (torch.Tensor, torch.Tensor, torch.Tensor):
        valid_mask = self.get_interactions(load_full=True) > 0
        rand_mask = torch.rand(valid_mask.shape)
        rand_mask[~valid_mask] = 0

        ref_mask = None
        if (masked_uid is not None) and (masked_iid is not None):
            ref_mask = self.vec_to_sparse(
                i=masked_uid, j=masked_iid, v=torch.ones(masked_uid.shape),
                size=(self.numIDs, self.numItems), load_full=True
            )>0

        if ref_mask is None:
            mask = rand_mask.ge(drop_ratio)
        else:
            ref_row_mask = torch.any(ref_mask, dim=1)
            rand_mask[~ref_row_mask] = 0
            mask = rand_mask.ge(drop_ratio)
            mask = torch.logical_and(mask, ref_mask)

        # Sanity check for rows with all `False`
        ill_masked_idx = ~torch.any(mask, dim=1)

        # Unmask the ill-masked values
        if ref_mask is None:
            mask[ill_masked_idx] = valid_mask[ill_masked_idx]
        else:
            mask[ill_masked_idx] = ref_mask[ill_masked_idx]

        print(f"Targeted drop ratio: {drop_ratio}")
        print(f"Actual drop ratio: {(valid_mask.sum() - mask.sum()) / valid_mask.sum()}")

        masked_uid, masked_iid, = mask.to_sparse().indices()
        masked_vid = mask[valid_mask]
        return masked_uid, masked_iid, masked_vid

    def save_mask(self, masked_uid: torch.Tensor, masked_iid: torch.Tensor, masked_vid: torch.Tensor, mode: str):
        if self.masked_vid is not None:
            raise TypeError("You should save your masks from the unmasked dataset!")

        if mode == 'val':
            prefix = UserDataset.VAL_PREFIX
        elif mode == 'train':
            prefix = UserDataset.TRAIN_PREFIX
        else:
            raise ValueError(f"Supported mode: {mode}")

        self.h5f = tb.open_file(self.path, 'a')
        cur_group = self.h5f.root[self.data_name]
        filters = tb.Filters(complib='zlib', complevel=5)
        self.h5f.create_carray(cur_group, prefix+'masked_uid', obj=masked_uid.numpy(), filters=filters)
        self.h5f.create_carray(cur_group, prefix+'masked_iid', obj=masked_iid.numpy(), filters=filters)
        self.h5f.create_carray(cur_group, prefix+'masked_vid', obj=masked_vid.numpy(), filters=filters)
        print(self.h5f)
        self.h5f.close()

    def __len__(self) -> int:
        return self.numIDs

    def __getitem__(self, idx) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        user_reviews_embedding = self.review_embeddings[idx].coalesce().values().mean(dim=0)
        user_ratings = self.interactions[idx].to_dense()

        return user_reviews_embedding, user_ratings, torch.tensor(idx)

class ItemDataset(UserDataset):
    """Implements Item Dataloader"""
    PATH = os.path.join(DATASET_DIR, HDF5_DATASET)
    VAL_PREFIX = 'validation_'
    TRAIN_PREFIX = 'training_'

    def __init__(self, data_name: str, path=PATH, masked_uid=None, masked_iid=None, masked_vid=None, mode='full'):
        self.path = path
        self.mode = mode
        self.data_name = data_name
        self.h5f = tb.open_file(path, 'r')
        cur_group = self.h5f.root[data_name]

        mask_prefix = None
        if self.mode == 'val':
            mask_prefix = ItemDataset.VAL_PREFIX
        elif self.mode == 'train':
            mask_prefix = ItemDataset.TRAIN_PREFIX
        elif self.mode != 'full':
            raise ValueError(f'Supported mode: {self.mode}')

        if mask_prefix is not None:
            self.userIdx = self.hdfarray_to_tensor(cur_group[mask_prefix+'masked_uid'])
            self.itemIdx = self.hdfarray_to_tensor(cur_group[mask_prefix+'masked_iid'])
            masked_vid = self.hdfarray_to_tensor(cur_group[mask_prefix+'masked_vid'])
        else:
            if masked_uid is not None:
                self.userIdx = masked_uid
            else:
                self.userIdx = self.hdfarray_to_tensor(cur_group['userIdx'])

            if masked_iid is not None:
                self.itemIdx = masked_iid
            else:
                self.itemIdx = self.hdfarray_to_tensor(cur_group['itemIdx'])

        if masked_vid is not None:
            self.masked_vid = masked_vid
            self.rating    = self.hdfarray_to_tensor(cur_group['rating'])[masked_vid]
            self.embedding = self.hdfarray_to_tensor(cur_group['embedding'])[masked_vid]
        else:
            self.masked_vid = None
            self.rating    = self.hdfarray_to_tensor(cur_group['rating'])
            self.embedding = self.hdfarray_to_tensor(cur_group['embedding'])

        self.idx = torch.stack((self.itemIdx, self.userIdx), axis=0)
        self.numIDs, self.numItems = int(torch.max(self.userIdx))+1, int(torch.max(self.itemIdx))+1
        self.interactions = self.vec_to_sparse(
            self.itemIdx, self.userIdx, self.rating,
            size=(self.numItems, self.numIDs),
            load_full=False,
            style='tensor'
        )
        self.review_embeddings = self.vec_to_sparse(
            self.itemIdx, self.userIdx, self.embedding,
            size=(self.numItems, self.numIDs, self.embedding.shape[1]),
            load_full=False,
            style='tensor'
        )

        self.h5f.close()

    def __len__(self) -> int:
        return self.numItems

    def get_indices(self, style='tensor') -> Union[torch.Tensor, np.ndarray]:
        if style == 'tensor':
            return self.idx
        elif style == 'numpy':
            return np.stack((self.itemIdx, self.userIdx), axis=0)
        else:
            raise NameError("style must be 'tensor' or 'numpy'!")

    def get_mask(self, drop_ratio: float, ref_mask=None) -> (torch.Tensor, torch.Tensor):
        raise NotImplementedError("Please use `get_mask` in UserDataset.")

    def save_mask(self, masked_uid: torch.Tensor, masked_iid: torch.Tensor, masked_vid: torch.Tensor, prefix: str):
        raise NotImplementedError("Please use `save_mask` in UserDataset.")

if __name__ == '__main__':
    """
    0. Flags

        load_full: bool
        if `load_full` is True, any function which takes this flag will return a full matrix
        otherwise, it will return in the form of a spared matrix

        style: ['tensor', 'numpy']
        if `style` is `tensor`, the return value will be a torch.Tensor
        if `style` is `numpy`, the return value will be a np.ndarray
    """

    """
    1. Load the unmodified dataset
    """
    user_dataset = UserDataset(data_name='food')

    """
    2. Access attributes within the Dataset object
    """
    # return indices for all valid entries within the dataset
    # returned value will be a 2d-matrix,
    # with 1st row as row indices and 2nd row as col indices
    indices = user_dataset.get_indices(style='tensor')

    # return all of the user review embeddings
    embeddings = user_dataset.get_reviewEmbeddings(load_full=False)

    # return the user-item Interactions
    interactions = user_dataset.get_interactions(load_full=True, style='tensor')

    # ...or you can access the dataset by index (through the getter function)
    user_reviews_embedding, user_ratings, idx = user_dataset[0]

    """
    3. Create masks for training/validation/testing
    """
    # Since we can use the unmodified dataset for testing, we need two more masks
    # for training and validation

    # We can generate a masking using `get_mask`
    # `drop_ratio` is the approximate percentage of rating/review we are dropping
    """
    validation_uid, validation_iid, validation_vid = user_dataset.get_mask(drop_ratio=0.3)
    """

    # To get the masking for training set, we can use the previously generated masks to get
    # a new set of masks with higher `drop_ratio`
    """
    training_uid, training_iid, training_vid = user_dataset.get_mask(
        drop_ratio=0.6, masked_uid=validation_uid, masked_iid=validation_iid
    )
    """

    # To save the generated mask into the h4 dataset, use the `save_mask` method:
    """
    user_dataset.save_mask(validation_uid, validation_iid, validation_vid, valset_prefix, mode='val')
    user_dataset.save_mask(training_uid, training_iid, training_vid, trainset_prefix, mode='train')
    """

    # Once we get the user_idx (uid), item_idx (iid), value_idx (vid), we can create
    # two new Dataset object using the masked ids.
    """
    training_dataset = UserDataset(
        data_name='food',
        masked_uid=training_uid,
        masked_iid=training_iid,
        masked_vid=training_vid
    )

    validation_dataset = UserDataset(
        data_name='food',
        masked_uid=validation_uid,
        masked_iid=validation_iid,
        masked_vid=validation_vid
    )
    """

    # ...or we can use the preprocessed masks to get training and validation set:
    training_dataset = UserDataset(
        data_name='food',
        mode='train'
    )
    validation_dataset = UserDataset(
        data_name='food',
        mode='val'
    )

    """
    4. Import Dataset object into a torch Dataloader
    """
    from torch.utils.data import DataLoader

    # In order to get data in batches, we can import our dataset into a torch dataloader
    train_loader = DataLoader(training_dataset, batch_size=1, shuffle=True, num_workers=16)
    val_loader = DataLoader(validation_dataset, batch_size=1, shuffle=True, num_workers=16)
    test_loader = DataLoader(user_dataset, batch_size=1, shuffle=True, num_workers=16)

    # Here is an example of how you can use the dataloader
    for i, batch in enumerate(train_loader):
        if i < 3:
            user_reviews_embedding, user_ratings, idx = batch
            print(f"user_reviews_embedding: {user_reviews_embedding.size()}")
            print(user_reviews_embedding)
            print(f"user_ratings: {user_ratings.size()}, # raings: {(user_ratings>0).sum()}")
            print(user_ratings)
            print()
        else:
            break

reason: import numpy,from scipy


text:
from __future__ import (absolute_import, division, print_function)

import numpy as np
from scipy import constants

planck_constant = constants.Planck / constants.e * 1E15  # meV*psec
hbar = planck_constant / (2 * np.pi)  # meV*psec

max_stars_repo_name: jmborr/QeF

max_issues_repo_name: jmborr/QeF

max_forks_repo_name: jmborr/QeF


text:
from scipy.signal import convolve2d
import numpy as np
import graphics
import random
import driver
import game
import time

rows = 15
cols = 112

class Life(game.Game):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.start = time.time()
        self.randomize()

    def randomize(self):
        self.graphics.buffer = [[random.getrandbits(1) for y in range(cols)] for x in range(rows)]

    def recv_input(self, evt):
        self.randomize()

    def loop(self):
        while not self.input_queue.empty():
            evt = self.input_queue.get()
            self.recv_input(evt)

        nbrs_count = convolve2d(self.graphics.buffer, np.ones((3, 3)), mode='same', boundary='wrap') - self.graphics.buffer
        self.graphics.buffer =  (nbrs_count == 3) | (self.graphics.buffer & (nbrs_count == 2))

        self.graphics.draw(self.serial)
        time.sleep(1 / self.framerate)
        if time.time() > self.start+20:
            self.randomize()
            self.start = time.time()

GAME = Life

max_stars_repo_name: umbc-hackafe/sign-drivers

max_issues_repo_name: umbc-hackafe/sign-drivers

max_forks_repo_name: umbc-hackafe/sign-drivers


text:
##
# @file ex_B3.py
# \brief Bordbar 2020 example 3
#

def ex_B3(method, nGG=4):

    import numpy as np

    from cyparallel_planes import parallel_planes
    #from parallel_planes import parallel_planes

    from pyrad import pyrad_planck_mean as rad_planck_mean
    from pyrad import pyrad_wsgg        as rad_wsgg
    from pyrad import pyrad_rcslw       as rad_rcslw

    ################################################################################

    #--------------------- parameters

    nGGa = nGG + 1

    P      = 101325.0
    L      = 1.0
    ntheta = 101
    nx     = 1001
    Twall  = 400.0

    xco2=0.0; xco=0.0; xh2o=0.0; xch4=0.0; fvs=0.0

    #---------------------

    xCO2   = np.full(nx, xco2, dtype=np.float64)
    xCO    = np.full(nx, xco,  dtype=np.float64)
    xH2O   = np.full(nx, xh2o, dtype=np.float64)
    xCH4   = np.full(nx, xch4, dtype=np.float64)
    fvsoot = np.full(nx, fvs,  dtype=np.float64)

    x = np.zeros(nx)
    T = np.full(nx,Twall, dtype=np.float64)

    x = np.linspace(0,L,nx)
    for i in range(nx):
        T[i]    = 400.0 + 1400.0*   np.sin(np.pi*x[i]/L)**2
        xH2O[i] = 1E-4 + (1.0-1E-4)*np.sin(np.pi*x[i]/L)**2
        xCO2[i] = 1.0-xH2O[i]
    xH2O_avg = np.average(xH2O)
    xCO2_avg = 1.0-xH2O_avg
    T_avg    = np.average(T)

    if method=='rcslw':
        rad = rad_rcslw(nGG, T_avg, P, fvs, xH2O_avg, xCO2_avg, xco)
    elif method=='wsgg':
        rad = rad_wsgg()
    elif method=='planckmean':
        rad = rad_planck_mean()
    else:
        raise ValueError('unknown radiation method')

    #------------------- get q, Q

    x, xQ, q, Q = parallel_planes(rad, L, ntheta, T, P, fvsoot, xH2O, xCO2, xCO, xCH4)

    print("# x (m), Q (kW/m3)")
    for i in range(len(xQ)):
        print(f'{xQ[i]:.5f}  {Q[i]/1000:.5f}')

    #print("# x (m), q (kW/m2)")
    #for i in range(len(x)):
    #    print(f'{x[i]:.5f}  {q[i]/1000:.5f}')

        #-------------------------------------------------------------------------

    print()

    return x, xQ, q, Q

################################################################################

if __name__=='__main__':
    x, xQ, q, Q = ex_B3('wsgg', nGG=4)

max_stars_repo_name: BYUignite/radlib

max_issues_repo_name: BYUignite/radlib

max_forks_repo_name: BYUignite/radlib


text:
""" Taken from https://gist.github.com/dennissergeev/60bf7b03443f1b2c8eb96ce0b1880150"""
import numpy as np
EARTH_RADIUS = 6371000.0  # m

def _guess_bounds(points, bound_position=0.5):
    """
    Guess bounds of grid cells.

    Simplified function from iris.coord.Coord.

    Parameters
    ----------
    points: numpy.array
        Array of grid points of shape (N,).
    bound_position: float, optional
        Bounds offset relative to the grid cell centre.
    Returns
    -------
    Array of shape (N, 2).
    """
    diffs = np.diff(points)
    diffs = np.insert(diffs, 0, diffs[0])
    diffs = np.append(diffs, diffs[-1])

    min_bounds = points - diffs[:-1] * bound_position
    max_bounds = points + diffs[1:] * (1 - bound_position)

    return np.array([min_bounds, max_bounds]).transpose()

def _quadrant_area(radian_lat_bounds, radian_lon_bounds, radius_of_earth):
    """
    Calculate spherical segment areas.
    Taken from SciTools iris library.
    Area weights are calculated for each lat/lon cell as:
        .. math::
            r^2 (lon_1 - lon_0) ( sin(lat_1) - sin(lat_0))
    The resulting array will have a shape of
    *(radian_lat_bounds.shape[0], radian_lon_bounds.shape[0])*
    The calculations are done at 64 bit precision and the returned array
    will be of type numpy.float64.
    Parameters
    ----------
    radian_lat_bounds: numpy.array
        Array of latitude bounds (radians) of shape (M, 2)
    radian_lon_bounds: numpy.array
        Array of longitude bounds (radians) of shape (N, 2)
    radius_of_earth: float
        Radius of the Earth (currently assumed spherical)
    Returns
    -------
    Array of grid cell areas of shape (M, N).
    """
    # ensure pairs of bounds
    if (
        radian_lat_bounds.shape[-1] != 2
        or radian_lon_bounds.shape[-1] != 2
        or radian_lat_bounds.ndim != 2
        or radian_lon_bounds.ndim != 2
    ):
        raise ValueError("Bounds must be [n,2] array")

    # fill in a new array of areas
    radius_sqr = radius_of_earth ** 2
    radian_lat_64 = radian_lat_bounds.astype(np.float64)
    radian_lon_64 = radian_lon_bounds.astype(np.float64)

    ylen = np.sin(radian_lat_64[:, 1]) - np.sin(radian_lat_64[:, 0])
    xlen = radian_lon_64[:, 1] - radian_lon_64[:, 0]
    areas = radius_sqr * np.outer(ylen, xlen)

    # we use abs because backwards bounds (min > max) give negative areas.
    return np.abs(areas)

def grid_cell_areas(lon1d, lat1d, radius=EARTH_RADIUS):
    """
    Calculate grid cell areas given 1D arrays of longitudes and latitudes
    for a planet with the given radius.

    Parameters
    ----------
    lon1d: numpy.array
        Array of longitude points [degrees] of shape (M,)
    lat1d: numpy.array
        Array of latitude points [degrees] of shape (M,)
    radius: float, optional
        Radius of the planet [metres] (currently assumed spherical)
    Returns
    -------
    Array of grid cell areas [metres**2] of shape (M, N).
    """
    lon_bounds_radian = np.deg2rad(_guess_bounds(lon1d))
    lat_bounds_radian = np.deg2rad(_guess_bounds(lat1d))
    area = _quadrant_area(lat_bounds_radian, lon_bounds_radian, radius)
    return area

def calc_spatial_mean(
    xr_data, lon_name="lon", lat_name="lat", radius=EARTH_RADIUS
):
    """
    Calculate spatial mean of xarray.DataArray with grid cell weighting.

    Parameters
    ----------
    xr_da: xarray.DataArray
        Data to average
    lon_name: str, optional
        Name of x-coordinate
    lat_name: str, optional
        Name of y-coordinate
    radius: float
        Radius of the planet [metres], currently assumed spherical (not important anyway)
    Returns
    -------
    Spatially averaged xarray.DataArray.
    """
    lon = xr_data[lon_name].values
    lat = xr_data[lat_name].values

    area_weights = grid_cell_areas(lon, lat, radius=radius)
    aw_factor = area_weights / area_weights.max()

    return (xr_data * aw_factor).mean(dim=[lon_name, lat_name], skipna=True)

def calc_spatial_integral(
    xr_da, lon_name="longitude", lat_name="latitude", radius=EARTH_RADIUS
):
    """
    Calculate spatial integral of xarray.DataArray with grid cell weighting.

    Parameters
    ----------
    xr_da: xarray.DataArray
        Data to average
    lon_name: str, optional
        Name of x-coordinate
    lat_name: str, optional
        Name of y-coordinate
    radius: float
        Radius of the planet [metres], currently assumed spherical (not important anyway)
    Returns
    -------
    Spatially averaged xarray.DataArray.
    """
    lon = xr_da[lon_name].values
    lat = xr_da[lat_name].values

    area_weights = grid_cell_areas(lon, lat, radius=radius)

    return (xr_da * area_weights).sum(dim=[lon_name, lat_name], skipna=True)

"""Example:
import xarray as xr

ds = xr.open_dataset("/path/to/data")

mean_precip = calc_spatial_mean(ds[precipitation_flux])
"""

max_stars_repo_name: mlcs/climnet

max_issues_repo_name: mlcs/climnet

max_forks_repo_name: mlcs/climnet