text:
# Copyright 2018 The Cirq Developers
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import Union, Sequence, Any

import numpy as np

from cirq import linalg, protocols
from cirq.ops import raw_types
from cirq.type_workarounds import NotImplementedType

class ControlledGate(raw_types.Gate):
    """Augments existing gates with a control qubit."""

    def __init__(self, sub_gate: raw_types.Gate) -> None:
        """Initializes the controlled gate.

        Args:
            sub_gate: The gate to add a control qubit to.
        """
        self.sub_gate = sub_gate

    def validate_args(self, qubits) -> None:
        if len(qubits) < 1:
            raise ValueError('No control qubit specified.')
        self.sub_gate.validate_args(qubits[1:])

    def __eq__(self, other):
        if not isinstance(other, type(self)):
            return NotImplemented
        return self.sub_gate == other.sub_gate

    def __ne__(self, other):
        return not self == other

    def __hash__(self):
        return hash((ControlledGate, self.sub_gate))

    def _apply_unitary_to_tensor_(self,
                                  target_tensor: np.ndarray,
                                  available_buffer: np.ndarray,
                                  axes: Sequence[int],
                                  ) -> np.ndarray:
        control = axes[0]
        rest = axes[1:]
        active = linalg.slice_for_qubits_equal_to([control], 1)
        sub_axes = [r - int(r > control) for r in rest]
        target_view = target_tensor[active]
        buffer_view = available_buffer[active]
        result = protocols.apply_unitary_to_tensor(
            self.sub_gate,
            target_view,
            buffer_view,
            sub_axes,
            default=NotImplemented)

        if result is NotImplemented:
            return NotImplemented

        if result is target_view:
            return target_tensor

        if result is buffer_view:
            inactive = linalg.slice_for_qubits_equal_to([control], 0)
            available_buffer[inactive] = target_tensor[inactive]
            return available_buffer

        # HACK: assume they didn't somehow escape the slice view and edit the
        # rest of target_tensor.
        target_tensor[active] = result
        return target_tensor

    def _unitary_(self) -> Union[np.ndarray, NotImplementedType]:
        sub_matrix = protocols.unitary(self.sub_gate, None)
        if sub_matrix is None:
            return NotImplemented
        return linalg.block_diag(np.eye(sub_matrix.shape[0]), sub_matrix)

    def __pow__(self, exponent: Any) -> 'ControlledGate':
        new_sub_gate = protocols.pow(self.sub_gate,
                                     exponent,
                                     NotImplemented)
        if new_sub_gate is NotImplemented:
            return NotImplemented
        return ControlledGate(new_sub_gate)

    def _phase_by_(self, phase_turns: float, qubit_index: int):
        if qubit_index == 0:
            return self
        phased_gate = protocols.phase_by(
            self.sub_gate, phase_turns, qubit_index-1, None)
        if phased_gate is None:
            return NotImplemented
        return ControlledGate(phased_gate)

    def _is_parameterized_(self):
        return protocols.is_parameterized(self.sub_gate)

    def _resolve_parameters_(self, param_resolver):
        new_sub_gate = protocols.resolve_parameters(self.sub_gate,
                                                    param_resolver)
        return ControlledGate(new_sub_gate)

    def _trace_distance_bound_(self):
        return protocols.trace_distance_bound(self.sub_gate)

    def _circuit_diagram_info_(self,
                               args: protocols.CircuitDiagramInfoArgs
                               ) -> protocols.CircuitDiagramInfo:
        sub_info = protocols.circuit_diagram_info(self.sub_gate, args, None)
        if sub_info is None:
            return NotImplemented
        return protocols.CircuitDiagramInfo(
            wire_symbols=('@',) + sub_info.wire_symbols,
            exponent=sub_info.exponent)

    def __str__(self):
        return 'C' + str(self.sub_gate)

    def __repr__(self):
        return 'cirq.ControlledGate(sub_gate={!r})'.format(self.sub_gate)

max_stars_repo_name: sboixo/Cirq

max_issues_repo_name: sboixo/Cirq

max_forks_repo_name: sboixo/Cirq


text:
import os
import sys
root_path = os.path.abspath("../../../../")
if root_path not in sys.path:
    sys.path.append(root_path)

import numpy as np
import matplotlib.pyplot as plt

from Util.Util import DataUtil
from _Dist.NeuralNetworks._Tests.Pruner.Advanced import Advanced

# (x, y), (x_test, y_test), *_ = DataUtil.get_dataset("mnist", "_Data/mnist.txt", n_train=1600, quantized=True)
(x, y), (x_test, y_test) = DataUtil.gen_noisy_linear(n_dim=100, n_valid=5, one_hot=False)

data_info = {
    "numerical_idx": [True] * 100 + [False],
    "categorical_columns": []
}

# nn = Advanced(
#     "NoisyLinear",
#     data_info=data_info,
#     model_param_settings={
#         "n_epoch": 40
#     },
#     model_structure_settings={
#         "use_wide_network": False,
#         "use_pruner": True,
#         "pruner_params": {
#             "prune_method": "surgery",
#             # "alpha": 1e-8,
#             # "beta": 1e8
#         },
#         "hidden_units": [512]
#     }
# ).fit(x, y, x_test, y_test, snapshot_ratio=0)
nn = Advanced("NoisyLinear", data_info=data_info).fit(x, y, x_test, y_test, snapshot_ratio=0)

def normalize(arr):
    arr = np.array(arr, np.float32)
    arr -= arr.mean()
    arr /= arr.std()
    return arr

plt.figure()
pruned_ratio = normalize(nn.log["pruned_ratio"])
recovered_ratio = normalize(nn.log["recovered_ratio"])
running_recovered = nn.log["running_recovered"]
w_abs_mean = nn.log["w_abs_mean"]
plt.plot(np.arange(len(pruned_ratio)), pruned_ratio, label="pruned_ratio")
plt.plot(np.arange(len(recovered_ratio)), recovered_ratio, label="recovered_ratio")
plt.plot(np.arange(len(w_abs_mean)), normalize(w_abs_mean), label="w_abs_mean")
plt.legend(loc=4, prop={'size': 14})
plt.show()
plt.figure()
plt.plot(np.arange(len(running_recovered)), running_recovered, label="running_recovered")
plt.legend(prop={'size': 14})
plt.show()
plt.figure()
w_abs_mean = nn.log["w_abs_mean"]
pruned_w_abs_mean = nn.log["pruned_w_abs_mean"]
org_survived_w_abs_mean = nn.log["org_w_abs_mean"]
org_pruned_w_abs_mean = nn.log["org_pruned_w_abs_mean"]
plt.plot(np.arange(len(w_abs_mean)), w_abs_mean, label="w_abs_mean")
plt.plot(np.arange(len(pruned_w_abs_mean)), pruned_w_abs_mean, label="pruned_w_abs_mean")
plt.plot(np.arange(len(org_pruned_w_abs_mean)), org_pruned_w_abs_mean, label="org_pruned_w_abs_mean")
plt.plot(np.arange(len(org_survived_w_abs_mean)), org_survived_w_abs_mean, label="org_survived_w_abs_mean")
plt.legend(prop={'size': 14})
plt.show()
plt.figure()
pruned_w_abs_residual = nn.log["pruned_w_abs_residual"]
plt.plot(np.arange(len(pruned_w_abs_residual)), pruned_w_abs_residual, label="pruned_w_abs_residual")
plt.legend(prop={'size': 14})
plt.show()
plt.figure()
target_ratio = nn.log["target_ratio"]
plt.plot(np.arange(len(target_ratio)), target_ratio, label="target_ratio")
plt.legend(prop={'size': 14})
plt.show()
plt.figure()
ptr = nn.log["pruned_target_ratio"]
plt.plot(np.arange(len(ptr)), ptr, label="pruned_target_ratio")
plt.legend(prop={'size': 14})
plt.show()
plt.figure()
rwwamr = np.array(nn.log["rwwamr"])
rwowamr = np.array(nn.log["rwowamr"])
rwnwamr = np.array(nn.log["rwnwamr"])
plt.plot(np.arange(len(rwwamr)), rwwamr, label="rwwamr")
plt.plot(np.arange(len(rwowamr)), rwowamr, label="rwowamr")
plt.plot(np.arange(len(rwnwamr)), rwnwamr, label="rwnwamr")
plt.legend(prop={'size': 14})
plt.show()
plt.figure()
accs = nn.log["acc"]
pwamr = np.array(nn.log["pruned_w_abs_mean"]) / np.array(nn.log["w_abs_mean"])
plt.plot(np.arange(len(pwamr)), pwamr, label="pruned_w_abs_mean_ratio")
plt.plot(np.arange(len(accs)), accs, label="acc")
plt.legend()
plt.show()
plt.figure()
pwamr = np.array(nn.log["pruned_w_abs_mean"]) / np.array(nn.log["w_abs_mean"])
plt.plot(np.arange(len(pwamr)), pwamr, label="pruned_w_abs_mean_ratio")
plt.legend(prop={'size': 14})
plt.show()

ext: py

max_stars_repo_name: leoatchina/MachineLearning

max_issues_repo_name: leoatchina/MachineLearning

max_forks_repo_name: leoatchina/MachineLearning

reason: import numpy


text:
from copy import deepcopy
from typing import Optional, List, Tuple

import numpy as np
import torch
from torch import Tensor
from tqdm import tqdm

from agent import Agent
from environment import Env
from memory import Memory

def test(agent: Agent, memory: Memory, mode: str, num_episodes: int, init_command: Optional[Tensor]=None) -> float:
  env = Env(mode)
  returns = []

  for setting_id in range(len(env.settings)):
    setting_returns = []

    for episode in tqdm(range(1, num_episodes + 1), leave=False):
      (observation, goal), terminal, total_reward = env.reset(setting=setting_id), False, 0
      command, hidden = agent.get_initial_command(goal, memory, mode) if init_command is None else deepcopy(init_command), None
      while not terminal:
        with torch.inference_mode(): policy, hidden = agent.observe(observation, command, hidden)
        action = policy.sample()
        next_observation, goal, reward, terminal = env.step(action)
        agent.update_command(observation, goal, action, reward, terminal, command, mode)
        total_reward += reward
        observation = next_observation
      setting_returns.append(total_reward)

    returns.append(np.mean(setting_returns))

  return returns

def calibration(agent: Agent, memory: Memory, mode: str, num_episodes: int) -> Tuple[List[float], List[float]]:
  achieved_returns, desired_returns = [], [100, 200, 300, 400, 500]
  command = agent.get_initial_command(torch.zeros(1, 1, 1), memory, mode)

  for desired_return in desired_returns:
    command['desired_horizon'], command['desired_return'] = torch.tensor([[[desired_return]]], dtype=torch.float32), torch.tensor([[[desired_return]]], dtype=torch.float32)  # Desired horizon is same as desired return for CartPole
    achieved_returns.append(test(agent, memory, mode, num_episodes, command)[0])

  return achieved_returns, desired_returns

max_stars_repo_name: Kaixhin/GUDRL

max_issues_repo_name: Kaixhin/GUDRL

max_forks_repo_name: Kaixhin/GUDRL


text:
from pathlib import Path

from networkx import relabel_nodes, DiGraph
import numpy as np
from pandas import read_csv

from benchmarks.utils.distance import distance

class AugeratNodePosition:
    """Stores coordinates of a node of Augerat's instances (set P)."""
    def __init__(self, values):
        # Node ID
        self.name = np.uint32(values[0]).item()
        if self.name == 1:
            self.name = "Source"
        # x coordinate
        self.x = np.float64(values[1]).item()
        # y coordinate
        self.y = np.float64(values[2]).item()

class AugeratNodeDemand:
    """Stores attributes of a node of Augerat's instances (set P)."""
    def __init__(self, values):
        # Node ID
        self.name = np.uint32(values[0]).item()
        if self.name == 1:
            self.name = "Source"
        # demand coordinate
        self.demand = np.float64(values[1]).item()

class AugeratDataSet:
    """Reads an Augerat instance and stores the network as DiGraph.

    Args:
        path (str) : Path to data folder.
        instance_name (str) : Name of instance to read.
    """
    def __init__(self, path: Path, instance_name):
        self.G: DiGraph = None
        self.best_known_solution: int = None
        self.best_value: float = None
        self.max_load: int = None

        path = Path(path)
        self._load(path, instance_name)

    def _load(self, path, instance_name):
        """Load Augerat instance into a DiGraph"""
        # Read vehicle capacity
        with open(path / instance_name) as fp:
            for i, line in enumerate(fp):
                if i == 1:
                    best = line.split()[-1][:-1]
                    self.best_known_solution = int(best)
                if i == 5:
                    self.max_load = int(line.split()[2])
        fp.close()
        # Create network and store name + capacity
        self.G = DiGraph(
            name=instance_name[:-4],
            vehicle_capacity=self.max_load,
        )

        # Read nodes from txt file
        if instance_name[5] == "-":
            n_vertices = int(instance_name[3:5])
        else:
            n_vertices = int(instance_name[3:6])
        df_augerat = read_csv(
            path / instance_name,
            sep="\t",
            skiprows=6,
            nrows=n_vertices,
        )
        # Scan each line of the file and add nodes to the network
        for line in df_augerat.itertuples():
            values = line[1].split()
            node = AugeratNodePosition(values)
            self.G.add_node(node.name, x=node.x, y=node.y, demand=0)
            # Add Sink as copy of Source
            if node.name == "Source":
                self.G.add_node("Sink", x=node.x, y=node.y, demand=0)

        # Read demand from txt file
        df_demand = read_csv(
            path / instance_name,
            sep="\t",
            skiprows=range(7 + n_vertices),
            nrows=n_vertices,
        )
        for line in df_demand.itertuples():
            values = line[1].split()
            node = AugeratNodeDemand(values)
            self.G.nodes[node.name]["demand"] = node.demand

        # Add the edges, the graph is complete
        for u in self.G.nodes():
            if u != "Sink":
                for v in self.G.nodes():
                    if v != "Source":
                        if u != v:
                            self.G.add_edge(u,
                                            v,
                                            cost=round(distance(self.G, u, v),
                                                       1))

        # relabel
        before = [v for v in self.G.nodes() if v not in ["Source", "Sink"]]
        after = [v - 1 for v in self.G.nodes() if v not in ["Source", "Sink"]]
        mapping = dict(zip(before, after))
        self.G = relabel_nodes(self.G, mapping)

max_stars_repo_name: Halvaros/vrpy

max_issues_repo_name: Halvaros/vrpy

max_forks_repo_name: Halvaros/vrpy

reason: import numpy,from networkx


text:
import pickle
import json
import gensim
import numpy
import scipy

BUCKET_NAME = 'ml-model-convergence'
MODEL_FILE_NAME = 'glove.6B.100d_model.sav'
MODEL_LOCAL_PATH = '/tmp/' + MODEL_FILE_NAME

def handler(event, context):
  print(event)
  positive = []
  negative = []
  if 'body' in event:
    event = json.loads(event['body'])

  for i in event["positive"]:
    positive.append(i)
  for i in event["negative"]:
    negative.append(i)
  print(positive)
  print(negative)
  try:
    response = load_model().most_similar(positive=positive,negative=negative, topn=5)
  except KeyError as e:
    return e
  return answer(response)

def load_model():
    return pickle.load(open(MODEL_FILE_NAME, 'rb'))

def answer (ans):
  response_object = {}
  response_object['statusCode'] = 200
  response_object['headers'] = {}
  response_object['headers']['Content-Type'] = "application/json"
  response_object['body'] = json.dumps(ans)
  return response_object

reason: import numpy,import scipy


text:
__author__ = 'Venkatarami Reddy'
import streamlit as st
import numpy as np
import pandas as pd
import cx_Oracle as Oracle

def db_config():
    engine = Oracle.makedsn(st.secrets['DB_HOST'], st.secrets['DB_PORT'], service_name=st.secrets['DB_SERVICE'])
    con = Oracle.connect(user=st.secrets['DB_USER'], password=st.secrets['DB_PASSWORD'], dsn=engine)
    return con

def first_app():
    connection = db_config()
    cur = connection.cursor()
    st.title('My First App')
    st.header('First App')
    st.subheader('My First App')
    st.button('Click to Submit', key='submit')
    st.checkbox('Check', value=False, key='check')
    st.radio('IDs', options=('ID1', 'ID2', 'ID3'))
    st.text('Employee records from database')
    db_nam = st.sidebar.selectbox('Database Names', ('Venkat', 'SMITH', 'KING'))
    st.write(db_nam)
    query = """SELECT 'EMPLOYEE NAME IS: ' || ENAME || ' AND HE IS EARNING ' || SAL || '$' FROM EMP
                WHERE ENAME = :db_nam"""
    res = cur.execute(query, (db_nam,)).fetchall()
    # st.write(res)
    for rec in res:
        st.write(rec)
    records = st.sidebar.selectbox('Employees', res)
    st.write(records)
    inp = st.text_input('Enter some value')
    st.write('You have entered: ', inp)
    inp_2 = st.text_area('Enter some text here:')
    st.write('You have entered: ', inp_2)
    names = st.sidebar.selectbox('Names', ('Venkat', 'Naga', 'Manishree'))
    st.write('U have selected', names)
    st.write('Data Below')
    st.write(pd.DataFrame({'First': [1, 2, 3],
                           'Second': [2, 3, 4],
                           'Third': [3, 4, 5],
                           }
                          )
             )

    data = pd.read_csv('C:\\Users\\Manishree\\OneDrive\\Desktop\\train.csv', header=0)
    st.write('Data from Desktop')
    st.write(data.loc[1:4, ['id', 'vendor_id']]
             )
    side = st.sidebar.selectbox('Which ID?', data.loc[2:5, 'id'].tolist())
    st.write(side)
    conditional_data = data.loc[data.loc[:, 'id'] == side, ['id', 'vendor_id', 'pickup_datetime', 'dropoff_datetime']]
    data_id, data_vendor_id, data_pick_date, data_drop_date = \
        conditional_data.id.values, conditional_data.vendor_id.values, conditional_data.pickup_datetime.values\
            , conditional_data.dropoff_datetime.values
    st.write(f'The ID You have selected {data_id} \n and it\'s vendor id is {data_vendor_id}\
                and pick up date time is {data_pick_date} and drop date time is {data_drop_date}')

    code = """def func():
    print('Its a function')
            """
    st.code(code, language='python')

    df = pd.DataFrame({'Col1': [1, 2, 3, 4], 'Col2': [2, 3, 4, 5]})
    st.dataframe(df)
    st.table(df)

first_app()


text:
"""Contains utility functions for the examples."""
import random

from typing import Any, Callable, Dict, Iterator, Optional, Tuple

import numpy as np

import spsa

def controlled_input_noise(f: Callable[[np.ndarray], float], /, shape: Tuple[int, ...], noise: float) -> Callable[[np.ndarray], float]:
    """Adds noise to the input before calling."""
    return spsa.with_input_noise(f, shape, noise)

def random_input_noise(f: Callable[[np.ndarray], float], /, shape: Tuple[int, ...], noise: float) -> Callable[[np.ndarray], float]:
    """Adds noise to the input before calling."""
    rng = np.random.default_rng()
    def wrapper(x: np.ndarray, /) -> float:
        return f(x + rng.uniform(-noise, noise, shape))
    return wrapper

HEADERS: Dict[Callable, str] = {
    controlled_input_noise: "{} with controlled input noise:",
    random_input_noise: "{} with random input noise:",
}

def run(f: Callable[[np.ndarray], float], x: Tuple[float], /, name: str, *args: Any, noise: float, **kwargs: Any) -> None:
    shape = np.shape(x)
    print(f"{name}:")
    x_ = spsa.minimize(f, x, *args, **kwargs)
    print(f"y = {f(x_)}")
    print(f"x = {x_}")
    print("-" * 80)
    del x_
    for wrapper, header in HEADERS.items():
        print(header.format(name))
        x_ = spsa.minimize(wrapper(f, shape, noise), x, *args, **kwargs)
        print(f"y = {f(x_)}")
        print(f"x = {x_}")
        print("-" * 80)
        del x_

max_stars_repo_name: SimpleArt/spsa

max_issues_repo_name: SimpleArt/spsa

max_forks_repo_name: SimpleArt/spsa


text:
#!/usr/bin/env python

#***********************************************************
#* Software License Agreement (BSD License)
#*
#*  Copyright (c) 2010, CSIRO Autonomous Systems Laboratory
#*  All rights reserved.
#*
#*  Redistribution and use in source and binary forms, with or without
#*  modification, are permitted provided that the following conditions
#*  are met:
#*
#*   * Redistributions of source code must retain the above copyright
#*     notice, this list of conditions and the following disclaimer.
#*   * Redistributions in binary form must reproduce the above
#*     copyright notice, this list of conditions and the following
#*     disclaimer in the documentation and/or other materials provided
#*     with the distribution.
#*   * Neither the name of the CSIRO nor the names of its
#*     contributors may be used to endorse or promote products derived
#*     from this software without specific prior written permission.
#*
#*  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#*  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#*  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
#*  FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
#*  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
#*  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
#*  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
#*  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
#*  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
#*  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
#*  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
#*  POSSIBILITY OF SUCH DAMAGE.
#***********************************************************

# Author: Brett Grandbois and Fred Pauling
#$Id: test_ldmrs.py 321 2010-10-28 08:44:15Z pau386 $

PKG = 'sick_ldmrs'

NODE = 'ldmrs'

import roslib; roslib.load_manifest(PKG)
import rostest

import sensor_msgs
from sensor_msgs.msg import PointCloud2, LaserScan,  PointField
from rospy.numpy_msg import numpy_msg # for numpy arrays in the messages
import dynamic_reconfigure.client

import unittest
import sys
import struct
import socket
import time
import math
import threading
import os
import numpy as np

from testhelpers import *
from sick_ldmrs.cfg import *
from sick_ldmrs.utils import *
from sick_ldmrs.params import *

class TestMsg(TestListenerHelper):
    """ Test that ROS messages from the sick ldmrs node
        are formatted correctly and contain the correct data.
        This is achieved by sending a message to the node via
        a socket and comparing the returned message to the original.
    """

    # stores the ros_message
    ros_msg = None
    ldmrs_msg = None
    timeout = 30.0 # seconds
    callback_event = threading.Event()

    # ROS subsriber callback
    def callback(self, data):
        self.ros_msg = data
        self.callback_event.set()

    # Test methods
    # would have like to use rospy.wait_for_message here
    # but need to subscribe to a topic before it will be published at all
    # so using traditional callbacks and thread events to achieve the same thing
    def test_cloud(self):
        cloud = rospy.Subscriber("/ldmrs/cloud", numpy_msg(PointCloud2), self.callback)
        self.get_msg()
        self.init_seq()
        self.send_ldmrs_msg()
        self.callback_event.wait(self.timeout)
        self.assertTrue(self.callback_event.isSet(),
                "Timeout waiting for cloud message")
        self.callback_event.clear()
        cloud.unregister()
        self.check_point_cloud()

    # 28 October 2010.
    # wanted to use numpy_msg deserialization for the scan messages
    # but there is a bug in the numpy.frombuffer method which
    # throws an exception when reading from a zero-length buffer
    # (even when count=0). This occurs because our intensities
    # field is empty. Using standard deserialization until
    # either numpy (ticket #1388) or ROS (ticket #4528) posts a fix.
    # Update 29 October: Looks like latest version of ROS fixes this error
    # but it is not available in the package repositories for Ubuntu yet

    def test_scan0(self):

        scan = rospy.Subscriber('/ldmrs/scan0', LaserScan, self.callback)
        #scan = rospy.Subscriber('/ldmrs/scan0', numpy_msg(LaserScan), self.callback)
        self.get_msg()
        self.init_seq()
        self.send_ldmrs_msg()
        self.callback_event.wait(self.timeout)
        self.assertTrue(self.callback_event.isSet(),
                "Timeout waiting for scan0 message")
        self.callback_event.clear()
        scan.unregister()
        self.check_scan(0)

    def test_scan1(self):

        scan = rospy.Subscriber('/ldmrs/scan1', LaserScan, self.callback)
        #scan = rospy.Subscriber('/ldmrs/scan1', numpy_msg(LaserScan), self.callback)
        self.get_msg()
        self.init_seq()
        self.send_ldmrs_msg()
        self.callback_event.wait(self.timeout)
        self.assertTrue(self.callback_event.isSet(),
                "Timeout waiting for scan1 message")
        self.callback_event.clear()
        scan.unregister()
        self.check_scan(1)

    def test_scan2(self):

        scan = rospy.Subscriber('/ldmrs/scan2', LaserScan, self.callback)
        #scan = rospy.Subscriber('/ldmrs/scan2', numpy_msg(LaserScan), self.callback)
        self.get_msg()
        self.init_seq()
        self.send_ldmrs_msg()
        self.callback_event.wait(self.timeout)
        self.assertTrue(self.callback_event.isSet(),
                "Timeout waiting for scan2 message")
        self.callback_event.clear()
        scan.unregister()
        self.check_scan(2)

    def test_scan3(self):

        scan = rospy.Subscriber('/ldmrs/scan3', LaserScan, self.callback)
        #scan = rospy.Subscriber('/ldmrs/scan3', numpy_msg(LaserScan), self.callback)
        self.get_msg()
        self.init_seq()
        self.send_ldmrs_msg()
        self.callback_event.wait(self.timeout)
        self.assertTrue(self.callback_event.isSet(),
                "Timeout waiting for scan3 message")
        self.callback_event.clear()
        scan.unregister()
        self.check_scan(3)

    @classmethod
    def get_msg(self):
        """ Get a single ldmrs scan data message from a file
            The file contains a single message only
        """
        msg_file = os.path.join(ldmrs_pkg_dir,'test_msg.ldmrs')
        f = open(msg_file,  'r')
        self.ldmrs_msg = f.read(-1)

    def send_ldmrs_msg(self):
        """ Send the test message to the LDMRS
        """
        self.conn.sendall(self.ldmrs_msg)

    def unpack_ldmrs_msg(self):
        """ Unpack the raw ldmrs message into a format suitable for comparison
            against the returned ROS message
        """

        self.assertTrue(self.ros_msg is not None, "Subscriber callback did not produce a message")

        # Extract LDMRS message fields for comparison
        #===================================================================
        self.data_msg_header_ld = struct.unpack_from('<HHHQQHhhHhhhhhhH', self.ldmrs_msg,  offset = 24)
        uint16le = np.dtype(np.uint16, align='<') # force little endian
        ldmrs_data = np.frombuffer(self.ldmrs_msg, dtype=uint16le, offset = 24 + 44)
        self.start_time_ld = self.data_msg_header_ld[3]
        self.end_time_ld = self.data_msg_header_ld[4]
        self.start_angle_ld = self.data_msg_header_ld[6]
        self.end_angle_ld = self.data_msg_header_ld[7]

        # check that the data has the appropriate size
        self.assertTrue(ldmrs_data.size%5.0 == 0.0,  "Message does not contain a whole number of points")

        ldmrs_data = ldmrs_data.reshape(-1,  5)

        self.num_points_ld = ldmrs_data.shape[0]
        self.layer_echo_flags_ld16 = ldmrs_data[:, 0] & np.array([0b0000101111111111])
        self.layer_ld = (self.layer_echo_flags_ld16 & np.array([0b0000000000001111])).astype(np.uint8)
        self.echo_ld = ((self.layer_echo_flags_ld16 & np.array([0b0000000011110000])) >> 4).astype(np.uint8)
        self.h_angle_ticks_ld = ldmrs_data[:, 1].astype(np.int16)
        self.ticknum_ld = -(self.h_angle_ticks_ld - self.start_angle_ld)
        self.r_dist_ld = ldmrs_data[:, 2]
        self.echo_w_ld = ldmrs_data[:, 3]

        #convert h_angle_ticks_ld to time deltas for comparison to pc message
        self.delta_angle_ld = abs(self.end_angle_ld - self.start_angle_ld)

        self.start_time_ld = NTP64_to_ROStime(self.start_time_ld)
        self.end_time_ld = NTP64_to_ROStime(self.end_time_ld)
        self.duration_ld = self.end_time_ld - self.start_time_ld

        self.time_increment_ld = self.duration_ld.to_sec()/float(self.delta_angle_ld)
        self.tick_num_ld = np.abs(self.h_angle_ticks_ld - self.start_angle_ld)
        self.time_deltas_ld = self.tick_num_ld * self.time_increment_ld

    def check_point_cloud(self):
        """ Check that an ldmrs scan message and the resulting
            PointCloud2 are the same (within some tolerance)
        """

        self.unpack_ldmrs_msg()
        point_cloud = self.ros_msg

        # Extract fields from point_cloud message
        #=================================================================
        pc_data = np.frombuffer(point_cloud.data, dtype=np.uint8)

        time_stamp = point_cloud.header.stamp

        # check that the data has the appropriate size
        self.assertTrue(pc_data.size%19.0 == 0.0,  "Message does not contain a whole number of points")

        pc_data = pc_data.reshape(-1, 19)
        num_points_pc = pc_data.shape[0]
        x = pc_data[:,0:4].copy().view(np.float32).T
        y = pc_data[:,4:8].copy().view(np.float32).T
        z = pc_data[:,8:12].copy().view(np.float32).T
        time_deltas_pc = pc_data[:, 12:16].copy().view(np.float32).T
        echo_w_pc = pc_data[:, 16:18].copy().view(np.uint16).T
        layer_echo_flags_pc = pc_data[:, 18].copy().T

        # Convert PointCloudMessage fields to LDMRS message format
        # =================================================================
        r_dist_pc = np.round(((x**2 + y**2 + z**2)**0.5)*100).astype(np.int16)
        theta_pc = np.arctan2(y, x)
        rad2ticks = 11520/(2*np.pi)
        h_angle_ticks_pc = np.round(theta_pc*rad2ticks).astype(np.int16)

        # Pull out echo_pc layer_pc and flags_pc fields from echo_layer_flags_pc
        # need to copy so we can view as two byte arrays
        layer_pc = layer_echo_flags_pc & np.array([0b00000011], dtype=np.uint8)
        echo_pc  = (layer_echo_flags_pc & np.array([0b00001100], dtype=np.uint8)) >> 2
        layer_echo_pc = layer_pc | (echo_pc << 4)

        flags_pc = (layer_echo_flags_pc & np.array([0b01110000], dtype=np.uint8)) >> 4
        flags_pc = flags_pc | ((flags_pc & np.array([0b00000100], dtype=np.uint8)) << 1)
        flags_pc = flags_pc & np.array([0b11111011], dtype=np.uint8)

        layer_echo_flags_pc16 = layer_echo_pc.astype(np.uint16) | (flags_pc.astype(np.uint16) << 8)

        # Compare LDMRS-message and point cloud, these should be (practically) identical
        #====================================================================

        # check point fields are formatted correctly
        fields = point_cloud.fields
        field_names = [f.name for f in fields]
        expected_field_names = ['x', 'y', 'z', 'timedelta', 'echowidth', 'layerechoflags']
        self.assertTrue(field_names == expected_field_names,
            "PointFields: Expected: field names == %s, but got %s"%(str(expected_field_names),  str(field_names)))
        offsets = [f.offset for f in fields]
        expected_offsets = [0, 4, 8, 12, 16, 18]
        self.assertTrue(offsets == expected_offsets,
            "PointFields: Expected: offsets == %s, but got %s"%(str(expected_offsets),  str(offsets)))
        counts = [f.count for f in fields]
        expected_counts = [1, 1, 1, 1, 1, 1]
        self.assertTrue(counts == expected_counts,
            "PointFields: Expected: counts == %s, but got %s"%(str(expected_counts),  str(counts)))

        # check point_step is correct
        self.assertEqual(point_cloud.point_step, 19,
            "Expected point_step to be 19, but got %s"%str(point_cloud.point_step))
        # check row_step is correct
        self.assertEqual(point_cloud.row_step, num_points_pc * 19,
            "Expected point_step to be %s, but got %s"%(str(num_points_pc * 19), str(point_cloud.point_step)))
        # check is_dense is false
        self.assertFalse(point_cloud.is_dense,
            "Expected is_dense to be False")

        # check width
        self.assertEqual(point_cloud.width, num_points_pc,
            "Expected width to be %d, but got %s"%(num_points_pc, point_cloud.width))
        #check height
        self.assertEqual(point_cloud.height, 1,
            "Expected width to be 1, but got %s"%str(point_cloud.height))

        #TODO: need to check for big-endian / little endian

        # Check number of points is OK first
        self.assertTrue(self.num_points_ld == num_points_pc,
            "Expected number of points to be equal but got ldmrs: %d, point_cloud: %d"%(self.num_points_ld, num_points_pc))

        # Check radial distances are identical
        self.assertFalse(np.any(r_dist_pc - self.r_dist_ld),
            "Radial distances do not match between ldmrs message and PointCloud message")

        # Check layer_echo_flags are identical
        self.assertFalse(np.any(layer_echo_flags_pc16 - self.layer_echo_flags_ld16),
            "Layer/Echo/Flags do not match between ldmrs message and PointCloud message")

        # Check echo widths are identical
        self.assertFalse(np.any(echo_w_pc - self.echo_w_ld),
            "Echo widths do not match between ldmrs message and PointCloud message")

        # Check h_angle ticks are identical
        self.assertFalse(np.any(h_angle_ticks_pc - self.h_angle_ticks_ld),
            "Sample angles do not match between ldmrs message and PointCloud message")

        # Check time deltas are within tolerance of 1 microsecond
        self.assertFalse(np.any(np.abs(self.time_deltas_ld - time_deltas_pc) > 1e-6),
            "Time deltas do not match between ldmrs message and PointCloud message")

        #check frame_id
        self.assertEqual(point_cloud.header.frame_id,  '/ldmrs')

    def check_scan(self,  scan_num):
        """ Check that the scan is formatted correctly and contains correct data
            as compared to the original message.
            @param scan_num: the number of the scan plane to check, valid values are in {0,1,2,3}
            @type scan_num: int
        """
        self.unpack_ldmrs_msg()
        laser_scan = self.ros_msg

        # for 50Hz sampling rate:
        # adjust start angle to reflect offset of two samples for first beam
        scan_freq = rospy.get_param('/ldmrs/scan_frequency')
        if(scan_freq == 2):
            tick_adjust = 2
            start_time_adjust = 2.0/(50.0 * 11520)
        else:
            tick_adjust = 0
            start_time_adjust = 0

        start_angle_ld = self.data_msg_header_ld[6] - tick_adjust
        end_angle_ld = self.data_msg_header_ld[7] - tick_adjust

        # Need to convert ldmrs message closer to LaserScan format
        # since we lose so much information compared to the PointCloud2 format
        # note that this is a different implementation of the layer/echo extraction
        # process than the dataproc module in order to help prevent any systemic errors
        layer_inds = (self.layer_ld == scan_num).nonzero()[0]
        echoes = self.echo_ld[layer_inds]
        ticknums = self.ticknum_ld[layer_inds]
        ranges = self.r_dist_ld[layer_inds]

        #No need to sort further here since device already sorts by ticknum and echo

        # get indices of first/last echoes
        use_first_echo = rospy.get_param('/ldmrs/use_first_echo')
        breaks = np.diff(ticknums).nonzero()[0]
        if use_first_echo:
            inds = np.hstack((np.array([0]), breaks + 1))
        else:
            inds = np.hstack((breaks,  np.array(ranges.size -1)))

        # extract first/last echo samples
        ticknums = ticknums[inds]
        ranges = ranges[inds]

        # downsample ticknum to index in array based on frequency
        angle_range = abs(start_angle_ld - end_angle_ld)
        if scan_freq == 2:
            # Device quirk, need to shift ticknum down by 2 before downsampling
            tick_inds = (ticknums - 2)/16
            array_size = angle_range/16
        elif scan_freq == 1:
            tick_inds = ticknums/8
            array_size = angle_range/8
        else:  # scan_freq S== 12.5
            tick_inds = ticknums/4
            array_size = angle_range/4

        scan_ranges = np.zeros(array_size)
        scan_ranges[tick_inds] = ranges/100.0  # convert to m

        # Compare ldmrs message data with scan data
        #====================================================

        n_points = scan_ranges.size

        # check that the ranges are identical and are in the right location within the array
        # this will work regardless of the
        ros_msg_ranges = np.array(laser_scan.ranges)
        self.assertFalse(np.any(np.abs(ros_msg_ranges - scan_ranges) > 1e-4),
                "LaserScan ranges do not match ranges from ldmrs for scan plane %d"%scan_num)

        # check that the intensity field  is empty (since the ldmrs does not have intensity values)
        self.assertEqual(np.array(laser_scan.intensities).size,  0,
                "Require intensities field to be empty, but got non_empty")

        # check frame_id
        expected_frame_id = "/ldmrs" + str(scan_num)
        self.assertEqual(laser_scan.header.frame_id,  expected_frame_id,
                "Expected frame id to be: %s, but got %s"%(expected_frame_id, laser_scan.header.frame_id))

        # check angles match up
        rad2tick = 11520/(2*np.pi)

        start_angle = (laser_scan.angle_min)* rad2tick  # max angle is start
        delta_start_angle = abs(start_angle_ld - start_angle)
        start_angle_tol = 1
        self.assertTrue(delta_start_angle < start_angle_tol,
                   "Start angles differ more than the allowed tolerance of %s ticks"%str(start_angle_tol))

        end_angle = laser_scan.angle_max * rad2tick  # min angle is end
        end_angle_tol = 1
        delta_end_angle = abs(end_angle_ld - end_angle)
        self.assertTrue(delta_end_angle < end_angle_tol,
                   "End angles differ more than the allowed tolerance of %s ticks"%str(start_angle_tol))

        end_angle_incremental = start_angle + n_points*laser_scan.angle_increment*rad2tick
        inc_angle_tol = 1
        delta_end_angle_inc = abs(end_angle_incremental - end_angle)
        self.assertTrue(delta_end_angle_inc < inc_angle_tol,
                    "Incrementally computed end angle differs more than the allowed tolerance of %s ticks from the true value"%str(inc_angle_tol))

if __name__ == '__main__':

    ldmrs_pkg_dir = os.path.join(roslib.packages.get_pkg_dir('sick_ldmrs'),  'test')
    rospy.init_node('ldmrs_test',  disable_rostime=True)
    rostest.rosrun(PKG, 'ldmrs_msg', TestMsg)
    listener.close()

max_stars_repo_name: davidtvs/calibration-old

max_issues_repo_name: davidtvs/calibration-old

max_forks_repo_name: davidtvs/calibration-old


text:
import glob
import logging
import numpy as np
from spexxy.utils.fits import bulk_read_header

from spexxy.data import FitsSpectrum, SpectrumFits

def add_parser(subparsers):
    """
    Adds 'spexxy tellurics mean' command

    :param subparsers:  Subparser to attach new one to.
    """

    # init parser
    parser = subparsers.add_parser('mean', help='Calculates the mean tellurics from a list of fits.')
    parser.set_defaults(func=run)

    # calculate mean
    parser.add_argument('spectra', help='List of processed spectra', type=str, nargs='+')
    parser.add_argument('-o', '--output', help='File to write tellurics to', type=str, default="tellurics.fits")
    parser.add_argument('-l', '--snlimit', help='minimum SNR to use', type=float)
    parser.add_argument('-f', '--snfrac', help='if snlimit is not given, calculate it from X percent best',
                        type=float, default=10)
    parser.add_argument('-w', '--weight', help='weight by SNR', action="store_true")
    parser.add_argument('-r', '--resample', help='resample to start,step,count', nargs=3,
                        default=(None, None, None), type=float)

def run(args):
    """
    Takes a list of spectra and calculates the mean of the fitted tellurics.

    :param args:    argparse namespace with
                    .spectra:   List of files containing spectra.
                    .output:    Output file for mean tellurics
                                (default: tellurics.fits)
                    .snlimit:   Only calculate mean tellurics for spectra
                                with a S/N higher than the given number.
                    .weight:    If set, tellurics are weightes by the S/N
                                of their spectra.
    """

    # get all spectra
    filenames = []
    for s in args.spectra:
        if '*' in s:
            filenames.extend(glob.glob(s))
        else:
            filenames.append(s)

    # read headers
    logging.info('Reading all FITS headers...')
    headers = bulk_read_header(filenames, ['HIERARCH SPECTRUM SNRATIO'])

    # get all snr values
    logging.info('Extracting all S/N values...')
    snrs = {}
    for i, row in headers.iterrows():
        try:
            snrs[row['FILE']] = float(row['HIERARCH SPECTRUM SNRATIO'])
        except ValueError:
            continue

    # no snlimit given?
    if args.snlimit is not None:
        snlimit = args.snlimit
    else:
        tmp = sorted(snrs.values())
        logging.info('Calculating S/N limit from %.2f%% highest S/N values...', args.snfrac)
        snlimit = tmp[-int(len(tmp)*args.snfrac/100)]
    logging.info('Using S/N limit of %.2f...', snlimit)

    # filter by snlimit
    logging.info('Filtering spectra by S/N limit...')
    spectra = [filename for filename, snr in snrs.items() if snr > snlimit]

    # tellurics spectrum
    tellurics = None
    weights = None
    count = None
    wave_start, wave_step, wave_count = args.resample
    if wave_count is not None:
        wave_count = int(wave_count)

    # loop files
    logging.info('Processing %d spectra...', len(spectra))
    for i, spec in enumerate(spectra, 1):
        # open file
        with FitsSpectrum(spec, 'r') as fs:
            # get signal to noise
            snr = fs.header["HIERARCH SPECTRUM SNRATIO"]

            # print
            logging.info('(%d/%d) %s %-5.2f', i, len(spectra), spec, snr)

            # weight
            weight = snr if args.weight else 1.

            # some more info
            if wave_start is None:
                # WAVE extension or CRVAL/CDELT?
                if 'CRVAL1' in fs.header and 'CDELT1' in fs.header:
                    wave_start = fs.header["CRVAL1"]
                    wave_step = fs.header["CDELT1"]
                    if fs.header['CUNIT1'] == 'm':
                        wave_start *= 1e10
                        wave_step *= 1e10
                    wave_count = fs.header['NAXIS1']
                elif 'WAVE' in fs.header and fs.header['WAVE'] in fs:
                    logging.error('Combining tellurics on PIXTABLE spectra not allowed without resampling.')
                    continue
                else:
                    logging.error('Could not determine wavelength grid.')
                    continue

            # get tellurics
            tell = fs.tellurics
            if not tell:
                continue

            # resample
            tell = tell.resample(wave_start=wave_start, wave_step=wave_step, wave_count=wave_count)

            # tellurics exist? on first iteration we create the array.
            if tellurics is None:
                tellurics = np.zeros((wave_count))
                weights = np.zeros((wave_count))
                count = np.zeros((wave_count))

            # add to sum
            w = np.where(~np.isnan(tell.flux))
            tellurics[w] += weight * tell.flux[w]
            weights[w] += weight
            count[w] += 1

    # divide by sum
    w = np.where(~np.isnan(tellurics) & ~np.isnan(weights))
    tellurics[w] /= weights[w]

    # create spectrum for tellurics and save it
    tell_spec = SpectrumFits(flux=tellurics, wave_start=wave_start, wave_step=wave_step, primary=True)
    tell_spec.save(args.output)

    # output
    logging.info("Finished successfully.")

max_stars_repo_name: thusser/spexxy

max_issues_repo_name: thusser/spexxy

max_forks_repo_name: thusser/spexxy


text:
# -*- coding: utf-8 -*-
import numpy as np
import chainer
from chainer import serializers
import chainerrl
from KEnv import KEnv
from KQFunc import QFunction

env = KEnv(True, ratio = 0.2)
obs_size = env.observation_space.n
n_actions = env.action_space.n
q_func = QFunction(obs_size, n_actions)

use_step = 2810013
if use_step > 0:
    serializers.load_npz(q_func.resDir + "/" + str(use_step) + "/model.npz", q_func)

optimizer = chainer.optimizers.Adam(eps=1e-2)
optimizer.setup(q_func) #設計したq関数の最適化にAdamを使う
gamma = 1
explorer = chainerrl.explorers.ConstantEpsilonGreedy(
    epsilon=0, random_action_func=env.action_space.sample)
replay_buffer = chainerrl.replay_buffer.ReplayBuffer(capacity = 10**6)
phi = lambda x:x.astype(np.float32, copy=False)##型の変換(chainerはfloat32型。float64は駄目)

agent = chainerrl.agents.DoubleDQN(
    q_func, optimizer, replay_buffer, gamma, explorer,
    replay_start_size=500, update_interval=1,
    target_update_interval=100, phi=phi)

back_vic = 0.
back_win = 0.
buy = 0.
nbuy = 0.
nvic = 0
nwin = 0
vic_list = []
win_list = []

observation = env.reset()
#while(env.race < env.races - 1):
for i in range(env.races * 15):
    action = agent.act(observation)
    observation, reward, done, info = env.step(action)
    if action == 1: #買う
        buy += 1
        if info["order"] == 1:
            back_vic += info["back_vic"]
            nvic += 1
            vic_list.append(info["back_vic"])

        if info["order"] <= 3:
            back_win += info["back_win"]
            nwin += 1
            win_list.append(info["back_win"])
    else:
        nbuy += 1

    if done:
        env.reset()

print "buy:{:d}/{:d}".format(int(buy), int(buy + nbuy))
print "vic back:{} ret:{:.2f}％ hit:{:.2f}％".format(back_vic, ((back_vic / buy) * 100) if buy != 0 else 100, ((nvic / buy) * 100) if buy != 0 else 0)
print "win back:{} ret:{:.2f}％ hit:{:.2f}％".format(back_win, ((back_win / buy) * 100) if buy != 0 else 100, ((nwin / buy) * 100) if buy != 0 else 0)
print vic_list
print win_list

max_stars_repo_name: ismale/knn

max_issues_repo_name: ismale/knn

max_forks_repo_name: ismale/knn


text:
from apertools import sario
import glob
import os
import numpy as np

def stitchtop(full, top):
    out = np.zeros((full.shape[0] + 600, full.shape[1]), dtype=full.dtype)
    out[-full.shape[0] :, :] = full
    out[: top.shape[0], 600:] = top
    return out

def stitchtop_bool(full, top):
    out = np.ones((full.shape[0] + 600, full.shape[1]), dtype=full.dtype)
    out[-full.shape[0] :, :] = full
    out[: top.shape[0], 600:] = top
    return out

def loop(ext):
    for topf in glob.glob("*" + ext):
        if os.path.exists("../../stitched/igrams/{}".format(topf)):
            print("skip", topf)
            continue

        try:
            top = sario.load(topf, return_amp=True)
            rest = sario.load("../../allpath85/igrams/{}".format(topf), return_amp=True)
        except FileNotFoundError:
            continue
        sario.save("../../stitched/igrams/{}".format(topf), stitchtop(rest, top))
        print("saved", topf)

def loop_cc():
    for topf in glob.glob("*cc"):
        if os.path.exists("../../stitched/igrams/{}".format(topf)):
            print("skip", topf)
            continue

        try:
            amptop, top = sario.load(topf, return_amp=True)
            amprest, rest = sario.load(
                "../../allpath85/igrams/{}".format(topf), return_amp=True
            )
        except FileNotFoundError:
            continue
        sario.save(
            "../../stitched/igrams/{}".format(topf),
            np.stack((stitchtop(amprest, amptop), stitchtop(rest, top)), axis=0),
        )
        print("saved", topf)

def loop_bool(ext=".geo.mask"):
    for topf in glob.glob("*" + ext):
        if os.path.exists("../../stitched/igrams/{}".format(topf)):
            print("skip", topf)
            continue

        try:
            top = sario.load(topf, return_amp=True)
            rest = sario.load("../../allpath85/igrams/{}".format(topf), return_amp=True)
        except FileNotFoundError:
            continue
        sario.save("../../stitched/igrams/{}".format(topf), stitchtop_bool(rest, top))
        print("saved", topf)

def loop_geo(ext=".geo"):
    for topf in glob.glob("*" + ext):
        if os.path.exists("../stitched/{}".format(topf)):
            print("skip", topf)
            continue

        try:
            top = sario.load(topf, return_amp=True)
            rest = sario.load("../allpath85/{}".format(topf))
        except FileNotFoundError:
            continue
        sario.save("../stitched/{}".format(topf), stitchtop(rest, top))
        print("saved", topf)

max_stars_repo_name: scottstanie/insar

max_issues_repo_name: scottstanie/insar

max_forks_repo_name: scottstanie/insar


text:
import cv2 as cv
import numpy as np

# loads images
haystack_img = cv.imread('starcraftBase.jpg', cv.IMREAD_UNCHANGED)
needle_img = cv.imread('commandCenter.jpg', cv.IMREAD_UNCHANGED)

# finds imagine in haystack
result = cv.matchTemplate(haystack_img, needle_img, cv.TM_CCOEFF_NORMED)

# best match positions
min_val, max_val, min_loc, max_loc = cv.minMaxLoc(result)
print('Best match top left position: ', str(max_loc))
print('Best match confidence: ', max_val)

threshold = 0.8
if max_val >= threshold:
    print('Needle found')

    # get the dimensions of the image
    needle_w = needle_img.shape[1]
    needle_h = needle_img.shape[0]

    top_left = max_loc
    bottom_right = (top_left[0] + needle_w, top_left[1] + needle_h)

    # draws box around object
    cv.rectangle(haystack_img, top_left, bottom_right, color = (0,255,0), thickness = 2, lineType = cv.LINE_4)
    cv.imshow('Result', haystack_img)
    cv.waitKey()
else:
    print('Needle not found')


text:
#-*-encoding:utf-8-*-
import re
import csv
import numpy as np
import jieba
import jieba.analyse
datas = []
with open("DXYNews.csv",'r',encoding='utf-8') as f:
    reader = csv.DictReader(f)
    for row in reader:
        date = re.search(r'(\d{4}/\d{1,2}/\d{1,2})',row['pubDate'])
        date = date.group(0)
        print(date)
        data = ([date,row['title'],row['summary']])
        datas.append(data)
    datas = np.array(datas)

np.save("datas.npy",datas)

datas = np.load('datas.npy')

date_title_dic = {}
date_summary_dic = {}

print(datas.shape)

titles = ''
summary = ''
index = 0
for i in range(datas.shape[0]):

    # print(index)

    if datas[index][0] != datas[i][0]:
        print('index: ' + str(index) + ' ' + datas[index][0])
        print('i: ' + str(i) + ' ' + datas[i][0])
        for j in range(index,i):
            titles = titles + datas[j][1] + ' '
            summary = summary + datas[j][2] + ' '
        print('titles: ' + titles)
        print('summary: ' + summary)
        date_title_dic[datas[index][0]] = titles
        date_summary_dic[datas[index][0]] = summary
        index = i
        titles = ''
        summary = ''
print('date_title_dic: ' , date_title_dic)
print('date_summary_dic: ' , date_summary_dic)
np.save('date_title_dic.npy',date_title_dic)
np.save('date_summary_dic.npy',date_summary_dic)


text:
import numpy as np
# Keras
from keras.applications.inception_v3 import InceptionV3
from keras.applications.vgg16 import VGG16
from keras.models import Model
from keras.layers import Input, Dense, Dropout, LSTM, Embedding, concatenate, RepeatVector, TimeDistributed, Bidirectional
from keras.preprocessing.sequence import pad_sequences
from tqdm import tqdm
# To measure BLEU Score
from nltk.translate.bleu_score import corpus_bleu

"""
	*Define the CNN model
"""
def CNNModel(model_type):
	if model_type == 'inceptionv3':
		model = InceptionV3()
	elif model_type == 'vgg16':
		model = VGG16()
	model.layers.pop()
	model = Model(inputs=model.inputs, outputs=model.layers[-1].output)
	return model

"""
	*Define the RNN model
"""
def RNNModel(vocab_size, max_len, rnnConfig, model_type):
	embedding_size = rnnConfig['embedding_size']
	if model_type == 'inceptionv3':
		# InceptionV3 outputs a 2048 dimensional vector for each image, which we'll feed to RNN Model
		image_input = Input(shape=(2048,))
	elif model_type == 'vgg16':
		# VGG16 outputs a 4096 dimensional vector for each image, which we'll feed to RNN Model
		image_input = Input(shape=(4096,))
	image_model_1 = Dropout(rnnConfig['dropout'])(image_input)
	image_model = Dense(embedding_size, activation='relu')(image_model_1)

	caption_input = Input(shape=(max_len,))
	# mask_zero: We zero pad inputs to the same length, the zero mask ignores those inputs. E.g. it is an efficiency.
	caption_model_1 = Embedding(vocab_size, embedding_size, mask_zero=True)(caption_input)
	caption_model_2 = Dropout(rnnConfig['dropout'])(caption_model_1)
	caption_model = LSTM(rnnConfig['LSTM_units'])(caption_model_2)

	# Merging the models and creating a softmax classifier
	final_model_1 = concatenate([image_model, caption_model])
	final_model_2 = Dense(rnnConfig['dense_units'], activation='relu')(final_model_1)
	final_model = Dense(vocab_size, activation='softmax')(final_model_2)

	model = Model(inputs=[image_input, caption_input], outputs=final_model)
	model.compile(loss='categorical_crossentropy', optimizer='adam')
	return model

"""
	*Define the RNN model with different architecture
"""
def AlternativeRNNModel(vocab_size, max_len, rnnConfig, model_type):
	embedding_size = rnnConfig['embedding_size']
	if model_type == 'inceptionv3':
		# InceptionV3 outputs a 2048 dimensional vector for each image, which we'll feed to RNN Model
		image_input = Input(shape=(2048,))
	elif model_type == 'vgg16':
		# VGG16 outputs a 4096 dimensional vector for each image, which we'll feed to RNN Model
		image_input = Input(shape=(4096,))
	image_model_1 = Dense(embedding_size, activation='relu')(image_input)
	image_model = RepeatVector(max_len)(image_model_1)

	caption_input = Input(shape=(max_len,))
	# mask_zero: We zero pad inputs to the same length, the zero mask ignores those inputs. E.g. it is an efficiency.
	caption_model_1 = Embedding(vocab_size, embedding_size, mask_zero=True)(caption_input)
	# Since we are going to predict the next word using the previous words
	# (length of previous words changes with every iteration over the caption), we have to set return_sequences = True.
	caption_model_2 = LSTM(rnnConfig['LSTM_units'], return_sequences=True)(caption_model_1)
	# caption_model = TimeDistributed(Dense(embedding_size, activation='relu'))(caption_model_2)
	caption_model = TimeDistributed(Dense(embedding_size))(caption_model_2)

	# Merging the models and creating a softmax classifier
	final_model_1 = concatenate([image_model, caption_model])
	# final_model_2 = LSTM(rnnConfig['LSTM_units'], return_sequences=False)(final_model_1)
	final_model_2 = Bidirectional(LSTM(rnnConfig['LSTM_units'], return_sequences=False))(final_model_1)
	# final_model_3 = Dense(rnnConfig['dense_units'], activation='relu')(final_model_2)
	# final_model = Dense(vocab_size, activation='softmax')(final_model_3)
	final_model = Dense(vocab_size, activation='softmax')(final_model_2)

	model = Model(inputs=[image_input, caption_input], outputs=final_model)
	model.compile(loss='categorical_crossentropy', optimizer='adam')
	# model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
	return model

"""
	*Map an integer to a word
"""
def int_to_word(integer, tokenizer):
	for word, index in tokenizer.word_index.items():
		if index == integer:
			return word
	return None

"""
	*Generate a caption for an image, given a pre-trained model and a tokenizer to map integer back to word
	*Uses simple argmax
"""
def generate_caption(model, tokenizer, image, max_length):
	# Seed the generation process
	in_text = 'startseq'
	# Iterate over the whole length of the sequence
	for _ in range(max_length):
		# Integer encode input sequence
		sequence = tokenizer.texts_to_sequences([in_text])[0]
		# Pad input
		sequence = pad_sequences([sequence], maxlen=max_length)
		# Predict next word
		# The model will output a prediction, which will be a probability distribution over all words in the vocabulary.
		yhat = model.predict([image,sequence], verbose=0)
		# The output vector representins a probability distribution where maximum probability is the predicted word position
		# Take output class with maximum probability and convert to integer
		yhat = np.argmax(yhat)
		# Map integer back to word
		word = int_to_word(yhat, tokenizer)
		# Stop if we cannot map the word
		if word is None:
			break
		# Append as input for generating the next word
		in_text += ' ' + word
		# Stop if we predict the end of the sequence
		if word == 'endseq':
			break
	return in_text

"""
	*Generate a caption for an image, given a pre-trained model and a tokenizer to map integer back to word
	*Uses BEAM Search algorithm
"""
def generate_caption_beam_search(model, tokenizer, image, max_length, beam_index=3):
	# in_text --> [[idx,prob]] ;prob=0 initially
	in_text = [[tokenizer.texts_to_sequences(['startseq'])[0], 0.0]]
	while len(in_text[0][0]) < max_length:
		tempList = []
		for seq in in_text:
			padded_seq = pad_sequences([seq[0]], maxlen=max_length)
			preds = model.predict([image,padded_seq], verbose=0)
			# Take top (i.e. which have highest probailities) `beam_index` predictions
			top_preds = np.argsort(preds[0])[-beam_index:]
			# Getting the top `beam_index` predictions and
			for word in top_preds:
				next_seq, prob = seq[0][:], seq[1]
				next_seq.append(word)
				# Update probability
				prob += preds[0][word]
				# Append as input for generating the next word
				tempList.append([next_seq, prob])
		in_text = tempList
		# Sorting according to the probabilities
		in_text = sorted(in_text, reverse=False, key=lambda l: l[1])
		# Take the top words
		in_text = in_text[-beam_index:]
	in_text = in_text[-1][0]
	final_caption_raw = [int_to_word(i,tokenizer) for i in in_text]
	final_caption = []
	for word in final_caption_raw:
		if word=='endseq':
			break
		else:
			final_caption.append(word)
	final_caption.append('endseq')
	return ' '.join(final_caption)

"""
	*Evaluate the model on BLEU Score using argmax predictions
"""
def evaluate_model(model, images, captions, tokenizer, max_length):
	actual, predicted = list(), list()
	for image_id, caption_list in tqdm(captions.items()):
		yhat = generate_caption(model, tokenizer, images[image_id], max_length)
		ground_truth = [caption.split() for caption in caption_list]
		actual.append(ground_truth)
		predicted.append(yhat.split())
	print('BLEU Scores :')
	print('A perfect match results in a score of 1.0, whereas a perfect mismatch results in a score of 0.0.')
	print('BLEU-1: %f' % corpus_bleu(actual, predicted, weights=(1.0, 0, 0, 0)))
	print('BLEU-2: %f' % corpus_bleu(actual, predicted, weights=(0.5, 0.5, 0, 0)))
	print('BLEU-3: %f' % corpus_bleu(actual, predicted, weights=(0.3, 0.3, 0.3, 0)))
	print('BLEU-4: %f' % corpus_bleu(actual, predicted, weights=(0.25, 0.25, 0.25, 0.25)))

"""
	*Evaluate the model on BLEU Score using BEAM search predictions
"""
def evaluate_model_beam_search(model, images, captions, tokenizer, max_length, beam_index=3):
	actual, predicted = list(), list()
	for image_id, caption_list in tqdm(captions.items()):
		yhat = generate_caption_beam_search(model, tokenizer, images[image_id], max_length, beam_index=beam_index)
		ground_truth = [caption.split() for caption in caption_list]
		actual.append(ground_truth)
		predicted.append(yhat.split())
	print('BLEU Scores :')
	print('A perfect match results in a score of 1.0, whereas a perfect mismatch results in a score of 0.0.')
	print('BLEU-1: %f' % corpus_bleu(actual, predicted, weights=(1.0, 0, 0, 0)))
	print('BLEU-2: %f' % corpus_bleu(actual, predicted, weights=(0.5, 0.5, 0, 0)))
	print('BLEU-3: %f' % corpus_bleu(actual, predicted, weights=(0.3, 0.3, 0.3, 0)))
	print('BLEU-4: %f' % corpus_bleu(actual, predicted, weights=(0.25, 0.25, 0.25, 0.25)))

max_stars_repo_name: thatrobotdev/Image-Caption-Generator

max_issues_repo_name: thatrobotdev/Image-Caption-Generator

max_forks_repo_name: thatrobotdev/Image-Caption-Generator


text:
# Mutual Information

import numpy as _np

from ._utils import (reshape as _reshape, get_bottom as _get_bottom)
from ..utils import Drawing as _Drawing

def _log2(a):
    return _np.log2(a+1e-15)

def calc_mutual_info(x, y, N, **options):
    H_x = calc_entropy(x, N, **options)
    H_y = calc_entropy(y, N, **options)
    H_xy = calc_joint_entropy(x, y, N, **options)
    return H_x + H_y - H_xy

def calc_entropy(x, N, **options):
    hist, _ = _np.histogram(x, **options)
    ps = hist/N
    H_x = -_np.sum(ps*_log2(ps))
    return H_x

def calc_joint_entropy(x, y, N, **options):
    hist, _, _ = _np.histogram2d(x, y, **options)
    ps = (hist/N).flatten()
    H_xy = -_np.sum(ps*_log2(ps))
    return H_xy

def mutual_info(u_seq, tau):
    u_seq = _reshape(u_seq)

    N, dim = u_seq.shape

    miss = []
    for i in range(dim):
        us = u_seq[:, i]
        mis = [calc_mutual_info(us[:-t], us[t:], N, bins=32)
               for t in range(1, tau+1)]
        miss.append(mis)

    return _np.array(miss).T

def est_lag_w_mi(u_seq, tau_max, plot=True, path_save_plot=None):
    mi_seq = mutual_info(u_seq, tau_max)
    bottom = _get_bottom(mi_seq)

    if plot:
        d = _Drawing()
        for i, idx in enumerate(bottom):
            d[0,0].plot(mi_seq[:, i], label=f'{i}',
                        marker='.', markersize=5, zorder=5)
            d[0,0].scatter(idx, mi_seq[idx, i], s=70, color='red', zorder=10)

        d[0,0].set_axis_label('Time~lag', 'Mutual~Information~[bit]')
        d[0,0].set_xlim(0, tau_max-1)
        d[0,0].legend()
        if path_save_plot is not None:
            d.save(path_save_plot)
        d.show()
        d.close()

    return list(bottom)

max_stars_repo_name: llbxg/hundun

max_issues_repo_name: llbxg/hundun

max_forks_repo_name: llbxg/hundun


text:
"""
    >>> from pybrain.tests.helpers import epsilonCheck
    >>> from pybrain.tools.functions import tanh
    >>> from pybrain.utilities import fListToString
    >>> from test_peephole_lstm import predictOutcome
    >>> from scipy import arctanh
    >>> from random import random

Test the MDLSTMLayer behavior when using peepholes.

    >>> N = buildMinimalMDLSTMNetwork()
    >>> N.params[:] = [.3,.4,.5]

    >>> s1 = 0.4
    >>> s2 = 0.414
    >>> s3 = -0.305
    >>> big = 10000

Set the state to s1
    >>> predictOutcome(N, [big, big, arctanh(s1), -big], 0)
    True

Verify that the state is conserved
    >>> predictOutcome(N, [-big, big, big*random(), big], tanh(s1))
    True

Add s2 to the state
    >>> predictOutcome(N, [big, big, arctanh(s2), big], tanh(s1+s2))
    True

Verify the peephole connection to the forgetgate (weight = .4) by neutralizing its contibution
and therefore dividing the state value by 2
    >>> predictOutcome(N, [-big, -(s1+s2) * .4, big*random(), big], tanh((s1+s2)/2))
    True

Verify the peephole connection to the inputgate (weight = .3) by neutralizing its contibution
and therefore dividing the provided input by 2. Also clearing the old state.
    >>> predictOutcome(N, [-(s1+s2)/2 * .3, -big, arctanh(s3), big], tanh(s3/2))
    True

Verify the peephole connection to the outputgate (weight = .5) by neutralizing its contibution
and therefore dividing the provided output by 2. Also clearing the old state.
    >>> predictOutcome(N, [-big, big, big*random(), -s3/2 * .5], tanh(s3/2)/2)
    True

List all the states again, explicitly (buffer size is 8 by now).
    >>> fListToString(N['mdlstm'].outputbuffer[:,1], 3)
    '[  0.400  0.400  0.814  0.407 -0.152 -0.152  0.000  0.000]'

"""

__author__ = 'Tom Schaul, tom@idsia.ch'

from pybrain3.tests import runModuleTestSuite
from pybrain3.structure import LinearLayer, IdentityConnection, MDLSTMLayer, RecurrentNetwork

def buildMinimalMDLSTMNetwork():
    N = RecurrentNetwork('simpleMdLstmNet')
    i = LinearLayer(4, name = 'i')
    h = MDLSTMLayer(1, peepholes = True, name = 'mdlstm')
    o = LinearLayer(1, name = 'o')
    N.addInputModule(i)
    N.addModule(h)
    N.addOutputModule(o)
    N.addConnection(IdentityConnection(i, h, outSliceTo = 4))
    N.addRecurrentConnection(IdentityConnection(h, h, outSliceFrom = 4, inSliceFrom = 1))
    N.addConnection(IdentityConnection(h, o, inSliceTo = 1))
    N.sortModules()
    return N

if __name__ == "__main__":
    runModuleTestSuite(__import__('__main__'))

reason: from scipy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Utilities regarding text placement in images.
Supplements OpenCV's putText()
"""
import cv2
import numpy as np

__all__ = ["putTextCenter", "putTextAutoscale"]

def putTextCenter(img, txt, coords, fontFace, fontScale=1.,
                  color=(0,0,0), thickness=2, hshift=0., vshift=0., baseline_shift=0.):
    """
    Like cv2.putText(), but auto-centers text on the given coordinates
    by computing size via cv2.getTextSize() and shifting the coordinates by that amount.

    Additional horizontal and vertical shifts can be performed by using nonzero
    hshift/vshift parameters.

    The actual text center is used as a reference, not the text baseline.
    However, baseline_shift, multiplied by the baseline, can be added to
    the value. Setting baseline_shift=1. will result in one (baseline - bottom)
    difference being added to the y coordinate, resulting in the image being shifted up.

    Parameters
    ----------
    img : numpy ndarray
        The image that is passed to OpenCV
    txt : str
        The text to render
    coords : (int, int)
        The coordinates where the text center should be placed
    fontFace
        The fontFace parameter that is passed to OpenCV
    fontScale
        The fontScale parameter that is passed to OpenCV
    color
        The font color that will be rendered
    thickness
        The rendering thickness
    hshift : float
        Horizontal shift, will be added to the x coordinate of the render position
    vshift : float
        Horizontal shift, will be added to the x coordinate of the render position
    baseline_shift : float
        Factor of the (baseline - bottom) y difference that will be added to the
        y coordinate of the rendering position
    """
    (w,h), baseline = cv2.getTextSize(txt, fontFace, fontScale, thickness)
    coords = (int(round(coords[0] - w/2) + hshift),
              int(round(coords[1] + h/2) + vshift + baseline_shift * baseline))
    cv2.putText(img, txt, coords, fontFace, fontScale, color, thickness)

def putTextAutoscale(img, txt, coords, fontFace, w, h, heightFraction=0.5, widthFraction=0.95,
                     maxHeight=60, color=(0,0,0), thickness=2, hshift=0., vshift=0., baseline_shift=0.):
    """
    Like cv2.putText(), but auto-centers text on the given coordinates and automatically
    chooses the text size for a (w*h) box around the center so that it consumes heightFraction*h height.
    Also ensures that the text consumes no more than widthFraction*w space horizontally,
    but in any case the text will not consume more than maxHeight pixels vertically.

    This is done by computing size via cv2.getTextSize() and shifting and scaling appropriately.

    Parameters
    ----------
    img : numpy ndarray
        The image that is passed to OpenCV
    txt : str
        The text to render
    coords : (int, int)
        The coordinates where the text center should be placed
    fontFace
        The fontFace parameter that is passed to OpenCV
    w : float
        The width of the box to place the text into
    h : float
        The height of the box to place the text into
    heightFraction : float
        Represents the maximum fraction of the height of the box
        that will be occupied by the text box
    widthFraction : float
        Represents the maximum fraction of the width of the box
        that will be occupied by the text box
    maxHeight : int
        The maximum height of the text box in pixels
    color
        The font color that will be rendered
    thickness
        The rendering thickness
    hshift : float
        Horizontal shift, will be added to the x coordinate of the render position
    vshift : float
        Horizontal shift, will be added to the x coordinate of the render position
    baseline_shift : float
        Factor of the (baseline - bottom) y difference that will be added to the
        y coordinate of the rendering position
    """
    # Get first size estimate. We will scale fractionally.
    (scale1Width, scale1Height), baseline = cv2.getTextSize(txt, fontFace, 1., thickness)
    # What are the different maximum scales that are mandated by the width and height
    heightFractionScale = (heightFraction * h) / scale1Height
    widthFractionScale = (widthFraction * w) / scale1Width
    absMaxScale = maxHeight / scale1Height
    # Combine height scale, width scale and absolute max height
    newScale = min(heightFractionScale, widthFractionScale, absMaxScale)
    # Now place text using putTextCenter()
    putTextCenter(img, txt, coords, fontFace, newScale,
                    color, thickness, hshift, vshift, baseline_shift)


text:
import numpy as np
import xarray as xr

# GD commented out the next line to avoid relative import issue.
#from utils.data_cube_utilities.dc_time import \
#    _n64_datetime_to_scalar, _scalar_to_n64_datetime
# GD added the next line to remove relative import.
from dc_time import \
    _n64_datetime_to_scalar, _scalar_to_n64_datetime

def get_bin_intervals(data, num_bins):
    """
    Returns bin intervals for 1D data.

    Parameters
    ----------
    data: np.ndarray
        A 1D NumPy array of values to get bin intervals for.
    num_bins: int
        The number of bins to create.

    Returns
    -------
    bin_intervals: np.ndarray of shape (num_bins, 2)
        A 2D NumPy array of bin intevals, with each row being one bin,
        with the first value being the lower bound for the bin and
        the second being the upper bound for the bin.
    """
    # Transition points between bins.
    bin_trans = np.linspace(data[0], data[-1], num_bins+1, endpoint=True)
    bin_intervals = np.empty((num_bins, 2), dtype=data.dtype)
    for i in range(num_bins):
        bin_intervals[i, :] = [bin_trans[i], bin_trans[i+1]]
    return bin_intervals

def xr_scale_res(dataset, x_coord='longitude', y_coord='latitude',
                 frac_res=None, abs_res=None):
    """
    Scales the resolution of an `xarray.Dataset` or `xarray.DataArray`
    to a fraction of its original resolution or an absolute resolution.

    Parameters
    ----------
    dataset: xarray.Dataset or xarray.DataArray
        The Dataset or DataArray to reduce the resolution of.
    x_coord, y_coord: str
        Names of the x and y coordinates in `dataset` to scale.
    frac_res: float
        The fraction of the original resolution to scale to. Must be postive.
        Note that this can be greater than 1.0, in which case the resolution
        is upsampled.
    abs_res: list-like
        A list-like of the number of pixels for the x and y axes, respectively.
        Overrides `frac_res` if specified.

    Returns
    -------
    dataset_scaled: xarray.Dataset or xarray.DataArray
        The result of scaling the resolution of `dataset`.

    Raises
    ------
    AssertionError: If neither `frac_res` nor `abs_res` is specified.
    """
    assert frac_res is not None or abs_res is not None, \
        "Either frac_res or abs_res must be specified (i.e. not None)."
    if frac_res is not None:
        x_px = y_px = np.sqrt(frac_res)
        interp_param = 'frac'
    elif abs_res is not None:
        interp_param = 'num'
        x_px, y_px = abs_res
    return xr_interp(dataset, {x_coord: ('interp', {interp_param: x_px}), \
                               y_coord: ('interp', {interp_param: y_px})})

def xr_sel_time_by_bin(dataset, num_bins, time_coord='time'):
    """
    Selects time coordinates by nearest neighbors of the means of bins.
    This is useful for plotting data with high variance in temporal
    spacing between acquisitions.

    Parameters
    ----------
    dataset: xarray.Dataset or xarray.DataArray
        The Dataset or DataArray to aggregate by binning.
        Must have a 'time' coordinate of type `datetime64`.
    num_bins: int
        The number of bins to use.
    time_coord: str
        The name of the time coordinate to bin.

    Returns
    -------
    result: xarray.Dataset or xarray.DataArray
        The result of aggregating within bins for the binned data.
    """
    return xr_interp(dataset, {time_coord: ('bin', {'num': num_bins})})

def xr_interp(dataset, interp_config):
    """
    Interpolates an `xarray.Dataset` or `xarray.DataArray`.
    This is often done to match dimensions between xarray objects or
    downsample to reduce memory consumption.

    First, coordinates are interpolated according to `interp_config`.
    Then the data values for those interpolated coordinates are obtained
    through nearest neighbors interpolation.

    Parameters
    ----------
    dataset: xarray.Dataset or xarray.DataArray
        The Dataset or DataArray to interpolate.
    interp_config: dict
        Mapping of names of coordinates to 2-tuples of the interpolation types
        to use for those coordinates and the parameters for those interpolation types.
        The supported coordinate interpolation types are 'interp' for
        linear interpolation and 'bin' for binning.
        The parameters, with supported interpolation types annotated to their
        left, are as follow:
        ('interp', 'bin'): 'frac':
            The fraction of the original size to use. Exclusive with 'num'.
        ('interp', 'bin'): 'num':
            The number of points in the output. Exclusive with 'frac'.
            Either 'frac' or 'num' must be in the interpolation parameters.
        The following is an example value:
        `{'latitude':('interp',{'frac':0.5}),'longitude':('interp',{'frac':0.5}),
          'time':('bin',{'num':20})}`.

    Returns
    -------
    interp_data: xarray.Dataset or xarray.DataArray
        The specified interpolation of `dataset`.

    :Authors:
        John Rattz (john.c.rattz@ama-inc.com)
    """
    # Create the new coordinates.
    new_coords = {}
    for dim, (interp_type, interp_kwargs) in interp_config.items():
        # Determine the number of points to use.
        num_pts = interp_kwargs.get('num', None)
        if num_pts is None:
            frac = interp_kwargs.get('frac', None)
            num_pts_orig = len(dataset[dim])
            num_pts = round(num_pts_orig * frac)
        dim_vals = dataset[dim].values
        dim_dtype = type(dim_vals[0])
        # Convert NumPy datetime64 objects to scalars.
        if dim_dtype == np.datetime64:
            dim_vals = np.array(list(map(_n64_datetime_to_scalar, dim_vals)))
        interp_vals = None
        # Interpolate coordinates.
        if interp_type == 'bin':
            bin_intervals = get_bin_intervals(dim_vals, num_pts)
            interp_vals = np.mean(bin_intervals, axis=1)
        if interp_type == 'interp':
            interp_inds = np.linspace(0, len(dim_vals) - 1, num_pts, dtype=np.int32)
            interp_vals = dim_vals[interp_inds]
        # Convert scalars to NumPy datetime64 objects.
        if dim_dtype == np.datetime64:
            interp_vals = np.array(list(map(_scalar_to_n64_datetime, interp_vals)))
        new_coords[dim] = interp_vals
    # Nearest-neighbor interpolate data values.
    interp_data = dataset.interp(coords=new_coords, method='nearest')
    # xarray.Dataset.interp() converts to dtype float64, so cast back to the original dtypes.
    if isinstance(dataset, xr.DataArray):
        interp_data = interp_data.astype(dataset.dtype)
    elif isinstance(dataset, xr.Dataset):
        for data_var_name in interp_data.data_vars:
            interp_data[data_var_name] = interp_data[data_var_name].astype(dataset[data_var_name].dtype)
    return interp_data


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Sep 18 16:37:00 2017

Ing.,Mgr. (MSc.) Jan Cimbálník, PhD.
Biomedical engineering
International Clinical Research Center
St. Anne's University Hospital in Brno
Czech Republic
&
Mayo systems electrophysiology lab
Mayo Clinic
200 1st St SW
Rochester, MN
United States
"""

# Std imports

# Third pary imports
from PyQt5.QtCore import pyqtSignal
from PyQt5.QtWidgets import (QDialog, QGridLayout, QLineEdit, QComboBox,
                             QLabel, QPushButton, QFrame, QHBoxLayout,
                             QVBoxLayout, QListWidget)
from PyQt5.QtGui import QIntValidator, QDoubleValidator

import numpy as np

# Local imports

class ConditionDialog(QDialog):

    condition_created = pyqtSignal(str, name='condition_created')

    def __init__(self, columns, condition):
        super().__init__()

        self.columns = columns

        # Strip df part
        condition = condition.replace('self.parent().df', '')

        # Strip brackets and quotes
        condition = condition.replace('["', '')
        condition = condition.replace('"]', '')
        condition = condition.replace('[\'', '')
        condition = condition.replace('\'[', '')

        self.layout = QGridLayout(self)

        line = QFrame()
        line.setFrameShape(QFrame.HLine)
        line.setFrameShadow(QFrame.Sunken)

        self.condition_le = QLineEdit(condition)

        self.layout.addWidget(QLabel('Pandas-like condition string'), 0, 0)
        self.layout.addWidget(self.condition_le, 0, 1, 1, 3)
        self.layout.addWidget(line, 1, 0, 1, 4)

        # ----- Labels -----
        self.layout.addWidget(QLabel('Column:'), 2, 0)
        self.layout.addWidget(QLabel('Math operation (optional):'), 2, 1)
        self.layout.addWidget(QLabel('Operator:'), 2, 2)
        self.layout.addWidget(QLabel('Condition value:'), 2, 3)

        # Rolldown column selector
        column_box = QComboBox(self)
        for col in self.columns:
            column_box.addItem(col)
        self.layout.addWidget(column_box, 3, 0)

        # Optional mathematical operation on column
        self.layout.addWidget(QLineEdit(), 3, 1)

        # Rolldown comparison oeprators
        operator_box = QComboBox(self)
        operator_box.addItem('<')
        operator_box.addItem('>')
        operator_box.addItem('==')
        operator_box.addItem('!=')
        self.layout.addWidget(operator_box, 3, 2)

        # Editline value input
        self.layout.addWidget(QLineEdit(), 3, 3)

        # OK / cancel button
        self.button_layout = QHBoxLayout()
        ok_btn = QPushButton('OK')
        ok_btn.clicked.connect(self.construct_condition_string)
        ok_btn.clicked.connect(self.accept)

        clc_btn = QPushButton('Cancel')
        clc_btn.clicked.connect(self.reject)

        self.button_layout.addWidget(ok_btn)
        self.button_layout.addWidget(clc_btn)

        self.layout.addLayout(self.button_layout, 4, 0, 1, 4)

        self.setLayout(self.layout)

    def construct_condition_string(self):
        if self.condition_le.text():
            cond_string = self.condition_le.text()

            for col in self.columns:

                col_i = 0
                cl_pos = cond_string.find(col)
                while cl_pos >= 0:
                    corr_pos = col_i + cl_pos
                    cond_string = (cond_string[:corr_pos+len(col)]
                                   + '"]'
                                   + cond_string[corr_pos+len(col):])
                    cond_string = (cond_string[:corr_pos]
                                   + 'self.parent().df["'
                                   + cond_string[corr_pos:])
                    col_i += corr_pos + len('self.parent().df["'+col+'"]') + 1
                    cl_pos = cond_string[col_i:].find(col)

        else:
            cond_string = ''

        print(cond_string)

        self.condition_created.emit(cond_string)

        return

class HistogramDialog(QDialog):

    histogram_strs_created = pyqtSignal(list, name='histogram_strs_created')

    def __init__(self, df):
        super().__init__()

        self.df = df
        columns = list(self.df.columns)
        self.edge_list = []

        self.layout = QGridLayout(self)

        # ----- Labels -----
        self.layout.addWidget(QLabel('Column:'), 0, 0)
        self.layout.addWidget(QLabel('Number of bins'), 0, 1)

        # Rolldown column selector
        self.column_box = QComboBox(self)
        for col in columns:
            self.column_box.addItem(col)
        self.column_box.currentIndexChanged.connect(self.set_bins)
        self.layout.addWidget(self.column_box, 1, 0)

        # Number of bins

        self.N_bins_le = QLineEdit(str(1))
        onlyInt = QIntValidator()
        self.N_bins_le.setValidator(onlyInt)
        self.N_bins_le.textEdited.connect(self.set_bins)
        self.layout.addWidget(self.N_bins_le, 1, 1)

        # Edge editor layout
        self.edge_edit_layout = QVBoxLayout()
#        self.edge_edit_layout.addWidget(QLineEdit())
        self.layout.addLayout(self.edge_edit_layout, 2, 0, 1, 2)

        # OK / cancel button
        self.button_layout = QHBoxLayout()
        ok_btn = QPushButton('OK')
        ok_btn.clicked.connect(self.construct_histogram_strings)
        ok_btn.clicked.connect(self.accept)

        clc_btn = QPushButton('Cancel')
        clc_btn.clicked.connect(self.reject)

        self.button_layout.addWidget(ok_btn)
        self.button_layout.addWidget(clc_btn)

        self.layout.addLayout(self.button_layout, 3, 0, 1, 2)

        self.setLayout(self.layout)
        self.set_bins()

    def set_bins(self):

        if not self.N_bins_le.text():
            return

        # Remove from layout
        for bin_le in self.edge_list:
            self.edge_edit_layout.removeWidget(bin_le)
            bin_le.setParent(None)

        # Get the data from the column
        series = self.df[self.column_box.currentText()]
        bins = np.histogram(series[~series.isnull()],
                            int(self.N_bins_le.text()))[1]
        self.edge_list = []
        self.layout.setRowStretch(2, len(bins))
        for bin_i in bins:
            le = QLineEdit(str(bin_i))
            onlyDouble = QDoubleValidator()
            le.setValidator(onlyDouble)
            self.edge_edit_layout.addWidget(le)
            self.edge_list.append(le)

    def construct_histogram_strings(self):

        col = self.column_box.currentText()

        # Get edges
        edge_arr = np.empty(len(self.edge_list))
        for i, edge in enumerate(self.edge_list[:-1]):
            edge_arr[i] = np.float(edge.text())

        eval_strs = self.construct_eval_strs(edge_arr, col)

        self.histogram_strs_created.emit(eval_strs)

        return

    def construct_eval_strs(self, edge_arr, col):

        # Sort
        edge_arr = np.sort(edge_arr)
        eval_strs = []
        for i in range(len(edge_arr)-1):
            bin_start = edge_arr[i]
            bin_stop = edge_arr[i+1]

            df_str = 'self.parent().df["'+col+'"]'
            left_edge = '('+df_str+' >= '+str(bin_start)+')'
            le_name = '['+str(bin_start)
            if i+1 < len(edge_arr):
                right_edge = '('+df_str + ' < '+str(bin_stop)+')'
                re_name = str(bin_stop) + ')'
            else:
                right_edge = '('+df_str + ' <= '+str(bin_stop)+')'
                re_name = str(bin_stop) + ']'

            eval_strs.append([left_edge + ' & ' + right_edge,
                              col+le_name+re_name])

        return eval_strs

class CathegoricalDialog(QDialog):

    categories_strs_created = pyqtSignal(list, name='categories_strs_created')

    def __init__(self, df):
        super().__init__()

        self.df = df
        columns = list(self.df.columns)
        self.edge_list = []

        self.layout = QGridLayout(self)

        # ----- Labels -----
        self.layout.addWidget(QLabel('Column:'), 0, 0)
        self.layout.addWidget(QLabel('Number of cathegories'), 1, 0)

        # Rolldown column selector
        self.column_box = QComboBox(self)
        for col in columns:
            self.column_box.addItem(col)
        self.column_box.currentIndexChanged.connect(self.set_cathegories)
        self.layout.addWidget(self.column_box, 0, 1)

        # Number of cathegories
        self.n_c_label = QLabel('')
        self.layout.addWidget(self.n_c_label, 1, 1)

        # List of cathegoreis
        self.cath_list = QListWidget(self)
        self.layout.addWidget(self.cath_list, 2, 0, 1, 2)

        # OK / cancel button
        self.button_layout = QHBoxLayout()
        ok_btn = QPushButton('OK')
        ok_btn.clicked.connect(self.construct_cathecoric_strings)
        ok_btn.clicked.connect(self.accept)

        clc_btn = QPushButton('Cancel')
        clc_btn.clicked.connect(self.reject)

        self.button_layout.addWidget(ok_btn)
        self.button_layout.addWidget(clc_btn)

        self.layout.addLayout(self.button_layout, 3, 0, 1, 2)

        self.setLayout(self.layout)
        self.set_cathegories()

    def set_cathegories(self):

        col = self.column_box.currentText()
        caths = self.df[col].unique()

        # Number of cathegories
        self.n_c_label.setText(str(len(caths)))

        # List of cathegories
        self.cath_list.clear()

        if len(caths) > 100:
            self.cath_list.addItem(str('N > 100, likely not cathegory'))
            return

        for i, cath in enumerate(caths):
            self.cath_list.addItem(str(cath))

    def construct_cathecoric_strings(self):

        col = self.column_box.currentText()

        if self.cath_list.count() < 2:
            return

        eval_strs = []
        for row_i in range(self.cath_list.count()):
            c_item = self.cath_list.item(row_i)
            cath = c_item.text()
            df_str = 'self.parent().df["'+col+'"]'

            try:
                float(cath)
                eval_strs.append([df_str + ' == ' + cath, cath])
            except ValueError:
                eval_strs.append([df_str + ' == "' + cath + '"', cath])

        self.categories_strs_created.emit(eval_strs)

max_stars_repo_name: ICRC-BME/PySigView

max_issues_repo_name: ICRC-BME/PySigView

max_forks_repo_name: ICRC-BME/PySigView