text:
"""
For License information see the LICENSE file.

Authors: Johannes Leupold, Amos Treiber

"""
from abc import ABC, abstractmethod
from logging import getLogger
from random import sample
from typing import Collection, Set, List, Iterator, Tuple, Any, Dict

import numpy as np

from .constants import Selectivity
from .dataset import Dataset, KeywordQueryLog
from .range_database import RangeDatabase

log = getLogger(__name__)

class QuerySpace(ABC, Collection):
    """
    A query space can be used to select a query sequence of arbitrary length (up to the size of the query space).

    It can be used to select multiple instances of queries.
    """

    @abstractmethod
    def create(self, *args, **kwargs):
        raise NotImplementedError

    @abstractmethod
    def select(self, n: int) -> Iterator:
        raise NotImplementedError

    @abstractmethod
    def __len__(self) -> int:
        raise NotImplementedError

    @abstractmethod
    def __iter__(self) -> Iterator:
        raise NotImplementedError

    @abstractmethod
    def __contains__(self, entry) -> bool:
        raise NotImplementedError

class KeywordQuerySpace(QuerySpace):
    __space: List[Set[Tuple[str, int]]]
    __allow_repetition: bool

    def __init__(self, full: Dataset, known: Dataset, selectivity: Selectivity, size: int,
                 query_log: KeywordQueryLog = None,
                 allow_repetition: bool = False):
        """
        Creates and populates the query space.

        Parameters
        ----------
        full : Dataset
            the full data set
        known : Dataset
            the known data set
        selectivity : Selectivity
            the selectivity of the keywords to use
        size : int
            the desired size of the query space
        query_log : QeryLog
            the query log of users
        allow_repetition : bool
            whether repetitions are allowed when drawing query sequences
        """
        self.__space: List[Set[Tuple[str, int]]] = []

        for i, candidate_keywords in enumerate(self._candidates(full, known, query_log)):
            if len(candidate_keywords) < size:
                log.warning(f"Set of candidate keywords with length {len(candidate_keywords)} at position {i} smaller "
                            f"than configured query space size of {size}. Requested selectivity ignored.")
                self.__space.append(candidate_keywords)
                continue
            if selectivity == Selectivity.High:
                self.__space.append(set(sorted(candidate_keywords, key=lambda item: full.selectivity(item[0]),
                                               reverse=True)[:size]))
            elif selectivity == Selectivity.Low:
                self.__space.append(set(sorted(candidate_keywords, key=lambda item: full.selectivity(item[0]))[:size]))
            elif selectivity == Selectivity.PseudoLow:
                self.__space.append(set(sorted(filter(lambda item: 10 <= full.selectivity(item[0]), candidate_keywords),
                                               key=lambda item: full.selectivity(item[0]))[:size]))
            elif selectivity == Selectivity.Independent:
                self.__space.append(set(sample(population=candidate_keywords, k=size)))

        self.__allow_repetition = allow_repetition

    @classmethod
    def create(cls, full: Dataset, known: Dataset, selectivity: Selectivity, size: int, query_log: KeywordQueryLog,
               allow_repetition: bool = False) -> 'KeywordQuerySpace':
        """
        Creates a query space.

        Parameters
        ----------
        full : Dataset
            the full data set
        known : Dataset
            the known data set
        selectivity : Selectivity
            the selectivity of the keywords to use
        size : int
            the desired size of the query space
        query_log : QeryLog
            the query log of users
        allow_repetition : bool
            whether repetitions are allowed when drawing query sequences

        Returns
        -------
        create : QuerySpace
            the created query space
        """
        return cls(full, known, selectivity, size, query_log, allow_repetition)

    def _get_space(self) -> Iterator[Set[Tuple[str, int]]]:
        yield from self.__space

    def select(self, n: int) -> Iterator[List[str]]:
        """
        Selects a query sequence of the desired length.

        Parameters
        ----------
        n : int
            the length of the query sequence

        Returns
        -------
        select : Iterator[List[str]]
            the selected queries per query space
        """
        # choice is expecting sequences with the same ordering here, hence the list(...)
        length = n
        for i, space in enumerate(self.__space):
            if len(space) == 0:
                log.warning(f"Encountered empty space ar position {i + 1} of {len(self.__space)}. "
                            f"Less evaluations than anticipated are performed.")
                continue
            if len(space) < n and not self.__allow_repetition:
                log.warning(
                    f"Encountered insufficiently large query space  with size {len(space)} at position {i + 1} of"
                    f" {len(self.__space)}.")
                length = len(space)
            space = list(space)
            p = np.array(list(map(lambda item: float(item[1]), space)))
            p /= p.sum()
            yield np.random.choice(list(map(lambda item: item[0], space)), length, p=p, replace=self.__allow_repetition)

    def __len__(self) -> int:
        return len(self.__space)

    def __iter__(self) -> Iterator[Iterator[str]]:
        for space in self.__space:
            yield from map(lambda item: item[0], iter(space))

    def __contains__(self, keyword: object) -> bool:
        return any([keyword in dict(space) for space in self.__space])

    @classmethod
    def is_multi_user(cls) -> bool:
        """Return True if multiple users are considered, False if a single user or queries aggregated from all users
        are considered."""
        return False

    @classmethod
    @abstractmethod
    def _candidates(cls, full: Dataset, known: Dataset, query_log: KeywordQueryLog) -> Iterator[Set[Tuple[str, int]]]:
        """
        Returns one or multiple sets of keyword candidates for populating the query space. Multiple sets can be used to,
        e.g., yield queries of individual users. Keyword candidates consist of the keyword and their frequency/weights
        for query selection.

        Parameters
        ----------
        full : Dataset
            the full data set
        known : Dataset
            the known data set
        query_log : QeryLog
            the query log of users
        """
        raise NotImplementedError

class RangeQuerySpace(QuerySpace):
    """
    A class to represent a QuerySpace for range queries.
    :param n: An upper bound on the number of queries to return. Returns all if = -1.
    The query space is re-created after each sampling if resample is set True
    """
    __queries: List[List[Tuple[int, int]]]
    __db: RangeDatabase
    __allow_repetition: bool
    __allow_empty: bool
    __kwargs: Dict[str, Any]
    __n: int
    __resample: bool

    def __init__(self, db: RangeDatabase, n: int = -1, allow_repetition: bool = True, allow_empty: bool = True,
                 resample: bool = True, **kwargs):
        self.__allow_repetition = allow_repetition
        self.__allow_empty = allow_empty
        self.__kwargs = kwargs
        self.__db = db
        self.__n = n
        self.__resample = resample

        self.__queries = self.gen_queries(self.__db, self.__n, self.__allow_repetition, self.__allow_empty,
                                          **self.__kwargs)

    @classmethod
    def create(cls, db: RangeDatabase, allow_repetition: bool = True, allow_empty: bool = True, **kwargs) \
            -> 'RangeQuerySpace':
        return cls(db, allow_repetition, allow_empty, **kwargs)

    def get_size(self) -> int:
        """
        Return the number of queries.
        :return: Number of queries
        """
        return self.__len__()

    def select(self, n: int = -1) -> Iterator[List[Tuple[int, int]]]:
        """
        Return n queries from the query space for each of its users.
        :param n: The number of queries to return. Returns all if = -1.
        :return: The queries
        """
        for queries in self.__queries:
            if n == -1 or n >= len(queries):
                res = queries
            else:
                res = sample(population=queries, k=n)
            yield res

        if self.__resample:
            self.__queries = self.gen_queries(self.__db, self.__n, self.__allow_repetition, self.__allow_empty,
                                              **self.__kwargs)

    @classmethod
    @abstractmethod
    def gen_queries(cls, db: RangeDatabase, n: int, allow_repetition: bool = False, allow_empty: bool = False,
                    **kwargs) -> List[List[Tuple[int, int]]]:
        """This implements the actual query space, creating a sequence of n queries according to a distribution,
        possibly sampled for multiple users"""
        raise NotImplementedError

    def __len__(self):
        return len(self.__queries)

    def __iter__(self):
        return iter(self.__queries)

    def __contains__(self, item):
        return item in self.__queries

max_stars_repo_name: encryptogroup/LEAKER

max_issues_repo_name: encryptogroup/LEAKER

max_forks_repo_name: encryptogroup/LEAKER


text:
import bisect
from collections import deque
from copy import deepcopy
from fractions import Fraction
from functools import reduce
import heapq as hq
import io
from itertools import combinations, permutations
import math
from math import factorial
import re
import sys
sys.setrecursionlimit(10000)

#from numba import njit
import numpy as np

_INPUT_1 = """\
3
4 1 5
3 10 100
"""
_INPUT_2 = """\
4
100 100 100 100
1 1 1 1
"""
_INPUT_3 = """\
4
1 2 3 4
1 2 4 7
"""

def solve():
    N = int(input())
    S = [int(x) for x in input().split()]
    T = [int(x) for x in input().split()]
    min_time = T[:]
    maxT = max(T)
    #distance[]

    cumsum = np.cumsum(S+S, axis=0).tolist()
    #print(cumsum)

    for i in range(N-1, -1, -1):
        for j in range(i-1, i-N,-1):
            if j < 1:
                diff_time = cumsum[N+i-1] - cumsum[N+j-1]
            else:
                diff_time = cumsum[i-1] - cumsum[j-1]
            #print(i, j, diff_time, T[i], T[j]+diff_time)
            min_time[i] = min(min_time[i], T[j]+diff_time)

    for i in range(N):
        print(min_time[i])

if __file__ != './Main.py':
    if '_INPUT_1' in globals():
        sys.stdin = io.StringIO(_INPUT_1)
        solve()
        print('====')
    if '_INPUT_2' in globals():
        sys.stdin = io.StringIO(_INPUT_2)
        solve()
        print('====')
    if '_INPUT_3' in globals():
        sys.stdin = io.StringIO(_INPUT_3)
        solve()
        print('====')
else:
    solve()

ext: py

reason: import numpy,from numba


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import glob
import os
import numpy as np

CALIBRATION_TYPES = ['convbike', 'ebike', 'etrike']
CALIBRATION_SPEEDS = np.array([0, 5, 10, 15, 20]) # kph
CALIBRATION_ANGLES = np.array([-60, -40, -20, 0, 20, 40, 60]) # degrees
CALIBRATION_STEER_SAMPLE_LENGTH = 120 # samples (1 sec @ 120 Hz)

def find_first(data, element):
    # find first element still not implemented in numpy
    i = np.argmax(data == element)
    if i == 0:
        assert data[i] == element
    return i

def find_last(data, element):
    return len(data) - 1 - find_first(data[::-1], element)

def load_file(filename):
    # line 2 missing time value in files:
    # data/calibration/ebike/speed/10kph.csv
    # data/calibration/etrike/steer_angle/3.csv
    return np.recfromcsv(filename, delimiter=',', invalid_raise=False)

def create_empty_config(calibration_types):
    config = {}
    for t in calibration_types:
        config[t] = {}
    return config

def get_calibration_config(data_path, calibration_types=None):
    # data_path is path to calibration data directory
    if calibration_types is None:
        ct = CALIBRATION_TYPES
    config = create_empty_config(ct)
    for t in ct:
        path = os.path.join(data_path, '{}/imu_static'.format(t))
        calibrate_imu(path, config[t])

        path = os.path.join(data_path, '{}/steer_angle'.format(t))
        calibrate_steer_angle(path, config[t])

        path = os.path.join(data_path, '{}/speed'.format(t))
        calibrate_speed(path, config[t])
    return config

def calibrate_imu(path, config):
    ''' path: path to directory containing csv file
        config: dictionary for a specific calibration type
                this dictionary is mutated
    '''
    # acceleration signals should show gravity in the +z direction
    pathname = os.path.join(path, '*.csv')
    filename = glob.glob(pathname)
    assert len(filename) == 1 # only one file for imu calibration
    r = load_file(filename[0])
    N = len(r) # number of samples
    names = r.dtype.names[3:]
    signals = ('accelerometer x',
               'accelerometer y',
               'accelerometer z',
               'gyroscope x',
               'gyroscope y',
               'gyroscope z')
    assert names == ('accx_g',
                     'accy_g',
                     'accz_g',
                     'gyrox_degs',
                     'gyroy_degs',
                     'gyroz_degs')
    for s, n in zip(signals, names):
        if n == 'accz_g':
            expected = 1 # in g
        else:
            expected = 0 # in g or deg/s
        if n.startswith('acc'):
            unit_factor = 9.81 # 1 g in m/s^2
        else:
            unit_factor = np.pi/180 # 1 deg/s in rad/s

        p0 = np.polyfit([expected] * N, r[n], 0)[0] # zero/1g offset
        if s == signals[2]:
            expected_values = [p0 - 1, p0] # [0, 1] g
        else:
            expected_values = [p0, p0 + 1] # [0, 1] g or deg/s
        print(s, expected_values)
        p = np.polyfit(expected_values, [0, unit_factor], 1)
        config[s] = p

def calibrate_steer_angle(path, config):
    ''' path: path to directory containing csv files
        config: dictionary for a specific calibration type
                this dictionary is mutated
    '''
    angles = CALIBRATION_ANGLES
    filenames = glob.glob(path)
    if len(filenames) < len(angles):
        # etrike configuration is missing the most extreme angles
        angles = angles[1:-1]
    steer_rad = angles * np.pi/180 # deg to rad
    steer_dac = []
    for i, a in enumerate(angles, 1):
        # steer angle not constant in files:
        # data/calibration/ebike/steer_angle/6.csv
        # data/calibration/ebike/steer_angle/7.csv
        # use last N samples
        filename = os.path.join(path, '{}.csv'.format(i))
        r = load_file(filename)
        data = r.steerangle_lsb
        assert len(data) > CALIBRATION_STEER_SAMPLE_LENGTH
        steer_dac.append(np.mean(data[-CALIBRATION_STEER_SAMPLE_LENGTH:]))
    p = np.polyfit(steer_dac, steer_rad, 1) # use linear fit
    config['steer angle'] = p

def calibrate_speed(path, config):
    ''' path: path to directory containing csv files
        config: dictionary for a specific calibration type
                this dictionary is mutated
    '''
    speed_mps = CALIBRATION_SPEEDS * 1000 / 3600 # km/hr to m/s
    speed_dac = [0] # start with 0 -> 0
    for v in CALIBRATION_SPEEDS[1:]:
        # We skip acceleration to the constant velocity value by using the
        # first index that matches the signal median
        # We also search the data in reverse in the event of deceleration at
        # the end of a run
        filename = os.path.join(path, '{}kph.csv'.format(v))
        r = load_file(filename)
        data = r.speed_lsb
        med = np.median(data)
        first = find_first(data, med)
        last = find_last(data, med)
        assert last >= first
        speed_dac.append(np.mean(data[first:last]))
    p = np.polyfit(speed_dac, speed_mps, 1) # use linear fit
    config['speed'] = p

if __name__ == '__main__':
    path = os.path.join(os.path.dirname(__file__),
                        r'../data/etrike/calibration')
    cfg1 = get_calibration_config(path)

    import pickle
    with open('config.p', 'wb') as f:
        pickle.dump(cfg1, f, protocol=pickle.HIGHEST_PROTOCOL)
    with open('config.p', 'rb') as f:
        cfg2 = pickle.load(f)

max_stars_repo_name: oliverlee/antlia

max_issues_repo_name: oliverlee/antlia

max_forks_repo_name: oliverlee/antlia

reason: import numpy


text:
from pyspark.mllib.clustering import KMeans
from numpy import array, random
from math import sqrt
from pyspark import SparkConf, SparkContext
from sklearn.preprocessing import scale

K = 5

# Boilerplate Spark stuff:
conf = SparkConf().setMaster("local").setAppName("SparkKMeans")
sc = SparkContext(conf = conf)

#Create fake income/age clusters for N people in k clusters
def createClusteredData(N, k):
    random.seed(10)
    pointsPerCluster = float(N)/k
    X = []
    for i in range (k):
        incomeCentroid = random.uniform(20000.0, 200000.0)
        ageCentroid = random.uniform(20.0, 70.0)
        for j in range(int(pointsPerCluster)):
            X.append([random.normal(incomeCentroid, 10000.0), random.normal(ageCentroid, 2.0)])
    X = array(X)
    return X

random.seed(0)

# Load the data; note I am normalizing it with scale() - very important!
data = sc.parallelize(scale(createClusteredData(100, K)))

# Build the model (cluster the data)
clusters = KMeans.train(data, K, maxIterations=10,
        runs=10, initializationMode="random")

# Print out the cluster assignments
resultRDD = data.map(lambda point: clusters.predict(point)).cache()

print("Counts by value:")
counts = resultRDD.countByValue()
print(counts)

print("Cluster assignments:")
results = resultRDD.collect()
print(results)

# Evaluate clustering by computing Within Set Sum of Squared Errors
def error(point):
    center = clusters.centers[clusters.predict(point)]
    return sqrt(sum([x**2 for x in (point - center)]))

WSSSE = data.map(lambda point: error(point)).reduce(lambda x, y: x + y)
print("Within Set Sum of Squared Error = " + str(WSSSE))

# Things to try:
# What happens to WSSSE as you increase or decrease K? Why?
# What happens if you don't normalize the input data before clustering?
# What happens if you change the maxIterations or runs parameters?

reason: from numpy


text:
# Piecewise-constant Neural ODEs
# Sam Greydanus, Stefan Lee, Alan Fern

import numpy as np
import torch

from .utils import ObjectView

def get_dataset_args(as_dict=False):
  arg_dict = {'num_samples': 10000,
              'train_split': 0.9,
              'r_range': [1., 2],
              'arcsize': 1.45 * 6.28,  # makes a sequence of length 30
              'dt': 0.3,
              'seed': 0}
  return arg_dict if as_dict else ObjectView(arg_dict)

def make_circle(radius=1, dt=0.3, domain=[0, 6.28]):
  start, stop = domain
  L = 2 * np.pi * radius * (stop-start)/(2*np.pi)
  theta = np.linspace(start,stop, int(L/dt))
  x = radius * np.stack([np.sin(theta), np.cos(theta)])
  return x

def get_dataset(args):
  np.random.seed(args.seed)  # random seed for reproducibility
  smallest_circle = make_circle(radius=args.r_range[0], dt=args.dt, domain=[0,args.arcsize])
  min_seq_len = smallest_circle.shape[-1]  # this is the shortest possible trajectory
  trajectories = []

  for i in range(args.num_samples):  # this loop generates the synthetic 'circles' dataset
    radius = args.r_range[0] + (args.r_range[1]-args.r_range[0]) * np.random.rand()  # random radius
    start_angle = 2 * np.pi * np.random.rand()                                       # random angle
    x = make_circle(radius=radius, dt=args.dt, domain=[start_angle, start_angle + args.arcsize])
    trajectories.append( x[...,:min_seq_len] )  # append trajectory to list

  xs = np.stack(trajectories).transpose(2,0,1)  # reshape tensor dimensions -> [time, batch, state]
  split_ix = int(args.num_samples*args.train_split) # train / test split
  dataset = {'x': xs[:, :split_ix], 'x_test': xs[:, split_ix:]}
  return dataset


text:
from __future__ import division
import torch
from torch import nn
import numpy as np
from demon_metrics import *

def compute_errors_test(gt, pred):
    gt = gt.numpy()
    pred = pred.numpy()
    # same scale
    scale = np.sum(gt) / np.sum(pred)
    pred = pred * scale
    n = float(np.size(gt))
    thresh = np.maximum((gt / pred), (pred / gt))
    a1 = (thresh < 1.25).mean()
    a2 = (thresh < 1.25 ** 2).mean()
    a3 = (thresh < 1.25 ** 3).mean()

    rmse = (gt - pred) ** 2
    rmse = np.sqrt(rmse.mean())

    rmse_log = (np.log(gt) - np.log(pred)) ** 2
    rmse_log = np.sqrt(rmse_log.mean())

    abs_diff = np.mean(np.abs(gt - pred))
    abs_rel = np.mean(np.abs(gt - pred) / gt)

    sq_rel = np.mean(((gt - pred) ** 2) / gt)
    pred = pred * scale
    l1_inv = np.mean(np.abs(np.reciprocal(gt) - np.reciprocal(pred)))
    pred = pred * scale
    log_diff = np.log(gt) - np.log(pred)
    sc_inv = np.sqrt(np.sum(np.square(log_diff)) / n - np.square(np.sum(log_diff)) / np.square(n))
    return abs_rel, abs_diff, sq_rel, rmse, rmse_log, a1, a2, a3, l1_inv, sc_inv

def compute_errors_train(gt, pred, valid):
    abs_diff, abs_rel, sq_rel, a1, a2, a3 = 0, 0, 0, 0, 0, 0
    batch_size = gt.size(0)

    for current_gt, current_pred, current_valid in zip(gt, pred, valid):
        valid_gt = current_gt[current_valid]
        valid_pred = current_pred[current_valid]

        if len(valid_gt) == 0:
            continue
        else:
            thresh = torch.max((valid_gt / valid_pred), (valid_pred / valid_gt))
            a1 += (thresh < 1.25).float().mean()
            a2 += (thresh < 1.25 ** 2).float().mean()
            a3 += (thresh < 1.25 ** 3).float().mean()

            abs_diff += torch.mean(torch.abs(valid_gt - valid_pred))
            abs_rel += torch.mean(torch.abs(valid_gt - valid_pred) / valid_gt)

            sq_rel += torch.mean(((valid_gt - valid_pred) ** 2) / valid_gt)

    return [metric / batch_size for metric in [abs_rel, abs_diff, sq_rel, a1, a2, a3]]

max_stars_repo_name: WestCityInstitute/DeepSFM

max_issues_repo_name: WestCityInstitute/DeepSFM

max_forks_repo_name: WestCityInstitute/DeepSFM


text:
# Copyright 2020-2021 AstroLab Software
# Author: Julien Peloton
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output, State
import dash_bootstrap_components as dbc
import dash_table
from dash.exceptions import PreventUpdate
import visdcc
import dash_trich_components as dtc

from app import server
from app import app
from app import client
from app import APIURL

from apps import home, summary, about, api
from apps import __version__ as portal_version

from apps.utils import markdownify_objectid
from apps.utils import isoify_time, validate_query, extract_query_url
from apps.plotting import draw_cutouts_quickview, draw_lightcurve_preview

import requests
import pandas as pd
import numpy as np
from astropy.time import Time, TimeDelta

simbad_types = pd.read_csv('assets/simbad_types.csv', header=None)[0].values
simbad_types = sorted(simbad_types, key=lambda s: s.lower())

tns_types = pd.read_csv('assets/tns_types.csv', header=None)[0].values
tns_types = sorted(tns_types, key=lambda s: s.lower())

message_help = """
##### Object ID

Enter a valid object ID to access its data, e.g. try:

* ZTF21abfmbix, ZTF21aaxtctv, ZTF21abfaohe, ZTF20aanxcpf, ZTF17aaaabte, ZTF18aafpcwm, ZTF21abujbqa, ZTF21abuipwb, ZTF18acuajcr

##### Conesearch

Perform a conesearch around a position on the sky given by (RA, Dec, radius).
The initializer for RA/Dec is very flexible and supports inputs provided in a number of convenient formats.
The following ways of initializing a conesearch are all equivalent (radius in arcsecond):

* 193.822, 2.89732, 5
* 193d49m18.267s, 2d53m50.35s, 5
* 12h55m17.218s, +02d53m50.35s, 5
* 12 55 17.218, +02 53 50.35, 5
* 12:55:17.218, 02:53:50.35, 5

Maximum radius length is 18,000 arcseconds (5 degrees). Note that in case of
several objects matching, the results will be sorted according to the angular
separation in degree between the input (ra, dec) and the objects found.

In addition, you can specify a starting date (UTC) and a window (in days) to refine your search.
Example, to refine your search starting at 2021-06-25 05:59:37.000 for 7 days:

* 193.822, 2.89732, 5, 2021-06-25 05:59:37.000, 7
* 193.822, 2.89732, 5, 2459391.7497338, 7

We encourage you to use the `startdate` and `window`, as your query will run much faster.

##### Date search

Choose a starting date and a time window to see all processed alerts in this period.
Dates are in UTC, and the time window in minutes.
Among several, you can choose YYYY-MM-DD hh:mm:ss, Julian Date, or Modified Julian Date. Example of valid search:

* 2021-07-01 05:59:37.000
* 2459396.7497337963
* 59396.2497337963

##### Class

Choose a class of interest using the drop-down menu to see the 100 latest alerts processed by Fink.

##### Solar System Objects (SSO)

Search for Solar System Object in the Fink database.
The numbers or designations are taken from the MPC archive.
When searching for a particular asteroid or comet, it is best to use the IAU number,
as in 4209 for asteroid "4209 Briggs". You can also try for numbered comet (e.g. 10P),
or interstellar object (none so far...). If the number does not yet exist, you can search for designation.
Here are some examples of valid queries:

* Asteroids by number (default)
  * Asteroids (Main Belt): 4209, 1922
  * Asteroids (Hungarians): 18582, 77799
  * Asteroids (Jupiter Trojans): 4501, 1583
  * Asteroids (Mars Crossers): 302530
* Asteroids by designation (if number does not exist yet)
  * 2010JO69, 2017AD19, 2012XK111
* Comets by number (default)
  * 10P, 249P, 124P
* Comets by designation (if number does no exist yet)
  * C/2020V2, C/2020R2

Note for designation, you can also use space (2010 JO69 or C/2020 V2).
"""

msg_info = """
![logoexp](/assets/Fink_PrimaryLogo_WEB.png)

Fill the search bar and hit the search button. You can access help by clicking on the Help button at the right of the bar.

By default, the table shows:

- i:objectId: Unique identifier for this object
- i:ra: Right Ascension of candidate; J2000 (deg)
- i:dec: Declination of candidate; J2000 (deg)
- v:lastdate: last date the object has been seen by Fink
- v:classification: Classification inferred by Fink (Supernova candidate, Microlensing candidate, Solar System, SIMBAD class, ...)
- i:ndethist: Number of spatially coincident detections falling within 1.5 arcsec going back to the beginning of the survey; only detections that fell on the same field and readout-channel ID where the input candidate was observed are counted. All raw detections down to a photometric S/N of ~ 3 are included.

You can also add more columns using the dropdown button above the result table. Full documentation of all available fields can be found at {}/api/v1/columns.

Finally, you can hit the button `Preview`. This will show you more information
about the first 10 alerts (science cutout, and basic information). Note you can
swipe between alerts (or use arrows on a laptop).
""".format(APIURL)

modal = html.Div(
    [
        dbc.Button(
            "Help",
            id="open",
            color='light',
            outline=True,
            style={
                "border": "0px black solid",
                'background': 'rgba(255, 255, 255, 0.0)',
                'color': 'grey'
            }
        ),
        dbc.Modal(
            [
                dbc.ModalHeader("Fink Science Portal"),
                dbc.ModalBody(dcc.Markdown(message_help)),
                dbc.ModalFooter(
                    dbc.Button("Close", id="close", className="ml-auto")
                ),
            ],
            id="modal", scrollable=True
        ),
    ]
)

@app.callback(
    Output("modal", "is_open"),
    [Input("open", "n_clicks"), Input("close", "n_clicks")],
    [State("modal", "is_open")],
)
def toggle_modal(n1, n2, is_open):
    """ Callback for the modal (open/close)
    """
    if n1 or n2:
        return not is_open
    return is_open

dropdown_menu_items = [
    dbc.DropdownMenuItem("ZTF Object ID", id="dropdown-menu-item-1"),
    dbc.DropdownMenuItem("Conesearch", id="dropdown-menu-item-2"),
    dbc.DropdownMenuItem("Date Search", id="dropdown-menu-item-3"),
    dbc.DropdownMenuItem("Class search", id="dropdown-menu-item-4"),
    dbc.DropdownMenuItem("SSO search", id="dropdown-menu-item-5")
]

fink_search_bar = dbc.InputGroup(
    [
        dbc.DropdownMenu(
            dropdown_menu_items,
            addon_type="append",
            id='dropdown-query',
            label="objectID",
            color='light',
            className='rcorners3',
            toggle_style={"border": "0px black solid", 'background': 'rgba(255, 255, 255, 0.0)', 'color': 'grey'}
        ),
        dbc.Input(
            id="search_bar_input",
            autoFocus=True,
            type='search',
            style={"border": "0px black solid", 'background': 'rgba(255, 255, 255, 0.0)', 'color': 'grey'},
            className='inputbar'
        ),
        dbc.Button(
            html.I(className="fas fa-search fa-1x"),
            id="submit",
            style={"border": "0px black solid", 'background': 'rgba(255, 255, 255, 0.0)', 'color': '#15284F90'}
        ),
        modal
    ], style={"border": "0.5px grey solid", 'background': 'rgba(255, 255, 255, .75)'}, className='rcorners2'
)

def print_msg_info():
    """ Display the explorer info message
    """
    h = html.Div([
        dbc.Row([
            dcc.Markdown(msg_info)
        ]),
    ])
    return h

def simple_card(
        name, finkclass, lastdate, fid,
        mag, jd, jdstarthist, ndethist, constellation, is_mobile):
    """ Preview card

    The laptop version shows Science cutout + metadata + lightcurve
    The mobile version shows Science cutout + metadata
    """
    dic_band = {1: 'g', 2: 'r'}
    fontsize = '75%'

    l1 = html.P(
        [
            html.Strong("Last emission date: ", style={'font-size': fontsize}),
            html.P(lastdate)
        ]
    )
    l2 = html.P(
        [
            html.Strong("Last magnitude (band {}): ".format(dic_band[fid]), style={'font-size': fontsize}),
            html.P("{:.2f}".format(mag))
        ]
    )

    l3 = html.P(
        [
            html.Strong("Days since first detection: ", style={'font-size': fontsize}),
            html.P('{}'.format(int(jd - jdstarthist)))
        ]
    )

    l4 = html.P(
        [
            html.Strong("Total number of detections: ", style={'font-size': fontsize}),
            html.P('{}'.format(ndethist))
        ]
    )

    l5 = html.P(
        [
            html.Strong("Constellation: ", style={'font-size': fontsize}),
            html.P('{}'.format(constellation))
        ]
    )

    if is_mobile:
        cardbody = dbc.CardBody(
            [
                html.H4("{}".format(finkclass), className="card-title"),
                l1,
                l2,
                l3,
                l4,
                l5
            ]
        )
        header = dbc.CardHeader(
            dbc.Row(
                draw_cutouts_quickview(name),
                id='stamps_quickview',
                justify='around'
            )
        )
    else:
        cardbody = dbc.CardBody(
            [
                html.H4("{}".format(finkclass), className="card-title"),
                html.P("Constellation: {}".format(constellation), className="card-title"),
                dcc.Graph(
                    figure=draw_lightcurve_preview(name),
                    config={'displayModeBar': False},
                    style={
                        'width': '100%',
                        'height': '15pc'
                    }
                )
            ]
        )

        header = dbc.CardHeader(
            dbc.Row(
                [
                    dbc.Col(draw_cutouts_quickview(name), width=2),
                    dbc.Col([l1, l2], width=5),
                    dbc.Col([l3, l4], width=5)
                ],
                id='stamps_quickview',
                justify='around'
            )
        )

    simple_card_ = dbc.Card(
        [
            header,
            cardbody,
            dbc.CardFooter(
                dbc.Button(
                    "Go to {}".format(name),
                    color="primary",
                    outline=True,
                    href='{}/{}'.format(APIURL, name)
                )
            )
        ], style={'background': '#000', 'background-image': 'linear-gradient(rgba(255,255,255,0.5), rgba(255,255,255,0.9)), url(/assets/background.png)'}
    )
    return simple_card_

@app.callback(
    Output('carousel', 'children'),
    [
        Input("open_modal_quickview", "n_clicks"),
        Input("result_table", "data"),
        Input('is-mobile', 'children')
    ],
)
def carousel(nclick, data, is_mobile):
    """ Carousel that shows alert preview
    """
    if nclick > 0:
        pdf = pd.DataFrame(data)
        names = pdf['i:objectId'].apply(lambda x: x.split('[')[1].split(']')[0]).values[0:10]
        finkclasses = pdf['v:classification'].values[0:10]
        lastdates = pdf['v:lastdate'].values[0:10]
        fids = pdf['i:fid'].values[0:10]
        mags = pdf['i:magpsf'].values[0:10]
        jds = pdf['i:jd'].values[0:10]
        jdstarthists = pdf['i:jdstarthist'].values[0:10]
        ndethists = pdf['i:ndethist'].values[0:10]
        constellations = pdf['v:constellation'][0:10]
        is_mobiles = [is_mobile] * 10
        carousel = dtc.Carousel(
            [
                html.Div(dbc.Container(simple_card(*args))) for args in zip(names, finkclasses, lastdates, fids, mags, jds, jdstarthists, ndethists, constellations, is_mobiles)
            ],
            slides_to_scroll=1,
            slides_to_show=1,
            swipe_to_slide=True,
            autoplay=False,
            speed=800,
            variable_width=False,
            center_mode=False
        )
    else:
        carousel = html.Div("")
    return carousel

modal_quickview = html.Div(
    [
        dbc.Button(
            "Preview",
            id="open_modal_quickview",
            n_clicks=0,
            outline=True,
            color="secondary"
        ),
        dbc.Modal(
            [
                dbc.ModalBody(
                    dbc.Container(
                        id='carousel',
                        fluid=True,
                        style={'width': '95%'}
                    ), style={
                        'background': '#000',
                        'background-image': 'linear-gradient(rgba(0,0,0,0.3), rgba(255,255,255,0.3)), url(/assets/background.png)'
                    }
                ),
                dbc.ModalFooter(
                    dbc.Button(
                        "Close", id="close_modal_quickview", className="ml-auto", n_clicks=0
                    ), style={'display': 'None'}
                ),
            ],
            id="modal_quickview",
            is_open=False,
            size="lg",
        ),
    ]
)

@app.callback(
    Output("modal_quickview", "is_open"),
    [
        Input("open_modal_quickview", "n_clicks"),
        Input("close_modal_quickview", "n_clicks")
    ],
    [State("modal_quickview", "is_open")],
)
def toggle_modal_preview(n1, n2, is_open):
    if n1 or n2:
        return not is_open
    return is_open

def display_table_results(table, is_mobile):
    """ Display explorer results in the form of a table with a dropdown
    menu on top to insert more data columns.

    The dropdown menu options are taken from the client schema (ZTF & Fink). It also
    contains other derived fields from the portal (fink_additional_fields).

    Parameters
    ----------
    table: dash_table.DataTable
        Dash DataTable containing the results. Can be empty.

    Returns
    ----------
    out: list of objects
        The list of objects contain:
          1. A dropdown menu to add new columns in the table
          2. Table of results
        The dropdown is shown only if the table is non-empty.
    """
    schema = client.schema()
    schema_list = list(schema.columnNames())
    fink_fields = [i for i in schema_list if i.startswith('d:')]
    ztf_fields = [i for i in schema_list if i.startswith('i:')]
    fink_additional_fields = ['v:constellation', 'v:g-r', 'v:rate(g-r)', 'v:classification', 'v:lastdate']

    dropdown = dcc.Dropdown(
        id='field-dropdown2',
        options=[
            {'label': 'Fink science module outputs', 'disabled': True, 'value': 'None'},
            *[{'label': field, 'value': field} for field in fink_fields],
            {'label': 'Fink additional values', 'disabled': True, 'value': 'None'},
            *[{'label': field, 'value': field} for field in fink_additional_fields],
            {'label': 'Original ZTF fields (subset)', 'disabled': True, 'value': 'None'},
            *[{'label': field, 'value': field} for field in ztf_fields]
        ],
        searchable=True,
        clearable=True,
        placeholder="Add more fields to the table",
    )

    if is_mobile:
        width_dropdown = 8
        width_preview = 4
    else:
        width_dropdown = 10
        width_preview = 2

    return dbc.Container([
        html.Br(),
        dbc.Row(
            [
                dbc.Col(dropdown, width=width_dropdown),
                dbc.Col(modal_quickview, width=width_preview)
            ]
        ),
        html.Br(),
        table
    ], fluid=True)

@app.callback(
    Output('aladin-lite-div-skymap', 'run'),
    [
        Input("validate_results", "value"),
        Input("result_table", "data"),
        Input("result_table", "columns"),
        Input('tabs', 'active_tab')
    ],
)
def display_skymap(validation, data, columns, activetab):
    """ Display explorer result on a sky map (Aladin lite). Limited to 1000 sources total.

    TODO: image is not displayed correctly the first time

    the default parameters are:
        * PanSTARRS colors
        * FoV = 360 deg
        * Fink alerts overlayed

    Callbacks
    ----------
    Input: takes the validation flag (0: no results, 1: results) and table data
    Output: Display a sky image around the alert position from aladin.
    """
    ctx = dash.callback_context
    button_id = ctx.triggered[0]["prop_id"].split(".")[0]
    if len(data) > 1000:
        msg = 'We cannot display {} objects on the sky map (limit at 1000). Please refine your query.
'.format(len(data))
        return """var container = document.getElementById('aladin-lite-div-skymap');var txt = '{}'; container.innerHTML = txt;""".format(msg)
    if validation and (activetab == 't2'):
        pdf = pd.DataFrame(data)

        # Coordinate of the first alert
        ra0 = pdf['i:ra'].values[0]
        dec0 = pdf['i:dec'].values[0]

        # Javascript. Note the use {{}} for dictionary
        # Force redraw of the Aladin lite window
        img = """var container = document.getElementById('aladin-lite-div-skymap');var txt = ''; container.innerHTML = txt;"""

        # Aladin lite
        img += """var a = A.aladin('#aladin-lite-div-skymap', {{target: '{} {}', survey: 'P/PanSTARRS/DR1/color/z/zg/g', showReticle: true, allowFullZoomout: true, fov: 360}});""".format(ra0, dec0)

        ras = pdf['i:ra'].values
        decs = pdf['i:dec'].values
        filts = pdf['i:fid'].values
        filts_dic = {1: 'g', 2: 'r'}
        times = pdf['v:lastdate'].values
        link = '{}'
        titles = [link.format(APIURL, i.split(']')[0].split('[')[1], i.split(']')[0].split('[')[1]) for i in pdf['i:objectId'].values]
        mags = pdf['i:magpsf'].values
        classes = pdf['v:classification'].values
        n_alert_per_class = pdf.groupby('v:classification').count().to_dict()['i:objectId']
        colors = {
            'Early SN Ia candidate': 'red',
            'SN candidate': 'orange',
            'Kilonova candidate': 'blue',
            'Microlensing candidate': 'green',
            'Solar System MPC': "rgb(254,224,144)",
            'Solar System candidate': "rgb(171,217,233)",
            'Ambiguous': 'rgb(116,196,118)',
            'Unknown': '#7f7f7f'
        }
        cats = []
        for ra, dec, fid, time_, title, mag, class_ in zip(ras, decs, filts, times, titles, mags, classes):
            if class_ in simbad_types:
                cat = 'cat_{}'.format(simbad_types.index(class_))
                color = '#3C8DFF'
            else:
                cat = 'cat_{}'.format(class_.replace(' ', '_'))
                color = colors[class_]
            if cat not in cats:
                img += """var {} = A.catalog({{name: '{}', sourceSize: 15, shape: 'circle', color: '{}', onClick: 'showPopup', limit: 1000}});""".format(cat, class_ + ' ({})'.format(n_alert_per_class[class_]), color)
                cats.append(cat)
            img += """{}.addSources([A.source({}, {}, {{objectId: '{}', mag: {:.2f}, filter: '{}', time: '{}', Classification: '{}'}})]);""".format(cat, ra, dec, title, mag, filts_dic[fid], time_, class_)

        for cat in sorted(cats):
            img += """a.addCatalog({});""".format(cat)

        # img cannot be executed directly because of formatting
        # We split line-by-line and remove comments
        img_to_show = [i for i in img.split('\n') if '// ' not in i]

        return " ".join(img_to_show)
    else:
        return ""

def display_skymap():
    """ Display the sky map in the explorer tab results (Aladin lite)

    It uses `visdcc` to execute javascript directly.

    Returns
    ---------
    out: list of objects
    """
    return [
        dbc.Container(
            html.Div(
                [
                    visdcc.Run_js(id='aladin-lite-div-skymap'),
                    dcc.Markdown('_Hit the Aladin Lite fullscreen button if the image is not displayed (we are working on it...)_'),
                ], style={
                    'width': '100%',
                    'height': '25pc'
                }
            )
        )
    ]

@app.callback(
    [
        Output("select", "style"),
        Output("select", "options"),
        Output("select", "placeholder"),
    ],
    [
        Input("dropdown-menu-item-1", "n_clicks"),
        Input("dropdown-menu-item-2", "n_clicks"),
        Input("dropdown-menu-item-3", "n_clicks"),
        Input("dropdown-menu-item-4", "n_clicks"),
        Input("dropdown-menu-item-5", "n_clicks")
    ]
)
def input_type(n1, n2, n3, n4, n5):
    """ Decide if the dropdown below the search bar should be shown

    Only some query types need to have a dropdown (Date & Class search). In
    those cases, we show the dropdown, otherwise it is hidden.

    In the case of class search, the options are derived from the
    Fink classification, and the SIMBAD labels.
    """
    ctx = dash.callback_context

    if not ctx.triggered:
        return {'display': 'none'}, [], ''
    else:
        button_id = ctx.triggered[0]["prop_id"].split(".")[0]

    if button_id == "dropdown-menu-item-3":
        options = [
            {'label': '1 minute', 'value': 1},
            {'label': '10 minutes', 'value': 10},
            {'label': '60 minutes (can be long)', 'value': 60}
        ]
        placeholder = "Choose a time window (default is 1 minute)"
        return {}, options, placeholder
    elif button_id == "dropdown-menu-item-4":
        options = [
            {'label': 'All classes', 'value': 'allclasses'},
            {'label': 'Unknown', 'value': 'Unknown'},
            {'label': 'Fink derived classes', 'disabled': True, 'value': 'None'},
            {'label': 'Early Supernova Ia candidates', 'value': 'Early SN candidate'},
            {'label': 'Supernova candidates', 'value': 'SN candidate'},
            {'label': 'Kilonova candidates', 'value': 'Kilonova candidate'},
            {'label': 'Microlensing candidates', 'value': 'Microlensing candidate'},
            {'label': 'Solar System (MPC)', 'value': 'Solar System MPC'},
            {'label': 'Solar System (candidates)', 'value': 'Solar System candidate'},
            {'label': 'Ambiguous', 'value': 'Ambiguous'},
            {'label': 'TNS classified data', 'disabled': True, 'value': 'None'},
            *[{'label': '(TNS) ' + simtype, 'value': '(TNS) ' + simtype} for simtype in tns_types],
            {'label': 'Simbad crossmatch', 'disabled': True, 'value': 'None'},
            *[{'label': '(SIMBAD) ' + simtype, 'value': '(SIMBAD) ' + simtype} for simtype in simbad_types]
        ]
        placeholder = "Start typing or choose a class. We will display the last 100 alerts for this class."
        return {}, options, placeholder
    else:
        return {'display': 'none'}, [], ''

@app.callback(
    [
        Output("search_bar_input", "placeholder"),
        Output("dropdown-query", "label"),
        Output("search_bar_input", "value")
    ],
    [
        Input("dropdown-menu-item-1", "n_clicks"),
        Input("dropdown-menu-item-2", "n_clicks"),
        Input("dropdown-menu-item-3", "n_clicks"),
        Input("dropdown-menu-item-4", "n_clicks"),
        Input("dropdown-menu-item-5", "n_clicks")
    ],
    State("search_bar_input", "value")
)
def on_button_click(n1, n2, n3, n4, n5, val):
    """ Change the placeholder value of the search bar based on the query type
    """
    ctx = dash.callback_context

    default = "Enter a valid ZTF object ID or choose another query type"
    if not ctx.triggered:
        return default, "objectID", ""
    else:
        button_id = ctx.triggered[0]["prop_id"].split(".")[0]

    if button_id == "dropdown-menu-item-1":
        return "Enter a valid ZTF object ID", "objectID", val
    elif button_id == "dropdown-menu-item-2":
        return "Conesearch around RA, Dec, radius(, startdate, window). See Help for the syntax", "Conesearch", val
    elif button_id == "dropdown-menu-item-3":
        return "Search alerts inside a time window. See Help for the syntax", "Date", val
    elif button_id == "dropdown-menu-item-4":
        return "Show last 100 alerts for a particular class", "Class", val
    elif button_id == "dropdown-menu-item-5":
        return "Enter a valid IAU number. See Help for more information", "SSO", val
    else:
        return "Valid ZTF object ID", "objectID", ""

@app.callback(
    Output("logo", "children"),
    [
        Input("submit", "n_clicks"),
        Input("url", "search")
    ],
)
def logo(ns, searchurl):
    """ Show the logo in the start page (and hide it otherwise)
    """
    ctx = dash.callback_context

    logo = [
        html.Br(),
        html.Br(),
        dbc.Row(
            dbc.Col(
                html.Img(
                    src="/assets/Fink_PrimaryLogo_WEB.png",
                    height='100%',
                    width='60%'
                )
            ), style={'textAlign': 'center'}
        ),
        html.Br()
    ]
    if not ctx.triggered and searchurl == '':
        return logo
    else:
        button_id = ctx.triggered[0]["prop_id"].split(".")[0]

    if button_id == "submit" or searchurl != '':
        return []
    else:
        return logo

def construct_results_layout(table, is_mobile):
    """ Construct the tabs containing explorer query results
    """
    results_ = [
        dbc.Tabs(
            [
                dbc.Tab(print_msg_info(), label='Info', tab_id='t0', label_style = {"color": "#000"}),
                dbc.Tab(display_table_results(table, is_mobile), label="Table", tab_id='t1', label_style = {"color": "#000"}),
                dbc.Tab(display_skymap(), label="Sky map", tab_id='t2', label_style = {"color": "#000"}),
            ],
            id="tabs",
            active_tab="t1",
        )
    ]
    return results_

def populate_result_table(data, columns, is_mobile):
    """ Define options of the results table, and add data and columns
    """
    if is_mobile:
        page_size = 5
        markdown_options = {'link_target': '_self'}
    else:
        page_size = 10
        markdown_options = {'link_target': '_blank'}
    table = dash_table.DataTable(
        data=data,
        columns=columns,
        id='result_table',
        page_size=page_size,
        style_as_list_view=True,
        sort_action="native",
        filter_action="native",
        markdown_options=markdown_options,
        fixed_columns={'headers': True, 'data': 1},
        style_data={
            'backgroundColor': 'rgb(248, 248, 248, .7)'
        },
        style_table={'maxWidth': '100%'},
        style_cell={'padding': '5px', 'textAlign': 'center', 'overflow': 'hidden'},
        style_data_conditional=[
            {
                'if': {'row_index': 'odd'},
                'backgroundColor': 'rgb(248, 248, 248, .7)'
            }
        ],
        style_header={
            'backgroundColor': 'rgb(230, 230, 230)',
            'fontWeight': 'bold'
        }
    )
    return table

@app.callback(
    [
        Output("result_table", "data"),
        Output("result_table", "columns"),
    ],
    [
        Input('field-dropdown2', 'value')
    ],
    [
        State("result_table", "data"),
        State("result_table", "columns"),
    ]
)
def update_table(field_dropdown, data, columns):
    """ Update table by adding new columns (no server call)
    """
    changed_id = [p['prop_id'] for p in dash.callback_context.triggered][0]
    # Adding new columns (no client call)
    if 'field-dropdown2' in changed_id:
        if field_dropdown is None or len(columns) == 0:
            raise PreventUpdate

        incolumns = any(c.get('id') == field_dropdown for c in columns)

        if incolumns is True:
            raise PreventUpdate

        columns.append({
            'name': field_dropdown,
            'id': field_dropdown,
            'type': 'text',
            'presentation': 'markdown'
            # 'hideable': True,
        })

        return data, columns
    else:
        raise PreventUpdate

@app.callback(
    [
        Output("results", "children"),
        Output("validate_results", "value"),
    ],
    [
        Input("submit", "n_clicks"),
        Input("search_bar_input", "value"),
        Input("dropdown-query", "label"),
        Input("select", "value"),
        Input("is-mobile", "children"),
        Input('url', 'search')
    ],
    State("results", "children")
)
def results(ns, query, query_type, dropdown_option, is_mobile, searchurl, results):
    """ Query the database from the search input

    Returns
    ---------
    out: list of Tabs
        Tabs containing info, table, and skymap with the results
    validation: int
        0: not results found, 1: results found
    """
    colnames_to_display = [
        'i:objectId', 'i:ra', 'i:dec', 'v:lastdate', 'v:classification', 'i:ndethist'
    ]

    ctx = dash.callback_context
    button_id = ctx.triggered[0]["prop_id"].split(".")[0]

    if button_id != "submit" and searchurl == '':
        raise PreventUpdate

    # catch parameters sent from URL
    # override any other options
    if searchurl != '':
        query, query_type, dropdown_option = extract_query_url(searchurl)

    is_ok = validate_query(query, query_type)
    if not is_ok['flag']:
        return dash_table.DataTable(
            data=[],
            columns=[],
            id='result_table'
        ), 0

    if query_type == 'objectID':
        r = requests.post(
            '{}/api/v1/explorer'.format(APIURL),
            json={
                'objectId': query,
            }
        )
    elif query_type == 'SSO':
        # strip from spaces
        query_ = str(query.replace(' ', ''))

        r = requests.post(
            '{}/api/v1/sso'.format(APIURL),
            json={
                'n_or_d': query_
            }
        )
    elif query_type == 'Conesearch':
        args = [i.strip() for i in query.split(',')]
        if len(args) == 3:
            ra, dec, radius = args
            startdate = None
            window = None
        elif len(args) == 5:
            ra, dec, radius, startdate, window = args
        r = requests.post(
            '{}/api/v1/explorer'.format(APIURL),
            json={
                'ra': ra,
                'dec': dec,
                'radius': float(radius),
                'startdate_conesearch': startdate,
                'window_days_conesearch': window
            }
        )
    elif query_type == 'Date':
        startdate = isoify_time(query)
        if dropdown_option is None:
            window = 1
        else:
            window = dropdown_option
        r = requests.post(
            '{}/api/v1/explorer'.format(APIURL),
            json={
                'startdate': startdate,
                'window': window
            }
        )
    elif query_type == 'Class':
        if dropdown_option is None:
            alert_class = 'allclasses'
        else:
            alert_class = dropdown_option
        r = requests.post(
            '{}/api/v1/latests'.format(APIURL),
            json={
                'class': alert_class,
                'n': '100'
            }
        )

    # Format output in a DataFrame
    pdf = pd.read_json(r.content)

    if pdf.empty:
        if searchurl != '':
            return dbc.Alert(
                "No alerts found using the {} type: {}".format(query_type, query),
                color="info",
                dismissable=True
            ), 0
        else:
            return dash_table.DataTable(
                data=[],
                columns=[],
                id='result_table'
            ), 0
    else:
        # Make clickable objectId
        pdf['i:objectId'] = pdf['i:objectId'].apply(markdownify_objectid)

        if query_type == 'Conesearch':
            data = pdf.sort_values(
                'v:separation_degree', ascending=True
            ).to_dict('records')
        else:
            data = pdf.sort_values('i:jd', ascending=False).to_dict('records')

        columns = [
            {
                'id': c,
                'name': c,
                'type': 'text',
                # 'hideable': True,
                'presentation': 'markdown',
            } for c in colnames_to_display
        ]
        validation = 1

    table = populate_result_table(data, columns, is_mobile)
    return construct_results_layout(table, is_mobile), validation

noresults_toast = html.Div(
    [
        dbc.Toast(
            "",
            header="",
            id="noresults-toast2",
            icon="danger",
            dismissable=True,
            is_open=False
        ),
        html.Br()
    ]
)

@app.callback(
    [
        Output("noresults-toast2", "is_open"),
        Output("noresults-toast2", "children"),
        Output("noresults-toast2", "header")
    ],
    [
        Input("submit", "n_clicks"),
        Input("validate_results", "value"),
        Input("search_bar_input", "value"),
        Input("dropdown-query", "label"),
        Input("select", "value"),
        Input("url", "search")
    ]
)
def open_noresults(n, results, query, query_type, dropdown_option, searchurl):
    """ Toast to warn the user about the fact that we found no results
    """
    # Trigger the query only if the submit button is pressed.
    changed_id = [p['prop_id'] for p in dash.callback_context.triggered][0]
    if ('submit' not in changed_id and searchurl == '') or results is None:
        raise PreventUpdate

    # catch parameters sent from URL
    # override any other options
    if searchurl != '':
        query, query_type, dropdown_option = extract_query_url(searchurl)

    validation = validate_query(query, query_type)
    if not validation['flag']:
        return (not validation['flag']), validation['header'], validation['text']

    # Good query, but no results
    # ugly hack
    if n and int(results) == 0:
        if query_type == 'objectID':
            header = "Search by Object ID"
            text = "{} not found".format(query)
        elif query_type == 'SSO':
            header = "Search by Solar System Object ID"
            text = "{} ({}) not found".format(query, str(query).replace(' ', ''))
        elif query_type == 'Conesearch':
            header = "Conesearch"
            text = "No alerts found for (RA, Dec, radius) = {}".format(
                query
            )
        elif query_type == 'Date':
            header = "Search by Date"
            if dropdown_option is None:
                window = 1
            else:
                window = dropdown_option
            jd_start = Time(query).jd
            jd_end = jd_start + TimeDelta(window * 60, format='sec').jd

            text = "No alerts found between {} and {}".format(
                Time(jd_start, format='jd').iso,
                Time(jd_end, format='jd').iso
            )
        elif query_type == 'Class':
            header = "Get latest 100 alerts by class"
            if dropdown_option is None:
                alert_class = 'allclasses'
            else:
                alert_class = dropdown_option
            # start of the Fink operations
            jd_start = Time('2019-11-01 00:00:00').jd
            jd_stop = Time.now().jd

            text = "No alerts for class {} in between {} and {}".format(
                alert_class,
                Time(jd_start, format='jd').iso,
                Time(jd_stop, format='jd').iso
            )
        return True, text, header
    return False, "", ""

# building the navigation bar
dropdown = dbc.DropdownMenu(
    children=[
        dbc.DropdownMenuItem("About", href="/about"),
        dbc.DropdownMenuItem("Fink Website", href="https://fink-broker.org/"),
        dbc.DropdownMenuItem(
            "Science Portal GitHub",
            href="https://github.com/astrolabsoftware/fink-science-portal"
        ),
    ],
    nav=True,
    in_navbar=True,
    label="Info",
)

navbar = dbc.Navbar(
    [
        html.A(
            dbc.Row(
                [
                    dbc.Col(
                        dbc.NavbarBrand(
                            "Fink Science portal {}".format(portal_version),
                            className="ml-2"
                        )
                    ),
                ],
                justify="start",
                no_gutters=True,
            ),
            href="/",
        ),
        dbc.NavbarToggler(id="navbar-toggler2"),
        dbc.Collapse(
            dbc.Nav(
                # right align dropdown menu with ml-auto className
                [
                    dbc.NavItem(dbc.NavLink('Search', href="{}".format(APIURL))),
                    dbc.NavItem(dbc.NavLink('API', href="{}/api".format(APIURL))),
                    dbc.NavItem(dbc.NavLink('Tutorials', href="https://github.com/astrolabsoftware/fink-notebook-template")),
                    dropdown
                ],
                navbar=True
            ),
            id="navbar-collapse2",
            navbar=True,
            style={'background-color': 'rgb(255,250,250)'}
        ),
    ],
    color="rgba(255,255,255,0.9)",
    dark=False,
    className="finknav",
    fixed='top'
)

# add callback for toggling the collapse on small screens
@app.callback(
    Output("navbar-collapse2", "is_open"),
    [Input("navbar-toggler2", "n_clicks")],
    [State("navbar-collapse2", "is_open")],
)
def toggle_navbar_collapse(n, is_open):
    if n:
        return not is_open
    return is_open

# embedding the navigation bar
app.layout = html.Div([
    dcc.Location(id='url', refresh=False),
    navbar,
    html.Div(id='page-content'),
    html.Div(children=False, id='is-mobile', hidden=True),
])

app.clientside_callback(
    """
    function(href) {
        var cond1 = ( ( window.innerWidth <= 820 ) && ( window.innerHeight <= 600 ) )
        var cond2 = ( ( window.innerWidth <= 600 ) && ( window.innerHeight <= 820 ) )
        return ( ( cond1 ) || ( cond2 ) );
    }
    """,
    Output('is-mobile', 'children'),
    Input('url', 'href')
)

@app.callback(
    Output('page-content', 'children'),
    [Input('url', 'pathname'), Input('is-mobile', 'children')]
)
def display_page(pathname, is_mobile):
    if is_mobile:
        width = '95%'
        style = {'background-image': 'linear-gradient(rgba(255,255,255,0.3), rgba(255,255,255,0.3)), url(/assets/background.png)'}
    else:
        width = '60%'
        style = {'background-image': 'linear-gradient(rgba(255,255,255,0.3), rgba(255,255,255,0.3)), url(/assets/background.png)', 'background-size': 'contain'}
    layout = html.Div(
        [
            html.Br(),
            html.Br(),
            dbc.Container(
                [
                    html.Div(id='logo'),
                    html.Br(),
                    dbc.Row(fink_search_bar),
                    html.Br(),
                    dcc.Dropdown(
                        id='select',
                        searchable=True,
                        clearable=True,
                    ),
                    html.Br(),
                    noresults_toast
                ], id='trash', fluid=True, style={'width': width}
            ),
            dbc.Container(id='results'),
            dbc.Input(id='validate_results', style={'display': 'none'}),
        ],
        className='home',
        style=style
    )
    if pathname == '/about':
        return about.layout
    elif pathname == '/api':
        return api.layout(is_mobile)
    elif 'ZTF' in pathname:
        return summary.layout(pathname, is_mobile)
    else:
        return layout

# register the API
try:
    from apps.api import api_bp
    server.register_blueprint(api_bp, url_prefix='/')
    server.config['JSONIFY_PRETTYPRINT_REGULAR'] = True
    server.config['JSON_SORT_KEYS'] = False
except ImportError as e:
    print('API not yet registered')

if __name__ == '__main__':
    app.run_server(debug=True)

max_stars_repo_name: astrolabsoftware/fink-science-portal

max_issues_repo_name: astrolabsoftware/fink-science-portal

max_forks_repo_name: astrolabsoftware/fink-science-portal

reason: import numpy,from astropy


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Aug 10 09:13:37 2017

@author: azunre
"""

# sumy imports
from __future__ import absolute_import
from __future__ import division, print_function, unicode_literals

from sumy.parsers.html import HtmlParser
from sumy.parsers.plaintext import PlaintextParser
from sumy.nlp.tokenizers import Tokenizer
from sumy.summarizers.lex_rank import LexRankSummarizer as Summarizer
from sumy.nlp.stemmers import Stemmer
from sumy.utils import get_stop_words

# other imports
import numpy as np
import pandas as pd
import scipy.sparse as ssp
from sklearn.feature_extraction.text import TfidfVectorizer
from typing import List

# Hyper settings...
LANGUAGE = "english"
SENTENCES_COUNT = 20
DEBUG = False
MAX_N_ROWS = 10000

class SubjectExplorer:
    def __init__(self):
        self.stemmer = Stemmer(LANGUAGE)
        self.summarizer = Summarizer(self.stemmer)
        self.summarizer.stop_words = get_stop_words(LANGUAGE)

    def ExtractivelySummarizeCorpus(self,corpus_path:str,HTML:bool = True)->List[str]:

        if(HTML):
            self.parser = HtmlParser.from_url(corpus_path, Tokenizer(LANGUAGE))
        else:
            # or for plain text files
            self.parser = PlaintextParser.from_file(corpus_path, Tokenizer(LANGUAGE))

        sentences = self.summarizer(self.parser.document, SENTENCES_COUNT)

        if(DEBUG):
            # print("DEBUG::ExtractivelySummarizeCorpus::these are all the parser.document.sentences")
            # print(self.parser.document.sentences)
            print("DEBUG::ExtractivelySummarizeCorpus::top n=%d sentences:"%SENTENCES_COUNT)
            for sentence in sentences:
                print(str(sentence))
        sentences = [str(sentence) for sentence in sentences]

        return sentences

    def ExtractTopics(self,sentences:List[str])->dict:

        TfidfVec = TfidfVectorizer(stop_words='english')

        tfidf = TfidfVec.fit_transform(sentences)
        features = TfidfVec.get_feature_names()

        if(DEBUG):
            print("DEBUG::ExtractTopics::all features (words)")
            print(features)

        cos_similarity_matrix = (tfidf * tfidf.T).toarray()
        dense_tfidf = tfidf.todense()

        if(DEBUG):
            print("DEBUG::ExtractTopics::tfidf matrix dimension:")
            print(tfidf.shape)
            print("DEBUG::ExtractTopics::dense tfidf matrix:")
            print(dense_tfidf)
            print("DEBUG::ExtractTopics::cos_similarity_matrix:")
            print(cos_similarity_matrix)

        topics = []
        all_importance_weights = np.empty(0)
        for i in np.arange(tfidf.shape[0]):
            for j in np.arange(tfidf.shape[0]):
                if((i!=j) and (cos_similarity_matrix[i,j]>0)):
                    importance_weights_vec = [dense_tfidf[i,k]*dense_tfidf[j,k] for k in np.arange(tfidf.shape[1])]
                    topic_indices = np.array(importance_weights_vec)>0
                    importance_weights = np.array(importance_weights_vec)[topic_indices]
                    tmp = np.array(features)[topic_indices]
                    topics.extend(list(tmp))
                    all_importance_weights = np.append(all_importance_weights,importance_weights)
                    if(DEBUG):
                        print(importance_weights)
                        print("DEBUG::ExtractTopics::extracted topics:")
                        print(tmp)
                        print("DEBUG::ExtractTopics::extracted topic weights:")
                        print(importance_weights)

        unique_topics = list(set(topics))
        if(DEBUG):
            print("DEBUG::ExtractTopics::concatenated extracted topics:")
            print(topics)
            print("DEBUG::ExtractTopics::concatenated extracted topic weights:")
            print(all_importance_weights)
            print("DEBUG::ExtractTopics::unique extracted topics:")
            print(unique_topics)

        importance_weights = {}
        for topic in unique_topics:
            importance_weights[topic]=np.sum(all_importance_weights[np.array(topics)==topic])

        # normalize
        tmp = np.array(list(importance_weights.values()))
        # factor=1.0/sum(np.exp(tmp)) # useful if softmax is required
        importance_weights = {k: 1.0/(1.0+np.exp(-v)) for k, v in importance_weights.items() }
        # sort in decreasing order
        import operator
        sorted_importance_weights = sorted(importance_weights.items(), key=operator.itemgetter(1),reverse=True)

        return sorted_importance_weights

if __name__ == "__main__":
    url = "http://newknowledge.io/team"
    print("DEBUG::main::starting sumy url summarization test...")
    TopicExtractor = SubjectExplorer()
    sentences = TopicExtractor.ExtractivelySummarizeCorpus(url,True)
    extractedTopics = TopicExtractor.ExtractTopics(sentences)
    print("These are the extracted topics, from url: (sorted, importance weights included)")
    print(extractedTopics)
    filename = "data/6_20_17_32_bp_content.txt"
    #filename = "data/pro_gun_text.csv"
    print("DEBUG::main::starting sumy file summarization test...")
    TopicExtractor = SubjectExplorer()
    df = pd.read_csv(filename, dtype='str', header=None)
    # print(df)
    df_list = df.ix[0:MAX_N_ROWS,0].tolist()
    df_list = [sentence + '.' for sentence in df_list]
    # print(df_list)
    (pd.DataFrame(df_list)).to_csv("data/truncated_frame.csv",index=False)
    sentences = TopicExtractor.ExtractivelySummarizeCorpus("data/truncated_frame.csv",False)
    #sentences = df_list
    extractedTopics = TopicExtractor.ExtractTopics(sentences)
    print("These are the extracted topics, from file: (sorted, importance weights included)")
    print(extractedTopics)

max_stars_repo_name: ghonk/ibex

max_issues_repo_name: ghonk/ibex

max_forks_repo_name: ghonk/ibex

reason: import numpy,import scipy


text:
import csv
import numpy as np

def convert_gpa_to_grade(hs_gpa):
    if hs_gpa > 3.7: return 8
    elif hs_gpa > 3.3: return 7
    elif hs_gpa > 3.0: return 6
    elif hs_gpa > 2.7: return 5
    elif hs_gpa > 2.3: return 4
    elif hs_gpa > 2.0: return 3
    elif hs_gpa > 1.7: return 2
    else: return 1

zip_pci_map = {}

for row in csv.DictReader(open('zip-pci.tsv', 'r'), dialect='excel-tab'):
  zip_pci_map[row['zip']] = row['pci']

pci = int(np.mean([int(r['pci']) for r in csv.DictReader(open('zip-pci.tsv', 'r'), dialect='excel-tab')]))

with open('data/data.csv','r') as fin:
  reader = csv.DictReader(fin)
  data = list(reader)

  print("Converting keys to lowercase...")
  for row in data:
      for k in row.keys():
        row[k.lower()] = row.pop(k)

  print("Removed rows without IDs")
  data = [r for r in data if r['people_code_id']]

  print("Calculating means for imputation...")
  years = set([r['entry_year'] for r in data])
  years = sorted(list(years))

  valid_gpa_data = [s for s in data if s['hs_gpa'] and float(s['hs_gpa']) > 0 and float(s['hs_gpa']) < 5]
  valid_sat_data = [s for s in data if s['sat_verbal'] and float(s['sat_verbal']) > 0]

  hs_gpa_mean = "%.2f" % (sum(float(r['hs_gpa']) for r in valid_gpa_data) / len(valid_gpa_data))
  sat_verb_mean = int(round(sum(float(r['sat_verbal']) for r in valid_sat_data) / len(valid_sat_data)))
  sat_math_mean = int(round(sum(float(r['sat_math']) for r in valid_sat_data) / len(valid_sat_data)))

  hs_gpa_means = {}
  for year in years:
    this_year = [s for s in valid_gpa_data if s['entry_year'] == str(year)]
    if len(this_year) > 0:
      hs_gpa_means[year] = "%.2f" % (sum(float(r['hs_gpa']) for r in this_year) / len(this_year))
    else:
      hs_gpa_means[year] = hs_gpa_mean
    print(year, len(this_year), hs_gpa_means[year])

  sat_verb_means = {}
  sat_math_means = {}
  for year in years:
    this_year = [s for s in valid_sat_data if s['entry_year'] == str(year)]
    if len(this_year) > 0:
      sat_verb_means[year] = int(round(sum(float(r['sat_verbal']) for r in this_year) / len(this_year)))
      sat_math_means[year] = int(round(sum(float(r['sat_math']) for r in this_year) / len(this_year)))
    else:
      sat_verb_means[year] = sat_verb_mean
      sat_math_means[year] = sat_math_mean
    print("%s SAT means: %d %d (n=%d)" % (year, sat_verb_means[year], sat_math_means[year], len(this_year)))

  print('Loading people...')
  people = {p['PEOPLE_CODE_ID']: p for p in csv.DictReader(open('data/people.csv'))}

  print('Loading addresses...')
  address = {p['PEOPLE_ORG_CODE_ID']: p for p in csv.DictReader(open('data/address.csv')) if p['ADDRESS_TYPE'] == 'PERM'}

  with open('data/clean_data.tsv','w') as fout:
    fieldnames = list(map(str.lower,reader.fieldnames))+['name','hs_grade','white','black','mexican_american','native_american','asian','hispanic', 'graduated_in_6', 'us_zip', 'country', 'zip_pci', 'international']
    fieldnames.remove('race')
    fieldnames.remove('sat_writing')
    fieldnames.remove('act_comp')
    fieldnames.remove('enrolled_in_entry_term')
    fieldnames.remove('lastupdated')

    writer = csv.DictWriter(fout,fieldnames=fieldnames,dialect='excel-tab',extrasaction='ignore')

    writer.writeheader()
    for row in data:
      if row['entry_year'] <= '2016' and row['entry_year'] >= '1991':
        row['name'] = "%s %s" % (people[row['people_code_id']]['FIRST_NAME'], people[row['people_code_id']]['LAST_NAME'],)
        try:
          row['country'] = address[row['people_code_id']]['COUNTRY']
          if row['country'] == 'US' or row['country'] == '':
            row['us_zip'] = address[row['people_code_id']]['ZIP_CODE']
          else:
            row['us_zip'] = ''
        except KeyError:
          row['us_zip'] = ''
          row['country'] = ''
        try:
          row['zip_pci'] = zip_pci_map[row['us_zip']]
        except KeyError:
          row['zip_pci'] = pci # Mean imputation
        row['hs_gpa'] = float(row['hs_gpa']) or hs_gpa_means[row['entry_year']] # Mean imputation
        row['hs_grade'] = convert_gpa_to_grade(float(row['hs_gpa']))

        row['sat_verbal'] = float(row['sat_verbal']) or sat_verb_mean # Mean imputation
        row['sat_math'] = float(row['sat_math']) or sat_math_mean # Mean imputation
        row['gender'] = 2 if row['gender'] == 'Female' else 1
        row['white'] = 2 if row['race'].startswith('White') else 1
        row['black'] = 2 if row['race'] == 'African American' else 1
        row['mexican_american'] = 2 if row['race'] == 'Mexican American' else 1
        row['native_american'] = 2 if row['race'] == 'American Indian/Alaskan Native' else 1
        row['asian'] = 2 if row['race'] == 'Asian' else 1
        row['hispanic'] = 2 if row['race'] == 'Hispanic' else 1
        row['international'] = 2 if row['country'] != '' and row['country'] != 'US' else 1

        row['graduated_in_6'] = int((row['undergrad_grad_date'] or False) and row['undergrad_grad_date'] < "%d/08" % (int(row['entry_year']) + 6))
        row['retained'] = int(bool(row['retained']))

        writer.writerow(row)

max_stars_repo_name: jncraton/retention-modeling

max_issues_repo_name: jncraton/retention-modeling

max_forks_repo_name: jncraton/retention-modeling


text:
#!/usr/bin/env python3
import numpy as np
import matplotlib.pyplot as plt

from logger_setup import logger

def transformation(gray, args):
    gray_flat = gray.flatten()
    image_out_flat = np.empty(gray_flat.shape, dtype=np.uint8)

   # Step 1. Define range of intensities
    r_i = np.arange(0,2**args.bits, dtype=np.uint8)
    r_i_count = np.zeros(r_i.shape, dtype=np.uint64)
    s_i = np.zeros(r_i.shape, dtype=np.uint8)

    # Step 2. Count occurences of pixels with same intensity value
    for pixel in gray_flat:
        r_i_count[pixel] += 1

    # Step 3. Calculate probability of occurrence for each intensity level
    p_r = r_i_count / sum(r_i_count)

    # Step 4. Cumulative sum
    r_cumsum = p_r.cumsum()

    # Step 5. Mapping of pixel intensities between the original image and the processed one
    s_i = r_cumsum * (2**args.bits-1)
    s_i = np.floor(s_i)

    # Apply transformation function
    for ii,pixel in enumerate(gray_flat):
        image_out_flat[ii] = s_i[np.where(r_i == pixel)]

    s_i_count = np.zeros(r_i.shape, dtype=np.uint64)
    for pixel in image_out_flat:
        s_i_count[pixel] += 1

    image_out = image_out_flat.reshape(gray.shape)

    # Save/show plots
    if args.kernel_size == -1:
        if args.save:
            save_dir = args.image.split('.tif')[0]
            logger.info(save_dir)

        fig = _plt_bar(r_i,r_i_count, title='Histogram of original image', xlabel='intensity value', ylabel='number of pixels')
        if args.save:
            fig.savefig(f'{save_dir}/2.png')
        if args.show:
            plt.show()

        fig = _plt_bar(r_i,s_i_count, title='Histogram of processed image', xlabel='intensity value', ylabel='number of pixels')
        if args.save:
            fig.savefig(f'{save_dir}/3.png')
        if args.show:
            plt.show()

        if args.save:
            fig = _plt_plot(r_i,s_i,title='Transformation function', xlabel='r_i', ylabel='s_i')
            fig.savefig(f'{save_dir}/4.png')
        if args.show:
            plt.show()

    return image_out

def _plt_plot(x, y, title='title', xlabel='x', ylabel='y'):
    fig = plt.figure()
    plt.title(title)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    plt.plot(x,y)
    return fig

def _plt_bar(x, height, title='title', xlabel='x', ylabel='y'):
    fig = plt.figure()
    plt.title(title)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    plt.bar(x,height)
    return fig


text:
import pandas as pd
from tqdm import tqdm

import dgp
import design
from plan import Plan
import estimator as est
import evaluator as evalr
import numpy as np

def make_plan(designs):
    plan = Plan()

    for name, dgn, estr in designs:
        plan.add_design(name, dgn, estr)

    plan.add_evaluator('ATEError', evalr.ATEError)
    plan.add_evaluator('ITEBias', evalr.ITEBias)
    plan.add_evaluator('ITEMSE', evalr.ITEMSE)
    plan.add_evaluator('CovariateMSE', evalr.CovariateMSE)
    plan.add_evaluator('ATECovers', evalr.ATECovers)
    plan.add_evaluator('CISize', evalr.CISize)
    return plan

NUM_ITERS = 100

plan = make_plan([
    ('QuickBlock-B', design.QuickBlock, est.Blocking),
    ('QuickBlock-RF', design.QuickBlock, est.BlockingRF),
    ('OptBlock-B', design.OptBlock, est.Blocking),
    ('OptBlock-RF', design.OptBlock, est.BlockingRF),
    ('Randomization-RF', design.Bernoulli, est.OLSandRFT),
    ('Fixed Margins Randomization-RF', design.Complete, est.OLSandRFT),
    ('Rerandomization-RF', design.ReRandomization, est.OLSandRFT),
    #('Rerandomization-KNN', design.ReRandomization, est.OLSandKNNT),
    ('Matched Pair-B', design.MatchedPair, est.MatchedPairBlocking),
    ('SoftBlock-L', design.SoftBlock, est.LaplacianNorm),
    ('SoftBlock-RF', design.SoftBlock, est.OLSandRFT),
    ('GreedyNeighbors-RF', design.GreedyNeighbors, est.OLSandRFT),
    ('GreedyNeighbors-L', design.GreedyNeighbors, est.LaplacianNorm),
    #('KallusHeuristic-KNN', design.Heuristic, est.DMandKNNT),
    ('KallusHeuristic-RF', design.Heuristic, est.DMandRFT),
    #('KallusPSOD-KNN', design.PSOD, est.DMandKNNT),
    ('KallusPSOD-RF', design.PSOD, est.DMandRFT),
])

dfs = []
dgp_factory_class_list = [dgp.LinearFactory, dgp.QuickBlockFactory, dgp.SinusoidalFactory, dgp.TwoCirclesFactory]
sample_sizes = [10, 20, 32, 50, 64, 72, 86, 100]
for sample_size in sample_sizes[::-1]:
    print(f"Sample Size: {sample_size}")
    dgp_factory_list = [factory(N=sample_size) for factory in dgp_factory_class_list]
    for dgp_factory in dgp_factory_list:
        dgp_name = type(dgp_factory.create_dgp()).__name__
        print(f"DGP name: {dgp_name}")
        for it in tqdm(range(NUM_ITERS)):
            result = plan.execute(dgp_factory, seed=it * 1001)
            result['iteration'] = it
            result['sample_size'] = sample_size
            result['dgp'] = dgp_name
            filename = f"results/{dgp_name}_n{sample_size}_i{it}.csv.gz"
            result.to_csv(filename, index=False)
            dfs.append(result)

plan = make_plan([
    ('QuickBlock-B', design.QuickBlock, est.Blocking),
    ('QuickBlock-RF', design.QuickBlock, est.BlockingRF),
    #('OptBlock-B', design.OptBlock, est.Blocking),
    ('OptBlock-RF', design.OptBlock, est.BlockingRF),
    ('SoftBlock-L', design.SoftBlock, est.LaplacianNorm),
    ('SoftBlock-RF', design.SoftBlock, est.OLSandRFT),
    ('GreedyNeighbors-RF', design.GreedyNeighbors, est.OLSandRFT),
    ('GreedyNeighbors-L', design.GreedyNeighbors, est.LaplacianNorm),
    ('Randomization-RF', design.Bernoulli, est.OLSandRFT),
    ('Fixed Margins Randomization-RF', design.Bernoulli, est.OLSandRFT),
    ('Rerandomization-RF', design.ReRandomization, est.OLSandRFT),
    #('Rerandomization-KNN', design.ReRandomization, est.OLSandKNNT),
    #('KallusPSOD-KNN', design.PSOD, est.DMandKNNT),
    ('KallusPSOD-RF', design.PSOD, est.DMandRFT),
])

sample_sizes = [128, 150, 200, 250, 500, 1000, 2000, 3000]
for sample_size in sample_sizes[::-1]:
    print(f"Sample Size: {sample_size}")
    dgp_factory_list = [factory(N=sample_size) for factory in dgp_factory_class_list]
    for dgp_factory in dgp_factory_list:
        dgp_name = type(dgp_factory.create_dgp()).__name__
        print(f"DGP name: {dgp_name}")
        for it in tqdm(range(NUM_ITERS)):
            result = plan.execute(dgp_factory, seed=it * 1001)
            result['iteration'] = it
            result['sample_size'] = sample_size
            result['dgp'] = dgp_name
            filename = f"results/{dgp_name}_n{sample_size}_i{it}.csv.gz"
            result.to_csv(filename, index=False)
            dfs.append(result)

plan = make_plan([
    ('SoftBlock-L', design.SoftBlock, est.LaplacianNorm),
    ('SoftBlock-RF', design.SoftBlock, est.OLSandRFT),
    ('GreedyNeighbors-RF', design.GreedyNeighbors, est.OLSandRFT),
    ('GreedyNeighbors-L', design.GreedyNeighbors, est.LaplacianNorm),
    ('QuickBlock-B', design.QuickBlock, est.Blocking),
    ('QuickBlock-RF', design.QuickBlock, est.BlockingRF),
    ('Randomization', design.Bernoulli, est.OLSandRFT),
    ('Fixed Margins Randomization-RF', design.Complete, est.OLSandRFT),
    ('Rerandomization-RF', design.ReRandomization, est.OLSandRFT),
    # ('Rerandomization-KNN', design.ReRandomization, est.OLSandKNNT),
])

NUM_ITERS = 25

sample_sizes = [4000, 5000, 7500, 10000, 12500, 15000, 20000, 25000, 30000, 50000, 100000]
for sample_size in sample_sizes:
    print(f"Sample Size: {sample_size}")
    dgp_factory_list = [factory(N=sample_size) for factory in dgp_factory_class_list]
    for dgp_factory in dgp_factory_list:
        dgp_name = type(dgp_factory.create_dgp()).__name__
        print(f"DGP name: {dgp_name}")
        for it in tqdm(range(NUM_ITERS)):
            result = plan.execute(dgp_factory, seed=it * 1001)
            result['iteration'] = it
            result['sample_size'] = sample_size
            result['dgp'] = dgp_name
            filename = f"results/{dgp_name}_n{sample_size}_i{it}.csv.gz"
            result.to_csv(filename, index=False)
            dfs.append(result)

NUM_ITERS = 5

sample_sizes = [50000, 100000, 500000, 1000000]
for sample_size in sample_sizes:
    print(f"Sample Size: {sample_size}")
    dgp_factory_list = [factory(N=sample_size) for factory in dgp_factory_class_list]
    for dgp_factory in dgp_factory_list:
        dgp_name = type(dgp_factory.create_dgp()).__name__
        print(f"DGP name: {dgp_name}")
        for it in tqdm(range(NUM_ITERS)):
            result = plan.execute(dgp_factory, seed=it * 1001)
            result['iteration'] = it
            result['sample_size'] = sample_size
            result['dgp'] = dgp_name
            filename = f"results/{dgp_name}_n{sample_size}_i{it}.csv.gz"
            result.to_csv(filename, index=False)
            dfs.append(result)

results = pd.concat(dfs)

filename = f"results/all_results.csv.gz"

print(f"""
\n**********************************************************************
***\tSAVING TO `{filename}`\t\t   ***
**********************************************************************""")

results.to_csv(filename, index=False)

max_stars_repo_name: ddimmery/softblock

max_issues_repo_name: ddimmery/softblock

max_forks_repo_name: ddimmery/softblock


text:
import os
import numpy as np
from queue import Queue
import math

from gen import *
import networkx as nx
import time
import numpy as np
import math
import argparse
import sys
import random
import numpy

class Table:
    #def __init__(self, src, dst, nxt):
    #   self.routes = [[src, dst, nxt]]

    def __init__(self, src, G, degree):

        self.routes = dict(nx.bfs_successors(G, src, depth_limit=degree))

    def has(self, x):

        for i in self.routes:
            for j in self.routes[i]:
                if j == x:
                    return True
        return False

        #for k in succs.keys():
        #    pred_weight = 0

        #    if k != src:
        #        pred_weight = G[src][k]

        #    for v in succs[k]:

        #        minw = math.inf
        #        try:
        #            minw = self.routes[src][v][1]
        #        except:
        #            pass
        #
        #        self.routes[src] = [v, min(pred_weight + G[k][v], minw)]

if __name__== "__main__":

    n = int(sys.argv[1])
    N = int(sys.argv[2])
    ds = int(sys.argv[3])
    de = int(sys.argv[4])

    print("Generating for")
    print(sys.argv)

    G = generate_graph(ds, de, N)

    # add random weights
    for i, j in G.edges():
        rand_weight = random.randint(1,10)
        G.remove_edge(i, j)
        #G.remove_edge(j, i)
        G.add_edge(i, j, weight=rand_weight)
        G.add_edge(j, i, weight=rand_weight)

    ##
    ##
    ##  SCENARIO 2a
    ##
    ##

    tables = []
    for i in G.nodes():

        tables.append(Table(i, G, None))

    # add a new node
    G.add_node(N)

    # pick a random node in the network and add and undirected edge to the new node
    random_node = random.randint(0, N)
    G.add_edge(random_node, N)
    G.add_edge(N, random_node)

    # update table of the random node
    tables[random_node].routes[random_node].append(N)

    # update new node's table
    tables.append(Table(N, G, 1))

    marktable = [0 for i in range(N+1)]
    marktable[N] = 1
    marktable[random_node] = 1

    # Flood until you find N
    for n in G.nodes():

        succ_list = list(G.neighbors(n))
        found = False

        messages = 0
        visited = [0 for i in range(N+1)]

        while succ_list:

            visited_node = succ_list.pop(0)

            if N == visited_node:
                found = True
                marktable[visited_node] = 1
                break

            if not visited[visited_node]:
                messages += 1
                succ_list.extend(list(G.neighbors(visited_node)))
                visited[visited_node] = True

        print("Node ", n, "found it after sending ", messages, " messages")

    ## Loop until everyone finds N
    #for time in range(100):
    #    for n in G.nodes():
    #        if n == N:
    #            continue
    #        # look at your immediate neighbor and see if there's N
    #        for neighbors in tables[n].routes[n]:
    #            if N in tables[neighbors].routes[neighbors]:
    #                tables[n].routes[neighbors].append(N)
    #                marktable[n] = 1
    #                break
    #            elif tables[neighbors].has(N):
    #                tables[n].routes[neighbors].append(N)
    #                marktable[n] = 1
    #                break

    #    if sum(marktable) == (N+1):
    #        print('Everyone found it! ', time)
    #        break

max_stars_repo_name: onai/hopxhop-discovery-modeler

max_issues_repo_name: onai/hopxhop-discovery-modeler

max_forks_repo_name: onai/hopxhop-discovery-modeler

reason: import numpy,import networkx


text:
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# (C) British Crown Copyright 2017-2019 Met Office.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
"""
Unit tests for the function "cube_manipulation.sort_coord_in_cube".
"""

import unittest

import iris
import numpy as np
from iris.coords import AuxCoord
from iris.tests import IrisTest

from improver.utilities.cube_manipulation import sort_coord_in_cube
from improver.utilities.warnings_handler import ManageWarnings

from ...set_up_test_cubes import set_up_variable_cube

class Test_sort_coord_in_cube(IrisTest):
    """Class to test the sort_coord_in_cube function."""

    def setUp(self):
        """Set up ascending and descending cubes"""
        self.ascending_height_points = np.array([5.0, 10.0, 20.0], dtype=np.float32)
        self.descending_height_points = np.flip(self.ascending_height_points)

        self.data = np.array(
            [np.ones((3, 3)), 2 * np.ones((3, 3)), 3 * np.ones((3, 3))],
            dtype=np.float32,
        )
        self.ascending_cube = set_up_variable_cube(self.data)
        self.ascending_cube.coord("realization").rename("height")
        self.ascending_cube.coord("height").points = self.ascending_height_points
        self.ascending_cube.coord("height").units = "m"

        self.descending_cube = self.ascending_cube.copy()
        self.descending_cube.coord("height").points = self.descending_height_points

    def test_ascending_then_ascending(self):
        """Test that the sorting successfully sorts the cube based
        on the points within the given coordinate. The points in the resulting
        cube should now be in ascending order."""
        expected_data = self.data
        coord_name = "height"
        result = sort_coord_in_cube(self.ascending_cube, coord_name)
        self.assertIsInstance(result, iris.cube.Cube)
        self.assertEqual(
            self.ascending_cube.coord_dims(coord_name), result.coord_dims(coord_name)
        )
        self.assertArrayAlmostEqual(
            self.ascending_height_points, result.coord(coord_name).points
        )
        self.assertArrayAlmostEqual(result.data, expected_data)

    def test_auxcoord(self):
        """Test that the above sorting is successful when an AuxCoord is
        used."""
        expected_data = self.data
        coord_name = "height_aux"
        height_coord = self.ascending_cube.coord("height")
        (height_coord_index,) = self.ascending_cube.coord_dims("height")
        new_coord = AuxCoord(height_coord.points, long_name=coord_name)
        self.ascending_cube.add_aux_coord(new_coord, height_coord_index)
        result = sort_coord_in_cube(self.ascending_cube, coord_name)
        self.assertIsInstance(result, iris.cube.Cube)
        self.assertEqual(
            self.ascending_cube.coord_dims(coord_name), result.coord_dims(coord_name)
        )
        self.assertArrayAlmostEqual(
            self.ascending_height_points, result.coord(coord_name).points
        )
        self.assertArrayAlmostEqual(result.data, expected_data)

    def test_ascending_then_descending(self):
        """Test that the sorting successfully sorts the cube based
        on the points within the given coordinate. The points in the resulting
        cube should now be in descending order."""
        expected_data = np.flip(self.data)
        coord_name = "height"
        result = sort_coord_in_cube(self.ascending_cube, coord_name, descending=True)
        self.assertIsInstance(result, iris.cube.Cube)
        self.assertEqual(
            self.descending_cube.coord_dims(coord_name), result.coord_dims(coord_name)
        )
        self.assertArrayAlmostEqual(
            self.descending_height_points, result.coord(coord_name).points
        )
        self.assertArrayAlmostEqual(result.data, expected_data)

    def test_descending_then_ascending(self):
        """Test that the sorting successfully sorts the cube based
        on the points within the given coordinate. The points in the resulting
        cube should now be in ascending order."""
        expected_data = np.flip(self.data)
        coord_name = "height"
        result = sort_coord_in_cube(self.descending_cube, coord_name)
        self.assertIsInstance(result, iris.cube.Cube)
        self.assertEqual(
            self.ascending_cube.coord_dims(coord_name), result.coord_dims(coord_name)
        )
        self.assertArrayAlmostEqual(
            self.ascending_height_points, result.coord(coord_name).points
        )
        self.assertArrayAlmostEqual(result.data, expected_data)

    def test_descending_then_descending(self):
        """Test that the sorting successfully sorts the cube based
        on the points within the given coordinate. The points in the resulting
        cube should now be in descending order."""
        expected_data = self.data
        coord_name = "height"
        result = sort_coord_in_cube(self.descending_cube, coord_name, descending=True)
        self.assertIsInstance(result, iris.cube.Cube)
        self.assertEqual(
            self.descending_cube.coord_dims(coord_name), result.coord_dims(coord_name)
        )
        self.assertArrayAlmostEqual(
            self.descending_height_points, result.coord(coord_name).points
        )
        self.assertArrayAlmostEqual(result.data, expected_data)

    def test_latitude(self):
        """Test that the sorting successfully sorts the cube based
        on the points within the given coordinate (latitude).
        The points in the resulting cube should now be in descending order."""
        expected_data = np.array(
            [
                [[1.00, 1.00, 1.00], [1.00, 1.00, 1.00], [6.00, 1.00, 1.00]],
                [[2.00, 2.00, 2.00], [2.00, 2.00, 2.00], [6.00, 2.00, 2.00]],
                [[3.00, 3.00, 3.00], [3.00, 3.00, 3.00], [6.00, 3.00, 3.00]],
            ]
        )
        self.ascending_cube.data[:, 0, 0] = 6.0
        expected_points = np.flip(self.ascending_cube.coord("latitude").points)
        coord_name = "latitude"
        result = sort_coord_in_cube(self.ascending_cube, coord_name, descending=True)
        self.assertIsInstance(result, iris.cube.Cube)
        self.assertEqual(
            self.ascending_cube.coord_dims(coord_name), result.coord_dims(coord_name)
        )
        self.assertArrayAlmostEqual(expected_points, result.coord(coord_name).points)
        self.assertArrayAlmostEqual(result.data, expected_data)

    @ManageWarnings(record=True)
    def test_warn_raised_for_circular_coordinate(self, warning_list=None):
        """Test that a warning is successfully raised when circular
        coordinates are sorted."""
        self.ascending_cube.data[:, 0, 0] = 6.0
        coord_name = "latitude"
        self.ascending_cube.coord(coord_name).circular = True
        result = sort_coord_in_cube(self.ascending_cube, coord_name, descending=True)
        self.assertTrue(any(item.category == UserWarning for item in warning_list))
        warning_msg = "The latitude coordinate is circular."
        self.assertTrue(any(warning_msg in str(item) for item in warning_list))
        self.assertIsInstance(result, iris.cube.Cube)

if __name__ == "__main__":
    unittest.main()


text:
#!/usr/bin/env python
import numpy as np
import cv2 as cv
"""
|-------------------------------------------------------------|
| BACKUP FILE FOR USING THIS PROJECT AS A SINGLE SCRIPT       |
| THE MAIN PROJECT IS RUN BY main.py                          |
| THIS FILE DOES NOT DEPEND ON ANY OTHER FILES THAN THE IMAGE |
|-------------------------------------------------------------|
"""

FEET_PER_PX = 8188 / 215  # ratio from the image scale

# open the image
prf = cv.imread('PartyRockFire.jpeg', 1)

if __name__ == "__main__":

    # set range for "red"
    redlo = np.array([0, 0, 118], np.uint8)
    redhi = np.array([100, 100, 255], np.uint8)

    # get red pixels
    red = cv.inRange(prf, redlo, redhi)

    # get indices of red cells then max and min
    indices = np.where(red > 0)
    col_max, col_min = max(indices[1]), min(indices[1])
    rows_with_red = np.unique(indices[0], return_counts=True)

    # basic data verification using a few test cases
    # for i in [(10, 55), (5761, 356), (8988, 525), (9078, 527), (9517, 555)]:
    #     print(f"index {i[0]} valid? ", indices[0][i[0]] == i[1])
    # print(indices[1][len(indices[1])-1])

    # now find the max horizontal distance between the white pixels on each row
    i = 0
    distances = []
    for count in rows_with_red[1]:
        distance = indices[1][i:i+count][-1] - indices[1][i:i+count][0]  # indices[1] is sorted
        distances.append(distance + 1)
        i += count

    # print the results
    print("the area is ", sum(distances), " pixels")
    print("which is ", round(sum(distances)*FEET_PER_PX*FEET_PER_PX, 2), " sq feet")

max_stars_repo_name: hunterjsb/CalculateArea

max_issues_repo_name: hunterjsb/CalculateArea

max_forks_repo_name: hunterjsb/CalculateArea