text:
import maml_rl.envs
import gym
import numpy as np
import torch
import json
import copy
import pickle

from maml_rl.metalearner import MetaLearner
from maml_rl.policies import CategoricalMLPPolicy, NormalMLPPolicy
from maml_rl.baseline import LinearFeatureBaseline
from maml_rl.sampler import BatchSampler
from maml_rl.lstm_tree import TreeLSTM

from teachers.teacher_controller import TeacherController

from tensorboardX import SummaryWriter

def total_rewards(episodes_rewards, aggregation=torch.mean):
    rewards = torch.mean(torch.stack([aggregation(torch.sum(rewards, dim=0))
        for rewards in episodes_rewards], dim=0))
    return rewards.item()

def main(args):
    continuous_actions = (args.env_name in ['AntVel-v1', 'AntDir-v1',
        'AntPos-v0', 'HalfCheetahVel-v1', 'HalfCheetahDir-v1',
        '2DNavigation-v0'])

    writer = SummaryWriter('./logs/{0}'.format(args.output_folder))
    save_folder = './saves/{0}'.format(args.output_folder)
    if not os.path.exists(save_folder):
        os.makedirs(save_folder)
    with open(os.path.join(save_folder, 'config.json'), 'w') as f:
        config = {k: v for (k, v) in vars(args).items() if k != 'device'}
        config.update(device=args.device.type)
        json.dump(config, f, indent=2)

    sampler = BatchSampler(args.env_name, batch_size=args.fast_batch_size,
        num_workers=args.num_workers)

    if args.env_name == 'AntVel-v1':
        param_bounds = {"goal": [0, 3]}

    if args.env_name == 'AntPos-v0':
        param_bounds = {"x": [-3, 3],
                        "y": [-3, 3]}

    teacher = TeacherController(args.teacher, args.nb_test_episodes, param_bounds, seed=args.seed, teacher_params={})
    tree = TreeLSTM(args.tree_hidden_layer, len(param_bounds.keys()), args.cluster_0, args.cluster_1, device=args.device)
    if continuous_actions:
        policy = NormalMLPPolicy(
            int(np.prod(sampler.envs.observation_space.shape) + args.tree_hidden_layer),
            int(np.prod(sampler.envs.action_space.shape)),
            hidden_sizes=(args.hidden_size,) * args.num_layers, tree=tree)
    else:
        policy = CategoricalMLPPolicy(
            int(np.prod(sampler.envs.observation_space.shape) + args.tree_hidden_layer),
            sampler.envs.action_space.n,
            hidden_sizes=(args.hidden_size,) * args.num_layers, tree=tree)
    baseline = LinearFeatureBaseline(
        int(np.prod(sampler.envs.observation_space.shape)) + args.tree_hidden_layer)

    metalearner = MetaLearner(sampler, policy, baseline, tree=tree, gamma=args.gamma,
        fast_lr=args.fast_lr, tau=args.tau, device=args.device)

    all_tasks = []
    for batch in range(args.num_batches):
        print("starting iteration {}".format(batch))
        tasks = []
        for _ in range(args.meta_batch_size):
            if args.env_name == 'AntPos-v0':
                tasks.append({"position": teacher.task_generator.sample_task()})
            if args.env_name == 'AntVel-v1':
                tasks.append({"velocity": teacher.task_generator.sample_task()[0]})
        all_tasks.append(tasks)
       # tasks = np.array(tasks)
        # tasks = sampler.sample_tasks(num_tasks=args.meta_batch_size)
        with open('./logs/{0}/task_list.pkl'.format(args.output_folder), 'wb') as pf:
            pickle.dump(all_tasks, pf)
        episodes = metalearner.sample(tasks, first_order=args.first_order)
        metalearner.step(episodes, max_kl=args.max_kl, cg_iters=args.cg_iters,
            cg_damping=args.cg_damping, ls_max_steps=args.ls_max_steps,
            ls_backtrack_ratio=args.ls_backtrack_ratio)

        # Tensorboard
        writer.add_scalar('total_rewards/before_update',
            total_rewards([ep.rewards for ep, _ in episodes]), batch)
        writer.add_scalar('total_rewards/after_update',
            total_rewards([ep.rewards for _, ep in episodes]), batch)

        tr = [ep.rewards for _, ep in episodes]
        tr = [torch.mean(torch.sum(rewards, dim=0)).item() for rewards in tr]
        print("rewards:", tr)
        for t in range(args.meta_batch_size):
            if args.env_name == 'AntPos-v0':
                teacher.task_generator.update(tasks[t]["position"], tr[t])
            if args.env_name == 'AntVel-v1':
                teacher.task_generator.update(np.array([tasks[t]["velocity"]]), tr[t])

        # Save policy network
        with open(os.path.join(save_folder,
                'policy-{0}.pt'.format(batch)), 'wb') as f:
            torch.save(policy.state_dict(), f)

        # Save tree
        torch.save(tree, os.path.join(save_folder, 'tree-{0}.pt'.format(batch)))

def eval(args):
    continuous_actions = (args.env_name in ['AntVel-v1', 'AntDir-v1',
                                            'AntPos-v0', 'HalfCheetahVel-v1', 'HalfCheetahDir-v1',
                                            '2DNavigation-v0'])

    # writer = SummaryWriter('./logs/{0}'.format(args.output_folder))
    save_folder = './saves/{0}'.format(args.output_folder)
    if not os.path.exists(save_folder):
        os.makedirs(save_folder)
    log_folder = './logs/{0}'.format(args.output_folder)
    if not os.path.exists(log_folder):
        os.makedirs(log_folder)

    sampler = BatchSampler(args.env_name, batch_size=args.fast_batch_size,
                           num_workers=args.num_workers)

    if args.env_name == 'AntPos-v0':
        param_bounds = {"x": [-3, 3],
                        "y": [-3, 3]}

    tree = TreeLSTM(args.tree_hidden_layer, len(param_bounds.keys()), args.cluster_0, args.cluster_1, device=args.device)

    if continuous_actions:
        policy = NormalMLPPolicy(
            int(np.prod(sampler.envs.observation_space.shape) + args.tree_hidden_layer),
            int(np.prod(sampler.envs.action_space.shape)),
            hidden_sizes=(args.hidden_size,) * args.num_layers)
    else:
        policy = CategoricalMLPPolicy(
            int(np.prod(sampler.envs.observation_space.shape)),
            sampler.envs.action_space.n,
            hidden_sizes=(args.hidden_size,) * args.num_layers)
    policy.eval()
    tree.eval()

    all_tasks = []
    # torch.autograd.set_detect_anomaly(True)
    reward_list = []
    for batch in range(args.num_batches + 1):
        print("starting iteration {}".format(batch))
        try:
            policy.load_state_dict(torch.load(os.path.join(save_folder,
                                                       'policy-{0}.pt'.format(batch))))
            tree = torch.load(os.path.join(save_folder,
                                                       'tree-{0}.pt'.format(batch)))
            tree.eval()
        except Exception:
            with open('./logs/{0}/reward_list_eval.pkl'.format(args.output_folder), 'wb') as pf:
                pickle.dump(reward_list, pf)

            print(reward_list)
            return

        # tree.load_state_dict(torch.load(os.path.join(save_folder,
        #                        'tree-{0}.pt'.format(batch))))

        tasks = sampler.sample_tasks(args.meta_batch_size)

        all_tasks.append(tasks)
        # tasks = np.array(tasks)
        # tasks = sampler.sample_tasks(num_tasks=args.meta_batch_size)
        with open('./logs/{0}/task_list_eval.pkl'.format(args.output_folder), 'wb') as pf:
            pickle.dump(all_tasks, pf)

        print("evaluating...".format(batch))
        all_rewards = []
        for task in tasks:
            print(task["position"])
            episodes = sampler.sample(policy, task, tree=tree)
        # print("training...".format(batch))

            # tr = [ep.rewards for ep in episodes]
            # tr = np.mean([torch.mean(torch.sum(rewards, dim=0)).item() for rewards in tr])
            all_rewards.append(total_rewards(episodes.rewards))

        reward_list.append(np.mean(all_rewards))

    with open('./logs/{0}/reward_list_eval.pkl'.format(args.output_folder), 'wb') as pf:
        pickle.dump(reward_list, pf)

    print(reward_list)

if __name__ == '__main__':
    import argparse
    import os
    import multiprocessing as mp

    parser = argparse.ArgumentParser(description='Reinforcement learning with '
        'Model-Agnostic Meta-Learning (MAML)')

    # General
    parser.add_argument('--env-name', type=str,
        help='name of the environment')
    parser.add_argument('--gamma', type=float, default=0.95,
        help='value of the discount factor gamma')
    parser.add_argument('--tau', type=float, default=1.0,
        help='value of the discount factor for GAE')
    parser.add_argument('--first-order', action='store_true',
        help='use the first-order approximation of MAML')

    # Policy network (relu activation function)
    parser.add_argument('--hidden-size', type=int, default=100,
        help='number of hidden units per layer')
    parser.add_argument('--num-layers', type=int, default=2,
        help='number of hidden layers')

    # Task-specific
    parser.add_argument('--fast-batch-size', type=int, default=20,
        help='batch size for each individual task')
    parser.add_argument('--fast-lr', type=float, default=0.5,
        help='learning rate for the 1-step gradient update of MAML')

    # Optimization
    parser.add_argument('--num-batches', type=int, default=200,
        help='number of batches')
    parser.add_argument('--meta-batch-size', type=int, default=40,
        help='number of tasks per batch')
    parser.add_argument('--max-kl', type=float, default=1e-2,
        help='maximum value for the KL constraint in TRPO')
    parser.add_argument('--cg-iters', type=int, default=10,
        help='number of iterations of conjugate gradient')
    parser.add_argument('--cg-damping', type=float, default=1e-5,
        help='damping in conjugate gradient')
    parser.add_argument('--ls-max-steps', type=int, default=15,
        help='maximum number of iterations for line search')
    parser.add_argument('--ls-backtrack-ratio', type=float, default=0.8,
        help='maximum number of iterations for line search')

    parser.add_argument('--teacher', type=str, default='ALP-GMM',
        help='teacher variant')
    parser.add_argument('--nb_test_episodes', type=int, default=50,
        help='number of test tasks')
    parser.add_argument('--tree_hidden_layer', type=int, default=40,
                        help='size of treelstm hidden layer')
    parser.add_argument('--cluster_0', type=int, default=4,
                        help='number of clusters in first tree layer')
    parser.add_argument('--cluster_1', type=int, default=2,
                        help='number of clusters in second tree layer')
    parser.add_argument('--seed', type=float, default=2,
        help='teacher seed')

    # Miscellaneous
    parser.add_argument('--output-folder', type=str, default='maml',
        help='name of the output folder')
    parser.add_argument('--num-workers', type=int, default=mp.cpu_count() - 1,
        help='number of workers for trajectories sampling')
    parser.add_argument('--device', type=str, default='cpu',
        help='set the device (cpu or cuda)')
    parser.add_argument('--do-eval', action='store_true',
                        help='do eval')

    args = parser.parse_args()

    # Create logs and saves folder if they don't exist
    if not os.path.exists('./logs'):
        os.makedirs('./logs')
    if not os.path.exists('./saves'):
        os.makedirs('./saves')
    # Device
    args.device = torch.device(args.device
                               if torch.cuda.is_available() else 'cpu')
    # Slurm
    if 'SLURM_JOB_ID' in os.environ:
        args.output_folder += '-{0}'.format(os.environ['SLURM_JOB_ID'])

    if not args.do_eval:
        main(args)
    else:
        eval(args)

max_stars_repo_name: henryzxu/pytorch-hsml-rl

max_issues_repo_name: henryzxu/pytorch-hsml-rl

max_forks_repo_name: henryzxu/pytorch-hsml-rl


text:
import os
import json
from netcdfswan import NetCDFSWAN
import numpy as np
import logging
from dataTest import elem,time,lat,lon,bed,slat,slon,freq,dir,spcgroup,variables, stations

def test_NetCDFSWAN_write():
  swanFolder="../s3/swandata"
  jsonFile='./test/json/demo.json'
  input=NetCDFSWAN.prepareInputJSON(jsonFile,swanFolder,year=2000,month=1)
  swan=NetCDFSWAN(input)

  # Write
  # swan.uploadStatic(year=2000)
  swan.uploadS()
  swan.uploadT()
  # swan.uploadSpc()

def test_NetCDFSWAN():

  input={
    "name":"swan-test1",
    "bucket":"uvic-bcwave",
    "cacheLocation":"../s3",
    "localOnly":True
  }

  swan=NetCDFSWAN(input)

  # Read
  np.testing.assert_array_equal(swan["nodes","bed"], bed)
  np.testing.assert_array_equal(swan["elem","elem"], elem)
  np.testing.assert_array_equal(swan["time","time"], time)
  np.testing.assert_array_equal(swan["nodes","lat"], lat)
  np.testing.assert_array_equal(swan["nodes","lon"], lon)

  np.testing.assert_array_equal(swan["freq","freq"], freq)
  np.testing.assert_array_equal(swan["dir","dir"], dir)
  np.testing.assert_array_equal(swan["snodes","slon",0:11], [0,1,2,3,4,5,6,6,7,7,8])
  np.testing.assert_array_equal(swan["snodes","slat",0:11], [0,0,0,0,0,0,0,1,0,1,0])
  np.testing.assert_array_equal(swan["snodes","stationid",0:11], [0,1,2,3,4,5,6,6,7,7,8])
  np.testing.assert_array_equal(swan["stations","name",0:2], ["beverly","brooks"])

  np.testing.assert_array_equal(swan["s","u10"], variables['WIND']['Windv_x'])
  np.testing.assert_array_equal(swan["s","v10"], variables['WIND']['Windv_y'])
  np.testing.assert_array_equal(swan["s","hs"], variables['HS']['Hsig'])
  np.testing.assert_array_equal(swan["s","tps"], variables['TPS']['TPsmoo'])
  np.testing.assert_array_equal(swan["s","tmm10"], variables['TMM10']['Tm_10'])
  np.testing.assert_array_equal(swan["s","tm01"], variables['TM01']['Tm01'])
  np.testing.assert_array_equal(swan["s","tm02"], variables['TM02']['Tm02'])
  np.testing.assert_array_equal(swan["s","pdir"], variables['PDIR']['Pdir'])
  np.testing.assert_array_equal(swan["s","dir"], variables['DIR']['Dir'])
  np.testing.assert_array_equal(swan["s","dspr"], variables['DSPR']['Dspr'])
  np.testing.assert_array_equal(swan["s","qp"], variables['QP']['Qp'])
  np.testing.assert_array_equal(swan["s","transpx"], variables['TRANSP']['Transp_x'])
  np.testing.assert_array_equal(swan["s","transpy"], variables['TRANSP']['Transp_y'])

  np.testing.assert_array_equal(swan["t","u10"], variables['WIND']['Windv_x'].T)
  np.testing.assert_array_equal(swan["t","v10"], variables['WIND']['Windv_y'].T)
  np.testing.assert_array_equal(swan["t","hs"], variables['HS']['Hsig'].T)

  np.testing.assert_array_equal(swan["t","tps"], variables['TPS']['TPsmoo'].T)
  np.testing.assert_array_equal(swan["t","tmm10"], variables['TMM10']['Tm_10'].T)
  np.testing.assert_array_equal(swan["t","tm01"], variables['TM01']['Tm01'].T)
  np.testing.assert_array_equal(swan["t","tm02"], variables['TM02']['Tm02'].T)
  np.testing.assert_array_equal(swan["t","pdir"], variables['PDIR']['Pdir'].T)
  np.testing.assert_array_equal(swan["t","dir"], variables['DIR']['Dir'].T)
  np.testing.assert_array_equal(swan["t","dspr"], variables['DSPR']['Dspr'].T)
  np.testing.assert_array_equal(swan["t","qp"], variables['QP']['Qp'].T)
  np.testing.assert_array_equal(swan["t","transpx"], variables['TRANSP']['Transp_x'].T)
  np.testing.assert_array_equal(swan["t","transpy"], variables['TRANSP']['Transp_y'].T)

  for name in swan.stations:

    id = swan.stations[name]['id']

    sIndex=swan.stations[name]['start']
    eIndex=swan.stations[name]['end']
    np.testing.assert_array_equal(swan["spc", "spectra",sIndex:eIndex], spcgroup["spectra"][sIndex:eIndex])

def test_NetCDFSWAN_logger():
  logging.basicConfig(
        filename=os.path.join('./data',"progress.log"),
        level=logging.DEBUG,
        format="%(levelname)s %(asctime)s   %(message)s"
        )
  logger = logging.getLogger()

  try:
    swanFolder="../s3/swandata"
    jsonFile='./test/json/demo.json'
    input=NetCDFSWAN.prepareInputJSON(jsonFile,swanFolder,year=2000,month=1)
    swan=NetCDFSWAN(input,logger=logger)
    swan.uploadStatic()
    swan.uploadS()
    swan.uploadT()
    swan.uploadSpc()
  except Exception as err:
    logger.error(err)

if __name__ == "__main__":
  test_NetCDFSWAN_write()
  test_NetCDFSWAN()

ext: py

max_stars_repo_name: meracan/netcdf-swan

max_issues_repo_name: meracan/netcdf-swan

max_forks_repo_name: meracan/netcdf-swan

reason: import numpy


text:
import copy
import json
import os
import random
import tempfile
import pathlib

import numpy as np
import torchvision

from lightly.api.utils import MAXIMUM_FILENAME_LENGTH
from lightly.data.dataset import LightlyDataset
from lightly.utils.io import COCO_ANNOTATION_KEYS

from tests.api_workflow.mocked_api_workflow_client import \
    MockedApiWorkflowSetup

import cv2

class TestApiWorkflowUploadCustomMetadata(MockedApiWorkflowSetup):

    def create_fake_dataset(self, n_data: int = 10, sample_names=None):
        self.dataset = torchvision.datasets.FakeData(size=n_data,
                                                     image_size=(3, 32, 32))

        self.folder_path = tempfile.mkdtemp()
        image_extension = '.jpg'
        sample_names = sample_names if sample_names is not None else [
            f'img_{i}{image_extension}' for i in range(n_data)]
        for sample_idx in range(n_data):
            data = self.dataset[sample_idx]
            sample_name = sample_names[sample_idx]
            path = os.path.join(self.folder_path, sample_name)
            data[0].save(path)

        coco_json = dict()
        coco_json[COCO_ANNOTATION_KEYS.images] = [{'id': i, 'file_name': fname} for i, fname in
                               enumerate(sample_names)]
        coco_json[COCO_ANNOTATION_KEYS.custom_metadata] = [{'id': i, 'image_id': i, 'custom_metadata': 0}
                                 for i, _ in enumerate(sample_names)]

        self.custom_metadata_file = tempfile.NamedTemporaryFile(mode="w+")
        json.dump(coco_json, self.custom_metadata_file)
        self.custom_metadata_file.flush()

    def test_upload_custom_metadata_one_step(self):
        self.create_fake_dataset()
        with open(self.custom_metadata_file.name, 'r') as f:
            custom_metadata = json.load(f)
            self.api_workflow_client.upload_dataset(input=self.folder_path, custom_metadata=custom_metadata)

    def test_upload_custom_metadata_two_steps_verbose(self):
        self.create_fake_dataset()
        self.api_workflow_client.upload_dataset(input=self.folder_path)
        with open(self.custom_metadata_file.name, 'r') as f:
            custom_metadata = json.load(f)
            self.api_workflow_client.upload_custom_metadata(custom_metadata, verbose=True)

    def test_upload_custom_metadata_two_steps(self):
        self.create_fake_dataset()
        self.api_workflow_client.upload_dataset(input=self.folder_path)
        with open(self.custom_metadata_file.name, 'r') as f:
            custom_metadata = json.load(f)
            self.api_workflow_client.upload_custom_metadata(custom_metadata)

    def test_upload_custom_metadata_before_uploading_samples(self):
        self.create_fake_dataset()
        with open(self.custom_metadata_file.name, 'r') as f:
            custom_metadata = json.load(f)
            with self.assertRaises(ValueError):
                self.api_workflow_client.upload_custom_metadata(custom_metadata)

max_stars_repo_name: umami-ware/lightly

max_issues_repo_name: umami-ware/lightly

max_forks_repo_name: umami-ware/lightly


text:
import ctypes
import typing as tp
import warnings
from typing import Any, Dict, Tuple, Union

import numpy as np
import pyarrow as pa
import pyarrow.compute

__all__ = [
    "coerce_arrow",
    "_process_null_values",
    "_ptr_to_numpy",
]

def coerce_arrow(array: pa.Array) -> pa.Array:
    # also coerces timezone to naive representation
    # units are accounted for by pyarrow
    if "timestamp" in str(array.type):
        warnings.warn(
            "Conversion of (potentially) timezone aware to naive datetimes. TZ information may be lost",
        )
        ts_ms = pa.compute.cast(array, pa.timestamp("ms"), safe=False)
        ms = pa.compute.cast(ts_ms, pa.int64())
        del ts_ms
        array = pa.compute.cast(ms, pa.date64())
        del ms
    # note: Decimal256 could not be cast to float
    elif isinstance(array.type, pa.Decimal128Type):
        array = pa.compute.cast(array, pa.float64())

    # simplest solution is to cast to (large)-string arrays
    # this is copy and expensive
    elif isinstance(array.type, pa.DictionaryType):
        if pa.types.is_string(array.type.value_type):
            array = pa.compute.cast(array, pa.large_utf8())
        else:
            raise ValueError(
                "polars does not support dictionary encoded types other than strings"
            )

    if hasattr(array, "num_chunks") and array.num_chunks > 1:
        if pa.types.is_string(array.type):
            array = pa.compute.cast(array, pa.large_utf8())
        elif pa.types.is_list(array.type):
            array = pa.compute.cast(array, pa.large_list())
        array = array.combine_chunks()
    return array

def _process_null_values(
    null_values: Union[None, str, tp.List[str], Dict[str, str]] = None,
) -> Union[None, str, tp.List[str], tp.List[Tuple[str, str]]]:
    if isinstance(null_values, dict):
        return list(null_values.items())
    else:
        return null_values

# https://stackoverflow.com/questions/4355524/getting-data-from-ctypes-array-into-numpy
def _ptr_to_numpy(ptr: int, len: int, ptr_type: Any) -> np.ndarray:
    """

    Parameters
    ----------
    ptr
        C/Rust ptr casted to usize.
    len
        Length of the array values.
    ptr_type
        Example:
            f32: ctypes.c_float)

    Returns
    -------
    View of memory block as numpy array.

    """
    ptr_ctype = ctypes.cast(ptr, ctypes.POINTER(ptr_type))
    return np.ctypeslib.as_array(ptr_ctype, (len,))


text:
# Read data, compute IFR probability distributions and produce pickle files.
#
#
# m.mieskolainen@imperial.ac.uk, 2020

import numpy as np
import matplotlib.pyplot as plt
import numba
import os
import pickle
import sys

sys.path.append('./analysis')
sys.path.append('./covidgen')
sys.path.append('./dataconfig')

import aux
import cstats
import estimators

# ** Generate delay kernels **
import run_generate_kernels

# ** Load datasets **
import load_datasets
CSETS,CSETS_sero = load_datasets.default()

# Font style
#import matplotlib; matplotlib.rcParams.update(aux.tex_fonts)

# =================================================================
### Parameters

analysis_param = {

    # Time interval considered from files. Make sure is large
    # enough for all the datasets.
    'FIRST_DATE' : '2020-03-01',
    'LAST_DATE'  : '2020-07-15',

    # How many days later the death count is read out
    # -1 is the optimal decision (convolution inverse estimate)
    'deltas'     : [0, 7, 14, 21, -1],

    # Bayesian estimator quantiles
    'qbayes'     : estimators.q68_q95,

    # Maximum IFR tail value to consider
    'RMAX' : 0.04 # between [0,1], where 1 is 100%
}

# =================================================================
### Deconvolution/time-delay inverse extraction parameters

unfold_param = {
    'ZEROPAD' : 31,    # Convolution domain pre-extension, also the maximum $\Delta t$ considered
    'BS'      : 300,   # Number of bootstrap/toy MC samples
    'TSPAN'   : 200,   # Convolution kernel span (days), long tails
    'alpha'   : 0.15,  # Regularization strength
    'mode'    : 'C',   # Inversion input ('C' for reported cases, 'F' for fatalities)

    'q'       : estimators.q95,   # bootstrap custom quantiles [0,1]

    'data_poisson' : True, # Statistical uncertainty on
    'kernel_syst'  : True  # Systematic uncertainty on
}

# =================================================================

# Analyze data
obj = cstats.covid_analyze(CSETS=CSETS, CSETS_sero=CSETS_sero, \
                analysis_param=analysis_param, unfold_param=unfold_param)

## Save output
os.makedirs('./output/', exist_ok=True)
filename = './output/computed_counts.pkl'

with open(filename, 'wb') as f:
    pickle.dump([obj, analysis_param, unfold_param], f)
print(f'Output data saved to {filename}')

print(__name__ + ' done!')

max_stars_repo_name: mieskolainen/covidgen

max_issues_repo_name: mieskolainen/covidgen

max_forks_repo_name: mieskolainen/covidgen

reason: import numpy,import numba


text:
import numpy as np
from sklearn.feature_extraction.text import HashingVectorizer
from tokenwiser.textprep import Cleaner, Identity, HyphenTextPrep
from tokenwiser.pipeline import PartialPipeline, PartialFeatureUnion

def test_shape_doubles():
    """If we concatenate using a partial union. It should increase in size."""
    pipe1 = PartialPipeline(
        [
            ("clean", Cleaner()),
            (
                "union",
                PartialFeatureUnion(
                    [
                        (
                            "full_text_pipe",
                            PartialPipeline(
                                [
                                    ("identity", Identity()),
                                    ("hash1", HashingVectorizer()),
                                ]
                            ),
                        )
                    ]
                ),
            ),
        ]
    )

    pipe2 = PartialPipeline(
        [
            ("clean", Cleaner()),
            (
                "union",
                PartialFeatureUnion(
                    [
                        (
                            "full_text_pipe",
                            PartialPipeline(
                                [
                                    ("identity", Identity()),
                                    ("hash1", HashingVectorizer()),
                                ]
                            ),
                        ),
                        (
                            "hyphen_pipe",
                            PartialPipeline(
                                [
                                    ("hyphen", HyphenTextPrep()),
                                    ("hash2", HashingVectorizer()),
                                ]
                            ),
                        ),
                    ]
                ),
            ),
        ]
    )

    X = [
        "i really like this post",
        "thanks for that comment",
        "i enjoy this friendly forum",
        "this is a bad post",
        "i dislike this article",
        "this is not well written",
    ]

    y = np.array([1, 1, 1, 0, 0, 0])

    p1 = pipe1.partial_fit(X, y, classes=[0, 1]).transform(X)
    p2 = pipe2.partial_fit(X, y, classes=[0, 1]).transform(X)

    assert p1.shape[1] * 2 == p2.shape[1]


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# File: train.py

import os
import argparse
import cv2
import shutil
import itertools
import tqdm
import numpy as np
import json
import six
import tensorflow as tf
try:
    import horovod.tensorflow as hvd
except ImportError:
    pass

assert six.PY3, "FasterRCNN requires Python 3!"

from tensorpack import *
from tensorpack.tfutils.summary import add_moving_summary
from tensorpack.tfutils import optimizer
from tensorpack.tfutils.common import get_tf_version_tuple
import tensorpack.utils.viz as tpviz

from coco import COCODetection
from basemodel import (
    image_preprocess, resnet_c4_backbone, resnet_conv5,
    resnet_fpn_backbone)

import model_frcnn
import model_mrcnn
from model_frcnn import (
    sample_fast_rcnn_targets, fastrcnn_outputs,
    fastrcnn_predictions, BoxProposals, FastRCNNHead)
from model_mrcnn import maskrcnn_upXconv_head, maskrcnn_loss
from model_rpn import rpn_head, rpn_losses, generate_rpn_proposals
from model_fpn import (
    fpn_model, multilevel_roi_align,
    multilevel_rpn_losses, generate_fpn_proposals)
from model_cascade import CascadeRCNNHead
from model_box import (
    clip_boxes, crop_and_resize, roi_align, RPNAnchors)

from data import (
    get_train_dataflow, get_eval_dataflow,
    get_all_anchors, get_all_anchors_fpn)
from viz import (
    draw_annotation, draw_proposal_recall,
    draw_predictions, draw_final_outputs)
from eval import (
    eval_coco, multithread_eval_coco,
    detect_one_image, print_coco_metrics, DetectionResult)
from config import finalize_configs, config as cfg

class DetectionModel(ModelDesc):
    def preprocess(self, image):
        image = tf.expand_dims(image, 0)
        image = image_preprocess(image, bgr=True)
        return tf.transpose(image, [0, 3, 1, 2])

    @property
    def training(self):
        return get_current_tower_context().is_training

    def optimizer(self):
        lr = tf.get_variable('learning_rate', initializer=0.003, trainable=False)
        tf.summary.scalar('learning_rate-summary', lr)

        # The learning rate is set for 8 GPUs, and we use trainers with average=False.
        lr = lr / 8.
        opt = tf.train.MomentumOptimizer(lr, 0.9)
        if cfg.TRAIN.NUM_GPUS < 8:
            opt = optimizer.AccumGradOptimizer(opt, 8 // cfg.TRAIN.NUM_GPUS)
        return opt

    def get_inference_tensor_names(self):
        """
        Returns two lists of tensor names to be used to create an inference callable.

        Returns:
            [str]: input names
            [str]: output names
        """
        out = ['output/boxes', 'output/scores', 'output/labels']
        if cfg.MODE_MASK:
            out.append('output/masks')
        return ['image'], out

    def build_graph(self, *inputs):
        inputs = dict(zip(self.input_names, inputs))

        image = self.preprocess(inputs['image'])     # 1CHW

        features = self.backbone(image)
        anchor_inputs = {k: v for k, v in inputs.items() if k.startswith('anchor_')}
        proposals, rpn_losses = self.rpn(image, features, anchor_inputs)  # inputs?

        targets = [inputs[k] for k in ['gt_boxes', 'gt_labels', 'gt_masks'] if k in inputs]
        head_losses = self.roi_heads(image, features, proposals, targets)

        if self.training:
            wd_cost = regularize_cost(
                '.*/W', l2_regularizer(cfg.TRAIN.WEIGHT_DECAY), name='wd_cost')
            total_cost = tf.add_n(
                rpn_losses + head_losses + [wd_cost], 'total_cost')
            add_moving_summary(total_cost, wd_cost)
            return total_cost

class ResNetC4Model(DetectionModel):
    def inputs(self):
        ret = [
            tf.placeholder(tf.float32, (None, None, 3), 'image'),
            tf.placeholder(tf.int32, (None, None, cfg.RPN.NUM_ANCHOR), 'anchor_labels'),
            tf.placeholder(tf.float32, (None, None, cfg.RPN.NUM_ANCHOR, 4), 'anchor_boxes'),
            tf.placeholder(tf.float32, (None, 4), 'gt_boxes'),
            tf.placeholder(tf.int64, (None,), 'gt_labels')]  # all > 0
        if cfg.MODE_MASK:
            ret.append(
                tf.placeholder(tf.uint8, (None, None, None), 'gt_masks')
            )   # NR_GT x height x width
        return ret

    def backbone(self, image):
        return [resnet_c4_backbone(image, cfg.BACKBONE.RESNET_NUM_BLOCKS[:3])]

    def rpn(self, image, features, inputs):
        featuremap = features[0]
        rpn_label_logits, rpn_box_logits = rpn_head('rpn', featuremap, cfg.RPN.HEAD_DIM, cfg.RPN.NUM_ANCHOR)
        anchors = RPNAnchors(get_all_anchors(), inputs['anchor_labels'], inputs['anchor_boxes'])
        anchors = anchors.narrow_to(featuremap)

        image_shape2d = tf.shape(image)[2:]     # h,w
        pred_boxes_decoded = anchors.decode_logits(rpn_box_logits)  # fHxfWxNAx4, floatbox
        proposal_boxes, proposal_scores = generate_rpn_proposals(
            tf.reshape(pred_boxes_decoded, [-1, 4]),
            tf.reshape(rpn_label_logits, [-1]),
            image_shape2d,
            cfg.RPN.TRAIN_PRE_NMS_TOPK if self.training else cfg.RPN.TEST_PRE_NMS_TOPK,
            cfg.RPN.TRAIN_POST_NMS_TOPK if self.training else cfg.RPN.TEST_POST_NMS_TOPK)

        if self.training:
            losses = rpn_losses(
                anchors.gt_labels, anchors.encoded_gt_boxes(), rpn_label_logits, rpn_box_logits)
        else:
            losses = []

        return BoxProposals(proposal_boxes), losses

    def roi_heads(self, image, features, proposals, targets):
        image_shape2d = tf.shape(image)[2:]     # h,w
        featuremap = features[0]

        gt_boxes, gt_labels, *_ = targets

        if self.training:
            # sample proposal boxes in training
            proposals = sample_fast_rcnn_targets(proposals.boxes, gt_boxes, gt_labels)
        # The boxes to be used to crop RoIs.
        # Use all proposal boxes in inference

        boxes_on_featuremap = proposals.boxes * (1.0 / cfg.RPN.ANCHOR_STRIDE)
        roi_resized = roi_align(featuremap, boxes_on_featuremap, 14)

        feature_fastrcnn = resnet_conv5(roi_resized, cfg.BACKBONE.RESNET_NUM_BLOCKS[-1])    # nxcx7x7
        # Keep C5 feature to be shared with mask branch
        feature_gap = GlobalAvgPooling('gap', feature_fastrcnn, data_format='channels_first')
        fastrcnn_label_logits, fastrcnn_box_logits = fastrcnn_outputs('fastrcnn', feature_gap, cfg.DATA.NUM_CLASS)

        fastrcnn_head = FastRCNNHead(proposals, fastrcnn_box_logits, fastrcnn_label_logits, gt_boxes,
                                     tf.constant(cfg.FRCNN.BBOX_REG_WEIGHTS, dtype=tf.float32))

        if self.training:
            all_losses = fastrcnn_head.losses()

            if cfg.MODE_MASK:
                gt_masks = targets[2]
                # maskrcnn loss
                # In training, mask branch shares the same C5 feature.
                fg_feature = tf.gather(feature_fastrcnn, proposals.fg_inds())
                mask_logits = maskrcnn_upXconv_head(
                    'maskrcnn', fg_feature, cfg.DATA.NUM_CATEGORY, num_convs=0)   # #fg x #cat x 14x14

                target_masks_for_fg = crop_and_resize(
                    tf.expand_dims(gt_masks, 1),
                    proposals.fg_boxes(),
                    proposals.fg_inds_wrt_gt, 14,
                    pad_border=False)  # nfg x 1x14x14
                target_masks_for_fg = tf.squeeze(target_masks_for_fg, 1, 'sampled_fg_mask_targets')
                all_losses.append(maskrcnn_loss(mask_logits, proposals.fg_labels(), target_masks_for_fg))
            return all_losses
        else:
            decoded_boxes = fastrcnn_head.decoded_output_boxes()
            decoded_boxes = clip_boxes(decoded_boxes, image_shape2d, name='fastrcnn_all_boxes')
            label_scores = fastrcnn_head.output_scores(name='fastrcnn_all_scores')
            final_boxes, final_scores, final_labels = fastrcnn_predictions(
                decoded_boxes, label_scores, name_scope='output')

            if cfg.MODE_MASK:
                roi_resized = roi_align(featuremap, final_boxes * (1.0 / cfg.RPN.ANCHOR_STRIDE), 14)
                feature_maskrcnn = resnet_conv5(roi_resized, cfg.BACKBONE.RESNET_NUM_BLOCKS[-1])
                mask_logits = maskrcnn_upXconv_head(
                    'maskrcnn', feature_maskrcnn, cfg.DATA.NUM_CATEGORY, 0)   # #result x #cat x 14x14
                indices = tf.stack([tf.range(tf.size(final_labels)), tf.cast(final_labels, tf.int32) - 1], axis=1)
                final_mask_logits = tf.gather_nd(mask_logits, indices)   # #resultx14x14
                tf.sigmoid(final_mask_logits, name='output/masks')
            return []

class ResNetFPNModel(DetectionModel):

    def inputs(self):
        ret = [
            tf.placeholder(tf.float32, (None, None, 3), 'image')]
        num_anchors = len(cfg.RPN.ANCHOR_RATIOS)
        for k in range(len(cfg.FPN.ANCHOR_STRIDES)):
            ret.extend([
                tf.placeholder(tf.int32, (None, None, num_anchors),
                               'anchor_labels_lvl{}'.format(k + 2)),
                tf.placeholder(tf.float32, (None, None, num_anchors, 4),
                               'anchor_boxes_lvl{}'.format(k + 2))])
        ret.extend([
            tf.placeholder(tf.float32, (None, 4), 'gt_boxes'),
            tf.placeholder(tf.int64, (None,), 'gt_labels')])  # all > 0
        if cfg.MODE_MASK:
            ret.append(
                tf.placeholder(tf.uint8, (None, None, None), 'gt_masks')
            )   # NR_GT x height x width
        return ret

    def slice_feature_and_anchors(self, p23456, anchors):
        for i, stride in enumerate(cfg.FPN.ANCHOR_STRIDES):
            with tf.name_scope('FPN_slice_lvl{}'.format(i)):
                anchors[i] = anchors[i].narrow_to(p23456[i])

    def backbone(self, image):
        c2345 = resnet_fpn_backbone(image, cfg.BACKBONE.RESNET_NUM_BLOCKS)
        p23456 = fpn_model('fpn', c2345)
        return p23456

    def rpn(self, image, features, inputs):
        assert len(cfg.RPN.ANCHOR_SIZES) == len(cfg.FPN.ANCHOR_STRIDES)

        image_shape2d = tf.shape(image)[2:]     # h,w
        all_anchors_fpn = get_all_anchors_fpn()
        multilevel_anchors = [RPNAnchors(
            all_anchors_fpn[i],
            inputs['anchor_labels_lvl{}'.format(i + 2)],
            inputs['anchor_boxes_lvl{}'.format(i + 2)]) for i in range(len(all_anchors_fpn))]
        self.slice_feature_and_anchors(features, multilevel_anchors)

        # Multi-Level RPN Proposals
        rpn_outputs = [rpn_head('rpn', pi, cfg.FPN.NUM_CHANNEL, len(cfg.RPN.ANCHOR_RATIOS))
                       for pi in features]
        multilevel_label_logits = [k[0] for k in rpn_outputs]
        multilevel_box_logits = [k[1] for k in rpn_outputs]
        multilevel_pred_boxes = [anchor.decode_logits(logits)
                                 for anchor, logits in zip(multilevel_anchors, multilevel_box_logits)]

        proposal_boxes, proposal_scores = generate_fpn_proposals(
            multilevel_pred_boxes, multilevel_label_logits, image_shape2d)

        if self.training:
            losses = multilevel_rpn_losses(
                multilevel_anchors, multilevel_label_logits, multilevel_box_logits)
        else:
            losses = []

        return BoxProposals(proposal_boxes), losses

    def roi_heads(self, image, features, proposals, targets):
        image_shape2d = tf.shape(image)[2:]     # h,w
        assert len(features) == 5, "Features have to be P23456!"
        gt_boxes, gt_labels, *_ = targets

        if self.training:
            proposals = sample_fast_rcnn_targets(proposals.boxes, gt_boxes, gt_labels)

        fastrcnn_head_func = getattr(model_frcnn, cfg.FPN.FRCNN_HEAD_FUNC)
        if not cfg.FPN.CASCADE:
            roi_feature_fastrcnn = multilevel_roi_align(features[:4], proposals.boxes, 7)

            head_feature = fastrcnn_head_func('fastrcnn', roi_feature_fastrcnn)
            fastrcnn_label_logits, fastrcnn_box_logits = fastrcnn_outputs(
                'fastrcnn/outputs', head_feature, cfg.DATA.NUM_CLASS)
            fastrcnn_head = FastRCNNHead(proposals, fastrcnn_box_logits, fastrcnn_label_logits,
                                         gt_boxes, tf.constant(cfg.FRCNN.BBOX_REG_WEIGHTS, dtype=tf.float32))
        else:
            def roi_func(boxes):
                return multilevel_roi_align(features[:4], boxes, 7)

            fastrcnn_head = CascadeRCNNHead(
                proposals, roi_func, fastrcnn_head_func,
                (gt_boxes, gt_labels), image_shape2d, cfg.DATA.NUM_CLASS)

        if self.training:
            all_losses = fastrcnn_head.losses()

            if cfg.MODE_MASK:
                gt_masks = targets[2]
                # maskrcnn loss
                roi_feature_maskrcnn = multilevel_roi_align(
                    features[:4], proposals.fg_boxes(), 14,
                    name_scope='multilevel_roi_align_mask')
                maskrcnn_head_func = getattr(model_mrcnn, cfg.FPN.MRCNN_HEAD_FUNC)
                mask_logits = maskrcnn_head_func(
                    'maskrcnn', roi_feature_maskrcnn, cfg.DATA.NUM_CATEGORY)   # #fg x #cat x 28 x 28

                target_masks_for_fg = crop_and_resize(
                    tf.expand_dims(gt_masks, 1),
                    proposals.fg_boxes(),
                    proposals.fg_inds_wrt_gt, 28,
                    pad_border=False)  # fg x 1x28x28
                target_masks_for_fg = tf.squeeze(target_masks_for_fg, 1, 'sampled_fg_mask_targets')
                all_losses.append(maskrcnn_loss(mask_logits, proposals.fg_labels(), target_masks_for_fg))
            return all_losses
        else:
            decoded_boxes = fastrcnn_head.decoded_output_boxes()
            decoded_boxes = clip_boxes(decoded_boxes, image_shape2d, name='fastrcnn_all_boxes')
            label_scores = fastrcnn_head.output_scores(name='fastrcnn_all_scores')
            final_boxes, final_scores, final_labels = fastrcnn_predictions(
                decoded_boxes, label_scores, name_scope='output')
            if cfg.MODE_MASK:
                # Cascade inference needs roi transform with refined boxes.
                roi_feature_maskrcnn = multilevel_roi_align(features[:4], final_boxes, 14)
                maskrcnn_head_func = getattr(model_mrcnn, cfg.FPN.MRCNN_HEAD_FUNC)
                mask_logits = maskrcnn_head_func(
                    'maskrcnn', roi_feature_maskrcnn, cfg.DATA.NUM_CATEGORY)   # #fg x #cat x 28 x 28
                indices = tf.stack([tf.range(tf.size(final_labels)), tf.cast(final_labels, tf.int32) - 1], axis=1)
                final_mask_logits = tf.gather_nd(mask_logits, indices)   # #resultx28x28
                tf.sigmoid(final_mask_logits, name='output/masks')
            return []

def visualize(model, model_path, nr_visualize=100, output_dir='output'):
    """
    Visualize some intermediate results (proposals, raw predictions) inside the pipeline.
    """
    df = get_train_dataflow()   # we don't visualize mask stuff
    df.reset_state()

    pred = OfflinePredictor(PredictConfig(
        model=model,
        session_init=get_model_loader(model_path),
        input_names=['image', 'gt_boxes', 'gt_labels'],
        output_names=[
            'generate_{}_proposals/boxes'.format('fpn' if cfg.MODE_FPN else 'rpn'),
            'generate_{}_proposals/scores'.format('fpn' if cfg.MODE_FPN else 'rpn'),
            'fastrcnn_all_scores',
            'output/boxes',
            'output/scores',
            'output/labels',
        ]))

    if os.path.isdir(output_dir):
        shutil.rmtree(output_dir)
    utils.fs.mkdir_p(output_dir)
    with tqdm.tqdm(total=nr_visualize) as pbar:
        for idx, dp in itertools.islice(enumerate(df), nr_visualize):
            img, gt_boxes, gt_labels = dp['image'], dp['gt_boxes'], dp['gt_labels']

            rpn_boxes, rpn_scores, all_scores, \
                final_boxes, final_scores, final_labels = pred(img, gt_boxes, gt_labels)

            # draw groundtruth boxes
            gt_viz = draw_annotation(img, gt_boxes, gt_labels)
            # draw best proposals for each groundtruth, to show recall
            proposal_viz, good_proposals_ind = draw_proposal_recall(img, rpn_boxes, rpn_scores, gt_boxes)
            # draw the scores for the above proposals
            score_viz = draw_predictions(img, rpn_boxes[good_proposals_ind], all_scores[good_proposals_ind])

            results = [DetectionResult(*args) for args in
                       zip(final_boxes, final_scores, final_labels,
                           [None] * len(final_labels))]
            final_viz = draw_final_outputs(img, results)

            viz = tpviz.stack_patches([
                gt_viz, proposal_viz,
                score_viz, final_viz], 2, 2)

            if os.environ.get('DISPLAY', None):
                tpviz.interactive_imshow(viz)
            cv2.imwrite("{}/{:03d}.png".format(output_dir, idx), viz)
            pbar.update()

def offline_evaluate(pred_config, output_file):
    num_gpu = cfg.TRAIN.NUM_GPUS
    graph_funcs = MultiTowerOfflinePredictor(
        pred_config, list(range(num_gpu))).get_predictors()
    predictors = []
    dataflows = []
    for k in range(num_gpu):
        predictors.append(lambda img,
                          pred=graph_funcs[k]: detect_one_image(img, pred))
        dataflows.append(get_eval_dataflow(shard=k, num_shards=num_gpu))
    if num_gpu > 1:
        all_results = multithread_eval_coco(dataflows, predictors)
    else:
        all_results = eval_coco(dataflows[0], predictors[0])
    with open(output_file, 'w') as f:
        json.dump(all_results, f)
    print_coco_metrics(output_file)

def predict(pred_func, input_file):
    img = cv2.imread(input_file, cv2.IMREAD_COLOR)
    results = detect_one_image(img, pred_func)
    final = draw_final_outputs(img, results)
    viz = np.concatenate((img, final), axis=1)
    cv2.imwrite("output.png", viz)
    logger.info("Inference output written to output.png")
    tpviz.interactive_imshow(viz)

class EvalCallback(Callback):
    """
    A callback that runs COCO evaluation once a while.
    It supports multi-gpu evaluation.
    """

    _chief_only = False

    def __init__(self, in_names, out_names):
        self._in_names, self._out_names = in_names, out_names

    def _setup_graph(self):
        num_gpu = cfg.TRAIN.NUM_GPUS
        if cfg.TRAINER == 'replicated':
            # TF bug in version 1.11, 1.12: https://github.com/tensorflow/tensorflow/issues/22750
            buggy_tf = get_tf_version_tuple() in [(1, 11), (1, 12)]

            # Use two predictor threads per GPU to get better throughput
            self.num_predictor = num_gpu if buggy_tf else num_gpu * 2
            self.predictors = [self._build_coco_predictor(k % num_gpu) for k in range(self.num_predictor)]
            self.dataflows = [get_eval_dataflow(shard=k, num_shards=self.num_predictor)
                              for k in range(self.num_predictor)]
        else:
            # Only eval on the first machine.
            # Alternatively, can eval on all ranks and use allgather, but allgather sometimes hangs
            self._horovod_run_eval = hvd.rank() == hvd.local_rank()
            if self._horovod_run_eval:
                self.predictor = self._build_coco_predictor(0)
                self.dataflow = get_eval_dataflow(shard=hvd.local_rank(), num_shards=hvd.local_size())

            self.barrier = hvd.allreduce(tf.random_normal(shape=[1]))

    def _build_coco_predictor(self, idx):
        graph_func = self.trainer.get_predictor(self._in_names, self._out_names, device=idx)
        return lambda img: detect_one_image(img, graph_func)

    def _before_train(self):
        eval_period = cfg.TRAIN.EVAL_PERIOD
        self.epochs_to_eval = set()
        for k in itertools.count(1):
            if k * eval_period > self.trainer.max_epoch:
                break
            self.epochs_to_eval.add(k * eval_period)
        self.epochs_to_eval.add(self.trainer.max_epoch)
        logger.info("[EvalCallback] Will evaluate every {} epochs".format(eval_period))

    def _eval(self):
        logdir = args.logdir
        if cfg.TRAINER == 'replicated':
            all_results = multithread_eval_coco(self.dataflows, self.predictors)
        else:
            filenames = [os.path.join(
                logdir, 'outputs{}-part{}.json'.format(self.global_step, rank)
            ) for rank in range(hvd.local_size())]

            if self._horovod_run_eval:
                local_results = eval_coco(self.dataflow, self.predictor)
                fname = filenames[hvd.local_rank()]
                with open(fname, 'w') as f:
                    json.dump(local_results, f)
            self.barrier.eval()
            if hvd.rank() > 0:
                return
            all_results = []
            for fname in filenames:
                with open(fname, 'r') as f:
                    obj = json.load(f)
                all_results.extend(obj)
                os.unlink(fname)

        output_file = os.path.join(
            logdir, 'outputs{}.json'.format(self.global_step))
        with open(output_file, 'w') as f:
            json.dump(all_results, f)
        try:
            scores = print_coco_metrics(output_file)
            for k, v in scores.items():
                self.trainer.monitors.put_scalar(k, v)
        except Exception:
            logger.exception("Exception in COCO evaluation.")

    def _trigger_epoch(self):
        if self.epoch_num in self.epochs_to_eval:
            logger.info("Running evaluation ...")
            self._eval()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--load', help='load a model for evaluation or training. Can overwrite BACKBONE.WEIGHTS')
    parser.add_argument('--logdir', help='log directory', default='train_log/maskrcnn')
    parser.add_argument('--visualize', action='store_true', help='visualize intermediate results')
    parser.add_argument('--evaluate', help="Run evaluation on COCO. "
                                           "This argument is the path to the output json evaluation file")
    parser.add_argument('--predict', help="Run prediction on a given image. "
                                          "This argument is the path to the input image file")
    parser.add_argument('--config', help="A list of KEY=VALUE to overwrite those defined in config.py",
                        nargs='+')

    if get_tf_version_tuple() < (1, 6):
        # https://github.com/tensorflow/tensorflow/issues/14657
        logger.warn("TF<1.6 has a bug which may lead to crash in FasterRCNN if you're unlucky.")

    args = parser.parse_args()
    if args.config:
        cfg.update_args(args.config)

    MODEL = ResNetFPNModel() if cfg.MODE_FPN else ResNetC4Model()

    if args.visualize or args.evaluate or args.predict:
        assert tf.test.is_gpu_available()
        assert args.load
        finalize_configs(is_training=False)

        if args.predict or args.visualize:
            cfg.TEST.RESULT_SCORE_THRESH = cfg.TEST.RESULT_SCORE_THRESH_VIS

        if args.visualize:
            visualize(MODEL, args.load)
        else:
            predcfg = PredictConfig(
                model=MODEL,
                session_init=get_model_loader(args.load),
                input_names=MODEL.get_inference_tensor_names()[0],
                output_names=MODEL.get_inference_tensor_names()[1])
            if args.predict:
                COCODetection(cfg.DATA.BASEDIR, 'val2014')   # Only to load the class names into caches
                predict(OfflinePredictor(predcfg), args.predict)
            elif args.evaluate:
                assert args.evaluate.endswith('.json'), args.evaluate
                offline_evaluate(predcfg, args.evaluate)
    else:
        is_horovod = cfg.TRAINER == 'horovod'
        if is_horovod:
            hvd.init()
            logger.info("Horovod Rank={}, Size={}".format(hvd.rank(), hvd.size()))

        if not is_horovod or hvd.rank() == 0:
            logger.set_logger_dir(args.logdir, 'd')

        finalize_configs(is_training=True)
        stepnum = cfg.TRAIN.STEPS_PER_EPOCH

        # warmup is step based, lr is epoch based
        init_lr = cfg.TRAIN.BASE_LR * 0.33 * min(8. / cfg.TRAIN.NUM_GPUS, 1.)
        warmup_schedule = [(0, init_lr), (cfg.TRAIN.WARMUP, cfg.TRAIN.BASE_LR)]
        warmup_end_epoch = cfg.TRAIN.WARMUP * 1. / stepnum
        lr_schedule = [(int(warmup_end_epoch + 0.5), cfg.TRAIN.BASE_LR)]

        factor = 8. / cfg.TRAIN.NUM_GPUS
        for idx, steps in enumerate(cfg.TRAIN.LR_SCHEDULE[:-1]):
            mult = 0.1 ** (idx + 1)
            lr_schedule.append(
                (steps * factor // stepnum, cfg.TRAIN.BASE_LR * mult))
        logger.info("Warm Up Schedule (steps, value): " + str(warmup_schedule))
        logger.info("LR Schedule (epochs, value): " + str(lr_schedule))
        train_dataflow = get_train_dataflow()
        # This is what's commonly referred to as "epochs"
        total_passes = cfg.TRAIN.LR_SCHEDULE[-1] * 8 / train_dataflow.size()
        logger.info("Total passes of the training set is: {:.5g}".format(total_passes))

        callbacks = [
            PeriodicCallback(
                ModelSaver(max_to_keep=10, keep_checkpoint_every_n_hours=1),
                every_k_epochs=20),
            # linear warmup
            ScheduledHyperParamSetter(
                'learning_rate', warmup_schedule, interp='linear', step_based=True),
            ScheduledHyperParamSetter('learning_rate', lr_schedule),
            EvalCallback(*MODEL.get_inference_tensor_names()),
            PeakMemoryTracker(),
            EstimatedTimeLeft(median=True),
            SessionRunTimeout(60000).set_chief_only(True),   # 1 minute timeout
        ]
        if not is_horovod:
            callbacks.append(GPUUtilizationTracker())

        if is_horovod and hvd.rank() > 0:
            session_init = None
        else:
            if args.load:
                session_init = get_model_loader(args.load)
            else:
                session_init = get_model_loader(cfg.BACKBONE.WEIGHTS) if cfg.BACKBONE.WEIGHTS else None

        traincfg = TrainConfig(
            model=MODEL,
            data=QueueInput(train_dataflow),
            callbacks=callbacks,
            steps_per_epoch=stepnum,
            max_epoch=cfg.TRAIN.LR_SCHEDULE[-1] * factor // stepnum,
            session_init=session_init,
            starting_epoch=cfg.TRAIN.STARTING_EPOCH
        )
        if is_horovod:
            trainer = HorovodTrainer(average=False)
        else:
            # nccl mode appears faster than cpu mode
            trainer = SyncMultiGPUTrainerReplicated(cfg.TRAIN.NUM_GPUS, average=False, mode='nccl')
        launch_train_with_config(traincfg, trainer)


text:
import sys
import cv2
import random
import numpy as np
import pandas as pd
from Ui_no5_ui import Ui_MainWindow
from matplotlib import pyplot as plt
from PyQt5.QtWidgets import QMainWindow, QApplication

import keras
from keras.datasets import cifar10
from keras.applications.vgg16 import VGG16
from keras.models import Sequential, load_model
from keras.layers import Dense, Activation, Flatten

_LABELS = ['airplane', 'automobile', 'bird', 'cat', 'deer',
           'dog', 'frog', 'horse', 'ship', 'truck']

class MainWindow(QMainWindow, Ui_MainWindow):

    label_11 = None
    x_train = None
    y_train = None
    x_test = None
    y_test = None

    def __init__(self, parent=None):
        super(MainWindow, self).__init__(parent)
        self.setupUi(self)
        self.onBindingUI()

    def onBindingUI(self):
        self.pushButton_13.clicked.connect(self.on_btn5_1_click)
        self.pushButton_14.clicked.connect(self.on_btn5_2_click)
        self.pushButton_15.clicked.connect(self.on_btn5_3_click)
        self.pushButton_16.clicked.connect(self.on_btn5_4_click)
        self.pushButton_17.clicked.connect(self.on_btn5_5_click)

    def on_btn5_1_click(self):
        self.label_11.clear()
        #Import dataset
        (x_train, y_train), (x_test, y_test) = cifar10.load_data()
        self.x_train = x_train.astype('float32')
        self.x_test = x_test.astype('float32')
        self.y_train = keras.utils.to_categorical(y_train, 10)
        self.y_test = keras.utils.to_categorical(y_test, 10)
        #print(self.x_train ,self.x_test ,'分隔', self.y_train , self.y_test)

        #plot_images_labels
        fig = plt.figure('5.1 Show Train Images',figsize=(10,5))
        fig.subplots_adjust(hspace=0.0,wspace=0.4)

        for i in range(0, 10):                                                      #依序顯示 num 個子圖
            ax = fig.add_subplot(2, 5, i+1)                                         #建立 2*5 個子圖中的第 i+1 個
            temp = random.randint(0,9999)
            x_train_resize=cv2.resize(self.x_train[temp], (128, 128))
            ax.imshow(np.uint8(x_train_resize))
            ax.set_title(_LABELS[list(self.y_train[temp]).index(1)],fontsize=10)    #設定標題
            ax.set_xticks([]);                                                      #不顯示 x 軸刻度
            ax.set_yticks([]);                                                      #不顯示 y 軸刻度
        plt.show()

    def on_btn5_2_click(self):
        print('hyperparameters:')
        print('batch size:', 32)
        print('learning rate:', 0.001)
        print('optimizer:','SGD')

    def on_btn5_3_click(self):
        input_shape = (32, 32, 3)
        model = Sequential()
        model.add(VGG16(weights=None, include_top=False,input_shape=input_shape))
        model.add(Flatten())
        model.add(Dense(4096, activation='relu'))
        model.add(Dense(4096, activation='relu'))
        model.add(Dense(10, activation='softmax'))

        model.summary()

    def on_btn5_4_click(self):
        Accurancy  = cv2.imread('Accurancy.png')
        loss = cv2.imread('loss.png')
        image = cv2.vconcat([Accurancy,loss])
        cv2.imshow('5.4 Show Accuracy', image)

    def on_btn5_5_click(self):
        self.label_11.clear()
        if self.x_train is None or self.y_train is None or self.x_test is None or  self.y_test is None :
            print('Please run "5.1 Show Train Images" first.')
            self.label_11.setText('Warning!! Please run "5.1 Show Train Images" first.')
            self.label_11.setStyleSheet("color: rgb(255, 0, 0);")
            #print(self.x_train ,self.y_train ,self.x_test , self.y_test)
        elif not len(self.lineEdit_5.text()) :
            print('Please enter (0~9999) first.')
            self.label_11.setText('Warning!! Please enter (0~9999) first.')
            self.label_11.setStyleSheet("color: rgb(255, 0, 0);")

        else :
            INDEX = int(self.lineEdit_5.text())

            model = load_model("cifar10_vgg16.h5")
            probabilities = model.predict(self.x_test)

            plt.figure('Estimation result',figsize=(10, 6), dpi=80)
            plt.bar(["airplain", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck"], list(probabilities[INDEX]), 0.5)
            plt.title('Estimation result')
            x_test = self.x_test.astype(np.uint8)
            plt.figure('image')
            plt.imshow(cv2.resize(x_test[INDEX], (128, 128)))
            title = _LABELS[list(self.y_test[INDEX]).index(1)]
            plt.title(title, fontsize=12)
            plt.xticks([])
            plt.yticks([])
            plt.show()

if __name__ == "__main__":
    app = QApplication(sys.argv)
    window = MainWindow()
    window.show()
    sys.exit(app.exec_())


text:
from scipy.io import loadmat
import numpy as np
from random import shuffle
from utils import calc_age

class DataManager(object):
    def __init__(self, dataset_name='imdb', dataset_path=None, image_size=(48, 48)):
        self.dataset_name = dataset_name
        self.dataset_path = dataset_path
        self.image_size = image_size
        if self.dataset_path is not None:
            self.dataset_path = dataset_path
        elif self.dataset_name == 'imdb':
            self.dataset_path = 'data/imdb_crop/imdb.mat'
        else:
            raise Exception('Invalid dataset')

    def get_data(self):
        if self.dataset_name == 'imdb':
            ground_truth_data = self._load_imdb()
        return ground_truth_data

    def _load_imdb(self):
        face_score_treshold = 3
        dataset = loadmat(self.dataset_path)
        image_names_array = dataset['imdb']['full_path'][0, 0][0]
        gender_classes = dataset['imdb']['gender'][0, 0][0]

        face_score = dataset['imdb']['face_score'][0, 0][0]
        second_face_score = dataset['imdb']['second_face_score'][0, 0][0]
        dob = dataset['imdb']['dob'][0, 0][0]
        photo_taken = dataset['imdb']['photo_taken'][0, 0][0]

        face_score_mask = face_score > face_score_treshold
        second_face_score_mask = np.isnan(second_face_score)
        unknown_gender_mask = np.logical_not(np.isnan(gender_classes))
        age_classes = np.array([calc_age(photo_taken[i], dob[i]) for i in range(len(dob))])

        # 0 <= age_classes <= 100
        valid_age_range = np.isin(age_classes, [x for x in range(101)])

        mask = np.logical_and(face_score_mask, second_face_score_mask)
        mask = np.logical_and(mask, unknown_gender_mask)
        mask = np.logical_and(mask, valid_age_range)

        image_names_array = image_names_array[mask]

        gender_classes = gender_classes[mask].tolist()
        age_classes = age_classes[mask].tolist()

        image_names = []
        for image_name_arg in range(image_names_array.shape[0]):
            image_name = image_names_array[image_name_arg][0]
            image_names.append(image_name)
        return dict(zip(image_names, zip(gender_classes, age_classes)))

def split_imdb_data(ground_truth_data, validation_split=.2, do_shuffle=False):
    ground_truth_keys = sorted(ground_truth_data.keys())
    if do_shuffle:
        shuffle(ground_truth_keys)
    training_split = 1 - validation_split
    num_train = int(training_split * len(ground_truth_keys))
    train_keys = ground_truth_keys[:num_train]
    validation_keys = ground_truth_keys[num_train:]
    return train_keys, validation_keys

reason: import numpy,from scipy


text:
from SpheroLib.config import sphero_config
from SpheroLib.sphero_library import SpheroLibrary
import numpy as np
import os
import cv2
import json
from pathlib import Path
import shutil
import time
from scipy.io.wavfile import write
import matplotlib.pyplot as plt
import sys

if __name__ == "__main__":
    print(sys.argv)
    import multiprocessing as mp
    mp.set_start_method("spawn")
    dataset_name = "dataset"
    num_samples = 40000
    dir_path = Path(f"{dataset_name}")
    if dir_path.exists() and dir_path.is_dir():
        start_num = len(os.listdir(dataset_name))
    else:
        start_num = 0
        os.mkdir(dataset_name)
    sphero_lib = SpheroLibrary(sphero_config)
    output = sphero_lib.get_sphero_states()

    # angles = [60, 0, 300]
    # speeds = [0, 100]
    # single_actions = [(x, y) for x in angles for y in speeds]
    # possible_actions = [(x, y) for x in single_actions for y in single_actions]

    for sample in range(start_num, num_samples + start_num):
        print(f"\rGathering Sample {sample}/{num_samples}", end=' ')
        sample_path = f"{dataset_name}/{sample}"
        [initial_state, _] = sphero_lib.get_sphero_states()
        # action_selection = np.random.randint(0, len(possible_actions))
        # action = possible_actions[action_selection]
        # heading_trajectory = [action[0][0], action[1][0]]
        # speed_trajectory = [action[0][1], action[1][1]]
        heading_trajectory = np.random.randint([0] * 4, [360] * 4)
        speed_trajectory = np.random.randint([0] * 4, [255] * 4)
        traj_start_time = time.time()
        for elt in range(4):
            action_start = time.time()
            sphero_lib.set_sphero_action(0, heading_trajectory[elt], speed_trajectory[elt])
            sphero_lib.set_sphero_action(1, heading_trajectory[elt], speed_trajectory[elt])
            time.sleep(max([.5 - (time.time() - action_start), 0]))
        [state, _] = sphero_lib.get_sphero_states()
        sphero_lib.set_sphero_action(0, heading_trajectory[-1], 0)
        time.sleep(1)
        """
        Save the data
        """
        os.mkdir(sample_path)
        cv2.imwrite(f"{sample_path}/rgb0.jpg", initial_state["rgb"][-1][:, 60:240])
        cv2.imwrite(f"{sample_path}/depth0.jpg", initial_state["depth"][-1][:, 70:310] / 64)
        write(f"{sample_path}/audio.wav", sphero_config["AUDIO_BYTES_PER_SECOND"], state["audio"].flatten())
        [cv2.imwrite(f"{sample_path}/rgb{rgb_elt + 1}.jpg", rgb[:, 60:240]) for rgb_elt, rgb in enumerate(state["rgb"])]
        [cv2.imwrite(f"{sample_path}/depth{d_elt + 1}.jpg", depth[:, 70:310] / 64) for d_elt, depth in
         enumerate(state["depth"])]
        sphero_data = {sensor: list(state["spheros"][0][:, sensor_elt]) for sensor_elt, sensor in
                       enumerate(sphero_config["SPHERO_OUTPUT_VARIABLES"])}
        sphero_data["angle_traj"] = list(heading_trajectory)
        sphero_data["speed_traj"] = list(speed_trajectory)
        with open(f"{sample_path}/data.json", 'w') as pkl_file:
            json.dump(repr(sphero_data), pkl_file)
        calm = False
        calm_start = time.time()
        while not calm:
            [state, timestamps] = sphero_lib.get_sphero_states()
            current_state = state["spheros"][0][0]
            speed = (current_state[3] ** 2 + current_state[4] ** 2) ** .5
            rotation = (current_state[5] ** 2 + current_state[6] ** 2 + current_state[7] ** 2) ** .5
            if (rotation < 15 and speed < 10) or time.time() - calm_start > 3:
                calm = True
        random_heading = np.random.randint(0, 360)
        sphero_lib.set_sphero_action(0, random_heading, 0)
        time.sleep(1)


text:
###############################################################################
# WaterTAP Copyright (c) 2021, The Regents of the University of California,
# through Lawrence Berkeley National Laboratory, Oak Ridge National
# Laboratory, National Renewable Energy Laboratory, and National Energy
# Technology Laboratory (subject to receipt of any required approvals from
# the U.S. Dept. of Energy). All rights reserved.
#
# Please see the files COPYRIGHT.md and LICENSE.md for full copyright and license
# information, respectively. These files are also available online at the URL
# "https://github.com/watertap-org/watertap/"
#
###############################################################################
import sys
import pytest
from io import StringIO

from pyomo.environ import ConcreteModel, assert_optimal_termination
from pyomo.util.check_units import assert_units_consistent
from idaes.core import FlowsheetBlock
from idaes.core.util import get_solver
from idaes.core.util.model_statistics import degrees_of_freedom
from idaes.generic_models.unit_models.heat_exchanger import (
    HeatExchanger,
    HeatExchangerFlowPattern,
)
import idaes.core.util.scaling as iscale

from watertap.unit_models.mvc.components.lmtd_chen_callback import (
    delta_temperature_chen_callback,
)
import watertap.property_models.seawater_prop_pack as props

solver = get_solver()

# -----------------------------------------------------------------------------
@pytest.mark.requires_idaes_solver
@pytest.mark.component
def test_heat_exchanger():
    m = ConcreteModel()
    m.fs = FlowsheetBlock(default={"dynamic": False})
    m.fs.properties = props.SeawaterParameterBlock()
    m.fs.unit = HeatExchanger(
        default={
            "hot_side_name": "hot",
            "cold_side_name": "cold",
            "hot": {"property_package": m.fs.properties},
            "cold": {"property_package": m.fs.properties},
            "delta_temperature_callback": delta_temperature_chen_callback,
            "flow_pattern": HeatExchangerFlowPattern.countercurrent,
        }
    )

    # scaling
    m.fs.properties.set_default_scaling("flow_mass_phase_comp", 1, index=("Liq", "H2O"))
    m.fs.properties.set_default_scaling(
        "flow_mass_phase_comp", 1e2, index=("Liq", "TDS")
    )
    iscale.set_scaling_factor(m.fs.unit.hot.heat, 1e-3)
    iscale.set_scaling_factor(m.fs.unit.cold.heat, 1e-3)
    iscale.set_scaling_factor(m.fs.unit.overall_heat_transfer_coefficient, 1e-3)
    iscale.set_scaling_factor(m.fs.unit.area, 1)
    iscale.calculate_scaling_factors(m)

    # ---specifications---
    # state variables
    m.fs.unit.hot_inlet.flow_mass_phase_comp[0, "Liq", "H2O"].fix(1)
    m.fs.unit.hot_inlet.flow_mass_phase_comp[0, "Liq", "TDS"].fix(0.01)
    m.fs.unit.hot_inlet.temperature[0].fix(350)
    m.fs.unit.hot_inlet.pressure[0].fix(2e5)

    m.fs.unit.cold_inlet.flow_mass_phase_comp[0, "Liq", "H2O"].fix(0.5)
    m.fs.unit.cold_inlet.flow_mass_phase_comp[0, "Liq", "TDS"].fix(0.01)
    m.fs.unit.cold_inlet.temperature[0].fix(298)
    m.fs.unit.cold_inlet.pressure[0].fix(2e5)

    m.fs.unit.area.fix(5)
    m.fs.unit.overall_heat_transfer_coefficient.fix(1000)

    # solving
    assert_units_consistent(m)
    degrees_of_freedom(m)

    m.fs.unit.initialize()

    solver = get_solver()
    results = solver.solve(m, tee=False)
    assert_optimal_termination(results)

    report_io = StringIO()
    m.fs.unit.report(ostream=report_io)
    output = """
====================================================================================
Unit : fs.unit                                                             Time: 0.0
------------------------------------------------------------------------------------
    Unit Performance

    Variables:

    Key            : Value  : Fixed : Bounds
           HX Area : 5.0000 :  True : (0, None)
    HX Coefficient : 1000.0 :  True : (0, None)
         Heat Duty : 89050. : False : (None, None)

    Expressions:

    Key             : Value
    Delta T Driving : 17.810
         Delta T In : 9.2239
        Delta T Out : 30.689

------------------------------------------------------------------------------------
    Stream Table
                                         Hot Inlet  Hot Outlet  Cold Inlet  Cold Outlet
    flow_mass_phase_comp ('Liq', 'H2O')     1.0000      1.0000     0.50000     0.50000
    flow_mass_phase_comp ('Liq', 'TDS')   0.010000    0.010000    0.010000    0.010000
    temperature                             350.00      328.69      298.00      340.78
    pressure                            2.0000e+05  2.0000e+05  2.0000e+05  2.0000e+05
====================================================================================
"""
    assert output == report_io.getvalue()

max_stars_repo_name: srikanthallu/watertap

max_issues_repo_name: srikanthallu/watertap

max_forks_repo_name: srikanthallu/watertap

reason: from pyomo


text:
#!/usr/bin/env python
import rospy
import time
import numpy as np
from Adafruit_LSM303 import Adafruit_LSM303
from Gyro_L3GD20 import Gyro_L3GD20
from sensor_msgs.msg import Imu
from sensor_msgs.msg import MagneticField

class AdafruitIMU(object):

    # Physical constants
    G = 9.80665 # Standart gravity at sea level (should be g, but
                # capitalization rules due to coding practices)
    DEG_TO_RAD = 0.0174533 # degrees to radians

    def __init__(self):
        self.node_name = rospy.get_name()
        rospy.loginfo("[%s] Initializing " %(self.node_name))

        # Setup compass and accelerometer
        self.compass_accel = Adafruit_LSM303()

        # Setup gyroscope
        self.gyro = Gyro_L3GD20()

        # Setup Parameters
        self.pub_timestep = self.setupParam("~pub_timestep", 0.02)

        # Publications
        self.pub_imu = rospy.Publisher("~adafruit_imu", Imu, queue_size=10)
        self.pub_mag = rospy.Publisher("~adafruit_mag", MagneticField, queue_size=10)

        # timer
        self.pub_timer = rospy.Timer(rospy.Duration.from_sec(self.pub_timestep),self.publish)

    def setupParam(self, param_name, default_value):
        value = rospy.get_param(param_name, default_value)
        # Write to parameter server for transparancy
        rospy.set_param(param_name, value)
        rospy.loginfo("[%s] %s = %s " % (self.node_name, param_name, value))
        return value

    def publish(self, event):
        compass_accel = self.compass_accel.read()
        compass = compass_accel[0:3]
        accel = compass_accel[3:6]
        gyro = self.gyro.read()

        # Put together an IMU message
        imu_msg = Imu()
        imu_msg.header.stamp = rospy.Time.now()
        imu_msg.orientation_covariance[0] = -1
        imu_msg.angular_velocity = gyro[0] * DEG_TO_RAD
        imu_msg.angular_velocity = gyro[1] * DEG_TO_RAD
        imu_msg.angular_velocity = gyro[2] * DEG_TO_RAD
        imu_msg.linear_acceleration.x = accel[0] * G
        imu_msg.linear_acceleration.y = accel[1] * G
        imu_msg.linear_acceleration.z = accel[2] * G

        self.pub_imu.publish(imu_msg)

        # Put together a magnetometer message
        mag_msg = MagneticField()
        mag_msg.header.stamp = rospy.Time.now()
        mag_msg.magnetic_field.x = compass[0]
        mag_msg.magnetic_field.y = compass[1]
        mag_msg.magnetic_field.z = compass[2]

        self.pub_mag.publish(mag_msg)

if __name__ == "__main__":
    rospy.init_node("Adafruit_IMU", anonymous=False)
    adafruit_IMU = AdafruitIMU()
    rospy.spin()


text:
import os
from utilities import get_spherical_distance
import numpy as np

def union(list1,list2):
	res = list1
	for i in list2:
		if not (i in res):
			res.append(i)
	return res

def get_neighbours(point,pointSet,eps):
	N = []
	for pt in pointSet:
		#if pointSet.index(point)>pointSet.index(pt):
			#continue
		#print (pt[0] - point[0]),(pt[1] - point[1])

		'''
		dist = math.pow(10000*(pt[1] - point[1]),2.0) + math.pow(10000*(pt[0] - point[0]),2.0)
		dist = math.sqrt(dist)
		dist_n = dist/10000
		#file.write(str(dist_n)+'\n')
		'''
		dist = get_spherical_distance(pt[0],pt[1],point[0],point[1])
		if dist<=eps:
			N.append(pt)
	return N

def findStops(folderName,tc,speed_threshold, stopRadi,stopFileName):
	if(len(stopFileName.split('.'))>1):
		stopFileName = stopFileName.split('.')[0] + '.stop'
	files = os.listdir(folderName)
	slow_points = []
	stopfilecount = 0
	for file in files:
		if(file.endswith('.stop')):
			stopfilecount += 1
			continue
		slows = []
		slowsets = []
		f = open(os.path.join(folderName,file),'r')
		lines = f.read().split('\n')
		f.close()
		pointSet = []
		for line in lines:
			components = line.split(',')
			try:
				point_x = float(components[0])
				point_y = float(components[1])
				#split time into hours, minutes and seconds
				timeunits = components[2].split(':')
				#convert the time into seconds
				point_t = ((int(timeunits[0])*60) + int(timeunits[1])) * 60 + int(timeunits[2])
				pointSet += [(point_x,point_y,point_t)]
			except Exception as e:
				# print(e)
				pass
		for i in range(len(pointSet)-1):
			d = get_spherical_distance(pointSet[i][0],pointSet[i][1],pointSet[i+1][0],pointSet[i+1][1])
			if (pointSet[i+1][2] - pointSet[i][2])==0:
				continue
			if (d/(pointSet[i+1][2] - pointSet[i][2])) <= speed_threshold:
				slows.append(pointSet[i+1])
		for i in range(len(slows)-1):
			if len(slowsets)==0:
				slowsets.append(slows[i])
			d = get_spherical_distance(slows[i][0],slows[i][1],slows[i+1][0],slows[i+1][1])
			if d<=stopRadi:
				slowsets.append(slows[i+1])
			else:
				slow_points.append(slowsets)
				slowsets = []
	# print(slow_points)
	# print(len(slow_points))
	slows = []
	centers = []
	for point in slow_points:
		# print(point)
		contribution = len(point)
		data = np.array(point)
		mean = np.mean(data[:,0:2],axis=0)
		std = np.std(data[:,0:2],axis=0)
		# print(contribution,mean)
		slows.append((contribution,mean,std))
		centers.append(mean.tolist())
		# break
	# print(len(centers))
	# centers = centers.tolist()
	pts = centers[:]

	ps = []
	for point in centers:
		if(point not in pts):
			continue
		pts.remove(point)
		#print len(pts), len(pointSet)
		N = get_neighbours(point,pts,stopRadi)
		C = [point]
		# print(C, N)
		C = union(C,N)
		for n in N:
			pts.remove(n)
		# print(len(pts))
		while len(N)>0:
			#print "Neighbour ", neighbour
			N = []
			for c in C:
				n = get_neighbours(c,pts,stopRadi)
				N = union(N,n)
				for i in n:
					pts.remove(i)
			C = union(C,N)
		ps.append(C)
	res = []
	for p in ps:
		if(len(p)>tc*(len(files)-stopfilecount)):
			print(len(p))
			mean = np.mean(p,axis=0)
			res.append(mean)
	print(len(res))
	# for r in res:
	# 	print(r[0],',',r[1])
	# return res

	file = open(os.path.join(folderName,stopFileName),'w')
	for i in range(len(res)):
		file.write(str(res[i][0])+','+str(res[i][1])+'\n')
		# res.find(s)==(len(res)-1)
		if(i==(len(res)-1)):
			file.write(str(res[i][0])+','+str(res[i][1]))
	file.close()
	return res

if __name__ == '__main__':
	detected = findStops('G:/BusStopFinderStuff/Code/input/GPS_Trails/8B_final/up_8B',0.25,0,30,'stop.txt')
	# actual = np.loadtxt("G:/BusStopFinderStuff/Code/groundtruth/8B_GT_SS(1).txt",delimiter=',',usecols=(1,2),skiprows=1)
	# actual = actual.tolist()
	# found = len(detected)
	# tobe = len(actual)
	# tp = 0
	# fp = 0
	# fn = 0
	# for d in detected:
	# 	match = False
	# 	for a in actual:
	# 		if (get_spherical_distance(d[0],d[1],a[0],a[1])<50):
	# 			tp += 1
	# 			actual.remove(a)
	# 			match = True
	# 			break
	# 	if not match:
	# 		fp += 1
	# fn = len(actual)

	# print(tobe,found,tp,fp,fn)


text:
import Neural_Net as NN
import utils
import numpy as np
from tensorflow.keras.datasets import mnist  # yes, a tf install, but just for the easy mnist import.
from matplotlib import pyplot as plt

(X_train, Y_train), (X_test, Y_test) = mnist.load_data()
# Flattening 28x28 pixel digits
X_train = X_train.reshape(X_train.shape[0], X_train.shape[1]*X_train.shape[2])
X_test = X_test.reshape(X_test.shape[0], X_test.shape[1]*X_test.shape[2])
# One hot encoding Y data
Y_train = utils.one_hot_encode(Y_train)
Y_test = utils.one_hot_encode(Y_test)
training_data = utils.np_to_list_tuples(X_train, Y_train)
test_data = utils.np_to_list_tuples(X_test, Y_test)

mnist_layers = [X_train.shape[1], 64, 16, 10]
mnist_net = NN.NeuralNet(mnist_layers, "logistic")
mnist_net.SGD(training_data, epochs=50, mini_batch_size=50, test_data=test_data[9500:])
print(mnist_net.one_hot_evaluate(X_test, Y_test))


text:
import numpy as np
from sklearn.metrics import precision_recall_fscore_support
from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_squared_error

def evaluation(args):
    '''
    We use f-score, accuracy, MSE to evaluation the performance of different models.
    Here, the best model is selected based on the averaged f-score.
    '''
    score_test = 0.0
    score_validate = 0.0
    mdx_test = 1
    mdx_validate = 1
    memo = []
    for epoch in range(1, args.n_epoch+1):
        print('='*50)
        print('Epoch: {}'.format(epoch))
        score_dict = {}

        mem_score = {'validate': [], 'test': []}

        pred_data = np.loadtxt('../nats_results/validate_pred_{}.txt'.format(epoch))
        true_data = np.loadtxt('../nats_results/validate_true_{}.txt'.format(epoch))
        accu = accuracy_score(true_data, pred_data)

        print('Accuracy={}'.format(np.round(accu, 4)))
        mem_score['validate']= accu
        if accu > score_validate:
            score_validate = accu
            mdx_validate = epoch

        pred_data = np.loadtxt('../nats_results/test_pred_{}.txt'.format(epoch))
        true_data = np.loadtxt('../nats_results/test_true_{}.txt'.format(epoch))

        accu = accuracy_score(true_data, pred_data)
        print('Accuracy={}'.format(np.round(accu, 4)))
        mem_score['test'] = accu

        if accu > score_test:
            score_test = accu
            mdx_test = epoch

        memo.append(mem_score)

    print('='*50)
    print('Best epoch {}'.format(mdx_validate))
    print('='*50)
    print('Val')
    accu = memo[mdx_validate-1]['validate']
    print('Accuracy={}'.format(np.round(accu, 4)))
    print('Test')
    accu = memo[mdx_validate-1]['test']
    print('Accuracy={}'.format(np.round(accu, 4)))
    print('='*50)
    print('Max epoch {}'.format(mdx_test))
    print('='*50)
    print('Val')
    accu = memo[mdx_test-1]['validate']
    print('Accuracy={}'.format(np.round(accu, 4)))
    print('Test')
    accu = memo[mdx_test-1]['test']
    print('Accuracy={}'.format(np.round(accu, 4)))

max_stars_repo_name: haophancs/TREQS

max_issues_repo_name: haophancs/TREQS

max_forks_repo_name: haophancs/TREQS


text:
import numpy as np

def indices_bool_arr(x, bool_arr):
    """
    Returns indices of an array x for which
    the corresponding value in bool_arr is true.
    """

    all_indices = np.arange(len(x), dtype=np.uint64)
    return all_indices[bool_arr]

def indices_cond(x, cond_func):
    """
    Returns indices of an array for which the supplied
    condition is true.
    """

    cond_res = cond_func(x)
    return indices_bool_arr(x, cond_res)

def close_to_zero(x, tol=1e-6):
    """
    Condition applied to an array that
    marks which elements of the array
    have values close to zero.
    """

    return np.bitwise_and(x > -tol, x < tol)

def not_inf(x):
    """
    Condition applied to an array that
    marks which elements of the array
    have values not in set {-inf, inf}.
    """

    return np.abs(x) != np.inf

def bool_arr_diff(a, b):
    """
    Returns a boolean array representing
    a set difference (a \ b).
    """

    return np.bitwise_and(a, np.bitwise_not(b))

def apply_multiple_conditions(x, *conds):
    """
    Applies multiple conditions to an array
    and returns bitwise_and of all the outputs.
    """

    cond_bool_arrays = [cnd(x) for cnd in conds]
    return np.bitwise_and(*cond_bool_arrays)

def arrays_are_identical(x1, x2):
    """
    Returns True of two arrays have identical values.
    Should be applied to arrays of booleans.
    """

    return np.all(x1 == x2)

max_stars_repo_name: semeniuta/visionfuncs

max_issues_repo_name: semeniuta/visionfuncs

max_forks_repo_name: semeniuta/visionfuncs