text:
"""
Code that wraps around Matlab to execute Feng Zhou's CTW/GTW code
http://www.f-zhou.com/ta_code.html
"""
import numpy as np
import scipy.io as sio
import subprocess
import matplotlib.pyplot as plt
import os
import sys
import inspect

def getPrefix():
    s = ''
    if 'Alignment.ctw' in sys.modules:
        #raise Exception('Must import Alignment.ctw before running Matlab code')
        s = "%s/"%os.path.dirname(inspect.getfile(sys.modules['Alignment.ctw']))
    return s

def getCTWAlignments(eng, X1, X2, doPCA = 1):
    """
    Wrap around the CTW library to compute warping paths between
    X1 and X2 using DTW, DDTW, IMW, CTW, and GTW
    :param eng: Matlab engine
    :param X1: N x d Euclidean vector
    :param X2: M x d Euclidean vector
    :param prefix: Relative path to help locate Matlab files
    :returns: dictionary of warping results
    """
    prefix = getPrefix()
    sio.savemat("%sXs.mat"%prefix, {"X1":X1, "X2":X2, "doPCA":doPCA})
    eng.extractAlignments(nargout=0)
    res = sio.loadmat("%smatlabResults.mat"%prefix)
    os.remove("%sXs.mat"%prefix)
    os.remove("%smatlabResults.mat"%prefix)
    toRemove = []
    for k in res.keys():
        if not k[0] == 'P':
            toRemove.append(k)
        else:
            res[k] = res[k] - 1 #Matlab is 1-indexed
    for k in toRemove:
        del res[k]
    return res

def initMatlabEngine():
    prefix = getPrefix()
    import matlab.engine
    beforePath = os.path.realpath(".")
    if len(prefix) > 0:
        os.chdir('%s'%prefix)
    eng = matlab.engine.start_matlab()
    os.chdir(beforePath)
    return eng

def testCTWAlignments(eng):
    N = 200
    t1 = np.linspace(0, 1, N)
    t2 = t1**2
    X1 = np.zeros((N, 2))
    X1[:, 0] = np.cos(2*np.pi*t1)
    X1[:, 1] = np.sin(4*np.pi*t1)
    X2 = np.zeros((N, 2))
    X2[:, 0] = np.cos(2*np.pi*t2)
    X2[:, 1] = np.sin(4*np.pi*t2)

    Ps = getCTWAlignments(eng, X1, X2)
    types = Ps.keys()
    plt.hold(True)
    for i in range(len(types)):
        print(types[i])
        P = Ps[types[i]]
        plt.plot(P[:, 0], P[:, 1])
    plt.legend([t[1::] for t in types])
    plt.show()

if __name__ == '__main__':
    eng = initMatlabEngine()
    testCTWAlignments(eng)

max_stars_repo_name: ctralie/IsometryBlindTimeWarping

max_issues_repo_name: ctralie/IsometryBlindTimeWarping

max_forks_repo_name: ctralie/IsometryBlindTimeWarping

reason: import numpy,import scipy


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from collections import defaultdict
import argparse
import cv2  # NOQA (Must import before importing caffe2 due to bug in cv2)
import glob
import logging
import os
import sys
from queue import Queue
import json
import yaml
import numpy as np
import time
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler

from caffe2.python import workspace

from core.config import assert_and_infer_cfg
from core.config import cfg
from core.config import merge_cfg_from_file
from utils.timer import Timer
from utils.io import cache_url
import core.test_engine as infer_engine
import datasets.dummy_datasets as dummy_datasets
import utils.c2 as c2_utils
import utils.logging
import utils.vis as vis_utils
import utils.segms as segms
import utils.videoutils as vid_utils

c2_utils.import_detectron_ops()
# OpenCL may be enabled by default in OpenCV3; disable it because it's not
# thread safe and causes unwanted GPU memory allocations.
cv2.ocl.setUseOpenCL(False)

MILLISECONDS_IN_SECOND= 1000.0
DIRECTORY_TO_WATCH = "/mnt/api_files/input/"
DIRECTORY_TEMP = "/mnt/api_files/tmp/"
DIRECTORY_TO_WRITE = "/mnt/api_files/output/"
GS_BUCKET = "gs://microapps-175405.appspot.com/srishti/"
OUTPUT_FILE_EXTENSION = '_output.png'
STICKER_SELFIE_HIT = "sticker"

def parse_args():
    parser = argparse.ArgumentParser(description='End-to-end inference')
    parser.add_argument(
        '--cfg',
        dest='cfg',
        help='cfg model file (/path/to/model_config.yaml)',
        default=None,
        type=str
    )
    parser.add_argument(
        '--wts',
        dest='weights',
        help='weights model file (/path/to/model_weights.pkl)',
        default=None,
        type=str
    )
    parser.add_argument(
        '--output-dir',
        dest='output_dir',
        help='directory for visualization pdfs (default: /tmp/infer_simple)',
        default='/tmp/infer_simple',
        type=str
    )
    parser.add_argument(
        '--image-ext',
        dest='image_ext',
        help='image file name extension (default: jpg)',
        default='jpg',
        type=str
    )
    parser.add_argument(
        '--class-label',
        dest='class_label',
        help='class label to extract',
        default='person',
        type=str
    )
    parser.add_argument(
        'im_or_folder', help='image or folder of images', default=None
    )
    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(1)
    return parser.parse_args()

def write_to_local(filename, filevalue):
    cv2.imwrite(filename, filevalue)
    logger.info("written to local: "+ filename)

def write_to_gcs(local_filepath, gcs_filename):
    cmd = "gsutil cp " + local_filepath + " " + GS_BUCKET + gcs_filename
    returned_value = os.system(cmd)
    logger.info("written to gcs: " + local_filepath + ", returned value: "+ str(returned_value))
    return returned_value

def image_resize(image, height = None, width = None, inter = cv2.INTER_AREA):
    # initialize the dimensions of the image to be resized and
    # grab the image size
    dim = None
    (h, w) = image.shape[:2]

    # if both the width and height are None, then return the
    # original image
    if width is None and height is None:
        return image

    # check to see if the width is None
    if width is None:
        # calculate the ratio of the height and construct the
        # dimensions
        r = height / float(h)
        dim = (int(w * r), height)

    # otherwise, the height is None
    else:
        # calculate the ratio of the width and construct the
        # dimensions
        r = width / float(w)
        dim = (width, int(h * r))

    # resize the image
    resized = cv2.resize(image, dim, interpolation = inter)

    # return the resized image
    return resized

def segment(im_list, filename):
    for i, im_name in enumerate(im_list):
        out_name = os.path.join(
            args.output_dir, '{}'.format(os.path.basename(im_name) + '.pdf')
        )
        logger.info('Processing {} -> {}'.format(im_name, out_name))
        im = cv2.imread(im_name)
        if (im.shape[0] > 650):
            im = image_resize(im, height=600)
        timers = defaultdict(Timer)
        t = time.time()
        with c2_utils.NamedCudaScope(0):
            cls_boxes, cls_segms, cls_keyps = infer_engine.im_detect_all(
                model, im, None, timers=timers
            )
        logger.info('Inference time: {:.3f}s'.format(time.time() - t))
        for k, v in timers.items():
            logger.info(' | {}: {:.3f}s'.format(k, v.average_time))
        if i == 0:
            logger.info(
                ' \ Note: inference on the first image will be slower than the '
                'rest (caches and auto-tuning need to warm up)'
            )
        segmented_images, classes, scores, segmented_binary_masks = vis_utils.segmented_images(
            im,
            im_name,
            args.output_dir,
            cls_boxes,
            cls_segms,
            cls_keyps,
            dataset=dummy_coco_dataset,
            box_alpha=0.3,
            show_class=True,
            thresh=0.7,
            kp_thresh=2
        )
        found = False
        if len(segmented_images) > 0:
            for index, value in enumerate(segmented_images):
                if classes[index] == args.class_label and not found:
                    found = True
                    bin_mask = segmented_binary_masks[index]
                    return found,  value, bin_mask

        return found, "", ""

def style_transfer(input_file_path, input_file_name, mask):
    logger.info("in function style_transfer")
    tmp_file_path = input_file_path
    output_file_name = input_file_name.rstrip(".png") + "styled.png"
    logger.info("calling style transfer model, output file in tmp folder: " + output_file_name)
    cmd = "cd /mnt/fast-style-transfer; " \
          +"python evaluate.py --checkpoint /mnt/fast-style-transfer/checkpoints/johnny2 --in-path "\
          + input_file_path + input_file_name + " --out-path " + tmp_file_path + output_file_name\
          + " --device '/gpu:1'; cd /mnt/Detectron"
    logger.info("running command: "+ cmd)
    returned_val = os.system(cmd)

    logger.info("drawing border, returned value from style transfer: " + str(returned_val))
    logger.info("segmented image path: "+ tmp_file_path + output_file_name)
    logger.info("styled image path tmp: "+ tmp_file_path + output_file_name)

    img = cv2.imread(input_file_path + input_file_name)
    styled_img = cv2.imread(tmp_file_path + output_file_name)
    img_final = vis_utils.add_sticker_border(img, styled_img, mask)
    img_final = vis_utils.adjust_gamma(img_final, 0.5)

    return returned_val, DIRECTORY_TO_WRITE,  output_file_name, img_final

class Watcher:
    def __init__(self):
        self.observer = Observer()

    def run(self):
        event_handler = Handler()
        self.observer.schedule(event_handler, DIRECTORY_TO_WATCH, recursive=True)
        self.observer.start()
        try:
            while True:
                time.sleep(50.0/ MILLISECONDS_IN_SECOND)
        except:
            self.observer.stop()
            logger.error("Error")

        self.observer.join()

class Handler(FileSystemEventHandler):
    @staticmethod
    def on_any_event(event):
        if event.is_directory:
            return None

        elif event.event_type == 'created':
            # Take any action here when a file is first created.
            logger.info("Received created event - %s." % event.src_path)
            im_list = [event.src_path]
            k = event.src_path.rfind("/")
            original_filename = event.src_path[k+1:]

        # Image segmentation
        logger.info("IMLIST size: " + str(len(im_list)))
        found, filevalue, binmask_value = segment(im_list, event.src_path[k+1:])
        style = STICKER_SELFIE_HIT in original_filename
        gcs_filename = original_filename.rstrip(".jpg") + OUTPUT_FILE_EXTENSION
        if found:
            final_local_file = DIRECTORY_TO_WRITE + original_filename.rstrip(".jpg") + OUTPUT_FILE_EXTENSION
            if style:
                tmp_local_file = DIRECTORY_TEMP + original_filename.rstrip(".jpg") + "segmented.png"
                write_to_local(tmp_local_file, filevalue)
                returned_value, output_file_path, output_file_name, output_file_value = style_transfer(DIRECTORY_TEMP, original_filename.rstrip(".jpg") + "segmented.png", binmask_value)
                write_to_local(output_file_path+output_file_name, output_file_value)
                write_to_gcs(output_file_path + output_file_name, gcs_filename)
            else:
                write_to_local(final_local_file, filevalue)
                write_to_gcs(final_local_file, gcs_filename)
        logger.info("segmentation done and saved: " + str(gcs_filename))

# Global variables on start
workspace.GlobalInit(['caffe2', '--caffe2_log_level=0'])
utils.logging.setup_logging(__name__)
args = parse_args()

logger = logging.getLogger(__name__)
merge_cfg_from_file(args.cfg)
cfg.NUM_GPUS = 1
args.weights = cache_url(args.weights, cfg.DOWNLOAD_CACHE)
assert_and_infer_cfg()
model = infer_engine.initialize_model_from_cfg(args.weights)
dummy_coco_dataset = dummy_datasets.get_coco_dataset()

if __name__ == '__main__':
    logger.info("Done initialising model")
    w = Watcher()
    w.run()

ext: py

max_stars_repo_name: srishti-hike/Detectron

max_issues_repo_name: srishti-hike/Detectron

max_forks_repo_name: srishti-hike/Detectron

reason: import numpy


text:
import numpy as np
import opensim

integrator_accuracy = 3e-3
stepsize = 0.00003074
model = opensim.Model('./gait14dof22musc_pros_20180507.osim')

# initialise model's simbody System object. System is a computational representation of the Model
model.initSystem()
# initialise the ModelVisualizer object
model.setUseVisualizer(True)

'''
Defines controller for controlling actuators.
A controller computes and sets the values of the controls for the actuators under its control
PrescribedController specifies functions that prescribe the control values of its actuators as a function of time.
So we will explicitly provide these functions.
'''
brain = opensim.PrescribedController()

muscleSet = model.getMuscles()

### set constant control on all muscles
for j in range(muscleSet.getSize()):
    func = opensim.Constant(1.0)
    brain.addActuator(muscleSet.get(j))
    brain.prescribeControlForActuator(j, func)

model.addController(brain)
model.initSystem()

########### System initialised

initial_state = model.initializeState()
initial_state.setTime(0)
istep = 0
manager = opensim.Manager(model)
state = initial_state

def reset_manager():
    manager.setIntegratorAccuracy(integrator_accuracy)
    manager.initialize(initial_state)

def reset():
    global istep
    istep = 0
    reset_manager()

reset()

def integrate(endtime=None):
    global istep, state
    # Define the new endtime of the simulation
    istep = istep + 1

    # Integrate till the new endtime
    try:
        if (endtime == None):
            endtime = stepsize * istep
        state = manager.integrate(endtime)
    except Exception as e:
        print(e)

def actuate(action):
    if np.any(np.isnan(action)):
        raise ValueError("NaN passed in the activation vector. Values in [0,1] interval are required.")

    brain = opensim.PrescribedController.safeDownCast(model.getControllerSet().get(0))
    functionSet = brain.get_ControlFunctions()

    for j in range(functionSet.getSize()):
        func = opensim.Constant.safeDownCast(functionSet.get(j))
        func.setValue(float(action[j]))

# action = [1.0] * muscleSet.getSize()
# actuate(action)
# for i in range(1):
# 	model.setStateVariableValue(state,"ground_pelvis/pelvis_tx/value",0.2*i)
# 	model.assemble(state)
# 	integrate()
# 	print(model.getStateVariableValue(state,"ground_pelvis/pelvis_tx/value"))
# time.sleep(1)
# time.sleep(3)

####################################################################################

# Dictionary containing mapping from .mot file headers to state variable names of osim
motFileHeader_to_stateVariableName_dict = {"pelvis_tx": "ground_pelvis/pelvis_tx/value",
                                           "pelvis_ty": "ground_pelvis/pelvis_ty/value",
                                           "pelvis_tz": "ground_pelvis/pelvis_tz/value",
                                           "pelvis_tilt": "ground_pelvis/pelvis_tilt/value",
                                           "pelvis_list": "ground_pelvis/pelvis_list/value",
                                           "pelvis_rotation": "ground_pelvis/pelvis_rotation/value",
                                           "hip_flexion_r": "hip_r/hip_flexion_r/value",
                                           "hip_adduction_r": "hip_r/hip_adduction_r/value",
                                           "hip_rotation_r": "hip_r/hip_rotation_r/value",
                                           "knee_angle_r": "knee_r/knee_angle_r/value",
                                           "ankle_angle_r": "ankle_r/ankle_angle_r/value",
                                           "hip_flexion_l": "hip_l/hip_flexion_l/value",
                                           "hip_adduction_l": "hip_l/hip_adduction_l/value",
                                           "hip_rotation_l": "hip_l/hip_rotation_l/value",
                                           "knee_angle_l": "knee_l/knee_angle_l/value",
                                           "ankle_angle_l": "ankle_l/ankle_angle_l/value",
                                           "lumbar_extension": "back/lumbar_extension/value"}

def getStateVariableName(motHeader):
    return motFileHeader_to_stateVariableName_dict[motHeader]

motHeader_distance_indices = [1, 2, 3]
motHeader_angle_indices = [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]

def toRadian(angle):
    return angle / 180 * np.pi

####################################################################################

# Simulate values from the mot file
with open("../../data/motion/subject01_Run_20002_cycle1_states.sto") as f:
    headers = f.readline().split()
    count = 0
    for line in f.readlines():
        # convert strings to floats
        values = [float(x) for x in line.split()]
        for motHeader_index in motHeader_distance_indices:
            model.setStateVariableValue(state, getStateVariableName(headers[motHeader_index]), values[motHeader_index])
        for motHeader_index in motHeader_angle_indices:
            model.setStateVariableValue(state, getStateVariableName(headers[motHeader_index]),
                                        toRadian(values[motHeader_index]))
        model.assemble(state)
        integrate(endtime=values[0])

####################################################################################


text:
import cfpq_data
import networkx as nx

from project.utils import graph_utils
from pathlib import Path

graph_env = {"test_graph": graph_utils.generate_two_cycles_graph("3", "4", "a", "b")}

def test_save_to_dot_path():
    path = graph_utils.save_to_dot(graph_env["test_graph"], "tests/data/test_graph.dot")
    assert path == Path("tests/data/test_graph.dot")

def test_graph_isomorphism(tmpdir):
    n, m = 52, 48
    edge_labels = ("a", "b")
    file = tmpdir.mkdir("test_dir").join("two_cycles.dot")

    graph = cfpq_data.labeled_two_cycles_graph(
        n, m, edge_labels=edge_labels, verbose=False
    )
    graph_utils.save_to_dot(graph, file)

    actual_graph_string = None
    with open(file, "r") as graph_file:
        actual_graph_string = graph_file.read()

    expected_graph = cfpq_data.labeled_two_cycles_graph(
        n, m, edge_labels=edge_labels, verbose=False
    )

    expected_graph_string = nx.drawing.nx_pydot.to_pydot(expected_graph).to_string()

    assert actual_graph_string == expected_graph_string

max_stars_repo_name: achains/formal-lang-course

max_issues_repo_name: achains/formal-lang-course

max_forks_repo_name: achains/formal-lang-course

reason: import networkx


text:
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
from __future__ import absolute_import
from unittest.mock import MagicMock

import pytest
from mock import Mock, patch

import sagemaker
from sagemaker.model import FrameworkModel, Model
from sagemaker.huggingface.model import HuggingFaceModel
from sagemaker.jumpstart.constants import JUMPSTART_BUCKET_NAME_SET, JUMPSTART_RESOURCE_BASE_NAME
from sagemaker.jumpstart.enums import JumpStartTag
from sagemaker.mxnet.model import MXNetModel
from sagemaker.pytorch.model import PyTorchModel
from sagemaker.sklearn.model import SKLearnModel
from sagemaker.tensorflow.model import TensorFlowModel
from sagemaker.xgboost.model import XGBoostModel
from sagemaker.workflow.properties import Properties

MODEL_DATA = "s3://bucket/model.tar.gz"
MODEL_IMAGE = "mi"
TIMESTAMP = "2017-10-10-14-14-15"
MODEL_NAME = "{}-{}".format(MODEL_IMAGE, TIMESTAMP)

INSTANCE_COUNT = 2
INSTANCE_TYPE = "ml.c4.4xlarge"
ROLE = "some-role"

REGION = "us-west-2"
BUCKET_NAME = "some-bucket-name"
GIT_REPO = "https://github.com/aws/sagemaker-python-sdk.git"
BRANCH = "test-branch-git-config"
COMMIT = "ae15c9d7d5b97ea95ea451e4662ee43da3401d73"
ENTRY_POINT_INFERENCE = "inference.py"
SCRIPT_URI = "s3://codebucket/someprefix/sourcedir.tar.gz"
IMAGE_URI = "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.9.0-gpu-py38"

class DummyFrameworkModel(FrameworkModel):
    def __init__(self, **kwargs):
        super(DummyFrameworkModel, self).__init__(
            **kwargs,
        )

@pytest.fixture()
def sagemaker_session():
    boto_mock = Mock(name="boto_session", region_name=REGION)
    sms = MagicMock(
        name="sagemaker_session",
        boto_session=boto_mock,
        boto_region_name=REGION,
        config=None,
        local_mode=False,
        s3_client=None,
        s3_resource=None,
    )
    sms.default_bucket = Mock(name="default_bucket", return_value=BUCKET_NAME)

    return sms

@patch("shutil.rmtree", MagicMock())
@patch("tarfile.open", MagicMock())
@patch("os.listdir", MagicMock(return_value=[ENTRY_POINT_INFERENCE]))
def test_prepare_container_def_with_model_src_s3_returns_correct_url(sagemaker_session):
    model = Model(
        entry_point=ENTRY_POINT_INFERENCE,
        role=ROLE,
        sagemaker_session=sagemaker_session,
        source_dir=SCRIPT_URI,
        image_uri=MODEL_IMAGE,
        model_data=Properties("Steps.MyStep"),
    )
    container_def = model.prepare_container_def(INSTANCE_TYPE, "ml.eia.medium")

    assert container_def["Environment"]["SAGEMAKER_SUBMIT_DIRECTORY"] == SCRIPT_URI

def test_prepare_container_def_with_model_data():
    model = Model(MODEL_IMAGE)
    container_def = model.prepare_container_def(INSTANCE_TYPE, "ml.eia.medium")

    expected = {"Image": MODEL_IMAGE, "Environment": {}}
    assert expected == container_def

def test_prepare_container_def_with_model_data_and_env():
    env = {"FOO": "BAR"}
    model = Model(MODEL_IMAGE, MODEL_DATA, env=env)

    expected = {"Image": MODEL_IMAGE, "Environment": env, "ModelDataUrl": MODEL_DATA}

    container_def = model.prepare_container_def(INSTANCE_TYPE, "ml.eia.medium")
    assert expected == container_def

    container_def = model.prepare_container_def()
    assert expected == container_def

def test_prepare_container_def_with_image_config():
    image_config = {"RepositoryAccessMode": "Vpc"}
    model = Model(MODEL_IMAGE, image_config=image_config)

    expected = {
        "Image": MODEL_IMAGE,
        "ImageConfig": {"RepositoryAccessMode": "Vpc"},
        "Environment": {},
    }

    container_def = model.prepare_container_def()
    assert expected == container_def

def test_model_enable_network_isolation():
    model = Model(MODEL_IMAGE, MODEL_DATA)
    assert model.enable_network_isolation() is False

    model = Model(MODEL_IMAGE, MODEL_DATA, enable_network_isolation=True)
    assert model.enable_network_isolation()

@patch("sagemaker.model.Model.prepare_container_def")
def test_create_sagemaker_model(prepare_container_def, sagemaker_session):
    container_def = {"Image": MODEL_IMAGE, "Environment": {}, "ModelDataUrl": MODEL_DATA}
    prepare_container_def.return_value = container_def

    model = Model(MODEL_DATA, MODEL_IMAGE, name=MODEL_NAME, sagemaker_session=sagemaker_session)
    model._create_sagemaker_model()

    prepare_container_def.assert_called_with(
        None, accelerator_type=None, serverless_inference_config=None
    )
    sagemaker_session.create_model.assert_called_with(
        MODEL_NAME, None, container_def, vpc_config=None, enable_network_isolation=False, tags=None
    )

@patch("sagemaker.model.Model.prepare_container_def")
def test_create_sagemaker_model_instance_type(prepare_container_def, sagemaker_session):
    model = Model(MODEL_DATA, MODEL_IMAGE, name=MODEL_NAME, sagemaker_session=sagemaker_session)
    model._create_sagemaker_model(INSTANCE_TYPE)

    prepare_container_def.assert_called_with(
        INSTANCE_TYPE, accelerator_type=None, serverless_inference_config=None
    )

@patch("sagemaker.model.Model.prepare_container_def")
def test_create_sagemaker_model_accelerator_type(prepare_container_def, sagemaker_session):
    model = Model(MODEL_IMAGE, MODEL_DATA, name=MODEL_NAME, sagemaker_session=sagemaker_session)

    accelerator_type = "ml.eia.medium"
    model._create_sagemaker_model(INSTANCE_TYPE, accelerator_type=accelerator_type)

    prepare_container_def.assert_called_with(
        INSTANCE_TYPE, accelerator_type=accelerator_type, serverless_inference_config=None
    )

@patch("sagemaker.model.Model.prepare_container_def")
def test_create_sagemaker_model_tags(prepare_container_def, sagemaker_session):
    container_def = {"Image": MODEL_IMAGE, "Environment": {}, "ModelDataUrl": MODEL_DATA}
    prepare_container_def.return_value = container_def

    model = Model(MODEL_IMAGE, MODEL_DATA, name=MODEL_NAME, sagemaker_session=sagemaker_session)

    tags = {"Key": "foo", "Value": "bar"}
    model._create_sagemaker_model(INSTANCE_TYPE, tags=tags)

    sagemaker_session.create_model.assert_called_with(
        MODEL_NAME, None, container_def, vpc_config=None, enable_network_isolation=False, tags=tags
    )

@patch("sagemaker.model.Model.prepare_container_def")
@patch("sagemaker.utils.name_from_base")
@patch("sagemaker.utils.base_name_from_image")
def test_create_sagemaker_model_optional_model_params(
    base_name_from_image, name_from_base, prepare_container_def, sagemaker_session
):
    container_def = {"Image": MODEL_IMAGE, "Environment": {}, "ModelDataUrl": MODEL_DATA}
    prepare_container_def.return_value = container_def

    vpc_config = {"Subnets": ["123"], "SecurityGroupIds": ["456", "789"]}

    model = Model(
        MODEL_IMAGE,
        MODEL_DATA,
        name=MODEL_NAME,
        role=ROLE,
        vpc_config=vpc_config,
        enable_network_isolation=True,
        sagemaker_session=sagemaker_session,
    )
    model._create_sagemaker_model(INSTANCE_TYPE)

    base_name_from_image.assert_not_called()
    name_from_base.assert_not_called()

    sagemaker_session.create_model.assert_called_with(
        MODEL_NAME,
        ROLE,
        container_def,
        vpc_config=vpc_config,
        enable_network_isolation=True,
        tags=None,
    )

@patch("sagemaker.model.Model.prepare_container_def")
@patch("sagemaker.utils.name_from_base", return_value=MODEL_NAME)
@patch("sagemaker.utils.base_name_from_image")
def test_create_sagemaker_model_generates_model_name(
    base_name_from_image, name_from_base, prepare_container_def, sagemaker_session
):
    container_def = {"Image": MODEL_IMAGE, "Environment": {}, "ModelDataUrl": MODEL_DATA}
    prepare_container_def.return_value = container_def

    model = Model(
        MODEL_IMAGE,
        MODEL_DATA,
        sagemaker_session=sagemaker_session,
    )
    model._create_sagemaker_model(INSTANCE_TYPE)

    base_name_from_image.assert_called_with(MODEL_IMAGE)
    name_from_base.assert_called_with(base_name_from_image.return_value)

    sagemaker_session.create_model.assert_called_with(
        MODEL_NAME,
        None,
        container_def,
        vpc_config=None,
        enable_network_isolation=False,
        tags=None,
    )

@patch("sagemaker.model.Model.prepare_container_def")
@patch("sagemaker.utils.name_from_base", return_value=MODEL_NAME)
@patch("sagemaker.utils.base_name_from_image")
def test_create_sagemaker_model_generates_model_name_each_time(
    base_name_from_image, name_from_base, prepare_container_def, sagemaker_session
):
    container_def = {"Image": MODEL_IMAGE, "Environment": {}, "ModelDataUrl": MODEL_DATA}
    prepare_container_def.return_value = container_def

    model = Model(
        MODEL_IMAGE,
        MODEL_DATA,
        sagemaker_session=sagemaker_session,
    )
    model._create_sagemaker_model(INSTANCE_TYPE)
    model._create_sagemaker_model(INSTANCE_TYPE)

    base_name_from_image.assert_called_once_with(MODEL_IMAGE)
    name_from_base.assert_called_with(base_name_from_image.return_value)
    assert 2 == name_from_base.call_count

@patch("sagemaker.session.Session")
@patch("sagemaker.local.LocalSession")
def test_create_sagemaker_model_creates_correct_session(local_session, session):
    model = Model(MODEL_IMAGE, MODEL_DATA)
    model._create_sagemaker_model("local")
    assert model.sagemaker_session == local_session.return_value

    model = Model(MODEL_IMAGE, MODEL_DATA)
    model._create_sagemaker_model("ml.m5.xlarge")
    assert model.sagemaker_session == session.return_value

@patch("sagemaker.model.Model._create_sagemaker_model")
def test_model_create_transformer(create_sagemaker_model, sagemaker_session):
    model_name = "auto-generated-model"
    model = Model(MODEL_IMAGE, MODEL_DATA, name=model_name, sagemaker_session=sagemaker_session)

    instance_type = "ml.m4.xlarge"
    transformer = model.transformer(instance_count=1, instance_type=instance_type)

    create_sagemaker_model.assert_called_with(instance_type, tags=None)

    assert isinstance(transformer, sagemaker.transformer.Transformer)
    assert transformer.model_name == model_name
    assert transformer.instance_type == instance_type
    assert transformer.instance_count == 1
    assert transformer.sagemaker_session == sagemaker_session
    assert transformer.base_transform_job_name == model_name

    assert transformer.strategy is None
    assert transformer.env is None
    assert transformer.output_path is None
    assert transformer.output_kms_key is None
    assert transformer.accept is None
    assert transformer.assemble_with is None
    assert transformer.volume_kms_key is None
    assert transformer.max_concurrent_transforms is None
    assert transformer.max_payload is None
    assert transformer.tags is None

@patch("sagemaker.model.Model._create_sagemaker_model")
def test_model_create_transformer_optional_params(create_sagemaker_model, sagemaker_session):
    model = Model(MODEL_IMAGE, MODEL_DATA, sagemaker_session=sagemaker_session)

    instance_type = "ml.m4.xlarge"
    strategy = "MultiRecord"
    assemble_with = "Line"
    output_path = "s3://bucket/path"
    kms_key = "key"
    accept = "text/csv"
    env = {"test": True}
    max_concurrent_transforms = 1
    max_payload = 6
    tags = [{"Key": "k", "Value": "v"}]

    transformer = model.transformer(
        instance_count=1,
        instance_type=instance_type,
        strategy=strategy,
        assemble_with=assemble_with,
        output_path=output_path,
        output_kms_key=kms_key,
        accept=accept,
        env=env,
        max_concurrent_transforms=max_concurrent_transforms,
        max_payload=max_payload,
        tags=tags,
        volume_kms_key=kms_key,
    )

    create_sagemaker_model.assert_called_with(instance_type, tags=tags)

    assert isinstance(transformer, sagemaker.transformer.Transformer)
    assert transformer.strategy == strategy
    assert transformer.assemble_with == assemble_with
    assert transformer.output_path == output_path
    assert transformer.output_kms_key == kms_key
    assert transformer.accept == accept
    assert transformer.max_concurrent_transforms == max_concurrent_transforms
    assert transformer.max_payload == max_payload
    assert transformer.env == env
    assert transformer.tags == tags
    assert transformer.volume_kms_key == kms_key

@patch("sagemaker.model.Model._create_sagemaker_model", Mock())
def test_model_create_transformer_network_isolation(sagemaker_session):
    model = Model(
        MODEL_IMAGE, MODEL_DATA, sagemaker_session=sagemaker_session, enable_network_isolation=True
    )

    transformer = model.transformer(1, "ml.m4.xlarge", env={"should_be": "overwritten"})
    assert transformer.env is None

@patch("sagemaker.model.Model._create_sagemaker_model", Mock())
def test_model_create_transformer_base_name(sagemaker_session):
    model = Model(MODEL_IMAGE, MODEL_DATA, sagemaker_session=sagemaker_session)

    base_name = "foo"
    model._base_name = base_name

    transformer = model.transformer(1, "ml.m4.xlarge")
    assert base_name == transformer.base_transform_job_name

@patch("sagemaker.session.Session")
@patch("sagemaker.local.LocalSession")
def test_transformer_creates_correct_session(local_session, session):
    model = Model(MODEL_IMAGE, MODEL_DATA, sagemaker_session=None)
    transformer = model.transformer(instance_count=1, instance_type="local")
    assert model.sagemaker_session == local_session.return_value
    assert transformer.sagemaker_session == local_session.return_value

    model = Model(MODEL_IMAGE, MODEL_DATA, sagemaker_session=None)
    transformer = model.transformer(instance_count=1, instance_type="ml.m5.xlarge")
    assert model.sagemaker_session == session.return_value
    assert transformer.sagemaker_session == session.return_value

def test_delete_model(sagemaker_session):
    model = Model(MODEL_IMAGE, MODEL_DATA, name=MODEL_NAME, sagemaker_session=sagemaker_session)

    model.delete_model()
    sagemaker_session.delete_model.assert_called_with(model.name)

def test_delete_model_no_name(sagemaker_session):
    model = Model(MODEL_IMAGE, MODEL_DATA, sagemaker_session=sagemaker_session)

    with pytest.raises(
        ValueError, match="The SageMaker model must be created first before attempting to delete."
    ):
        model.delete_model()
    sagemaker_session.delete_model.assert_not_called()

@patch("time.strftime", MagicMock(return_value=TIMESTAMP))
@patch("sagemaker.utils.repack_model")
def test_script_mode_model_same_calls_as_framework(repack_model, sagemaker_session):
    t = Model(
        entry_point=ENTRY_POINT_INFERENCE,
        role=ROLE,
        sagemaker_session=sagemaker_session,
        source_dir=SCRIPT_URI,
        image_uri=IMAGE_URI,
        model_data=MODEL_DATA,
    )
    t.deploy(instance_type=INSTANCE_TYPE, initial_instance_count=INSTANCE_COUNT)

    assert len(sagemaker_session.create_model.call_args_list) == 1
    assert len(sagemaker_session.endpoint_from_production_variants.call_args_list) == 1
    assert len(repack_model.call_args_list) == 1

    generic_model_create_model_args = sagemaker_session.create_model.call_args_list
    generic_model_endpoint_from_production_variants_args = (
        sagemaker_session.endpoint_from_production_variants.call_args_list
    )
    generic_model_repack_model_args = repack_model.call_args_list

    sagemaker_session.create_model.reset_mock()
    sagemaker_session.endpoint_from_production_variants.reset_mock()
    repack_model.reset_mock()

    t = DummyFrameworkModel(
        entry_point=ENTRY_POINT_INFERENCE,
        role=ROLE,
        sagemaker_session=sagemaker_session,
        source_dir=SCRIPT_URI,
        image_uri=IMAGE_URI,
        model_data=MODEL_DATA,
    )
    t.deploy(instance_type=INSTANCE_TYPE, initial_instance_count=INSTANCE_COUNT)

    assert generic_model_create_model_args == sagemaker_session.create_model.call_args_list
    assert (
        generic_model_endpoint_from_production_variants_args
        == sagemaker_session.endpoint_from_production_variants.call_args_list
    )
    assert generic_model_repack_model_args == repack_model.call_args_list

@patch("sagemaker.git_utils.git_clone_repo")
@patch("sagemaker.model.fw_utils.tar_and_upload_dir")
def test_git_support_succeed_model_class(tar_and_upload_dir, git_clone_repo, sagemaker_session):
    git_clone_repo.side_effect = lambda gitconfig, entrypoint, sourcedir, dependency: {
        "entry_point": "entry_point",
        "source_dir": "/tmp/repo_dir/source_dir",
        "dependencies": ["/tmp/repo_dir/foo", "/tmp/repo_dir/bar"],
    }
    entry_point = "entry_point"
    source_dir = "source_dir"
    dependencies = ["foo", "bar"]
    git_config = {"repo": GIT_REPO, "branch": BRANCH, "commit": COMMIT}
    model = Model(
        sagemaker_session=sagemaker_session,
        entry_point=entry_point,
        source_dir=source_dir,
        dependencies=dependencies,
        git_config=git_config,
        image_uri=IMAGE_URI,
    )
    model.prepare_container_def(instance_type=INSTANCE_TYPE)
    git_clone_repo.assert_called_with(git_config, entry_point, source_dir, dependencies)
    assert model.entry_point == "entry_point"
    assert model.source_dir == "/tmp/repo_dir/source_dir"
    assert model.dependencies == ["/tmp/repo_dir/foo", "/tmp/repo_dir/bar"]

@patch("sagemaker.utils.repack_model")
def test_script_mode_model_tags_jumpstart_models(repack_model, sagemaker_session):

    jumpstart_source_dir = f"s3://{list(JUMPSTART_BUCKET_NAME_SET)[0]}/source_dirs/source.tar.gz"
    t = Model(
        entry_point=ENTRY_POINT_INFERENCE,
        role=ROLE,
        sagemaker_session=sagemaker_session,
        source_dir=jumpstart_source_dir,
        image_uri=IMAGE_URI,
        model_data=MODEL_DATA,
    )
    t.deploy(instance_type=INSTANCE_TYPE, initial_instance_count=INSTANCE_COUNT)

    assert sagemaker_session.create_model.call_args_list[0][1]["tags"] == [
        {
            "Key": JumpStartTag.INFERENCE_SCRIPT_URI.value,
            "Value": jumpstart_source_dir,
        },
    ]
    assert sagemaker_session.endpoint_from_production_variants.call_args_list[0][1]["tags"] == [
        {
            "Key": JumpStartTag.INFERENCE_SCRIPT_URI.value,
            "Value": jumpstart_source_dir,
        },
    ]

    non_jumpstart_source_dir = "s3://blah/blah/blah"
    t = Model(
        entry_point=ENTRY_POINT_INFERENCE,
        role=ROLE,
        sagemaker_session=sagemaker_session,
        source_dir=non_jumpstart_source_dir,
        image_uri=IMAGE_URI,
        model_data=MODEL_DATA,
    )
    t.deploy(instance_type=INSTANCE_TYPE, initial_instance_count=INSTANCE_COUNT)

    assert {
        "Key": JumpStartTag.INFERENCE_SCRIPT_URI.value,
        "Value": non_jumpstart_source_dir,
    } not in sagemaker_session.create_model.call_args_list[0][1]["tags"]

    assert {
        "Key": JumpStartTag.INFERENCE_SCRIPT_URI.value,
        "Value": non_jumpstart_source_dir,
    } not in sagemaker_session.create_model.call_args_list[0][1]["tags"]

@patch("sagemaker.utils.repack_model")
@patch("sagemaker.fw_utils.tar_and_upload_dir")
def test_all_framework_models_add_jumpstart_tags(
    repack_model, tar_and_uload_dir, sagemaker_session
):
    framework_model_classes_to_kwargs = {
        PyTorchModel: {"framework_version": "1.5.0", "py_version": "py3"},
        TensorFlowModel: {
            "framework_version": "2.3",
        },
        HuggingFaceModel: {
            "pytorch_version": "1.7.1",
            "py_version": "py36",
            "transformers_version": "4.6.1",
        },
        MXNetModel: {"framework_version": "1.7.0", "py_version": "py3"},
        SKLearnModel: {
            "framework_version": "0.23-1",
        },
        XGBoostModel: {
            "framework_version": "1.3-1",
        },
    }
    jumpstart_model_dir = f"s3://{list(JUMPSTART_BUCKET_NAME_SET)[0]}/model_dirs/model.tar.gz"
    for framework_model_class, kwargs in framework_model_classes_to_kwargs.items():
        framework_model_class(
            entry_point=ENTRY_POINT_INFERENCE,
            role=ROLE,
            sagemaker_session=sagemaker_session,
            model_data=jumpstart_model_dir,
            **kwargs,
        ).deploy(instance_type="ml.m2.xlarge", initial_instance_count=INSTANCE_COUNT)

        assert {
            "Key": JumpStartTag.INFERENCE_MODEL_URI.value,
            "Value": jumpstart_model_dir,
        } in sagemaker_session.create_model.call_args_list[0][1]["tags"]

        assert {
            "Key": JumpStartTag.INFERENCE_MODEL_URI.value,
            "Value": jumpstart_model_dir,
        } in sagemaker_session.endpoint_from_production_variants.call_args_list[0][1]["tags"]

        sagemaker_session.create_model.reset_mock()
        sagemaker_session.endpoint_from_production_variants.reset_mock()

@patch("sagemaker.utils.repack_model")
def test_script_mode_model_uses_jumpstart_base_name(repack_model, sagemaker_session):

    jumpstart_source_dir = f"s3://{list(JUMPSTART_BUCKET_NAME_SET)[0]}/source_dirs/source.tar.gz"
    t = Model(
        entry_point=ENTRY_POINT_INFERENCE,
        role=ROLE,
        sagemaker_session=sagemaker_session,
        source_dir=jumpstart_source_dir,
        image_uri=IMAGE_URI,
        model_data=MODEL_DATA,
    )
    t.deploy(instance_type=INSTANCE_TYPE, initial_instance_count=INSTANCE_COUNT)

    assert sagemaker_session.create_model.call_args_list[0][0][0].startswith(
        JUMPSTART_RESOURCE_BASE_NAME
    )

    assert sagemaker_session.endpoint_from_production_variants.call_args_list[0].startswith(
        JUMPSTART_RESOURCE_BASE_NAME
    )

    sagemaker_session.create_model.reset_mock()
    sagemaker_session.endpoint_from_production_variants.reset_mock()

    non_jumpstart_source_dir = "s3://blah/blah/blah"
    t = Model(
        entry_point=ENTRY_POINT_INFERENCE,
        role=ROLE,
        sagemaker_session=sagemaker_session,
        source_dir=non_jumpstart_source_dir,
        image_uri=IMAGE_URI,
        model_data=MODEL_DATA,
    )
    t.deploy(instance_type=INSTANCE_TYPE, initial_instance_count=INSTANCE_COUNT)

    assert not sagemaker_session.create_model.call_args_list[0][0][0].startswith(
        JUMPSTART_RESOURCE_BASE_NAME
    )

    assert not sagemaker_session.endpoint_from_production_variants.call_args_list[0][1][
        "name"
    ].startswith(JUMPSTART_RESOURCE_BASE_NAME)

@patch("sagemaker.utils.repack_model")
@patch("sagemaker.fw_utils.tar_and_upload_dir")
def test_all_framework_models_add_jumpstart_base_name(
    repack_model, tar_and_uload_dir, sagemaker_session
):
    framework_model_classes_to_kwargs = {
        PyTorchModel: {"framework_version": "1.5.0", "py_version": "py3"},
        TensorFlowModel: {
            "framework_version": "2.3",
        },
        HuggingFaceModel: {
            "pytorch_version": "1.7.1",
            "py_version": "py36",
            "transformers_version": "4.6.1",
        },
        MXNetModel: {"framework_version": "1.7.0", "py_version": "py3"},
        SKLearnModel: {
            "framework_version": "0.23-1",
        },
        XGBoostModel: {
            "framework_version": "1.3-1",
        },
    }
    jumpstart_model_dir = f"s3://{list(JUMPSTART_BUCKET_NAME_SET)[0]}/model_dirs/model.tar.gz"
    for framework_model_class, kwargs in framework_model_classes_to_kwargs.items():
        framework_model_class(
            entry_point=ENTRY_POINT_INFERENCE,
            role=ROLE,
            sagemaker_session=sagemaker_session,
            model_data=jumpstart_model_dir,
            **kwargs,
        ).deploy(instance_type="ml.m2.xlarge", initial_instance_count=INSTANCE_COUNT)

        assert sagemaker_session.create_model.call_args_list[0][0][0].startswith(
            JUMPSTART_RESOURCE_BASE_NAME
        )

        assert sagemaker_session.endpoint_from_production_variants.call_args_list[0].startswith(
            JUMPSTART_RESOURCE_BASE_NAME
        )

        sagemaker_session.create_model.reset_mock()
        sagemaker_session.endpoint_from_production_variants.reset_mock()

@patch("sagemaker.utils.repack_model")
def test_script_mode_model_uses_proper_sagemaker_submit_dir(repack_model, sagemaker_session):

    source_dir = "s3://blah/blah/blah"
    t = Model(
        entry_point=ENTRY_POINT_INFERENCE,
        role=ROLE,
        sagemaker_session=sagemaker_session,
        source_dir=source_dir,
        image_uri=IMAGE_URI,
        model_data=MODEL_DATA,
    )
    t.deploy(instance_type=INSTANCE_TYPE, initial_instance_count=INSTANCE_COUNT)

    assert (
        sagemaker_session.create_model.call_args_list[0][0][2]["Environment"][
            "SAGEMAKER_SUBMIT_DIRECTORY"
        ]
        == "/opt/ml/model/code"
    )

reason: import sage,from sage


text:
import torch
import random
import rdkit
import rdkit.Chem as Chem
import networkx as nx
from fuseprop.chemutils import *
from fuseprop.nnutils import *
from fuseprop.vocab import common_atom_vocab
from collections import deque

add = lambda x,y : x + y if type(x) is int else (x[0] + y, x[1] + y)

class MolGraph(object):

    BOND_LIST = [None, Chem.rdchem.BondType.SINGLE, Chem.rdchem.BondType.DOUBLE, Chem.rdchem.BondType.TRIPLE, Chem.rdchem.BondType.AROMATIC]
    MAX_POS = 40

    def __init__(self, smiles, init_atoms, root_atoms=None, shuffle_roots=True):
        self.smiles = smiles
        self.mol = get_mol(smiles)
        self.mol_graph = self.build_mol_graph()
        self.init_atoms = set(init_atoms)
        self.root_atoms = self.get_root_atoms() if root_atoms is None else root_atoms
        if len(self.root_atoms) > 0:
            if shuffle_roots: random.shuffle(self.root_atoms)
            self.order = self.get_bfs_order()

    def debug(self):
        for atom in self.mol.GetAtoms():
            if atom.GetIdx() in self.init_atoms:
                atom.SetAtomMapNum(atom.GetIdx())
        print( Chem.MolToSmiles(self.mol) )
        print('root', self.root_atoms)
        print('init', self.init_atoms)
        for x in self.order:
            print(x)

    def get_root_atoms(self):
        roots = []
        for idx in self.init_atoms:
            atom = self.mol.GetAtomWithIdx(idx)
            bad_neis = [y for y in atom.GetNeighbors() if y.GetIdx() not in self.init_atoms]
            if len(bad_neis) > 0:
                roots.append(idx)
        return roots

    def build_mol_graph(self):
        mol = self.mol
        graph = nx.DiGraph(Chem.rdmolops.GetAdjacencyMatrix(mol))
        for atom in mol.GetAtoms():
            graph.nodes[atom.GetIdx()]['label'] = (atom.GetSymbol(), atom.GetFormalCharge())

        for bond in mol.GetBonds():
            a1 = bond.GetBeginAtom().GetIdx()
            a2 = bond.GetEndAtom().GetIdx()
            btype = MolGraph.BOND_LIST.index( bond.GetBondType() )
            graph[a1][a2]['label'] = btype
            graph[a2][a1]['label'] = btype

        return graph

    def get_bfs_order(self):
        order = []
        visited = set(self.init_atoms)
        queue = deque( [self.mol.GetAtomWithIdx(k) for k in self.root_atoms] )

        for a in self.init_atoms:
            self.mol_graph.nodes[a]['pos'] = 0

        for i,root in enumerate(self.root_atoms):
            self.mol_graph.nodes[root]['pos'] = i + 1  # interior atoms are 0, boundary atoms are 1,2,...

        pos_id = len(self.root_atoms)
        while len(queue) > 0:
            x = queue.popleft()
            x_idx = x.GetIdx()
            for y in x.GetNeighbors():
                y_idx = y.GetIdx()
                if y_idx in visited: continue

                frontier = [x_idx] + [a.GetIdx() for a in list(queue)]
                bonds = [0] * len(frontier)
                y_neis = set([z.GetIdx() for z in y.GetNeighbors()])

                for i,z_idx in enumerate(frontier):
                    if z_idx in y_neis:
                        bonds[i] = self.mol_graph[y_idx][z_idx]['label']

                order.append( (x_idx, y_idx, frontier, bonds) )
                pos_id += 1
                self.mol_graph.nodes[y_idx]['pos'] = min(MolGraph.MAX_POS - 1, pos_id)
                visited.add( y_idx )
                queue.append(y)

            order.append( (x_idx, None, None, None) )

        return order

    @staticmethod
    def tensorize(mol_batch, avocab=common_atom_vocab):
        graph_tensors, graph_batchG = MolGraph.tensorize_graph([x.mol_graph for x in mol_batch], avocab)
        graph_scope = graph_tensors[-1]

        add = lambda a,b : None if a is None else a + b
        add_list = lambda alist,b : None if alist is None else [a + b for a in alist]

        all_orders = []
        all_init_atoms = []
        for i,hmol in enumerate(mol_batch):
            offset = graph_scope[i][0]
            order = [(x + offset, add(y, offset), add_list(z, offset), t) for x,y,z,t in hmol.order]
            init_atoms = [x + offset for x in hmol.init_atoms]
            all_orders.append(order)
            all_init_atoms.append(init_atoms)

        return graph_batchG, graph_tensors, all_init_atoms, all_orders

    @staticmethod
    def tensorize_graph(graph_batch, vocab):
        fnode,fmess = [None],[(0,0,0)]
        agraph,bgraph = [[]], [[]]
        scope = []
        edge_dict = {}
        all_G = []

        for bid,G in enumerate(graph_batch):
            offset = len(fnode)
            scope.append( (offset, len(G)) )
            G = nx.convert_node_labels_to_integers(G, first_label=offset)
            all_G.append(G)
            fnode.extend( [None for v in G.nodes] )

            for v, attr in G.nodes(data='label'):
                G.nodes[v]['batch_id'] = bid
                fnode[v] = (vocab[attr], G.nodes[v]['pos'])
                agraph.append([])

            for u, v, attr in G.edges(data='label'):
                fmess.append( (u, v, attr) )
                edge_dict[(u, v)] = eid = len(edge_dict) + 1
                G[u][v]['mess_idx'] = eid
                agraph[v].append(eid)
                bgraph.append([])

            for u, v in G.edges:
                eid = edge_dict[(u, v)]
                for w in G.predecessors(u):
                    if w == v: continue
                    bgraph[eid].append( edge_dict[(w, u)] )

        fnode[0] = fnode[1]
        fnode = torch.LongTensor(fnode)
        fmess = torch.LongTensor(fmess)
        agraph = create_pad_tensor(agraph)
        bgraph = create_pad_tensor(bgraph)
        return (fnode, fmess, agraph, bgraph, scope), nx.union_all(all_G)


text:
from torch.utils.data import Dataset, DataLoader
from abc import abstractmethod
import numpy as np
import os
from PIL import Image

'''
 data format:
 0001.jpg, 0002.jpg, 1,  0
 0034.jpg, 0023.jpg, -1, 1
 0022.jpg, 0002.jpg, 0,  1
 0001.jpg, 0045.jpg, 1,  2
 if binary == True, label1==-1 will be set 0.
 '''

class PairwiseDataset(Dataset):
    def __init__(self, txt, transform=None, target_transform=None, binary=False,
                 load_img=True, load_rgb=False, attr_num=10, root_dir='./data'):
        self.transform = transform
        self.target_transform = target_transform
        self.binary = binary
        self.root_dir = root_dir
        self.load_img = load_img
        self.load_rgb = load_rgb
        self.attr_num = attr_num
        self.lens = 0
        self.lines = list()

    @abstractmethod
    def _parse_each_item(self, line):
        pass

    @abstractmethod
    def _get_img_name(self, img_id):
        pass

    def __getitem__(self, index):
        img_id0, img_id1, label, attr_id, pair_id, strength = self._parse_each_item(
            self.lines[index])
        label, attr_id, pair_id, strength = np.float32(label), np.float32(
            attr_id), np.float32(pair_id), np.float32(strength)
        if self.binary:  # default is False
            if int(label) == -1:
                label = np.float32(0.)

        if self.load_img:
            img0 = Image.open(self._get_img_name(img_id0))
            img1 = Image.open(self._get_img_name(img_id1))
            if self.load_rgb:
                img0 = img0.convert('RGB')
                img1 = img1.convert('RGB')

            if self.transform is not None:
                img0 = self.transform(img0)
                img1 = self.transform(img1)
            img_id0, img_id1 = np.float32(img_id0), np.float32(img_id1)
            return img_id0, img_id1, img0, img1, label, attr_id, pair_id, strength
        else:
            img_id0, img_id1 = np.float32(img_id0), np.float32(img_id1)
            return img_id0, img_id1, label, attr_id, pair_id, strength

    def __len__(self):
        return self.lens

class AgeDataSet(PairwiseDataset):
    def __init__(self, txt, file2id_dict, id2file_dict, transform=None,
                 target_transform=None, binary=False, load_img=True, load_rgb=True, attr_num=1, root_dir='./data/'):
        super(AgeDataSet, self).__init__(txt, transform,
                                         target_transform, binary, load_img, load_rgb, attr_num, root_dir)
        self.lines = [line.rstrip() for line in open(txt, 'r')]
        self.lens = len(self.lines)
        self.file2id_dict = file2id_dict
        self.id2file_dict = id2file_dict

    def _parse_each_item(self, line):
        split = line.split(',')
        img_id0, img_id1 = self.file2id_dict[split[0]
                                             ], self.file2id_dict[split[1]]
        label = int(split[2])  # comp_value
        pair_id = int(split[3])  # edge(x, y)_id
        strength = int(split[4])  # strength
        attr_id = 0  # attr_id
        return img_id0, img_id1, label, attr_id, pair_id, strength

    def _get_img_name(self, img_id):
        return os.path.join(self.root_dir, self.id2file_dict[img_id])

    def get_img_file_name(self, img_id):
        return self.id2file_dict[img_id]

class PairwiseImgDataSet(Dataset):

    def __init__(self, txt, load_rgb=False, transform=None, target_transform=None, root_dir='./data'):
        self.load_rgb = load_rgb
        self.transform = transform
        self.target_transform = target_transform
        self.root_dir = root_dir
        self.lines = list()
        self.lens = 0

    @abstractmethod
    def _parse_each_item(self, line):
        pass

    @abstractmethod
    def _get_img_name(self, img_id):
        pass

    def _get_all_img_id(self):
        img_ids = set()
        for i in range(len(self.lines)):
            img_id1, img_id2, attr_id = self._parse_each_item(self.lines[i])
            img_ids.add((img_id1, attr_id))
            img_ids.add((img_id2, attr_id))
        self.img_ids = list(img_ids)

    def __getitem__(self, index):
        img_id, attr_id = self.img_ids[index]
        img = Image.open(self._get_img_name(img_id))
        if self.load_rgb:
            img = img.convert('RGB')
        img_id, attr_id = np.float32(img_id), np.float32(attr_id)

        if self.transform is not None:
            img = self.transform(img)

        return img, img_id, attr_id

    def __len__(self):
        return self.lens

class AgeImgDataSet(PairwiseImgDataSet):

    def __init__(self, txt, file2id_dict, id2file_dict, load_rgb=True,
                 transform=None, target_transform=None, root_dir='./data'):
        super(AgeImgDataSet, self).__init__(
            txt, load_rgb, transform, target_transform, root_dir)
        self.lines = [line.rstrip() for line in open(txt, 'r')]
        self.file2id_dict = file2id_dict
        self.id2file_dict = id2file_dict
        self._get_all_img_id()
        self.lens = len(self.img_ids)

    def _parse_each_item(self, line):
        split = line.split(',')
        attr_id = 0
        img_id1, img_id2 = self.file2id_dict[split[0]], self.file2id_dict[split[1]]
        return img_id1, img_id2, attr_id

    def _get_img_name(self, img_id):
        return os.path.join(self.root_dir, self.id2file_dict[img_id])

def load_age(**kwargs):
    id2file_dict = dict()
    file2id_dict = dict()
    for file in os.listdir(kwargs['img_path']):
        img_id = file.split('.')[0].replace('A', '').replace(
            'a', '1').replace('b', '2').replace('c', '3')
        id2file_dict[int(img_id)] = file
        file2id_dict[file] = int(img_id)

    train_dataset = AgeDataSet(txt=kwargs['train_path'],
                            root_dir=kwargs['img_path'],
                            file2id_dict=file2id_dict,
                            id2file_dict=id2file_dict,
                            binary=False,
                            load_img=True,
                            load_rgb=True,
                            transform=kwargs['transform'])
    train_loader = DataLoader(dataset=train_dataset,
                                batch_size=kwargs['batch_size'],
                                shuffle=kwargs['shuffle'], **kwargs['others'])

    img_dataset = AgeImgDataSet(txt=kwargs['test_path'],
                                root_dir=kwargs['img_path'],
                                file2id_dict=file2id_dict,
                                id2file_dict=id2file_dict,
                                load_rgb=True,
                                transform=kwargs['transform'])
    img_loader = DataLoader(dataset=img_dataset,
                            batch_size=kwargs['batch_size'],
                            shuffle=False, **kwargs['others'])

    test_dataset_wo_img = AgeDataSet(txt=kwargs['test_path'],
                            root_dir=kwargs['img_path'],
                            file2id_dict=file2id_dict,
                            id2file_dict=id2file_dict,
                            binary=False,
                            load_img=False)
    test_loader_wo_img = DataLoader(dataset=test_dataset_wo_img,
                                batch_size=kwargs['batch_size'],
                                shuffle=False, **kwargs['others'])

    return train_dataset, train_loader, img_dataset, img_loader, test_dataset_wo_img, test_loader_wo_img

def get_age(img_id):
    splitor = None
    for i in img_id:
        if i < '0' or i > '9':
            splitor = i
            break
    _, left = img_id.split(splitor, 1)
    for i in left:
        if i < '0' or i > '9':
            splitor = i
            break
    age, left = left.split(splitor, 1)
    return int(age)

max_stars_repo_name: statusrank/Sever

max_issues_repo_name: statusrank/Sever

max_forks_repo_name: statusrank/Sever


text:
import HP
import numpy as np
# the class is desinged to get pre-trained embedding

class Embedding:
    embedding = None

    @staticmethod
    def get_embedding():
        """Get the embedding."""
        if Embedding.embedding is None:
            Embedding.embedding = Embedding.load_embedding()
        return Embedding.embedding

    @staticmethod
    def load_embedding():
        """Load the embedding from the pretrained file."""
        embedding_file = open(HP.embedding_file)
        embedding_map = {}
        for line in embedding_file:
            values = line.split()
            word = values[0]
            coef = np.asarray(values[1:], dtype='float32')
            embedding_map[word] = coef
        embedding_file.close()
        return embedding_map

max_stars_repo_name: ruiwangcl/deepmimiciii

max_issues_repo_name: ruiwangcl/deepmimiciii

max_forks_repo_name: ruiwangcl/deepmimiciii


text:
import sys
import numpy as np
def readData(filename):
    labels, features, allFeatures, = [], [], set()
    with open(filename, 'r') as f:
        for l in f:
            l = l.strip().split(' ')
            instanceFeatures = set()
            labels.append(l[0])
            for feature in l[1:]:
                if len(feature) > 0:
                    word, count = feature.split(':')
                    allFeatures.add(word)
                    instanceFeatures.add(word)
            features.append(instanceFeatures)
    allLabels = set(labels)
    guessLabel = np.full(shape=len(features),fill_value=list(allLabels)[0]) #initialize our first guess to a random class
    return [np.array([labels, guessLabel, features]), allFeatures, allLabels]
def findAndApplyNextTransformation(data, transformation2index, index2transformation):
    #loop through data and model every possible transformation and choose the one that produces the most gain
    gains = np.zeros(len(transformation2index))
    for i in range(len(data[0][0])):
        gold, candidate = data[0][0][i], data[0][1][i]
        for feature in data[0][2][i]:
            for label in data[2]:
                if candidate == gold:
                    gains[transformation2index[(feature, candidate, label)]] -= 1 #bad change so loss
                if label == gold:
                    gains[transformation2index[(feature, candidate, label)]] += 1 #good change so gain
    bestIndex = np.argmax(gains) #index of the highest gain value
    transformation = index2transformation[bestIndex]
    targetFeature, fromClass, toClass = transformation
    for i in range(len(data[0][0])):
        if targetFeature in data[0][2][i] and data[0][1][i] == fromClass: #apply the best transformation to entire dataset
            data[0][1][i] = toClass
    return data, transformation, gains[bestIndex]
def buildTables(features, classes):
    #build tansformation2index and index2tranformation per suggestion in hw7 slides
    transformation2index, index2transformation = {}, {}
    index = 0
    for feature in features:
        for fromClass in classes:
            for toClass in classes:
                transformation = (feature, fromClass, toClass)
                transformation2index[transformation] = index
                index2transformation[index] = transformation
                index += 1
    return transformation2index, index2transformation
def trainTBL(data, modelFilename, minGain):
    transformation2index, index2transformation = buildTables(data[1],data[2]) #make loopup tables
    gain = minGain + 1
    initClass = ''
    with open(modelFilename,'w') as w:
        while gain >= minGain: #as long as we get gains we keep going
            data, transformation, gain = findAndApplyNextTransformation(data, transformation2index, index2transformation)
            targetFeature, fromClass, toClass = transformation
            if initClass == '':
                initClass = fromClass #we make our first transform the default init class
                w.write('{}\n'.format(initClass))  #init_classname
            w.write('{} {} {} {}\n'.format(targetFeature, fromClass, toClass, int(gain))) #featName from class to class net gain
if __name__ == "__main__":
    if len(sys.argv) != 4:
        print("Usage: TBL_train.sh <train_data> <model_file> <min_gain>")
        exit(-1)
    else:
        data = readData(sys.argv[1])
        trainTBL(data, sys.argv[2], int(sys.argv[3]))


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""Predict with Mars in R/rpy2
"""

import mars

import pandas as pd
import numpy as np
import numpy.linalg as LA

x1 = np.linspace(-1,1,10)
x2 = np.linspace(-1,1,10)
x2, x1=np.meshgrid(x2,x1)
y1 = np.abs(x1) + np.abs(x2)
y2= np.abs(x1) + np.abs(x2)*np.abs(x1)

Xtrain = np.column_stack([x1.flatten(), x2.flatten()])
Ytrain=np.column_stack([y1.flatten(), y2.flatten()])
model = mars.Mars(degree=2, thresh=0.0000001)
model.fit(Xtrain, Ytrain)

Ypred = model.predict(Xtrain)

print(LA.norm(Ypred-Ytrain)/LA.norm(Ytrain))

max_stars_repo_name: Freakwill/pymars

max_issues_repo_name: Freakwill/pymars

max_forks_repo_name: Freakwill/pymars


text:
############################################################
# -*- coding: utf-8 -*-
#
#       #   #  #   #   #  ####
#      ##  ##  #  ##  #     #
#     # # # #  # # # #     ###
#    #  ##  #  ##  ##        #
#   #   #   #  #   #     ####
#
# Python-based Tool for interaction with the 10micron mounts
# GUI with PyQT5 for python
# Python  v3.6.5
#
# Michael Würtenberger
# (c) 2016, 2017, 2018
#
# Licence APL2.0
#
###########################################################
import logging
import math
import datetime
import PyQt5
from astropy import _erfa

class Transform:
    logger = logging.getLogger(__name__)

    def __init__(self, app):
        self.app = app
        self.ERFA = _erfa
        self.mutexERFA = PyQt5.QtCore.QMutex()
        self.mutexTopocentric = PyQt5.QtCore.QMutex()
        # if nothing is present, use the coordinates of greenwich
        self.siteLat = 51.476852
        self.siteLon = 0
        self.siteHeight = 46
        # date of 01.05.2018
        self.julianDate = 2458240
        # connect data transfer
        self.app.signalMountSiteData.connect(self.setSiteData)
        self.app.signalJulianDate.connect(self.setJulianDate)

    def setSiteData(self, lat, lon, height):
        self.siteLat = self.degStringToDecimal(lat)
        self.siteLon = self.degStringToDecimal(lon)
        self.siteHeight = float(height)

    def setJulianDate(self, jd):
        self.julianDate = jd

    def topocentricToAzAlt(self, ra, dec):
        self.mutexTopocentric.lock()
        ra = (ra * 360 / 24 + 360.0) % 360.0
        dec = math.radians(dec)
        ra = math.radians(ra)
        lat = math.radians(self.siteLat)
        alt = math.asin(math.sin(dec) * math.sin(lat) + math.cos(dec) * math.cos(lat) * math.cos(ra))
        value = (math.sin(dec) - math.sin(alt) * math.sin(lat)) / (math.cos(alt) * math.cos(lat))
        # we have to check for rounding error, which could happen
        if value > 1:
            value = 1
        elif value < -1:
            value = -1
        A = math.acos(value)
        A = math.degrees(A)
        alt = math.degrees(alt)
        if math.sin(ra) >= 0.0:
            az = 360.0 - A
        else:
            az = A
        self.mutexTopocentric.unlock()
        return az, alt

    def degStringToDecimal(self, value, splitter=':'):
        returnValue = 0
        sign = 1
        if '-' in value:
            value = value.replace('-', '')
            sign = -1
        elif '+' in value:
            value = value.replace('+', '')
        else:
            # just for formal understanding
            pass
        try:
            if len(value.split(splitter)) == 3:
                hour, minute, second = value.split(splitter)
                returnValue = (float(hour) + float(minute) / 60 + float(second) / 3600) * sign
            elif len(value.split(splitter)) == 2:
                hour, minute = value.split(splitter)
                returnValue = (float(hour) + float(minute) / 60) * sign
        except Exception as e:
            self.logger.error('Error in conversion of:{0} with splitter:{1}, e:{2}'.format(value, splitter, e))
            returnValue = 0
        finally:
            pass
        return returnValue

    @staticmethod
    def decimalToDegree(value, with_sign=True, with_decimal=False, spl=':'):
        if value >= 0:
            sign = '+'
        else:
            sign = '-'
        value = abs(value)
        hour = int(value)
        minute = int((value - hour) * 60)
        second = int(((value - hour) * 60 - minute) * 60)
        if with_decimal:
            second_dec = '.{0:1d}'.format(int((((value - hour) * 60 - minute) * 60 - second) * 10))
        else:
            second_dec = ''
        if with_sign:
            returnValue = '{0}{1:02d}{5}{2:02d}{5}{3:02d}{4}'.format(sign, hour, minute, second, second_dec, spl)
        else:
            returnValue = '{0:02d}{4}{1:02d}{4}{2:02d}{3}'.format(hour, minute, second, second_dec, spl)
        return returnValue

    @staticmethod
    def decimalToDegreeMountSr(value):
        degree = int(value)
        minute = int((value - degree) * 60)
        second = int(((value - degree) * 60 - minute) * 60)
        second_dec = int((((value - degree) * 60 - minute) * 60 - second) * 100)
        returnValue = ':Sr{0:02d}:{1:02d}:{2:02d}.{3:02d}#'.format(degree, minute, second, second_dec)
        return returnValue

    @staticmethod
    def decimalToDegreeMountSd(value):
        if value >= 0:
            sign = '+'
        else:
            sign = '-'
        value = abs(value)
        degree = int(value)
        minute = int((value - degree) * 60)
        second = int(((value - degree) * 60 - minute) * 60)
        second_dec = int((((value - degree) * 60 - minute) * 60 - second) * 10)
        returnValue = ':Sd{0}{1:02d}*{2:02d}:{3:02d}.{4:01d}#'.format(sign, degree, minute, second, second_dec)
        return returnValue

    def transformERFA(self, ra, dec, transform=1):
        self.mutexERFA.lock()
        ts = datetime.datetime.utcnow()
        dut1_prev = self.ERFA.dat(ts.year, ts.month, ts.day, 0)
        dut1 = 37 + 4023.0 / 125.0 - dut1_prev
        # suc, tai1, tai2 = self.ERFA.eraUtctai(self.julianDate, 0)
        tai1, tai2 = self.ERFA.utctai(self.julianDate, 0)
        # tt1, tt2 = self.ERFA.eraTaitt(tai1, tai2)
        tt1, tt2 = self.ERFA.taitt(tai1, tai2)
        jdtt = tt1 + tt2
        date1 = self.julianDate
        date2 = 0

        if transform == 1:  # J2000 to Topo Az /Alt
            ra = ra % 24
            aob, zob, hob, dob, rob, eo = self.ERFA.atco13(ra * self.ERFA.D2PI / 24,
                                                           dec * self.ERFA.D2PI / 360,
                                                           0.0,
                                                           0.0,
                                                           0.0,
                                                           0.0,
                                                           date1 + date2,
                                                           0.0,
                                                           dut1,
                                                           self.siteLon * self.ERFA.DD2R,
                                                           self.siteLat * self.ERFA.DD2R,
                                                           self.siteHeight,
                                                           0.0,
                                                           0.0,
                                                           0.0,
                                                           0.0,
                                                           0.0,
                                                           0.0)
            val1 = aob * 360 / self.ERFA.D2PI
            val2 = 90.0 - zob * 360 / self.ERFA.D2PI

        elif transform == 2:                                                                                # Topo to J2000
            rc, dc, eo = self.ERFA.atic13(self.ERFA.anp(ra * self.ERFA.D2PI / 24 + self.ERFA.eo06a(jdtt, 0.0)),
                                          dec * self.ERFA.D2PI / 360,
                                          date1 + date2,
                                          0.0)
            val1 = rc * 24.0 / self.ERFA.D2PI
            val2 = dc * self.ERFA.DR2D

        elif transform == 3:                                                                                # J2000 to Topo
            ri, di, eo = self.ERFA.atci13(ra * self.ERFA.D2PI / 24,
                                          dec * self.ERFA.D2PI / 360,
                                          0,
                                          0,
                                          0,
                                          0,
                                          date1 + date2,
                                          0)
            val1 = self.ERFA.anp(ri - eo) * 24 / self.ERFA.D2PI
            val2 = di * 360 / self.ERFA.D2PI
        else:
            val1 = ra
            val2 = dec
        self.mutexERFA.unlock()
        return val1, val2

reason: from astropy


text:
# encoding: utf-8
# author: liaochangzeng
# e-mail: 1207338829@qq.com

import os
import sys
import json
import thulac
import numpy as nps
import tensorflow as tf
from flask import Flask, render_template
from flask import request
from random import randint
from util import Vocabulary

# load thulac and initializing
thu = thulac.thulac(seg_only=True)
thu.cut("hello world")

# load validation data
with open("data/validation.data", encoding="utf-8") as fd:
	sen_list = fd.read().strip().split("\n")

# load dictionary
with open("../CNN/data/train.data", encoding="utf-8") as fd:
    txt = fd.read().strip().split("\n")
    txt = [item.split("\t") for item in txt]
    en = [item[0] for item in txt]
    zh = [item[1] for item in txt]
vocab_en = Vocabulary(100, 5)
vocab_zh = Vocabulary(100, 5)
vocab_en.fit_transform(en)
vocab_zh.fit_transform(zh)

# load model parameters
TEXTCNN_PATH = "../CNN/runs/"
def load_model(model_root_path):
	model_dict = {}
	for file in os.listdir(model_root_path):
		_file_path = os.path.join(model_root_path, file)
		if os.path.isdir(_file_path):
			model_dict[file] = os.path.join(_file_path, "checkpoints")
	return model_dict
TEXTCNN_model = load_model(TEXTCNN_PATH)

app = Flask(__name__)

@app.route('/')
def index():
	return render_template("index.html")

@app.route('/random')
def random_select():
	random_index = randint(0, len(sen_list)-1)
	sen = sen_list[random_index]
	sen = sen.split("\t")
	sen_en = sen[0]
	sen_zh = "".join(sen[1].split(" "))
	label = sen[2]
	if label == "1":
		label = u"相似"
	else:
		label = u"不相似"
	return json.dumps({"sen_zh": sen_zh, "sen_en": sen_en, "label": label}, ensure_ascii=False)

@app.route('/calculation', methods=['POST', 'GET'])
def calculation():
	if request.method != 'POST':
		return "Request Method Must Be POST!"

	zh_val = request.form['zh']
	en_val = request.form['en']
	model_type = request.form['model_type']
	model_parameter = request.form['model_parameter']
	zh_val = [item[0] for item in thu.cut(zh_val)]
	zh_val = " ".join(zh_val)
	zh_val = list(vocab_zh.transform([zh_val, ]))
	en_val = list(vocab_zh.transform([en_val, ]))

	model_path = "../CNN/runs/" + model_parameter + "/checkpoints/"
	with tf.Session() as sess:
		ckpt = tf.train.get_checkpoint_state(model_path)
		saver = tf.train.import_meta_graph(ckpt.model_checkpoint_path +'.meta')
		saver.restore(sess,ckpt.model_checkpoint_path)
		graph = tf.get_default_graph()
		feed_dict = {
			graph.get_tensor_by_name("input_x_en:0"): en_val,
			graph.get_tensor_by_name("input_x_zh:0"): zh_val,
			graph.get_tensor_by_name("dropout_keep_prob:0"): 0.5,
			graph.get_tensor_by_name("is_training:0"): False
		}
		predictions = graph.get_tensor_by_name("output/predictions:0")
		predict = sess.run(predictions, feed_dict=feed_dict)
	print(predict)
	predict = predict[0]
	if predict == 1:
		result = "相似"
	else:
		result = "不相似"
	return json.dumps({"result": result})

# 获得保存模型的关键参数
@app.route('/model_parameter', methods=['POST', 'GET'])
def model_parameter():
	if request.method != 'POST':
		return "Request Method Must Be POST!"
	model_type = request.form['model_type']
	result = {}
	if model_type == "TextCNN":
		result = TEXTCNN_model
	return json.dumps(result)

app.run(debug = True)

max_stars_repo_name: changzeng/Typhoon

max_issues_repo_name: changzeng/Typhoon

max_forks_repo_name: changzeng/Typhoon


text:
'''
This library is used to preprocess raw images (resizing, denoising).
This library also stores statistical parameters of images for automated image quality filtering.
The input for the library is relative path for raw image folder and resized image folder.
'''
#import all libraries
import os, glob
import pandas as pd
import numpy as np
import cv2, csv, json, math, time, argparse
from pathlib import Path
from PIL import Image, ImageOps,ImageFilter
from scipy import ndimage
import imageio
from pylab import *

# Pandas column names for storing resized and preprocessed images
logo_folder_columns = ['true_logo','folder_name','folder_img_count']
# Pandas column names for storing statistical parameters for image quality filtering
resized_image_columns = ['true_logo','logo_img_name','original_image_name','company_brand','mean_mean_val','mean_std_val','mscn_val']
# Sampling images for balancing the images from each class
SAMPLE_SIZE = 200
EXPECTED_DIMENSION=120
SIZE_THRESHOLD=17

#Get folder information and logo brand names
def Get_folder_name_labels(dir_path):

    df_logos_folder = pd.DataFrame(columns=logo_folder_columns)
    mn = 0
    folders = ([name for name in sorted(os.listdir(dir_path), key=str.casefold)]) # get all directories
    num_folders =1

    for company_name in folders:
        contents = os.listdir(os.path.join(dir_path,company_name)) # get list of contents
        if len(contents) <= mn: # if greater than the limit, print folder and number of contents
            continue
        row = pd.Series({logo_folder_columns[0] :num_folders,
                         logo_folder_columns[1] :company_name,
                        logo_folder_columns[2]: len(contents),
                        'folder_path': os.path.join(dir_path,company_name),})
        print(os.path.join(dir_path,company_name))
        print("length:",str(len(contents)))
        num_folders = num_folders+1
        df_logos_folder = df_logos_folder.append(row,ignore_index=True)

    return df_logos_folder

#Image processing for resizing images
def fix_aspect_ratio(img):
    original_max_dim = max(float(img.size[0]),float(img.size[1]))
    original_min_dim = min(float(img.size[0]),float(img.size[1]))
    wpercent = (EXPECTED_DIMENSION/float(original_max_dim))
    hsize = int((original_min_dim*float(wpercent)))
    new_im = img.resize((EXPECTED_DIMENSION,hsize), Image.ANTIALIAS)
    return new_im

# Make square images
def make_square(img, min_size=120, fill_color=(0, 0, 0, 0)):
    x, y = img.size
    size = max(min_size, x, y)
    new_im = Image.new('RGBA', (size, size), fill_color)
    new_im.paste(img, (int((size - x) / 2), int((size - y) / 2)))
    return new_im

# Sharpen the edges
def sharpen_filter(img):
    sharp_im = img.filter(ImageFilter.SHARPEN)
    return sharp_im

# Statistical parameters for images
def calculate_MSCN_val(img):
    C=3.0/255.0
    blurred_img=cv2.GaussianBlur(img, (7, 7), 1.166)
    blurred_sq = blurred_img * blurred_img
    sigma = cv2.GaussianBlur(img * img, (7, 7), 1.166)
    sigma = (sigma - blurred_sq) ** 0.5
    sigma = sigma + C
    MCSN_value = (img - blurred_img)/sigma
    return MCSN_value

# Load dataset of logos in a dataframe
def resize_images_df(folder_path_global, company_name,dir_litw_resized,folder_logo_num):
    df_logos_pickle = pd.DataFrame(columns=resized_image_columns)

     # in each folder, find image file and resize-scale them without distortion
    index_image= 1;
    glob_path_complete_path_inter=os.path.join(folder_path_global,company_name)
    glob_path_complete_path=Path(glob_path_complete_path_inter)
    mod_folder_name = 'Mod-' + str(company_name)
    mod_full_dir = os.path.join(dir_litw_resized,mod_folder_name)
    brand_count_row=1
    for filename_logo in glob_path_complete_path.glob('**/*.jpg'):
         with open(filename_logo) as imagefile:
            # Choose the classes with enough samples
            if brand_count_row > SAMPLE_SIZE:
                return df_logos_pickle
            image_original = Image.open(filename_logo)
            dir_name,file_orig_name = os.path.split(filename_logo)
            get_file_data_shape_x, get_file_data_shape_y = image_original.size
            max_size_dimension = max(float(get_file_data_shape_x),float(get_file_data_shape_y))
            if max_size_dimension < SIZE_THRESHOLD: # size should be above the threshold
                continue
            img_mod_name = company_name +'-AB-'+ str(index_image) +'.png'

            os.makedirs(mod_full_dir, exist_ok=True)
            path_to_image = os.path.join(mod_full_dir,img_mod_name)

            company_brand_name = company_name

            # Image preprocessing
            image_aspect=fix_aspect_ratio(image_original)
            img_new = make_square(image_aspect)
            sharpen_image=sharpen_filter(img_new)
            im_med = ndimage.median_filter(sharpen_image, 3)

            # Statistical information
            norm_image=cv2.normalize(im_med,None,0,1, cv2.NORM_MINMAX)
            mean_values, std_values = cv2.meanStdDev(norm_image)
            mscn_values= calculate_MSCN_val(norm_image)
            mean_mean_val = np.mean(mean_values)
            mean_std_val = np.mean(std_values)
            mean_mscn_val =  np.mean(mscn_values)
            company_brand_name = company_name

            row = pd.Series({resized_image_columns[0] :folder_logo_num,
                             resized_image_columns[1] :img_mod_name,
                             resized_image_columns[2] :file_orig_name,
                             resized_image_columns[3] :company_brand_name,
                            resized_image_columns[4]: mean_mean_val,
                            resized_image_columns[5]: mean_std_val,
                            resized_image_columns[6]: mean_mscn_val,})

            # save information and save resized images
            df_logos_pickle = df_logos_pickle.append(row,ignore_index=True)
            brand_count_row=brand_count_row+1
            file_name=os.path.join(mod_full_dir,img_mod_name)
            imageio.imwrite(file_name, im_med)
            index_image =index_image+1

    return df_logos_pickle

# Access all the logo folders and process the image files
def get_all_images(dir_litw,dir_litw_resized):
    df_list=[]
    df_folder_Stat_list=[]
    folder_logo_num=0
    folders = ([name for name in sorted(os.listdir(dir_litw), key=str.casefold)])
    for company_name in folders:
        folder_path_global=dir_litw
        df_list.append(resize_images_df(folder_path_global, company_name,dir_litw_resized, folder_logo_num))
        tp_df = pd.concat(df_list, ignore_index=True)
        folder_logo_num=folder_logo_num+1
    return tp_df

reason: import numpy,from scipy


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Sep  9 12:12:35 2019

@author: jaehooncha

@email: chajaehoon79@gmail.com

autoencoder-generative model
"""
import tensorflow as tf
from tensorflow.contrib import slim
import networks
import numpy as np
tfd = tf.contrib.distributions
tfb = tfd.bijectors

def xavier_init(fan_in, fan_out, constant = 1):
    low = -constant*np.sqrt(6.0/(fan_in + fan_out))
    high = constant*np.sqrt(6.0/(fan_in + fan_out))
    return tf.random_uniform((fan_in, fan_out),
                             minval = low, maxval = high,
                             dtype = tf.float32)

'''
Autoencoder
'''
class AE(object):
    def __init__(self, n_input, n_output, archi):
        self.n_input = n_input
        self.n_output = n_output
        self.archi = archi

        self.x = tf.placeholder(tf.float32, shape=(None, 784),  name = "input")

    def Encoder(self, inputs, name):
        with tf.name_scope(name):
            layer_1 = slim.layers.linear(inputs, num_outputs = self.archi['h1'], activation_fn = tf.nn.softplus, scope = 'encoder1')
            layer_2 = slim.layers.linear(layer_1, num_outputs = self.archi['h2'], activation_fn = tf.nn.softplus, scope = 'encoder2')
            return layer_2

    def To_latent(self, inputs, name):
        with tf.name_scope(name):
            encoder_out = slim.layers.linear(inputs, num_outputs = self.archi['latent'])
            return encoder_out

    def Decoder(self, inputs, name):
        with tf.name_scope(name):
            layer_1 = slim.layers.linear(inputs, num_outputs = self.archi['h2'], activation_fn = tf.nn.softplus, scope = 'decoder1')
            layer_2 = slim.layers.linear(layer_1, num_outputs = self.archi['h1'], activation_fn = tf.nn.softplus, scope = 'decoder2')
            decoder_out = slim.layers.linear(layer_2, num_outputs = self.n_output, activation_fn = tf.nn.sigmoid, scope = 'decoder_out')
            return decoder_out

    def Forward(self):
        self.encoder_out = self.Encoder(self.x, 'encoder')
        self.z = self.To_latent(self.encoder_out, 'latent')
        self.reconstr = self.Decoder(self.z, 'decoder')

        with tf.name_scope('loss'):
            self.loss = tf.reduce_mean(-tf.reduce_sum(self.x*tf.math.log(tf.clip_by_value(self.reconstr, 1e-10, 1.0))
                                            + (1-self.x)*tf.math.log(tf.clip_by_value(1-self.reconstr,1e-10,1.0)), 1))
        def Summaray():
            tf.summary.scalar('loss', self.loss)

        Summaray()

        return self.z, self.reconstr, self.loss

'''
Variational Autoencoder
'''
class VAE(AE):
    def __init__(self, n_input, n_output, archi):
        super().__init__(n_input, n_output, archi)
        self.prior = networks.DiagonalGaussian(mu = tf.zeros(1,1), logvar = tf.zeros(1,1))

    def To_latent(self, inputs, name):
        with tf.name_scope(name):
            mu = slim.layers.linear(inputs, num_outputs = self.archi['latent'], scope = 'mean')
            logvar = slim.layers.linear(inputs, num_outputs = self.archi['latent'], scope = 'logvar')
            return networks.DiagonalGaussian(mu, logvar)

    def Forward(self):
        self.encoder_out = self.Encoder(self.x, 'encoder')

        self.encode_dist = self.To_latent(self.encoder_out, 'encoder_dist')

        with tf.name_scope('latent'):
            self.z = self.encode_dist.sample()

        self.reconstr = self.Decoder(self.z, 'decoder')

        with tf.name_scope('loss'):
            self.reconst_loss = tf.reduce_mean(-tf.reduce_sum(self.x*tf.math.log(tf.clip_by_value(self.reconstr, 1e-10, 1.0))
                                                + (1-self.x)*tf.math.log(tf.clip_by_value(1-self.reconstr,1e-10,1.0)), 1))

            self.kl_loss = tf.reduce_mean(-self.prior.log_probability(self.z))

            self.lat_loss = tf.reduce_mean(self.encode_dist.log_probability(self.z))

            self.loss = tf.reduce_mean(self.reconst_loss + self.kl_loss + self.lat_loss)

        def Summaray():
            tf.summary.scalar('reconst_loss', self.reconst_loss)
            tf.summary.scalar('kl_loss', self.kl_loss)
            tf.summary.scalar('lat_loss', self.lat_loss)
            tf.summary.scalar('loss', self.loss)

        Summaray()

        return self.z, self.reconstr, self.loss


text:
#from utils import *
import torch
from tqdm import tqdm
import numpy as np
import io
import lmdb
import os
from collections import defaultdict
import pandas as pd
from scipy.special import erfinv
import argparse

parser = argparse.ArgumentParser()
parser.add_argument(
    "-c",
    "--city",
    default="BERLIN",
    type=str,
)

parser.add_argument(
    "-p",
    "--path",
    default="/home/omnisky/processed_data/",
    type=str,
)

args = parser.parse_args()

def get_item(idx):
    idx = str(idx).encode("ascii")
    try:
        with env.begin() as txn:data = txn.get(idx)
        data = np.load(io.BytesIO(data))
        x = np.zeros(495 * 436 * 9, dtype=np.uint8)
        x[data["x"]] = data["y"]
        x = x.reshape([495, 436, 9])
        x = np.moveaxis(x, -1, 0)
    except:
        x = np.zeros([9, 495, 436], dtype=np.uint8)
    return x

counter = defaultdict(lambda:defaultdict(lambda:0))
env = lmdb.open(os.path.join(args.path, args.city.upper()), readonly=True)

if __name__=="__main__":
    for idx in tqdm(range(52104)):
        array = get_item(idx)
        for i in range(9):
            t = array[i]
            for x ,y in zip(*np.unique(t,return_counts=True)):
                counter[i][x] +=y

    all_arr = []
    for i in range(9):
        df = pd.DataFrame(counter[i],index=[0]).T
        df = df.reset_index()
        df = df.sort_values('index')
        df.set_index('index',inplace=True)

        df = df[1:-1]

        df['cum'] = df.cumsum().shift().fillna(0)

        df['rank'] = erfinv(df['cum']/(df['cum'].max()+1e3))
        df['rank'] = df['rank']/df['rank'].max()
        df['rank'] = (df['rank']+1)/3
        if i!=8:
            arr = df.reset_index()['rank'].values
            arr = np.append(np.append(0,arr),1)
        else:
            arr = np.zeros(256)
            for idx,row in df.iterrows():
                arr[idx] = row['rank']
            arr[-1] = 1
        all_arr.append(arr)
    np.savez(f'{args.city.upper()}_value_map.npz',array=all_arr)


text:
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

from tfbldr.nodes import Linear
from tfbldr.nodes import LSTMCell
from tfbldr.nodes import VqEmbedding
from tfbldr.nodes import Sigmoid
from tfbldr.nodes import ReLU
from tfbldr.nodes import BernoulliCrossEntropyCost
from tfbldr.nodes import Bilinear
from tfbldr import get_params_dict
from tfbldr import run_loop
from tfbldr import scan

import numpy as np
import tensorflow as tf
from collections import namedtuple

# from Rithesh
ct_random = np.random.RandomState(1290)
def copytask_loader(batch_size, seq_width, min_len, max_len):
    while True:
        # All batches have the same sequence length, but varies across batch
        if min_len == max_len:
            seq_len = min_len
        else:
            seq_len = ct_random.randint(min_len, max_len)
        seq = ct_random.binomial(1, 0.5, (seq_len, batch_size, seq_width))
        #seq = Variable(torch.from_numpy(seq)).cuda()

        # The input includes an additional channel used for the delimiter
        inp = np.zeros((2 * seq_len + 2, batch_size, seq_width + 1))
        inp[:seq_len, :, :seq_width] = seq
        inp[seq_len, :, seq_width] = 1.0 # start output delimiter in our control channel

        outp = np.zeros((2 * seq_len + 2, batch_size, seq_width + 1))
        outp[seq_len + 1:-1, :, :seq_width] = seq
        outp[-1, :, seq_width] = 1.0 # end output delimiter in our control channel
        yield inp.astype("float32"), outp.astype("float32")

batch_size = 10
seq_length_min = 5
seq_length_max = 5
seq_width = 8
copy_itr = copytask_loader(batch_size, seq_width, seq_length_min, seq_length_max)
a, b = next(copy_itr)
"""
plt.matshow(a[:, 0, :])
plt.savefig("a")
plt.matshow(b[:, 0, :])
plt.savefig("b")
"""

random_state = np.random.RandomState(1177)

n_hid = 1024
n_emb = 4096
n_inputs = a.shape[-1]
rnn_init = None#"truncated_normal"
forward_init = None#"truncated_normal"

def create_model(inp, h1_init, c1_init, h1_q_init, c1_q_init):
    p = Linear([inp], [n_inputs], n_hid, random_state=random_state, name="proj_in",
               init=forward_init)
    def step(x_t, h1_tm1, c1_tm1, h1_q_tm1, c1_q_tm1):
        output, s = LSTMCell([x_t, h1_q_tm1], [n_hid, n_hid], h1_tm1, c1_tm1, n_hid,
                             random_state=random_state,
                             name="rnn1", init=rnn_init)
        h1_t = s[0]
        c1_t = s[1]

        output, s = LSTMCell([h1_t], [n_hid], h1_q_tm1, c1_q_tm1, n_hid,
                             random_state=random_state,
                             name="rnn1_q", init=rnn_init)
        h1_cq_t = s[0]
        c1_q_t = s[1]

        h1_q_t, h1_i_t, h1_nst_q_t, h1_emb = VqEmbedding(h1_cq_t, n_hid, n_emb,
                                                         random_state=random_state,
                                                         name="h1_vq_emb")

        output_q_t, output_i_t, output_nst_q_t, output_emb = VqEmbedding(output, n_hid, n_emb,
                                                                         random_state=random_state,
                                                                         name="out_vq_emb")

        # not great
        h1_i_t = tf.cast(h1_i_t, tf.float32)
        output_i_t = tf.cast(h1_i_t, tf.float32)

        lf_output = Bilinear(h1_q_t, n_hid, output_emb, n_hid,
                             random_state=random_state, name="out_mix",
                             init=forward_init)
        rf_output = Bilinear(output_q_t, n_hid, h1_emb, n_hid,
                             random_state=random_state, name="h_mix",
                             init=forward_init)
        f_output = Linear([lf_output, rf_output], [n_emb, n_emb], n_hid,
                          random_state=random_state, name="out_f",
                          init=forward_init)

        # r[0]
        rets = [output_q_t]
        # r[1:3]
        rets += [h1_t, c1_t]
        # r[3:9]
        rets += [h1_q_t, c1_q_t, h1_nst_q_t, h1_cq_t, h1_i_t, h1_emb]
        # r[9:]
        rets += [output_q_t, output_nst_q_t, output, output_i_t, output_emb]
        return rets

    outputs_info = [None,
                    h1_init, c1_init,
                    h1_q_init, c1_q_init, None, None, None, None,
                    None, None, None, None, None]
    r = scan(step, [p], outputs_info)
    out = r[0]
    hiddens = r[1]
    cells = r[2]
    q_hiddens = r[3]
    q_cells = r[4]
    q_nst_hiddens = r[5]
    q_nvq_hiddens  = r[6]
    i_hiddens = r[7]
    emb_hiddens = r[8]
    q_out = r[9]
    q_nst_out = r[10]
    q_nvq_out = r[11]
    i_out = r[12]
    emb_out = r[13]

    l1 = Linear([out, q_hiddens], [n_hid, n_hid], n_hid, random_state=random_state, name="l1",
                init=forward_init)
    r_l1 = ReLU(l1)
    pred = Linear([r_l1], [n_hid], n_inputs, random_state=random_state, name="out",
                  init=forward_init)
    pred_sig = Sigmoid(pred)
    outs_names = ["pred_sig",
                  "pred",
                  "hiddens",
                  "cells",
                  "q_hiddens",
                  "q_cells",
                  "q_nst_hiddens",
                  "q_nvq_hiddens",
                  "i_hiddens",
                  "emb_hiddens",
                  "q_out",
                  "q_nst_out",
                  "q_nvq_out",
                  "i_out",
                  "emb_out"]
    outs_tf = [eval(name) for name in outs_names]
    c = namedtuple("Core", outs_names)(*outs_tf)
    return c

def create_graph():
    graph = tf.Graph()
    with graph.as_default():
        inputs = tf.placeholder(tf.float32, shape=[None, batch_size, n_inputs])
        targets = tf.placeholder(tf.float32, shape=[None, batch_size, n_inputs])
        init_hidden = tf.placeholder(tf.float32, shape=[batch_size, n_hid])
        init_cell = tf.placeholder(tf.float32, shape=[batch_size, n_hid])
        init_q_hidden = tf.placeholder(tf.float32, shape=[batch_size, n_hid])
        init_q_cell = tf.placeholder(tf.float32, shape=[batch_size, n_hid])
        r = create_model(inputs, init_hidden, init_cell, init_q_hidden, init_q_cell)

        # reconstruction loss
        rec_loss = tf.reduce_mean(BernoulliCrossEntropyCost(tf.reshape(r.pred_sig, (-1, 1)), tf.reshape(targets, (-1, 1))))

        # h2h embedding losses
        alpha = 1.
        beta = 0.25
        vq_h_loss = tf.reduce_mean(tf.square(tf.stop_gradient(r.q_nvq_hiddens) - r.q_nst_hiddens))
        commit_h_loss = tf.reduce_mean(tf.square(r.q_nvq_hiddens - tf.stop_gradient(r.q_nst_hiddens)))

        # output embedding losses
        vq_o_loss = tf.reduce_mean(tf.square(tf.stop_gradient(r.q_nvq_out) - r.q_nst_out))
        commit_o_loss = tf.reduce_mean(tf.square(r.q_nvq_out - tf.stop_gradient(r.q_nst_out)))

        loss_r = rec_loss
        loss_h = alpha * vq_h_loss + beta * commit_h_loss
        loss_o = alpha * vq_o_loss + beta * commit_o_loss
        loss = loss_r + loss_h + loss_o

        params = get_params_dict()
        grads = tf.gradients(loss, params.values())
        learning_rate = 0.0001
        optimizer = tf.train.AdamOptimizer(learning_rate, use_locking=True)
        assert len(grads) == len(params)
        grads = [tf.clip_by_value(g, -10., 10.) if g is not None else None for g in grads]
        j = [(g, p) for g, p in zip(grads, params.values())]
        train_step = optimizer.apply_gradients(j)

    things_names = ["inputs",
                    "targets",
                    "init_hidden",
                    "init_cell",
                    "init_q_hidden",
                    "init_q_cell",
                    "loss",
                    "rec_loss",
                    "train_step"]
    things_tf = [eval(name) for name in things_names]
    things_names += r._asdict().keys()
    things_tf += r._asdict().values()
    train_model = namedtuple('Model', things_names)(*things_tf)
    for tn, tt in zip(things_names, things_tf):
        graph.add_to_collection(tn, tt)
    return graph, train_model

g, vs = create_graph()

def loop(sess, itr, extras, stateful_args):
    inps, targets = next(itr)
    init_h = np.zeros((batch_size, n_hid)).astype("float32")
    init_c = np.zeros((batch_size, n_hid)).astype("float32")
    init_q_h = np.zeros((batch_size, n_hid)).astype("float32")
    init_q_c = np.zeros((batch_size, n_hid)).astype("float32")
    if extras["train"]:
        feed = {vs.inputs: inps,
                vs.targets: targets,
                vs.init_hidden: init_h,
                vs.init_cell: init_c,
                vs.init_q_hidden: init_q_h,
                vs.init_q_cell: init_q_c}
        outs = [vs.rec_loss, vs.loss, vs.train_step]
        r = sess.run(outs, feed_dict=feed)
        l = r[0]
        t_l = r[1]
        step = r[2]
    else:
        raise ValueError("No valid, no cry")
    return l, None, stateful_args

with tf.Session(graph=g) as sess:
    run_loop(sess,
             loop, copy_itr,
             loop, copy_itr,
             n_steps=200000,
             n_train_steps_per=10000,
             n_valid_steps_per=0)

max_stars_repo_name: kastnerkyle/tfbldr

max_issues_repo_name: kastnerkyle/tfbldr

max_forks_repo_name: kastnerkyle/tfbldr


text:
import pytest
from astropy.table import Table

from b_to_zooniverse.new_subjects.find_new_subjects import find_new_catalog_images

TEST_EXAMPLES_DIR = 'python/test_examples'

@pytest.fixture()
def nsa_decals_dr3():
    return Table([
        # galaxy a is a new DR3 image, only in this catalog
        {'iauname': 'gal_a',
         'ra': 146.714208787,
         'dec': -1.04128156958,
         'petroth50': 3.46419,
         'petroth90': 10.4538,
         'fits_loc': '{}/example_a.fits'.format(TEST_EXAMPLES_DIR)},

        # galaxy b is in dr2, but with a different fits file
        # it should be included in the gz dr3 upload
        {'iauname': 'gal_b',
         'ra': 146.631735209,
         'dec': -0.988354858412,
         'petroth50': 2.28976,
         'petroth90': 5.20297,
         'fits_loc': '{}/example_b.fits'.format(TEST_EXAMPLES_DIR)},

        # galaxy c is in dr2 with an identical fits file
        # it should not be included in the gz dr3 upload (already classified)
        {'iauname': 'gal_c',
         'ra': 147.176446949,
         'dec': -0.354030416643,
         'petroth50': 7.16148,
         'petroth90': 24.7535,
         'fits_loc': '{}/example_c.fits'.format(TEST_EXAMPLES_DIR)},

        # same properties as dr3 including fits file
        {'iauname': 'gal_missing_in_dr2',
         'ra': 147.176446949,
         'dec': -0.354030416643,
         'petroth50': 7.16148,
         'petroth90': 24.7535,
         'fits_loc': '{}/example_c.fits'.format(TEST_EXAMPLES_DIR)}  # good path, unlike in dr2
    ])

@pytest.fixture()
def nsa_decals_dr2():
    # same properties as dr3 except fits file
    return Table([
        {'iauname': 'gal_b',
         'ra': 146.631735209,
         'dec': -0.988354858412,
         'petroth50': 2.28976,
         'petroth90': 5.20297,
         'fits_loc': '{}/example_c.fits'.format(TEST_EXAMPLES_DIR)},  # points to dif. fits file than dr3

        # same properties as dr3 including fits file
        {'iauname': 'gal_c',
         'ra': 147.176446949,
         'dec': -0.354030416643,
         'petroth50': 7.16148,
         'petroth90': 24.7535,
         'fits_loc': '{}/example_c.fits'.format(TEST_EXAMPLES_DIR)},  # points to same fits file as dr3

        # same properties as dr3 including fits file
        {'iauname': 'gal_missing_in_dr2',
         'ra': 147.176446949,
         'dec': -0.354030416643,
         'petroth50': 7.16148,
         'petroth90': 24.7535,
         'fits_loc': 'no_image_here'}  # points nowhere
    ])

def test_find_new_catalog_images(nsa_decals_dr2, nsa_decals_dr3):
    only_new_galaxies = find_new_catalog_images(old_catalog=nsa_decals_dr2, new_catalog=nsa_decals_dr3)
    new_galaxy_names = set(only_new_galaxies['iauname'])
    assert 'gal_a' in new_galaxy_names
    assert 'gal_b' not in new_galaxy_names  # TODO checking for different fits files is disabled
    assert 'gal_missing_in_dr2' not in new_galaxy_names  # TODO checking for different fits files is disabled
    assert 'gal_c' not in new_galaxy_names

max_stars_repo_name: zooniverse/decals

max_issues_repo_name: zooniverse/decals

max_forks_repo_name: zooniverse/decals