text:
from collections import namedtuple
import itertools as it
import os
from random import sample as rsample

import numpy as np

from keras.models import Sequential
from keras.layers.normalization import BatchNormalization
from keras.layers.convolutional import Convolution2D
from keras.layers.core import Dense, Flatten
from keras.optimizers import SGD, RMSprop

from matplotlib import pyplot as plt

GRID_SIZE = 10

Fruit = namedtuple('Fruit', ['x', 'y'])

def game(snake_length=3):
    """
    Coroutine of single snake game.

    An action tuple (dx, dy) has to be provided each call to send.
    """
    actions = [(-1, 0)] * snake_length  # An action for each snake segment
    head_x = GRID_SIZE // 2 - snake_length // 2
    snake = [(x, GRID_SIZE // 2) for x in range(head_x, head_x + snake_length)]
    grow = -1  # Don't start growing snake yet
    fruit = Fruit(-1, -1)

    while True:
        # Draw borders
        screen = np.zeros((GRID_SIZE, GRID_SIZE))
        screen[[0, -1]] = 1
        screen[:, [0, -1]] = 1
        sum_of_borders = screen.sum()

        # Draw snake
        for segm in snake:
            x, y = segm
            screen[y, x] = 1

        # Snake hit into wall or ate itself
        end_of_game = len(snake) > len(set(snake)) or \
            screen.sum() < sum_of_borders + len(snake)
        reward = -1 * end_of_game

        # Draw fruit
        if screen[fruit.y, fruit.x] > .5:
            grow += 1
            reward = len(snake)
            while True:
                fruit = Fruit(*np.random.randint(1, GRID_SIZE - 1, 2))
                if screen[fruit.y, fruit.x] < 1:
                    break

        screen[fruit.y, fruit.x] = .5

        action = yield screen, reward

        step_size = sum([abs(act) for act in action])
        if not step_size:
            action = actions[0]  # Repeat last action
        elif step_size > 1:
            raise ValueError, 'Cannot move more than 1 unit at a time'

        actions.insert(0, action)
        actions.pop()

        # For as long as the snake needs to grow,
        # copy last segment, and add (0, 0) action
        if grow > 0:
            snake.append(snake[-1])
            actions.append((0, 0))
            grow -= 1

        # Update snake segments
        for ix, act in enumerate(actions):
            x, y = snake[ix]
            delta_x, delta_y = act
            snake[ix] = x + delta_x, y + delta_y

        if end_of_game:
            break

def experience_replay(batch_size):
    """
    Coroutine of experience replay.

    Provide a new experience by calling send, which in turn yields
    a random batch of previous replay experiences.
    """
    memory = []
    while True:
        experience = yield rsample(memory, batch_size) if batch_size <= len(memory) else None
        memory.append(experience)

nb_epochs = 10000
batch_size = 64
epsilon = 1.
gamma = .8

all_possible_actions = ((0, 0), (-1, 0), (1, 0), (0, -1), (0, 1))
nb_actions = len(all_possible_actions)

nb_frames = 4  # Number of frames (i.e., screens) to keep in history

# Recipe of deep reinforcement learning model
model = Sequential()
model.add(BatchNormalization(axis=1, input_shape=(nb_frames, GRID_SIZE, GRID_SIZE)))
model.add(Convolution2D(16, nb_row=3, nb_col=3, activation='relu'))
model.add(Convolution2D(32, nb_row=3, nb_col=3, activation='relu'))
model.add(Flatten())
model.add(Dense(256, activation='relu'))
model.add(Dense(nb_actions))
model.compile(RMSprop(), 'MSE')

exp_replay = experience_replay(batch_size)
exp_replay.next()  # Start experience replay coroutine

for i in xrange(nb_epochs):
    g = game()
    screen, _ = g.next()
    S = np.asarray([screen] * nb_frames)
    try:
        # Decrease epsilon over the first half of training
        if epsilon > .1:
            epsilon -= .9 / (nb_epochs / 2)

        loss = 0.
        while True:
            ix = np.random.randint(nb_actions)
            if np.random.random() > epsilon:
                ix = np.argmax(model.predict(S[np.newaxis]), axis=-1)[0]

            action = all_possible_actions[ix]
            screen, reward = g.send(action)
            S_prime = np.zeros_like(S)
            S_prime[1:] = S[:-1]
            S_prime[0] = screen
            experience = (S, action, reward, S_prime)
            S = S_prime

            batch = exp_replay.send(experience)
            if batch:
                inputs = []
                targets = []
                for s, a, r, s_prime in batch:
                    # The targets of unchosen actions are set to the q-values of the model,
                    # so that the corresponding errors are 0. The targets of chosen actions
                    # are set to either the rewards, in case a terminal state has been reached,
                    # or future discounted q-values, in case episodes are still running.
                    t = model.predict(s[np.newaxis]).flatten()
                    ix = all_possible_actions.index(a)
                    if r < 0:
                        t[ix] = r
                    else:
                        t[ix] = r + gamma * model.predict(s_prime[np.newaxis]).max(axis=-1)
                    targets.append(t)
                    inputs.append(s)

                loss += model.train_on_batch(np.array(inputs), np.array(targets))
                # print(loss)

    except StopIteration:
       pass

    if (i + 1) % 100 == 0:
        print 'Epoch %6i/%i, loss: %.6f, epsilon: %.3f' % (i + 1, nb_epochs, loss, epsilon)

def save_img():
    if 'images' not in os.listdir('.'):
        os.mkdir('images')
    frame_cnt = it.count()
    while True:
        screen = (yield)
        plt.imshow(screen, interpolation='none')
        plt.savefig('images/%04i.png' % (frame_cnt.next(), ))

img_saver = save_img()
img_saver.next()

game_cnt = it.count(1)
for _ in xrange(10):
    g = game()
    screen, _ = g.next()
    img_saver.send(screen)
    frame_cnt = it.count()
    try:
        S = np.asarray([screen] * nb_frames)
        while True:
            frame_cnt.next()
            ix = np.argmax(model.predict(S[np.newaxis]), axis=-1)[0]
            screen, _ = g.send(all_possible_actions[ix])
            S[1:] = S[:-1]
            S[0] = screen
            img_saver.send(screen)

    except StopIteration:
        print 'Saved %3i frames for game %3i' % (frame_cnt.next(), game_cnt.next())

img_saver.close()

ext: py

max_stars_repo_name: gopala-kr/ds-notebooks

max_issues_repo_name: gopala-kr/ds-notebooks

max_forks_repo_name: gopala-kr/ds-notebooks

reason: import numpy


text:
import numpy as np

# 문법 파일 불러오기
class Grammar :
    def __init__(self):
        self.contents = None
        self.len = 0

    def load(self, filePath):
        try :
            self.contents = np.loadtxt(filePath, dtype='str', delimiter=' -> ', encoding='utf-8')
            self.len = len(self.contents)
            print("문법 정보를 불러왔습니다.")
            print("문법의 개수는 {}개 입니다".format(self.len))

        # 파일 없을 때 예외 처리
        except:
            print("문법 정보가 없습니다.")

# 사전 파일 불러오기
class Dict :
    def __init__(self):

        self.contents = None
        self.numberOfContents = 0

    def load(self, filePath):
        try:
            # 첫 문자에 ufeff가 붙는 현상 제거 : utf-8-sig로 해결
            self.contents = np.loadtxt(filePath, dtype='str', delimiter='   ', encoding='utf-8-sig')
            print("사전을 불러왔습니다. (utf-8 인코딩)")
            self.numberOfContents = len(self.contents)
            print("사전에 들어있는 단어 개수는 " + str(self.numberOfContents) + "개 입니다.")

        # 사전 파일 없을 때 예외 처리
        except:
            print("사전이 없습니다. ")

# 문장 파일 불러오기
class Sentence :
    def __init__(self):

        self.contents = None
        self.len = 0

    def load(self, filePath):
        try:
        # 첫 문자에 ufeff가 붙는 현상 제거 : utf-8-sig로 해결
            self.contents = np.loadtxt(filePath, dtype='str', delimiter='\n')
            print("문장을 불러왔습니다. (utf-8 인코딩)")
            self.len = len(self.contents)
            print("불러온 문장은 " + str(self.len) + "개 입니다.")

        # 문장 파일 없을 때 예외 처리
        except:
            print("불러올 문장이 없습니다.")
#

max_stars_repo_name: henniekim/cky-parser-with-tree

max_issues_repo_name: henniekim/cky-parser-with-tree

max_forks_repo_name: henniekim/cky-parser-with-tree


text:
import logging
import os
import random
import string
import tarfile
from time import gmtime, strftime

import sagemaker
from sagemaker.debugger import TensorBoardOutputConfig
from sagemaker.inputs import TrainingInput
from sagemaker.processing import (
    ProcessingInput,
    ProcessingOutput,
    Processor,
    ScriptProcessor,
)
from sagemaker.pytorch.estimator import PyTorch
from sagemaker.tensorflow.estimator import TensorFlow

from . import VERSION, constants
from .s3_sync import S3Sync

logger = logging.getLogger(__name__)

class SageMakerTask:
    """
    An easy to use wrapper around SageMaker jobs into Tasks, a collection of jobs sharing
        the same input data and state (between runs).
    """

    def __init__(
        self,
        boto3_session,
        task_name,
        image_uri,
        prefix,
        bucket_name=None,
        smSession=None,
        local_mode=False,
        task_type=None,
    ):
        """
        Initializes a task

        Arguments:
            task_name -
            image_uri -
            bucket_name - The default SageMaker's bucker is used if not provided
            smSession -

        Data is maintained on [bucket_name]/[task_name]
        """
        self.boto3_session = boto3_session
        self.sm_client = boto3_session.client("sagemaker")
        self.task_name = task_name
        self.image_uri = image_uri
        self.estimators = list()
        self.jobNames = list()
        self.descriptions = list()
        self.local_mode = local_mode
        self.task_type = task_type
        self.prefix = prefix

        if smSession is None:
            smSession = sagemaker.Session(boto_session=boto3_session)
        self.smSession = smSession

        if not bucket_name:
            bucket_name = self.smSession.default_bucket()
        self.bucket_name = bucket_name

        self.baseTaskS3Uri = SageMakerTask.getBaseTaskS3Uri(
            bucket_name, prefix, task_name
        )
        self.stateS3Uri = None
        self.stateLocalPath = None
        if not local_mode:
            self.stateS3Uri = SageMakerTask.getStateS3Uri(
                bucket_name, prefix, task_name
            )
            self.stateLocalPath = constants.LOCAL_STATE_PATH
        self.inputS3Uri = None

        self.internalDependencies = [
            os.path.abspath(os.path.join(os.path.split(__file__)[0], "worker_toolkit"))
        ]

    @staticmethod
    def getBaseTaskS3Uri(bucket_name, prefix, task_name):
        return sagemaker.s3.s3_path_join("s3://", bucket_name, prefix, task_name)

    @staticmethod
    def getStateS3Uri(bucket_name, prefix, task_name):
        baseTaskS3Uri = SageMakerTask.getBaseTaskS3Uri(bucket_name, prefix, task_name)
        return sagemaker.s3.s3_path_join(baseTaskS3Uri, "state")

    def _getJobName(self):
        timestamp_prefix = strftime("%Y-%m-%d-%H-%M-%S", gmtime())
        # Add a 8-bytes random string to avoid collisions
        randString = "".join(random.choices(string.ascii_letters + string.digits, k=8))
        job_name = f"{self.task_name}-{timestamp_prefix}-{randString}"
        return job_name

    def runProcessing(
        self,
        entrypoint=None,
        command=None,
        env=None,
        code=None,
        arguments=None,
        inputs=list(),
        outputs=list(),
        instance_type=constants.DEFAULT_INSTANCE_TYPE_TRAINING,
        instance_count=constants.DEFAULT_INSTANCE_COUNT,
        role_name=constants.DEFAULT_IAM_ROLE,
        volume_size=constants.DEFAULT_VOLUME_SIZE,
        max_run_mins=constants.DEFAULT_MAX_RUN,
        tags=dict(),
        input_distribution="FullyReplicated",
        dependencies=list(),
    ):
        logger.info(
            f"===== Running a processing job {self.task_name} entrypoint={entrypoint} "
            f"command={command} code={code} arguments={arguments}... ====="
        )
        job_name = self._getJobName()

        # ## Outputs

        # state - continuesly updated
        state_path = "/opt/ml/processing/state"
        outputs.append(
            ProcessingOutput(state_path, self.stateS3Uri, "state", "Continuous")
        )
        env["SSM_STATE"] = state_path

        # output - copied by end of job
        output_path = "/opt/ml/processing/output"
        output_s3_uri = sagemaker.s3.s3_path_join(
            self.baseTaskS3Uri, job_name, "output"
        )
        outputs.append(
            ProcessingOutput(output_path, output_s3_uri, "output", "EndOfJob")
        )
        env["SSM_OUTPUT"] = output_path

        # ## Inputs

        # prev state
        bucket, prefix = sagemaker.s3.parse_s3_url(self.stateS3Uri)
        if self.smSession.list_s3_files(bucket, prefix):
            prev_state_path = "/opt/ml/processing/state_prev"
            inputs.append(
                ProcessingInput(
                    self.stateS3Uri,
                    prev_state_path,
                    "state_prev",
                    s3_data_distribution_type="FullyReplicated",
                )
            )

        # dependencies

        # append the internal dependencies
        dependencies.extend(self.internalDependencies)
        for dep in dependencies:
            dep = os.path.abspath(dep)
            basename = os.path.basename(dep)
            local_path = f"/opt/ml/processing/input/code/{basename}"
            inputs.append(
                ProcessingInput(
                    dep,
                    local_path,
                    "DEP_" + basename,
                    s3_data_distribution_type="FullyReplicated",
                )
            )

        # input data
        if self.inputS3Uri:
            data_path = "/opt/ml/processing/data"
            inputs.append(
                ProcessingInput(
                    self.inputS3Uri,
                    data_path,
                    "data",
                    s3_data_distribution_type=input_distribution,
                )
            )
            env["SM_CHANNEL_DATA"] = data_path

        tags["SimpleSagemakerTask"] = self.task_name
        tags["SimpleSagemakerVersion"] = VERSION
        tags = [{"Key": k, "Value": v} for k, v in tags.items()]

        additional_args = dict()
        if code:
            processor_class = ScriptProcessor
            additional_args["command"] = command
        else:
            assert (
                not command
            ), "Command can't be given where code isn't given (for the `Processor` class)"
            processor_class = Processor
            additional_args["entrypoint"] = entrypoint

        processor = processor_class(
            role=role_name,
            image_uri=self.image_uri,
            instance_count=instance_count,
            instance_type=instance_type,
            volume_size_in_gb=volume_size,
            max_runtime_in_seconds=max_run_mins * 60,
            sagemaker_session=self.smSession,
            tags=tags,
            env=env,
            **additional_args,
        )
        if code:
            processor.run(
                code=code,
                inputs=inputs,
                outputs=outputs,
                arguments=arguments,
                job_name=job_name,
            )
        else:
            processor.run(
                inputs=inputs,
                outputs=outputs,
                arguments=arguments,
                job_name=job_name,
            )

        proecessing_job_description = self.smSession.describe_processing_job(job_name)

        self.estimators.append(processor)
        self.jobNames.append(job_name)
        self.descriptions.append(proecessing_job_description)
        # print(proecessing_job_description)
        # if "Completed" != proecessing_job_description["TrainingJobStatus"]:
        #    logger.error(
        #        f"Task failed with status: {proecessing_job_description['TrainingJobStatus']}"
        #    )
        return job_name

    def runTrainingJob(
        self,
        framework,
        source_dir,
        entry_point,
        dependencies,
        hyperparameters,
        instance_type=constants.DEFAULT_INSTANCE_TYPE_TRAINING,
        instance_count=constants.DEFAULT_INSTANCE_COUNT,
        role_name=constants.DEFAULT_IAM_ROLE,
        additional_inputs=dict(),
        model_uri=None,
        use_spot_instances=constants.DEFAULT_USE_SPOT,
        max_wait_mins=constants.DEFAULT_MAX_WAIT,
        volume_size=constants.DEFAULT_VOLUME_SIZE,
        max_run_mins=constants.DEFAULT_MAX_RUN,
        tags=dict(),
        input_distribution="FullyReplicated",
        metric_definitions=dict(),
        enable_sagemaker_metrics=False,
        **additionalEstimatorArgs,
    ):
        """
        Runs a training job

        Arguments:
            source_dir - local/s3
            entry_point - entry point
            dependencies - additional local dependencies (directories) to be copied to the code path
            hyperparameters -
            instance_type -
            instance_count -
            model_uri - local/s3
            ...

        Returns estimator object
        """
        logger.info(
            f"===== Running a training job {self.task_name} source_dir={source_dir} "
            f"entry_point={entry_point} hyperparameters={hyperparameters}... ====="
        )
        job_name = self._getJobName()

        # append the internal dependencies
        dependencies.extend(self.internalDependencies)

        tags["SimpleSagemakerTask"] = self.task_name
        tags["SimpleSagemakerVersion"] = VERSION
        tags = [{"Key": k, "Value": v} for k, v in tags.items()]

        metric_definitions = [
            {"Name": k, "Regex": v} for k, v in metric_definitions.items()
        ]

        # zero max_wait_mins if not using spot instances
        if not use_spot_instances:
            max_wait_mins = 0
        # if using spot, and max_wait_mins isn't specified -> set it to max_run_mins
        elif not max_wait_mins:
            max_wait_mins = max_run_mins

        classes = {
            "pytorch": PyTorch,
            "tensorflow": TensorFlow,
        }
        estimator_class = classes[framework]

        # Configure TensorBoard
        tensorboard_output_config = TensorBoardOutputConfig(
            s3_output_path=self.baseTaskS3Uri,
            container_local_output_path="/opt/ml/output/tensorboard",
        )

        estimator = estimator_class(
            entry_point=entry_point,
            source_dir=source_dir,
            hyperparameters=hyperparameters,
            image_uri=self.image_uri,
            role=role_name,
            instance_count=instance_count,
            instance_type=instance_type,
            sagemaker_session=self.smSession,
            checkpoint_s3_uri=self.stateS3Uri,
            checkpoint_local_path=self.stateLocalPath,
            output_path=self.baseTaskS3Uri,
            code_location=self.baseTaskS3Uri,
            dependencies=dependencies,
            container_log_level=logging.INFO,
            volume_size=volume_size,
            max_run=max_run_mins * 60,
            model_uri=model_uri,
            use_spot_instances=use_spot_instances,
            max_wait=max_wait_mins * 60,
            tags=tags,
            metric_definitions=metric_definitions,
            enable_sagemaker_metrics=enable_sagemaker_metrics,
            tensorboard_output_config=tensorboard_output_config,
            debugger_hook_config=False,
            **additionalEstimatorArgs,
        )
        inputs = dict()
        if self.inputS3Uri:
            inputs.update(
                {
                    "data": TrainingInput(
                        self.inputS3Uri, distribution=input_distribution
                    )
                }
            )
        if additional_inputs:
            inputs.update(additional_inputs)

        estimator.fit(inputs=inputs if inputs else None, job_name=job_name)
        # training_job_description = estimator.latest_training_job.describe()
        # logging.info(f"Job is done: {training_job_description}")
        training_job_description = self.smSession.describe_training_job(job_name)

        self.estimators.append(estimator)
        self.jobNames.append(job_name)
        self.descriptions.append(training_job_description)
        if "Completed" != training_job_description["TrainingJobStatus"]:
            logger.error(
                f"Task failed with status: {training_job_description['TrainingJobStatus']}"
            )
        return job_name

    def _getJobByName(self, name_contains):
        funcs_type = (
            (
                self.sm_client.list_training_jobs,
                constants.TASK_TYPE_TRAINING,
                "TrainingJobSummaries",
            ),
            (
                self.sm_client.list_processing_jobs,
                constants.TASK_TYPE_PROCESSING,
                "ProcessingJobSummaries",
            ),
        )
        for (list_func, task_type, results_field_name) in funcs_type:
            extra_args = {}
            while True:
                resp = list_func(
                    NameContains=name_contains, MaxResults=100, **extra_args
                )

                if resp.get(results_field_name, None):
                    assert (
                        len(resp[results_field_name]) == 1
                    ), f"More than one options for {name_contains}"
                    return resp[results_field_name][0], task_type

                if "NextToken" in resp:
                    extra_args["NextToken"] = resp["NextToken"]
                else:
                    break
        return None, None

    @staticmethod
    def getLastTrainingJob(sm_client, project_name, task_name):
        # look for training jobs
        search_res = sm_client.search(
            Resource="TrainingJob",
            SearchExpression={
                "Filters": [
                    {
                        "Name": "Tags.SimpleSagemakerTask",
                        "Operator": "Equals",
                        "Value": task_name,
                    },
                    {
                        "Name": "Tags.SimpleSagemakerProject",
                        "Operator": "Equals",
                        "Value": project_name,
                    },
                ]
            },
            SortBy="CreationTime",
            SortOrder="Descending",
            MaxResults=1,
        )
        if search_res["Results"]:
            training_job = search_res["Results"][0]["TrainingJob"]
            status = training_job["TrainingJobStatus"]
            name = training_job["TrainingJobName"]
            time = training_job["CreationTime"]
            return name, status, time
        return None, None, None

    @staticmethod
    def getLastProcessingJob(boto3_session, sm_client, project_name, task_name):
        # look for processing jobs
        extra_args = {}
        arn_tags = {}
        rt_client = boto3_session.client("resourcegroupstaggingapi")
        while True:
            resp = rt_client.get_resources(
                ResourcesPerPage=100,
                TagFilters=[
                    {
                        "Key": "SimpleSagemakerProject",
                        "Values": [project_name],
                    },
                    {
                        "Key": "SimpleSagemakerTask",
                        "Values": [task_name],
                    },
                ],
                ResourceTypeFilters=["sagemaker:processing-job"],
                **extra_args,
            )

            for res_tags in resp["ResourceTagMappingList"]:
                tags = {x["Key"]: x["Value"] for x in res_tags["Tags"]}
                arn_tags[res_tags["ResourceARN"]] = tags

            if resp.get("PaginationToken", None):
                extra_args["PaginationToken"] = resp["PaginationToken"]
            else:
                break

        extra_args = {}
        while True:
            resp = sm_client.list_processing_jobs(
                NameContains=task_name,
                MaxResults=100,
                SortBy="CreationTime",
                SortOrder="Descending",
                **extra_args,
            )

            if resp.get("ProcessingJobSummaries", None):
                for job_summary in resp["ProcessingJobSummaries"]:
                    tags = arn_tags.get(job_summary["ProcessingJobArn"])
                    if (
                        tags
                        and tags.get("SimpleSagemakerProject", None) == project_name
                        and tags.get("SimpleSagemakerTask", None) == task_name
                    ):
                        return (
                            job_summary["ProcessingJobName"],
                            job_summary["ProcessingJobStatus"],
                            job_summary["CreationTime"],
                        )

            if "NextToken" in resp:
                extra_args["NextToken"] = resp["NextToken"]
            else:
                break
        return None, None, None

    @staticmethod
    def getLastJob(boto3_session, project_name, task_name, task_type=None):
        # Look for training job first and return it if it's there
        from botocore.config import Config

        config = Config(retries={"max_attempts": 10, "mode": "standard"})
        client = boto3_session.client("sagemaker", config=config)

        name, status, job_type = None, None, None

        if not task_type or task_type == constants.TASK_TYPE_TRAINING:
            name, status, time = SageMakerTask.getLastTrainingJob(
                client, project_name, task_name
            )
            if name:
                job_type = constants.TASK_TYPE_TRAINING

        if not task_type or task_type == constants.TASK_TYPE_PROCESSING:
            name2, status2, time2 = SageMakerTask.getLastProcessingJob(
                boto3_session, client, project_name, task_name
            )
            if name2:
                if not name or time2 > time:
                    name, status, time = name2, status2, time2
                    job_type = constants.TASK_TYPE_PROCESSING

        return name, job_type, status

    def bindToLastJob(self, job_name, task_type):
        self.task_type = task_type
        self.jobNames.append(job_name)

    def clean_state(self):
        uri = self.getOutputTargetUri(state=True)
        bucket, prefix = sagemaker.s3.parse_s3_url(uri)
        s3c = self.boto3_session.client("s3")
        for file in self.smSession.list_s3_files(bucket, prefix):
            s3c.delete_object(Bucket=bucket, Key=file)

    def uploadOrSetInputData(self, input_data_path):
        """
        Use a local/s3 path as input data, uploads/sync to Task's input path if local path is given

        Arguments:
            input_data_path - local/s3 path of input data
        """
        if input_data_path.lower().startswith("s3://"):
            logger.info(f"Setting input data to {input_data_path}...")
            self.inputS3Uri = input_data_path
        else:
            # uploadedUri = sagemaker_session.upload_data(path='data', bucket=bucket, key_prefix=prefix)
            sync = S3Sync(self.boto3_session)
            self.inputS3Uri = sagemaker.s3.s3_path_join(self.baseTaskS3Uri, "input")
            logger.info(f"Syncing data from {input_data_path} to {self.inputS3Uri}...")
            sync.syncFolderToS3(
                input_data_path,
                self.bucket_name,
                sagemaker.s3.parse_s3_url(self.inputS3Uri)[1],
            )

    def _downloadData(self, path, uri, extra_args):
        bucket, prefix = sagemaker.s3.parse_s3_url(uri)
        try:
            if self.smSession.list_s3_files(bucket, prefix):
                self.smSession.download_data(path, bucket, prefix, extra_args)
        except:  # noqa: E722
            logger.info(f"Couldn't download from {uri}", exc_info=True)

    def getOutputUri(self, job_name=None):
        """
        Get the output URI for a given / the last job
        """
        if job_name is None:
            job_name = self.jobNames[-1]
        return sagemaker.s3.s3_path_join(self.baseTaskS3Uri, job_name)

    def _extractTars(self, path):
        if not os.path.isdir(path):
            return
        for file_name in os.listdir(path):
            if file_name.endswith(".tar.gz"):
                tarFileName = os.path.join(path, file_name)
                if os.path.isfile(tarFileName):
                    tar = tarfile.open(tarFileName)
                    tar.extractall(path)
                    os.remove(tarFileName)

    def getOutputTargetUri(self, model=False, output=False, state=False, source=False):
        assert (
            model + output + state + source == 1
        ), "Only one output type flag should be set"
        uri = None
        # Bug? in SageMaker local mode - model and output are saved directly to output dir, not in a sub folder
        output_dir = "" if self.local_mode else "output"
        if model:
            uri = sagemaker.s3.s3_path_join(
                self.getOutputUri(), output_dir, "model.tar.gz"
            )
        elif output:
            additional_path_param = ""
            if self.task_type == constants.TASK_TYPE_TRAINING:
                additional_path_param = "output.tar.gz"
            uri = sagemaker.s3.s3_path_join(
                self.getOutputUri(), output_dir, additional_path_param
            )
        elif state:
            uri = self.stateS3Uri
        elif source:
            uri = sagemaker.s3.s3_path_join(
                self.getOutputUri(), "source", "source_dir.tar.gz"
            )
        return uri

    def getInputConfig(
        self, output_type, distribution="FullyReplicated", subdir="", return_s3uri=False
    ):
        uri = self.getOutputTargetUri(**{output_type: True})
        if subdir:
            uri = sagemaker.s3.s3_path_join(uri, subdir)
        if return_s3uri:
            return uri
        return TrainingInput(uri, distribution=distribution)

    def downloadResults(
        self,
        output_base,
        logs=True,
        state=True,
        model=True,
        output=True,
        source=True,
        extractTars=True,
        extra_args=None,
    ):
        logger.info(f"Downloading results to {output_base}")
        os.makedirs(output_base, exist_ok=True)

        if logs:
            # get and save the logs
            logs = self.getLogs()
            logsPath = os.path.join(output_base, "logs")
            os.makedirs(logsPath, exist_ok=True)
            for channel_name in logs.keys():
                ff = open(os.path.join(logsPath, f"logs{channel_name}"), "wt")
                for line in logs[channel_name]:
                    ff.write(line)
                    ff.write("\n")
                ff.close()

        # download and extract state, output, model, source

        for (shouldDownload, argName) in zip(
            [state, model, output, source], ["state", "model", "output", "source"]
        ):
            if shouldDownload:
                output_path = os.path.join(output_base, argName)
                uri = self.getOutputTargetUri(**{argName: True})
                logger.debug(f"Downloading {argName} from {uri} to {output_path}")
                self._downloadData(output_path, uri, extra_args)
                if extractTars:
                    if uri.endswith(".tar.gz"):
                        self._extractTars(output_path)

    # Based on logs_for_job in https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/session.py
    def getLogs(self, job_name=None):
        """
        Get the logs for a given / the last job
        """
        if job_name is None:
            job_name = self.jobNames[-1]
        wait = False

        class logsWrapper(object):
            def __init__(self):
                self.logsChannels = dict()

            def __call__(self, index, s):
                self.logsChannels.setdefault(index, list()).append(s)

            def getChannels(self):
                return self.logsChannels.keys()

            def getLogsForChannel(self, channel):
                return self.logsChannels[channel]

        if self.task_type == constants.TASK_TYPE_TRAINING:
            description = self.smSession.describe_training_job(job_name)
        elif self.task_type == constants.TASK_TYPE_PROCESSING:
            description = self.smSession.describe_processing_job(job_name)
        (
            instance_count,
            stream_names,
            positions,
            client,
            log_group,
            dot,
            color_wrap,
        ) = sagemaker.session._logs_init(
            self.smSession, description, job=self.task_type
        )
        lw = logsWrapper()  # raplace with our own class
        state = sagemaker.session._get_initial_job_state(  # noqa: F841
            description, f"{self.task_type}JobStatus", wait
        )
        sagemaker.session._flush_log_streams(
            stream_names,
            instance_count,
            client,
            log_group,
            job_name,
            positions,
            dot,
            lw,
        )
        return lw.logsChannels

def main():
    import boto3

    s = boto3.session.Session()
    job_name, task_type, status = SageMakerTask.getLastJob(
        s, "tests/ssm-example-processing_2020-10-09-09-47-57_py37", "cli-bash"
    )

    print("...")

if __name__ == "__main__":
    main()

# Relevant documnets:
# https://docs.aws.amazon.com/sagemaker/latest/dg/processing-container-run-scripts.html
# Docker SDK - https://docker-py.readthedocs.io/en/stable/images.html
# Amazon SageMaker ML Instance Types - https://aws.amazon.com/sagemaker/pricing/instance-types/
# Pricing, availability per processing type and region - https://aws.amazon.com/sagemaker/pricing/
# Instance Advisor - https://aws.amazon.com/ec2/spot/instance-advisor/

reason: import sage,from sage


text:
import numpy as np

from sample_model.constants import MAX_TOKENS, CHARS

def clean_data_onehot(sentence, max_tokens=MAX_TOKENS, sup_chars=CHARS):
    sentence = str(sentence).lower()
    x = np.zeros((max_tokens, len(sup_chars)), dtype=np.float32)
    for i, c in enumerate(sentence):
        if i >= max_tokens:
            break
        try:
            x[i][sup_chars.index(c)] = 1
        except ValueError:
            pass
    return x

def clean_data_encoded(sentence, max_tokens=MAX_TOKENS, sup_chars=CHARS):
    sentence = str(sentence).lower()
    x = np.zeros((max_tokens, ), dtype=np.float32)
    for i, c in enumerate(sentence):
        if i >= max_tokens:
            break
        try:
            x[i] = sup_chars.index(c)
        except ValueError:
            pass
    return x

def y_onehot(y_value):
    y = np.zeros((5), dtype=np.int8)
    y[y_value - 1] = 1
    return y


text:
#!/usr/bin/env python
# coding: utf-8

# # Factors of Flexibility
#
# Investigate what factors influence whether a word is flexible or not.

# In[1]:

import sys
sys.path.append('../')

import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import pandas as pd

import src.corpus
import src.partial

get_ipython().run_line_magic('matplotlib', 'inline')
get_ipython().run_line_magic('load_ext', 'autoreload')
get_ipython().run_line_magic('autoreload', '2')

# ## Parse the corpus

# In[2]:

data_file = "../data/wiki/processed/en.pkl"

corpus = src.corpus.POSCorpus.create_from_pickle(data_file_path=data_file)

# In[3]:

lemma_count_df = corpus.get_per_lemma_stats()
lemma_count_df = lemma_count_df[lemma_count_df.total_count >= 100]
lemma_count_df.sort_values('total_count', ascending=False).head(20)

# ## LR to predict flexibility

# In[4]:

lemma_count_df['log_freq'] = np.log(lemma_count_df.total_count)
lemma_count_df['length'] = lemma_count_df.lemma.apply(lambda x: len(x))

# In[5]:

import statsmodels.discrete.discrete_model
model = statsmodels.discrete.discrete_model.Logit(
  lemma_count_df.is_flexible,
  pd.get_dummies(lemma_count_df[['majority_tag', 'log_freq', 'length']], drop_first=True)
)
lr = model.fit()

# In[6]:

lr.summary()

# In[7]:

lr.params.tolist()

# ## Partial correlation

# In[8]:

partials = src.partial.calculate_partial_correlation(pd.get_dummies(lemma_count_df[['majority_tag', 'log_freq', 'length', 'is_flexible']], drop_first=True))
print(partials['is_flexible']['log_freq'])
print(partials['is_flexible']['length'])
print(partials['is_flexible']['majority_tag_VERB'])
partials

max_stars_repo_name: SPOClab-ca/word-class-flexibility

max_issues_repo_name: SPOClab-ca/word-class-flexibility

max_forks_repo_name: SPOClab-ca/word-class-flexibility

reason: import numpy,import statsmodels


text:
import numpy as np

def update_iou_stat(predict, gt, TP, P, T, num_classes = 21):
    """
    :param predict: the pred of each batch,  should be numpy array, after take the argmax   b,h,w
    :param gt: the gt label of the batch, should be numpy array     b,h,w
    :param TP: True positive
    :param P: positive prediction
    :param T: True seg
    :param num_classes: number of classes in the dataset
    :return: TP, P, T
    """
    cal = gt < 255

    mask = (predict == gt) * cal

    for i in range(num_classes):
        P[i] += np.sum((predict == i) * cal)
        T[i] += np.sum((gt == i) * cal)
        TP[i] += np.sum((gt == i) * mask)

    return TP, P, T

def iter_iou_stat(predict, gt, num_classes = 21):
    """
    :param predict: the pred of each batch,  should be numpy array, after take the argmax   b,h,w
    :param gt: the gt label of the batch, should be numpy array     b,h,w
    :param TP: True positive
    :param P: positive prediction
    :param T: True seg
    :param num_classes: number of classes in the dataset
    :return: TP, P, T
    """
    cal = gt < 255

    mask = (predict == gt) * cal

    TP = np.zeros(num_classes)
    P = np.zeros(num_classes)
    T = np.zeros(num_classes)

    for i in range(num_classes):
        P[i] = np.sum((predict == i) * cal)
        T[i] = np.sum((gt == i) * cal)
        TP[i] = np.sum((gt == i) * mask)

    return np.array([TP, P, T])

def compute_iou(TP, P, T, num_classes = 21):
    """
    :param TP:
    :param P:
    :param T:
    :param num_classes: number of classes in the dataset
    :return: IoU
    """
    IoU = []
    for i in range(num_classes):
        IoU.append(TP[i] / (T[i] + P[i] - TP[i] + 1e-10))
    return IoU

def update_fraction_batchwise(mask, gt, fraction, num_classes = 21):
    """
    :param mask: True when belong to subgroup (memorized, correct, others) which we want to calculate fraction on
    :param gt: the gt label of the batch, numpy array
    :param fraction: fraction of pixels in the subgroup
    :param num_classes: number of classes in the dataset
    :return: updated fraction
    """
    cal = gt < 255

    for i in range(num_classes):
        fraction[i] += np.sum((mask * (gt == i) * cal))/np.sum((gt == i) * cal)

    return fraction

def update_fraction_instancewise(mask, gt, fraction, num_classes = 21):
    """
    :param mask: True when belong to subgroup (memorized, correct, others) which we want to calculate fraction on
    :param gt: the gt label of the batch, numpy array
    :param fraction: fraction of pixels in the subgroup
    :param num_classes: number of classes in the dataset
    :return: updated fraction
    """
    # np.sum((gt == i) * cal maybe a nan value, can't do that
    cal = gt < 255

    for i in range(num_classes):
        fraction[i] += np.mean(np.sum((mask * (gt == i) * cal), axis= (-2,-1))/np.sum((gt == i) * cal, axis= (-2,-1)))

    return fraction

def update_fraction_pixelwise(mask, gt, abs_num_and_total, num_classes = 21):
    """
    :param mask: True when belong to subgroup (memorized, correct, others) which we want to calculate fraction on
    :param gt: the gt label of the batch, numpy array
    :param abs_num_and_total: the absolute number of pixel belong to the mask and the total num of pixels [abs_num, pixel_num]
    :param num_classes: number of classes in the dataset
    :return: updated fraction
    """
    cal = gt < 255

    for i in range(num_classes):
        abs_num_and_total[i][0] += np.sum(mask * (gt == i) * cal)
        abs_num_and_total[i][1] += np.sum((gt == i) * cal)

    return abs_num_and_total

def iter_fraction_pixelwise(mask, gt, num_classes = 21):
    """
    :param mask: True when belong to subgroup (memorized, correct, others) which we want to calculate fraction on
    :param gt: the gt label of the batch, numpy array
    :param num_classes: number of classes in the dataset
    :return: updated fraction
    """
    cal = gt < 255

    abs_num_and_total = np.zeros((num_classes,2))

    for i in range(num_classes):
        abs_num_and_total[i][0] += np.sum(mask * (gt == i) * cal)
        abs_num_and_total[i][1] += np.sum((gt == i) * cal)

    return abs_num_and_total

def get_mask(gt_np, label_np, pred_np):
    """

    Args:
        gt_np: the GT label
        label_np: the CAM pseudo label
        pred_np: the prediction

    Returns: the mask of different type

    """
    wrong_mask_correct = (gt_np != label_np) & (pred_np == gt_np)
    wrong_mask_memorized = (gt_np != label_np) & (pred_np == label_np)
    wrong_mask_others = (gt_np != label_np) & (pred_np != gt_np) & (pred_np != label_np)
    clean_mask_correct = (gt_np == label_np) & (pred_np == gt_np)
    clean_mask_incorrect = (gt_np == label_np) & (pred_np != gt_np)

    return (wrong_mask_correct,wrong_mask_memorized,wrong_mask_others,clean_mask_correct,clean_mask_incorrect)

max_stars_repo_name: Kangningthu/ADELE

max_issues_repo_name: Kangningthu/ADELE

max_forks_repo_name: Kangningthu/ADELE


text:
import numpy as np

class Player:
    """ Play randomly """

    def __init__(self, train_mode):
        """
        :param train_mode: bool
        """
        self.train_mode = train_mode

    def start(self, state, valid_actions):
        """
        :param state: np.array
        :param valid_actions: np.array 1D
        :returns: float
        """
        return np.random.choice(valid_actions)

    def step(self, state, valid_actions, prev_reward):
        """
        :param state: np.array
        :param valid_actions: np.array 1D
        :param prev_reward: float
        :returns: float
        """
        return np.random.choice(valid_actions)

    def end(self, state, prev_reward):
        """
        :param state: np.array
        :param prev_reward: float
        """
        pass

max_stars_repo_name: school-of-ai-angers/rl-arena

max_issues_repo_name: school-of-ai-angers/rl-arena

max_forks_repo_name: school-of-ai-angers/rl-arena


text:
import copy
from collections import OrderedDict
from datetime import datetime, timedelta

from circum.utils.state.tracking import ObjectTracker, TrackedObject

import mock

import numpy as np

def test_TrackedObject_creation():
    time_thresh = timedelta(milliseconds=100)
    to = TrackedObject(np.array([0, 1, 2]))
    dt = datetime.now()
    assert np.array_equal(to.pos, np.array([0, 1, 2]))
    assert dt - to.last_seen < time_thresh
    assert to.id is None
    assert to.tracking_ctx is None
    assert dt - to.created < time_thresh
    assert to.history == []

def test_TrackedObject_equal():
    to1 = TrackedObject(np.array([0, 1, 2]))
    to2 = TrackedObject(np.array([0, 1, 2]))

    to1.id = 1
    to2.id = 1

    assert to1 == to2

def test_TrackedObject_notequal_different_ids():
    to1 = TrackedObject(np.array([0, 1, 2]))
    to2 = TrackedObject(np.array([0, 1, 2]))

    to1.id = 1
    to2.id = 2

    assert to1 != to2

def test_TrackedObject_notequal_different_types():
    to1 = TrackedObject(np.array([0, 1, 2]))
    to2 = object()

    assert to1 != to2

def test_ObjectTracker_creation():
    ot = ObjectTracker(5)
    assert isinstance(ot._objects, OrderedDict)
    assert len(ot._objects) == 0
    assert ot._next == 0
    assert ot._deletion_threshold == 5

def test_ObjectTracker_update():
    ot = ObjectTracker(5)
    tos = [TrackedObject(np.array([0, 1, 2]))]
    ot.update(tos)
    assert len(ot._objects) == 1
    assert ot._objects[0].id == 0

def test_ObjectTracker_update_multiple():
    ot = ObjectTracker(5)
    tos = [TrackedObject(np.array([0, 1, 2])), TrackedObject(np.array([0, 1, 2]))]
    ot.update(tos)
    assert len(ot._objects) == 2
    assert ot._objects[0].id == 0
    assert ot._objects[1].id == 1

def test_ObjectTracker_prune_too_old():
    dt = datetime.now()
    ot = ObjectTracker(5)
    tos = [TrackedObject(np.array([0, 1, 2])), TrackedObject(np.array([0, 1, 2]))]
    ot.update(tos)

    assert len(ot._objects) == 2

    ot._objects[1].created = dt - timedelta(seconds=100)
    ot._objects[1].last_seen = dt - timedelta(seconds=6)

    ot.update([])

    assert len(ot._objects) == 1

@mock.patch("circum.utils.state.tracking.ObjectTracker._now")
@mock.patch("circum.utils.state.tracking.TrackedObject._now")
def test_ObjectTracker_prune_too_invisible(object_now, tracker_now):
    now = datetime.now()

    object_now.side_effect = [
        now,  # create
        now,  # create
        now,  # create
        now,  # create
    ]

    tracker_now.side_effect = [
        # update 1
        now,  # _prune
        # update 2
        now + timedelta(seconds=10),  # _prune
    ]

    ot = ObjectTracker(5)
    tos = [TrackedObject(np.array([0, 1, 2])), TrackedObject(np.array([0, 1, 2]))]

    ot.update(copy.deepcopy(tos))
    assert len(ot._objects) == 2

    ot._objects[0].last_seen = now + timedelta(seconds=9)
    ot._objects[1].last_seen = now + timedelta(seconds=3.9)

    ot.update([])
    assert len(ot._objects) == 1

max_stars_repo_name: LumineerLabs/circum

max_issues_repo_name: LumineerLabs/circum

max_forks_repo_name: LumineerLabs/circum


text:
from flask_restful import Resource
from flask import stream_with_context, request, Response
import json
from processor import *
from ImageExporter import *
from Connections import MongoDB
from bson.objectid import ObjectId
import pymongo
from six.moves import cPickle as pickle
from numpy import *

def getJSON(request):
	return json.loads(json.dumps(request.get_json(force = True)))

class imageCollector(Resource):
	def get(self):
		def generate():
			db = MongoDB()
			cursor = db.cursor.imageLibrary.find({'label': {'$ne': '0,0,0,0,0,0,0,0,0,0,0,0'}})
			while cursor.alive:
				for imagePackage in cursor:
					img, label = imagePackage['img'], imagePackage['label']
					exporter = ImageExporter(image = img, label = label)
					images, labels = exporter.downloadImagesFromServer()
					for image, label in zip(images, labels):
						yield pickle.dumps((image, label), pickle.HIGHEST_PROTOCOL)
		return Response(stream_with_context(generate()), mimetype = 'application/pickle', headers = {'Content-Disposition': 'attachment; filename=data.pickle', 'Content-type': 'application/pickle'})

	def post(self):
		content = getJSON(request)
		processor = Processor()
		try:
			id = processor.processImgAtServer(image = content['img'], label = content['label'], mode = content['mode'], id = content['id'])
		except:
			return {'Result': 'Failed processing image'}
		return {'Result': str(id)}

	def put(self):
		content = getJSON(request)
		print(content)
		db = MongoDB()
		db.cursor.imageLibrary.remove({'_id': ObjectId(content['_id'])})
		return {'Result': 'Removed'}

max_stars_repo_name: YaxinCheng/ImageProcessor

max_issues_repo_name: YaxinCheng/ImageProcessor

max_forks_repo_name: YaxinCheng/ImageProcessor

reason: from numpy


text:
# To add a new cell, type '# '
# To add a new markdown cell, type '#  [markdown]'
#
#from IPython import get_ipython

#  [markdown]
# # This notebook shows how to fit a resonator with Kerr nonlinearity in the shunt configuration.

#
#from __future__ import division, absolute_import, print_function
#get_ipython().run_line_magic('matplotlib', 'inline')

import matplotlib.pyplot as plt
import numpy as np

from resonator import background, see, shunt

#  [markdown]
# Create data using the model plus gaussian noise. The model is the product of the nonlinear shunt resonator model and a background model, which in this case assumes that the magnitude and phase are independent of frequency.
#
# When the input photon rate exceeds a critical value, the photon number in the resonator has two stable states (and one unstable state that cannot be observed) that correpond to the roots of a cubic polynomial. A typical VNA sweep with increasing frequency is expected to cause the resonator to track the branch corresponding to lower photon number. When fitting, the model for the scattering data must select one of these branches. The `choose` function below is used to choose one of the roots.

#
# Background parameters
background_model = background.MagnitudePhase()
magnitude = 0.01
phase = -np.pi / 2

# Select the root corresponding to max detuning, not minimum photon number -- these may differ.
choose = np.max

# Foreground parameters
foreground_model = shunt.KerrShunt(choose=choose)
f_r = 1e9  # resonance frequency
df = 2e6  # frequency width
f = np.linspace(f_r - df / 2, f_r + df / 2, 2001)
internal_loss = 1e-5  # Q_i = 100 000
coupling_loss = 5e-5  # Q_c = 20 000
asymmetry = 0
# This value of the parameter kerr_input implies a negative Kerr coefficient and a value of the input photon rate that is well above the point
# where the resonator has two stable states with different photon number.
critical_kerr_input = foreground_model.absolute_kerr_input_at_bifurcation(
    coupling_loss=coupling_loss, internal_loss=internal_loss)
kerr_input = -1.5 * critical_kerr_input

# Create the data by evaluating the model functions with the given parameters
bg = background_model.func(frequency=f, magnitude=magnitude, phase=phase)
fg = foreground_model.func(frequency=f, resonance_frequency=f_r, coupling_loss=coupling_loss, internal_loss=internal_loss, asymmetry=0,
                           kerr_input=kerr_input)

# The ideal data is the product of the background and foreground.
s21 = bg * fg

# The measured data will include noise
noise = 0.01 * magnitude
s21n = s21 + noise * (np.random.randn(s21.size) +
                      1j * np.random.randn(s21.size))

#
fig, ax = plt.subplots()
ax.set_aspect('equal')
ax.set_xlabel('Re $S_{21}$')
ax.set_ylabel('Im $S_{21}$')
ax.plot(fg.real, fg.imag, '.')
ax.plot([0, 1], [0, 0], marker='x', linestyle='none', color='gray')
ax.set_title('foreground without noise')

#
fig, ax = plt.subplots()
ax.set_aspect('equal')
ax.set_xlabel('Re $S_{21}$')
ax.set_ylabel('Im $S_{21}$')
ax.plot(s21n.real, s21n.imag, '.', color='gray', label='with noise')
ax.plot(s21.real, s21.imag, '.', label='ideal')
ax.legend(fontsize='xx-small')
ax.set_title('foreground times background')

#
fig, ax = plt.subplots()
ax.set_xlabel('frequency / GHz')
ax.set_ylabel('magnitude / dB')
ax.plot(1e-9 * f, 20 * np.log10(np.abs(s21n)),
        '.', color='gray', label='with noise')
ax.plot(1e-9 * f, 20 * np.log10(np.abs(s21)), '.', label='ideal')
ax.legend(fontsize='xx-small')
ax.set_title('foreground times background')

#
fig, ax = plt.subplots()
ax.set_xlabel('frequency / GHz')
ax.set_ylabel('phase / radians')
ax.plot(1e-9 * f, np.angle(s21n), '.', color='gray', label='with noise')
ax.plot(1e-9 * f, np.angle(s21), '.', label='ideal')
ax.legend(fontsize='xx-small')
ax.set_title('foreground times background')

#  [markdown]
# ## Fit the model to the data using a KerrShuntFitter.
#
# The `choose` function provided by the user must select the correct root for the fit to succeed.

#
resonator = shunt.KerrShuntFitter(frequency=f, data=s21n, choose=choose)
# result is a lmfit.model.ModelResult object
print(resonator.result.fit_report())

#  [markdown]
# ## Plot the data
#
# Use the `see.py` fitting module to plot the data, model, and resonance.

#
fig, (ax_raw, ax_norm) = plt.subplots(1, 2, figsize=(6, 3),
                                      dpi=300)  # Adjust these values for your plot settings
ax_raw.set_title('measurement plane')
ax_norm.set_title('resonator plane')
see.real_and_imaginary(resonator=resonator, axes=ax_raw, normalize=False)
see.real_and_imaginary(resonator=resonator, axes=ax_norm, normalize=True)
ax_raw.legend(handles=ax_raw.lines, labels=(
    'data', 'fit', 'resonance'), fontsize='xx-small')

#
# Adjust these values for your plot settings
fig, axes = plt.subplots(2, 2, sharex='all', figsize=(6, 6), dpi=300)
ax_raw_mag, ax_norm_mag, ax_raw_phase, ax_norm_phase = axes.flatten()
ax_raw_mag.set_title('measurement plane')
ax_norm_mag.set_title('resonator plane')
ax_raw_phase.set_title('measurement plane')
ax_norm_phase.set_title('resonator plane')
see.magnitude_vs_frequency(
    resonator=resonator, axes=ax_raw_mag, normalize=False, frequency_scale=1e-9)
see.magnitude_vs_frequency(
    resonator=resonator, axes=ax_norm_mag, normalize=True, frequency_scale=1e-9)
see.phase_vs_frequency(resonator=resonator, axes=ax_raw_phase,
                       normalize=False, frequency_scale=1e-9)
see.phase_vs_frequency(resonator=resonator, axes=ax_norm_phase,
                       normalize=True, frequency_scale=1e-9)

#  [markdown]
# The plot below shows the effect of selecting different roots in the bifurcation region. In this case, because the Kerr coefficient is negative, the Kerr detuning is also negative and the `np.max` function selects the maximum root, which corresponds to minimal photon number in the resonator.

#
fig, ax = plt.subplots()
ax.set_xlabel('frequency / Hz')
ax.set_ylabel(r'photon number $\bar{n}$')
ax.plot(f, resonator.photon_number(input_frequency=f,
                                   input_rate=resonator.f_r), label='max')
# Calculate the photon number using another root-finding function that selects the minimum Kerr detuning.
# Since the Kerr coefficient is negative here the Kerr detuning is also negative, so choosing the minimum
# (most negative) detuning corresponds to choosing the maximum photon number.
ax.plot(f, resonator.photon_number(input_frequency=f,
                                   input_rate=resonator.f_r, choose=np.min), label='min')
ax.legend()
ax.set_xlim(1e9 - 1e5, 1e9 + 1e5)

plt.show()

max_stars_repo_name: tnakaicode/PyResonator

max_issues_repo_name: tnakaicode/PyResonator

max_forks_repo_name: tnakaicode/PyResonator


text:
import numpy as np
import matplotlib.pyplot as plt
GHz = 1e9

path = r'D:\data\20190307\193346_detuned_2w_drive-2dbm_probe_power_4.66mA'

data_name = path+path[16:]+r'.dat'

data = np.loadtxt(data_name, unpack=True)

pump_freq= np.array_split(data[0],111)
freq = np.array_split(data[1],111)[0]
absol = np.array_split(data[4],111)

# plt.plot(freq, absol[0])
# plt.plot(freq, absol[-1])
plt.title(path[8:])
plt.imshow(np.rot90(absol), aspect='auto',extent=[(pump_freq[0][0]-2*7.414667*GHz)/1e6, (pump_freq[-1][0]-2*7.414667*GHz)/1e6, freq[0]/1e9, freq[-1]/1e9], cmap = 'jet')
plt.xlabel(r'Pump frequency detuned from 2$w_{c}$(MHz)')
plt.ylabel('Frequency (GHz)')
plt.colorbar()
plt.show()

max_stars_repo_path: scripts/Qubit/Analysis/two omega drive/two omega drive_dynamic_freq.py

max_stars_repo_name: sourav-majumder/qtlab

max_issues_repo_name: sourav-majumder/qtlab

max_forks_repo_name: sourav-majumder/qtlab


text:
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 31 11:50:34 2021

@author: shrav
"""

import pandas as pd
import numpy as np
import sys
import os
import json
from sys import platform as _platform
import math

if _platform == "win32":
    sys.path.insert(1, '..\..\prediction\lstm\\')
else:
    sys.path.insert(1, '../../prediction/lstm//')

# from LSTMpredict import LSTMpredict
from LSTMpredict_copy import LSTMpredict

if _platform == "win32":
    sys.path.insert(1, '..\..\\')
else:
    sys.path.insert(1, '../../')
from utils import fileutils

if _platform == "win32":
    predict_obj = LSTMpredict(model_file_path='..\..\data\model_lstm\\', ground_truth_file_path='..\..\data\ground_truth_info\\', topk=5)
    test_data_path = '..\..\data\csv\\'
    ground_truth_label_path = '..\..\data\ground_truth_info\\'
else:
    predict_obj = LSTMpredict(model_file_path='../../data/model_lstm/', ground_truth_file_path='../../data/ground_truth_info/', topk=5)
    test_data_path = '../../data/csv/'
    ground_truth_label_path = '../../data/ground_truth_info/'

df_test = pd.read_csv(fileutils.get_latest_file_with_path(test_data_path, 'lipdverse_test_*.csv'))
# df_test = pd.read_csv('D:\\annotating_paleoclimate_data\\paleorec\\data\\csv\\lipdverse_test_1234.csv')
df_test = df_test.replace(np.nan, 'NA', regex=True)
df_test = df_test.replace(',', '', regex=True)
df_test_list = df_test.values.tolist()
len_dict = predict_obj.len_dict

inverse_ref_dict = {v:k for k,v in predict_obj.reference_dict.items()}

ground_truth_label_file = fileutils.get_latest_file_with_path(ground_truth_label_path, 'ground_truth_label_*.json')
with open(ground_truth_label_file, 'r') as json_file:
    ground_truth = json.load(json_file)

ground_truth_label_dict = ground_truth['ground_truth']
accuracy_list = []
precision_list = []
recall_list = []
good_list = []
total_acc, total_pres, total_rec = 0,0,0

avg_mrr = {"1": [], "2": [], "3": [], "3_units": [], "4": [], "5": [], "6": []}
avg_rec = {"1": [], "2": [], "3": [], "3_units": [], "4": [], "5": [], "6": []}
avg_dcg = {"1": [], "2": [], "3": [], "3_units": [], "4": [], "5": [], "6": []}

# Yi = ground truth set for input
# hxi = predicted set for the input
# xi = input label for which next word is predicted
# resi = ground truth value from chain

def get_mrr_score(resi,hxi):

    if hxi[0] == 'NA' or hxi[0] == 'NotApplicable':
            hxi = hxi[:1]

    if resi == hxi[0]:
        return 1
    elif resi in hxi:
        return 1/hxi.index(resi)
    else:
        return 0

def get_dcg_score(resi, hxi):

    if hxi[0] == 'NA' or hxi[0] == 'NotApplicable':
            hxi = hxi[:1]

    if resi in hxi:
        val = 1 if resi == hxi[0] else hxi.index(resi)
        return 1/math.log((val + 1),2)
    else:
        return 0

def get_recall_score(resi, hxi):

    if hxi[0] == 'NA' or hxi[0] == 'NotApplicable':
            hxi = hxi[:1]

    if resi in hxi:
        return 1
    else:
        return 0

def calculate_score_for_test_data_chain():

    global accuracy_list, precision_list, recall_list, df_test_list
    global total_acc, total_pres, total_rec

    df = pd.DataFrame(np.nan, index = [0,1,2,3],columns=['archiveType', 'proxyObsType', 'units', 'interpretation/variable', 'interpretation/variableDetail', 'inferredVariable', 'inferredVarUnits', 'accuracy_score', 'precision_score', 'recall_score'])

    c = 0
    chain_count = 1
    for lis in df_test_list:
        lis = [val.replace(" ", "") for val in lis]
        i = 1

        while i < 7:

            ref_word = inverse_ref_dict[lis[i-1]] if lis[i-1] in inverse_ref_dict else ''
            if c < 12:
                print('ref word {} for lis item {}'.format(ref_word, lis[i-1]))
            actual_len = len(ground_truth_label_dict[ref_word]) if ref_word in ground_truth_label_dict else -1

            if i < 2:
                input_sent_list = lis[:i]
                results = predict_obj.predictForSentence(sentence=','.join(input_sent_list))['0']
                received_len = len(results)
                if not results:
                    results = ['NA']

                avg_mrr['2'].append(get_mrr_score(lis[i], results))
                avg_rec['2'].append(get_recall_score(lis[i], results))
                avg_dcg['2'].append(get_dcg_score(lis[i], results))

                values_to_add = {'archiveType': lis[i-1], 'proxyObsType': lis[i], 'Recall' : avg_rec['2'][-1], 'MRR' : avg_mrr['2'][-1], 'NDCG' : avg_dcg['2'][-1]}
                row_to_add = pd.Series(values_to_add, name = chain_count)
                df = df.append(row_to_add)
                chain_count += 1

            elif i == 2:
                input_sent_list = lis[:i]
                results_units =  predict_obj.predictForSentence(sentence=','.join(input_sent_list))['0']
                results = predict_obj.predictForSentence(sentence=','.join(input_sent_list))['1']

                received_len = len(results)

                if not results_units:
                    results_units = ['NA']
                if not results:
                    results = ['NA']

                avg_mrr['3_units'].append(get_mrr_score(lis[i], results_units))
                avg_rec['3_units'].append(get_recall_score(lis[i], results_units))
                avg_dcg['3_units'].append(get_dcg_score(lis[i], results_units))

                values_to_add = {'archiveType': lis[i-2], 'proxyObsType': lis[i-1], 'units': lis[i], 'Recall' : avg_rec['3_units'][-1], 'MRR' : avg_mrr['3_units'][-1], 'NDCG' : avg_dcg['3_units'][-1]}
                row_to_add = pd.Series(values_to_add, name = chain_count)
                df = df.append(row_to_add)
                chain_count += 1

                avg_mrr['3'].append(get_mrr_score(lis[i+1], results))
                avg_rec['3'].append(get_recall_score(lis[i+1], results))
                avg_dcg['3'].append(get_dcg_score(lis[i+1], results))

                values_to_add = {'archiveType': lis[i-2], 'proxyObsType': lis[i-1],'interpretation/variable' : lis[i+1], 'Recall' : avg_rec['2'][-1], 'MRR' : avg_mrr['2'][-1], 'NDCG' : avg_dcg['2'][-1]}
                row_to_add = pd.Series(values_to_add, name = chain_count)
                df = df.append(row_to_add)
                chain_count += 1

                i += 1
            else:
                temp = lis[:2]
                temp.extend(lis[3:i])
                input_sent_list = temp
                results = predict_obj.predictForSentence(sentence=','.join(input_sent_list))['0']
                received_len = len(results)
                if not results:
                    results = ['NA']

                avg_mrr[str(i)].append(get_mrr_score(lis[i], results))
                avg_rec[str(i)].append(get_recall_score(lis[i], results))
                avg_dcg[str(i)].append(get_dcg_score(lis[i], results))

                values_to_add = {'archiveType': lis[0], 'proxyObsType': lis[1], 'interpretation/variable': lis[3], 'Recall' : avg_rec[str(i)][-1], 'MRR' : avg_mrr[str(i)][-1], 'NDCG' : avg_dcg[str(i)][-1]}
                if i == 4:
                    values_to_add['interpretation/variableDetail'] = lis[i]
                elif i == 5:
                    values_to_add['interpretation/variableDetail'] = lis[4]
                    values_to_add['inferredVariable'] = lis[i]
                elif i == 6:
                    values_to_add['interpretation/variableDetail'] = lis[4]
                    values_to_add['inferredVariable'] = lis[5]
                    values_to_add['inferredVarUnits'] = lis[i]

                row_to_add = pd.Series(values_to_add, name = chain_count)
                df = df.append(row_to_add)
                chain_count += 1

            i += 1
            c += 1
            if c < 13:
                print('actual length = {}, received length = {}'.format(actual_len, received_len))

    print('**************************************************************************************')
    for val in ['2', '3', '3_units', '4', '5', '6']:

        print('Chain Length = {} Recall = {}'.format(val, sum(avg_rec[val])/len(avg_rec[val])))
        print('Chain Length = {} MRR = {}'.format(val, sum(avg_mrr[val])/len(avg_mrr[val])))
        print('Chain Length = {} NDCG = {}'.format(val, sum(avg_dcg[val])/len(avg_dcg[val])))
        print('**************************************************************************************')

    accuracy_data_path = os.path.join(test_data_path, 'accuracy_prediction_fang_metrics_1.csv')
    df = df.replace(np.nan, '', regex=True)
    df.to_csv(accuracy_data_path, sep = ',', encoding = 'utf-8',index = False)

def store_results_to_csv():
    '''
    Append the accuracy score for each row.
    Also append the information which signifies whether LSTM is a fit for predictions for this archiveType.
    Store this information back to a csv file.

    Returns
    -------
    None.

    '''
    global df_test

    accuracy_data_path = os.path.join(test_data_path, 'accuracy_prediction_fang_metrics.csv')
    df_test = df_test.assign(accuracy_score=pd.Series(accuracy_list).values)
    df_test = df_test.assign(precision_score=pd.Series(precision_list).values)
    df_test = df_test.assign(recall_score=pd.Series(recall_list).values)
    df_test.to_csv(accuracy_data_path, sep = ',', encoding = 'utf-8',index = False)

if __name__ == "__main__":
    calculate_score_for_test_data_chain()

max_stars_repo_name: paleopresto/recommender

max_issues_repo_name: paleopresto/recommender

max_forks_repo_name: paleopresto/recommender


text:
from sympy.core.singleton import S
from sympy.core.basic import Basic
from sympy.core.containers import Tuple
from sympy.core.function import Lambda, BadSignatureError
from sympy.core.logic import fuzzy_bool
from sympy.core.relational import Eq
from sympy.core.symbol import Dummy
from sympy.core.sympify import _sympify
from sympy.logic.boolalg import And, as_Boolean
from sympy.utilities.iterables import sift, flatten, has_dups
from sympy.utilities.exceptions import SymPyDeprecationWarning
from .contains import Contains
from .sets import Set, Union, FiniteSet

adummy = Dummy('conditionset')

class ConditionSet(Set):
    r"""
    Set of elements which satisfies a given condition.

    .. math:: \{x \mid \textrm{condition}(x) = \texttt{True}, x \in S\}

    Examples
    ========

    >>> from sympy import Symbol, S, ConditionSet, pi, Eq, sin, Interval
    >>> from sympy.abc import x, y, z

    >>> sin_sols = ConditionSet(x, Eq(sin(x), 0), Interval(0, 2*pi))
    >>> 2*pi in sin_sols
    True
    >>> pi/2 in sin_sols
    False
    >>> 3*pi in sin_sols
    False
    >>> 5 in ConditionSet(x, x**2 > 4, S.Reals)
    True

    If the value is not in the base set, the result is false:

    >>> 5 in ConditionSet(x, x**2 > 4, Interval(2, 4))
    False

    Notes
    =====

    Symbols with assumptions should be avoided or else the
    condition may evaluate without consideration of the set:

    >>> n = Symbol('n', negative=True)
    >>> cond = (n > 0); cond
    False
    >>> ConditionSet(n, cond, S.Integers)
    EmptySet

    Only free symbols can be changed by using `subs`:

    >>> c = ConditionSet(x, x < 1, {x, z})
    >>> c.subs(x, y)
    ConditionSet(x, x < 1, {y, z})

    To check if ``pi`` is in ``c`` use:

    >>> pi in c
    False

    If no base set is specified, the universal set is implied:

    >>> ConditionSet(x, x < 1).base_set
    UniversalSet

    Only symbols or symbol-like expressions can be used:

    >>> ConditionSet(x + 1, x + 1 < 1, S.Integers)
    Traceback (most recent call last):
    ...
    ValueError: non-symbol dummy not recognized in condition

    When the base set is a ConditionSet, the symbols will be
    unified if possible with preference for the outermost symbols:

    >>> ConditionSet(x, x < y, ConditionSet(z, z + y < 2, S.Integers))
    ConditionSet(x, (x < y) & (x + y < 2), Integers)

    """
    def __new__(cls, sym, condition, base_set=S.UniversalSet):
        sym = _sympify(sym)
        flat = flatten([sym])
        if has_dups(flat):
            raise BadSignatureError("Duplicate symbols detected")
        base_set = _sympify(base_set)
        if not isinstance(base_set, Set):
            raise TypeError(
                'base set should be a Set object, not %s' % base_set)
        condition = _sympify(condition)

        if isinstance(condition, FiniteSet):
            condition_orig = condition
            temp = (Eq(lhs, 0) for lhs in condition)
            condition = And(*temp)
            SymPyDeprecationWarning(
                feature="Using {} for condition".format(condition_orig),
                issue=17651,
                deprecated_since_version='1.5',
                useinstead="{} for condition".format(condition)
                ).warn()

        condition = as_Boolean(condition)

        if condition is S.true:
            return base_set

        if condition is S.false:
            return S.EmptySet

        if base_set is S.EmptySet:
            return S.EmptySet

        # no simple answers, so now check syms
        for i in flat:
            if not getattr(i, '_diff_wrt', False):
                raise ValueError('`%s` is not symbol-like' % i)

        if base_set.contains(sym) is S.false:
            raise TypeError('sym `%s` is not in base_set `%s`' % (sym, base_set))

        know = None
        if isinstance(base_set, FiniteSet):
            sifted = sift(
                base_set, lambda _: fuzzy_bool(condition.subs(sym, _)))
            if sifted[None]:
                know = FiniteSet(*sifted[True])
                base_set = FiniteSet(*sifted[None])
            else:
                return FiniteSet(*sifted[True])

        if isinstance(base_set, cls):
            s, c, b = base_set.args
            def sig(s):
                return cls(s, Eq(adummy, 0)).as_dummy().sym
            sa, sb = map(sig, (sym, s))
            if sa != sb:
                raise BadSignatureError('sym does not match sym of base set')
            reps = dict(zip(flatten([sym]), flatten([s])))
            if s == sym:
                condition = And(condition, c)
                base_set = b
            elif not c.free_symbols & sym.free_symbols:
                reps = {v: k for k, v in reps.items()}
                condition = And(condition, c.xreplace(reps))
                base_set = b
            elif not condition.free_symbols & s.free_symbols:
                sym = sym.xreplace(reps)
                condition = And(condition.xreplace(reps), c)
                base_set = b

        # flatten ConditionSet(Contains(ConditionSet())) expressions
        if isinstance(condition, Contains) and (sym == condition.args[0]):
            if isinstance(condition.args[1], Set):
                return condition.args[1].intersect(base_set)

        rv = Basic.__new__(cls, sym, condition, base_set)
        return rv if know is None else Union(know, rv)

    sym = property(lambda self: self.args[0])
    condition = property(lambda self: self.args[1])
    base_set = property(lambda self: self.args[2])

    @property
    def free_symbols(self):
        cond_syms = self.condition.free_symbols - self.sym.free_symbols
        return cond_syms | self.base_set.free_symbols

    @property
    def bound_symbols(self):
        return flatten([self.sym])

    def _contains(self, other):
        def ok_sig(a, b):
            tuples = [isinstance(i, Tuple) for i in (a, b)]
            c = tuples.count(True)
            if c == 1:
                return False
            if c == 0:
                return True
            return len(a) == len(b) and all(
                ok_sig(i, j) for i, j in zip(a, b))
        if not ok_sig(self.sym, other):
            return S.false

        # try doing base_cond first and return
        # False immediately if it is False
        base_cond = Contains(other, self.base_set)
        if base_cond is S.false:
            return S.false

        # Substitute other into condition. This could raise e.g. for
        # ConditionSet(x, 1/x >= 0, Reals).contains(0)
        lamda = Lambda((self.sym,), self.condition)
        try:
            lambda_cond = lamda(other)
        except TypeError:
            return Contains(other, self, evaluate=False)
        else:
            return And(base_cond, lambda_cond)

    def as_relational(self, other):
        f = Lambda(self.sym, self.condition)
        if isinstance(self.sym, Tuple):
            f = f(*other)
        else:
            f = f(other)
        return And(f, self.base_set.contains(other))

    def _eval_subs(self, old, new):
        sym, cond, base = self.args
        dsym = sym.subs(old, adummy)
        insym = dsym.has(adummy)
        # prioritize changing a symbol in the base
        newbase = base.subs(old, new)
        if newbase != base:
            if not insym:
                cond = cond.subs(old, new)
            return self.func(sym, cond, newbase)
        if insym:
            pass  # no change of bound symbols via subs
        elif getattr(new, '_diff_wrt', False):
            cond = cond.subs(old, new)
        else:
            pass  # let error about the symbol raise from __new__
        return self.func(sym, cond, base)

max_stars_repo_name: utkarshdeorah/sympy

max_issues_repo_name: utkarshdeorah/sympy

max_forks_repo_name: utkarshdeorah/sympy

reason: from sympy


text:
import numpy as np
import pandas as pd
import pickle
import matplotlib.pyplot as plt

import torch
from torch.utils.data import Dataset, DataLoader

from sklearn.metrics import accuracy_score, ConfusionMatrixDisplay, confusion_matrix

from .LogisticRegression import LogisticRegression
from .HeartDiseaseDataset import HeartDiseaseDataset
from .Trainer import Trainer
from .HeartDiseaseDataset import HeartDiseaseDataset
from .create_dataloaders import create_dataloaders

def save_accuracies_pkl(pkl_filepath, accuracies):
    accuracies_file = open(pkl_filepath, "wb")
    pickle.dump(accuracies, accuracies_file)
    accuracies_file.close()

def prediction_analyse_labels(columns_names, trainers, all_labels, data_path, split_proportions, batch_size=1, display_confusion=True):
    accuracies = {}

    for i, column in enumerate(all_labels):

        if not column in columns_names:
            continue

        for trainer in trainers:
            if column == trainer.label_name:
                dataset = HeartDiseaseDataset(data_path, any_disease=False, label_indexes=[23 + i, 24 + i])
                _, _, test_dataloader = create_dataloaders(dataset, split_proportions, batch_size=batch_size)

                accuracies[column] = prediction_analyse(test_dataloader, trainer, display_confusion=display_confusion)
                print("\n\n")
                break
    return accuracies

def prediction_analyse(dataloader, trainer, display_confusion=True):
    accuracy = trainer.evaluate(dataloader, display=False)
    print("accuracy", accuracy)

    all_predictions = np.array([])
    all_labels = np.array([])

    for i, (inputs, labels) in enumerate(dataloader):
        pred = trainer.model(inputs)
        pred = np.where(pred[0].detach().numpy() < 0., 0, 1)
        pred = np.argmax(pred)

        label = np.argmax(labels[0].detach().numpy())

        all_predictions = np.append(all_predictions, pred)
        all_labels = np.append(all_labels, label)

    cm = confusion_matrix(all_labels, all_predictions)

    if display_confusion:
        display = ConfusionMatrixDisplay(confusion_matrix=cm)
        display.plot()
        plt.show()

    TP = cm[1,1] # true positive
    TN = cm[0,0] # true negatives
    FP = cm[0,1] # false positives
    FN = cm[1,0] # false negatif

    sensitivity = TP / (TP + FN)
    specificity = TN / (TN + FP)
    #positive predictive value
    PPV = TP / (TP + FP)
    #negative predictive value
    NPV = TN / (TN + FN)

    PPV = TP / (TP + FP) if (TP + FP) != 0 else 'no positives values'
    NPV = TN / (TN + FN) if (TN + FN) != 0 else 'no negatives values'

    print(f'sensitivity : {sensitivity}, specificity : {specificity}, PPV : {PPV}, NPV : {NPV}')
    return accuracy

max_stars_repo_name: ninous-dev/heart-prediction

max_issues_repo_name: ninous-dev/heart-prediction

max_forks_repo_name: ninous-dev/heart-prediction


text:
import os
import glob
import sys
import numpy as np
import math
import matplotlib.pyplot as plt
import xml.dom.minidom

class point:
    def __init__(self,tag,r,proj,Eint):
        self.tag = tag
        self.r = [s for s in r]
        self.proj = proj
        self.Eint = Eint

    def dis(self,other):
        return np.sum( (np.array(self.r)-np.array(other.r))**2 )**0.5
    def __lt__(self,other):
        return NotImplemented
    def __gt__(self,other):
        return NotImplemented
    def __le__(self,other):
        return NotImplemented
    def __ge__(self,other):
        return NotImplemented

class step_plist:
    def __init__(self,plist):
        self.plist = [s for s in plist]

    def average(self):
        x_average, y_average, z_average, p_average = 0.0, 0.0, 0.0, 0.0
        for s in self.plist:
            x_average += s.r[0]
            y_average += s.r[1]
            z_average += s.r[2]
            p_average += s.proj
        x_average /= len(plist)
        y_average /= len(plist)
        z_average /= len(plist)
        p_average /= len(plist)
        return point(99,[x_average,y_average,z_average],p_average,0.0)

def compute_msd(position, shifts):
    msds = np.zeros(len(shifts))

    for i,shift in enumerate(shifts):
        diffs = position[:-shift if shift else None] - position[shift:]
        sqdlist = np.square(diffs)
        msds[i] = sqdlist.mean()
    return msds

#---------------------------------------------------------------------------------------#
#			   Get name of inputfile
#---------------------------------------------------------------------------------------#
inpu_name = raw_input('Please give the name of the input file (incluede_):')

res_in = 'res'+inpu_name

#---------------------------------------------------------------------------------------#
#				  Get time
#---------------------------------------------------------------------------------------#

tt = []    #in ns
tt.append(0.0)    #time zero

f_name = res_in + '/SIGEPS'
fcon = open(f_name)

line = fcon.readline()    #skip first line

line = fcon.readline()

while line :
    line = line.strip()
    if line == "":
        line = fcon.readline()
        continue

    else:
        line = line.split()
        tt.append(float(line[1]))
        line = fcon.readline()

'''
#---------------------------------------------------------------------------------------#
#				  Get initial position
#---------------------------------------------------------------------------------------#
fplist = []    #list to all plist for different steps

f_name = 'graph' + inpu_name + '.xml'

dom = xml.dom.minidom.parse(f_name)

root = dom.documentElement

itemlist = root.getElementsByTagName('object')

plist = []

for p_number in range(len(itemlist)):
    item = itemlist[p_number]
    plist.append(point(float(p_number),[float(item.getAttribute('cx')),float(item.getAttribute('cy')),float(item.getAttribute('cz'))],float(item.getAttribute('cz'))))

plist.append( step_plist(plist).average() )
fplist.append(step_plist(plist))
'''

#---------------------------------------------------------------------------------------#
#				  Get points
#---------------------------------------------------------------------------------------#
fplist = []    #list to all plist for different steps

f_name = res_in + '/COP*.dat'

flist = glob.glob(f_name)
flist.sort(key = str.lower)

for ff in flist:
    fcon = open(ff)
    plist = []
    line = fcon.readline()    #skip first line
    line = fcon.readline()

    while line :
        line = line.strip()
        if line == "":
            line = fcon.readline()
            continue

        else:
            line = line.split()
            plist.append(point(int(line[0]),[float(s) for s in line[2:5]],float(line[5]),1.e6*1.e-30/1.6e-19*float(line[6])))
            line = fcon.readline()

    plist.append( step_plist(plist).average() )
    fplist.append(step_plist(plist))

    if ( len(plist) != len(fplist[0].plist) ) :
        print("The number of loops is chaning in file %s " %ff )

dr = np.zeros([len(tt),len(fplist[0].plist)])
dr2 = np.zeros(dr.shape)

for step in range(len(tt)):
    for ii in range(len(fplist[0].plist)):
      dr[step,ii] = fplist[step].plist[ii].proj - fplist[0].plist[ii].proj
      dr2[step,ii] = (fplist[step].plist[ii].proj - fplist[0].plist[ii].proj)**2.0

#---------------------------------------------------------------------------------------#
#	 Drift method in: S. L. Dudarev, C. R. Phys. 9, 409 (2009).
#---------------------------------------------------------------------------------------#
dt_mean = 0.0
sum_z = np.zeros(len(fplist[0].plist))

for step in range(1,len(tt)):
    dt_mean += tt[step]-tt[step-1]
    for ii in range(len(fplist[0].plist)):
        sum_z[ii] += (tt[step]-tt[step-1])*dr[step][ii]

dt_mean /= 1.0*(len(tt)-1.0)
print dt_mean

dr_cor = np.zeros(dr.shape)
dr_cor[0,:] = dr[0,:]

for step in range(1,len(tt)):
    for ii in range(len(fplist[0].plist)):
        dr_cor[step,ii] = dr[step,ii] - 2.0*tt[step]/tt[-1]/tt[-1]*sum_z[ii]     # R_drift(t) = R(t) - 2t/tau^2 * int_0^tau {R(t')dt'}

#---------------------------------------------------------------------------------------#
#	                 Calculation of MSD cut in nt pieces
#---------------------------------------------------------------------------------------#

nt = 8.001

dshift = [10,30,50]
tau = np.array([s*dt_mean for s in dshift])

msd_cor_total = np.zeros([int(nt),len(fplist[0].plist),len(dshift)])
t_D = []
D_t = np.zeros([len(fplist[0].plist),int(nt)])

icompt = 1

for jj,kk in enumerate(tt):
    if kk >= tt[-1]/nt*1.0*icompt:
        msd_cor = np.zeros([len(fplist[0].plist),len(dshift)])

        for ii in range(len(fplist[0].plist)):  # ii th loop
            msd_cor[ii,:] = compute_msd(dr[:jj,ii],dshift)
        for ii in range(len(fplist[0].plist)):  # ii th loop
            msd_cor_total[icompt-1,ii,:] = msd_cor[ii,:]
        for ii in range(len(fplist[0].plist)):  # ii th loop
            msd_cor[ii,:] /= (2.0*tau[:])       #A^2/ns  D = MSD/(2*tau)
        D_t[:,icompt-1] = np.mean(msd_cor,axis=1)[:]
        print msd_cor
        t_D.append(kk)
        icompt += 1

'''
#---------------------------------------------------------------------------------------#
#	                 Calculation of MSD total in 1 piece
#---------------------------------------------------------------------------------------#

dshift = [5,10,20,25,50,55,75,80,90,100,150]
tau = np.array([s*dt_mean for s in dshift])

msd_t = np.zeros([len(fplist[0].plist),len(dshift)])
msd_cor = np.zeros(msd_t.shape)

for ii in range(len(fplist[0].plist)):
    msd_t[ii,:] = compute_msd(dr[:,ii],dshift)
    msd_cor[ii,:] = compute_msd(dr_cor[:,ii],dshift)
'''

#---------------------------------------------------------------------------------------#
#	                     Single D for reference at 600K
#---------------------------------------------------------------------------------------#
# for simulation case
D0 = 128.25    #A^2/ns  D = MSD/(2*tau)
# for real case
D00 = D0 *3    #A^2/ns

ref_D0 = []
ref_D00 = []

for s in tau:
    ref_D0.append(D0*2.0*s)
    ref_D00.append(D00*2.0*s)

#---------------------------------------------------------------------------------------#
#	                    Calculating velocity
#---------------------------------------------------------------------------------------#
velocity_loop1 = []   #velocity of loop1

for i in range(len(tt)-1):
    velocity_loop1.append((fplist[i+1].plist[0].proj - fplist[i].plist[0].proj)/(tt[i+1]-tt[i]))

mean_v = 0.0
for i,s in enumerate(tt):
    if s < 0.62:
        mean_v += velocity_loop1[i]*1.0

#---------------------------------------------------------------------------------------#
#	             Calculating distance between two loops
#---------------------------------------------------------------------------------------#
distance = []   #distance between two loops

for i,_ in enumerate(fplist):
    distance.append(abs(fplist[i].plist[1].proj - fplist[i].plist[0].proj))

#---------------------------------------------------------------------------------------#
#	                     Plotting......
#---------------------------------------------------------------------------------------#
plt.figure()
plt.plot(tt,distance)

plt.figure()
plt.plot(tt[:-1],velocity_loop1)
plt.xlim(0,0.62)

plt.figure()
for ii in range(len(fplist[0].plist)):
    plt.plot(tt,dr[:,ii])

'''
for s in range(int(nt)):
    plt.figure()
    for ii in range(len(fplist[0].plist)):
        plt.plot(tau,msd_cor_total[s,ii,:])
    plt.plot(tau,ref_D0,'k--')
'''
plt.figure()
for ii in range(len(fplist[0].plist)):
    plt.plot(t_D,D_t[ii,:])

plt.plot(t_D,[D0 for s in t_D],'k--')

plt.figure()
plt.plot(tt,[s.plist[0].Eint for s in fplist])

plt.figure()
plt.plot(distance,[s.plist[0].Eint for s in fplist],'o')

'''
plt.figure()
for ii in range(len(fplist[0].plist)):
    plt.plot(tau,msd_cor[ii,:])
plt.plot(tau,ref_D0,'k--')

plt.figure()
for ii in range(len(fplist[0].plist)):
    plt.plot(tau,msd_t[ii,:])
plt.plot(tau,ref_D00,'k--')
'''
#---------------------------------------------------------------------------------------#
#	              Reference E_int
#---------------------------------------------------------------------------------------#
sys.path.append('/scratch/yli_scratch/m_loopp/doule/intE/')
import commu

plt.plot(commu.out1,commu.out2)    #ref_Eint
plt.plot(commu.out3,commu.out4)    #re_orient the normal of habit plan
plt.xlim(0,300)
plt.show()

#---------------------------------------------------------------------------------------#
#	                Output
#---------------------------------------------------------------------------------------#
flag = raw_input('Output? (y or n)')
if ( flag == 'n' or flag == 'N' ):
    sys.exit(0)

Cv_out = np.zeros(len(tt)*14)
Cv_out = Cv_out.reshape(len(tt),14)

for s in range(len(tt)):
    Cv_out[s][0] = tt[s]                     #time
    for ii in range(3):
        Cv_out[s][ii+1] = fplist[s].plist[ii].proj     #Z_cordinate
    Cv_out[s][4] = t_D[s] if s<len(t_D) else None      #t_cut for calculate D
    for ii in range(3):
        Cv_out[s][ii+5] = D_t[ii,s] if s<len(t_D) else None    #coeff_D
    Cv_out[s][8] = distance[s]                       #distance between 2 loop along z
    Cv_out[s][9] = fplist[s].plist[0].Eint           #intE from simulation
    Cv_out[s][10] = commu.out1[s] if s<len(commu.out1) else None
    Cv_out[s][11] = commu.out2[s] if s<len(commu.out1) else None         #ref_intE
    Cv_out[s][12] = commu.out3[s] if s<len(commu.out3) else None
    Cv_out[s][13] = commu.out4[s] if s<len(commu.out3) else None         #re_orient the normal of habit plan

outfile_name = raw_input("Enter the file name: ")

outfile_name+= '.txt'

f = open(outfile_name,'w')
f.write("#time(ns) #Z_cordinate_loop1(A) #Z_cordinate_loop2(A) #Z_cordinate_COM(A) #t_D(ns) #D_T_loop1(A^2/ns) #D_T_loop2(A^2/ns) #D_T_COM(A^2/ns) #distance_loop(A) #intE(eV) #zz(A) #intE_ref(eV) #zz(A) #intE_normal(eV)\n")
for s in range(len(tt)):
    for i in range(14):
        f.write('%f '%(Cv_out[s][i]))
    f.write(' \n')


text:
import pandas as pd
import collections
import json
import numpy as np

from autoscalingsim.application.application_model_conf import ApplicationModelConfiguration
from autoscalingsim.infrastructure_platform.platform_model_configuration_parser import PlatformModelConfigurationParser
from autoscalingsim.infrastructure_platform.node_information.system_resource_usage import SystemResourceUsage

class DeploymentGenerator:

    @classmethod
    def generate(cls,
                 application_model_config_path : str, platform_model_config_path : str,
                 provider_name : str = 'aws', regions : list = ['eu'],
                 reqs_fraction_expected_to_serve : float = 0.7,
                 simulation_step : pd.Timedelta = pd.Timedelta(100, unit = 'ms'),
                 load_magnitude : int = 10000, load_batch_size : int = 1000):

        app_conf = ApplicationModelConfiguration(application_model_config_path)
        providers_configs = PlatformModelConfigurationParser.parse(platform_model_config_path)
        time_of_req_processing_by_service = collections.defaultdict(lambda: pd.Timedelta(0, unit ='s'))
        for rpi in app_conf.reqs_processing_infos.values():
            for service_name, proc_times_by_dir in rpi.processing_times.items():
                for proc_time in proc_times_by_dir.values():
                    time_of_req_processing_by_service[service_name] += pd.Timedelta(proc_time['mean'], unit = proc_time['unit'])

        services_reqs = { service_conf.service_name : service_conf.system_requirements for service_conf in app_conf.service_confs }

        res_req_joint = sum(rpi.resource_requirements for rpi in app_conf.reqs_processing_infos.values())

        estimated_count_of_service_instances_needed_per_second = dict()
        for service_name, proc_time in time_of_req_processing_by_service.items():
            real_time = np.ceil(proc_time / simulation_step) * simulation_step
            estimated_count_of_service_instances_needed_per_second[service_name] = int(reqs_fraction_expected_to_serve * load_magnitude * (real_time / pd.Timedelta(1, unit = 's')))

        options = collections.defaultdict(list)
        for service_name, service_inst_cnt in estimated_count_of_service_instances_needed_per_second.items():
            res_required = service_inst_cnt * services_reqs[service_name] + load_magnitude * reqs_fraction_expected_to_serve * res_req_joint
            for node_type, node_info in providers_configs[provider_name]:
                if node_info.fits_service_instance(services_reqs[service_name]):
                    options[service_name].append({'node_type': node_type, 'cnt': res_required.how_many_nodes_needed(node_info, service_inst_cnt, services_reqs[service_name])})

        depl_output = list()
        for service_name, options_list in options.items():
            selected_option = options_list[np.random.randint(len(options_list))]
            depl_output.append({'service_name': service_name,
                                'deployment': {region_name: {'init_aspects': {'count': max(estimated_count_of_service_instances_needed_per_second[service_name], 1)}, 'platform': {'provider': provider_name, 'node_type': selected_option['node_type'], 'count': max(selected_option['cnt'], 1)}} for region_name in regions}})

        print(json.dumps(depl_output))

max_stars_repo_name: Remit/autoscaling-simulator

max_issues_repo_name: Remit/autoscaling-simulator

max_forks_repo_name: Remit/autoscaling-simulator


text:
import tensorflow as tf
import numpy as np
import math
import sys
import os
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(BASE_DIR)
sys.path.append(os.path.join(BASE_DIR, '../utils'))
sys.path.append(os.path.join(BASE_DIR, '../../utils'))

ROOT_DIR = os.path.dirname(BASE_DIR)
sys.path.append(os.path.join(ROOT_DIR, 'utils'))
sys.path.append(os.path.join(ROOT_DIR, 'tf_ops/sampling'))
sys.path.append(os.path.join(ROOT_DIR, 'tf_ops/grouping'))
sys.path.append(os.path.join(ROOT_DIR, 'tf_ops/3d_interpolation'))
from tf_sampling import farthest_point_sample, gather_point
from tf_grouping import query_ball_point, group_point, knn_point
from tf_interpolate import three_nn, three_interpolate

import tf_util
import extra_loss
import pan_util

def placeholder_inputs(batch_size, num_point, normal_flag=False):
	if normal_flag:
		pointclouds_pl = tf.placeholder(tf.float32, shape=(batch_size, num_point, 6))
	else:
		pointclouds_pl = tf.placeholder(tf.float32, shape=(batch_size, num_point, 3))
	labels_pl = tf.placeholder(tf.int32, shape=(batch_size))
	return pointclouds_pl, labels_pl

def get_model(point_input, is_training, pfs_flag=False, bn_decay=None):
	""" Classification PointNet, input is BxNxC, output Bx40 """
	batch_size = point_input.get_shape()[0].value
	num_point1 = point_input.get_shape()[1].value
	num_point2 = int(np.floor(num_point1 / 4.0))
	num_point3 = int(np.floor(num_point2 / 4.0))
	# num_point4 = num_point3 / 4
	num_features = point_input.get_shape()[2].value

	end_points = {}
	k = 10
	pk = 10

	de1_1 = 1
	de1_2 = 2
	de2_1 = 1
	de2_2 = 2
	de3_1 = 1
	de3_2 = 2

	if num_features > 3:
		point_cloud1 = tf.slice(point_input, [0, 0, 0], [-1, -1, 3])
	else:
		point_cloud1 = point_input

	r1_11 = 0
	r1_12 = 0.1
	r1_21 = 0
	r1_22 = 0.2

	r2_11 = 0
	r2_12 = 0.4
	r2_21 = 0
	r2_22 = 0.8

	r3_11 = 0
	r3_12 = 1.6
	r3_21 = 0
	r3_22 = 3.2

	p1_1 = 0
	p1_2 = 0.8

	p2_1 = 0
	p2_2 = 1.6

	# activation_fn = tf.math.softplus
	activation_fn = tf.nn.relu

##################################################################################################
	# Hierarchy 1

	hie_matrix1 = tf.math.maximum(tf.sqrt(tf_util.pairwise_distance(point_cloud1)), 1e-20)
	# dist_matrix1_1 = hie_matrix1
	# dist_matrix1_2 = hie_matrix1
	dist_matrix1_1 = None
	dist_matrix1_2 = None

	adj_matrix = tf_util.pairwise_distance(point_cloud1)
	net1_1 = pan_util.point_atrous_conv(point_input, adj_matrix, dist_matrix1_1, k, de1_1, r1_11, r1_12, 64,
		scope='page1_1', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn)

	adj_matrix = tf_util.pairwise_distance(net1_1)
	net1_2 = pan_util.point_atrous_conv(net1_1, adj_matrix, dist_matrix1_2, k, de1_2, r1_21, r1_22, 64,
		scope='page1_2', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn)

	net1_1 = tf.squeeze(net1_1)
	net1_2 = tf.squeeze(net1_2)

##################################################################################################
	# Hierarchy 2

	dist_threshold1 = False
	net, p1_idx, pn_idx, point_cloud2 = pan_util.edge_preserve_sampling(net1_2, point_cloud1, num_point2, hie_matrix1,
		dist_threshold1, pk, 1, p1_1, p1_2, PFS_flag=pfs_flag, atrous_flag=False)

	net1_1 = tf.squeeze(tf.reduce_max(group_point(net1_1, pn_idx), axis=-2, keepdims=True))
	net1_2 = tf.squeeze(net)

	# point_cloud2 = gather_point(point_cloud1, p1_idx)
	hie_matrix2 = tf.math.maximum(tf.sqrt(tf_util.pairwise_distance(point_cloud2)), 1e-20)
	# dist_matrix2_1 = hie_matrix2
	# dist_matrix2_2 = hie_matrix2
	dist_matrix2_1 = None
	dist_matrix2_2 = None

	adj_matrix = tf_util.pairwise_distance(net)
	net2_1 = pan_util.point_atrous_conv(net, adj_matrix, dist_matrix2_1, k, de2_1, r2_11, r2_12, 128,
		scope='page2_1', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn)

	adj_matrix = tf_util.pairwise_distance(net2_1)
	net2_2 = pan_util.point_atrous_conv(net2_1, adj_matrix, dist_matrix2_2, k, de2_2, r2_21, r2_22, 128,
		scope='page2_2', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn)

	net2_1 = tf.squeeze(net2_1)
	net2_2 = tf.squeeze(net2_2)

##################################################################################################
	# Hierarchy 3

	dist_threshold2 = False
	net, p2_idx, pn_idx, point_cloud3 = pan_util.edge_preserve_sampling(net2_2, point_cloud2, num_point3, hie_matrix2,
		dist_threshold2, pk, 1, p2_1, p2_2, PFS_flag=pfs_flag, atrous_flag=False)

	net1_1 = tf.reduce_max(group_point(net1_1, pn_idx), axis=-2, keepdims=True)
	net1_2 = tf.reduce_max(group_point(net1_2, pn_idx), axis=-2, keepdims=True)
	net2_1 = tf.reduce_max(group_point(net2_1, pn_idx), axis=-2, keepdims=True)
	net2_2 = net

	# point_cloud3 = gather_point(point_cloud2, p2_idx)
	hie_matrix3 = tf.math.maximum(tf.sqrt(tf_util.pairwise_distance(point_cloud3)), 1e-20)
	# dist_matrix3_1 = hie_matrix3
	# dist_matrix3_2 = hie_matrix3
	dist_matrix3_1 = None
	dist_matrix3_2 = None

	adj_matrix = tf_util.pairwise_distance(net)
	net3_1 = pan_util.point_atrous_conv(net, adj_matrix, dist_matrix3_1, k, de3_1, r3_11, r3_12, 256,
		scope='page3_1', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn)

	adj_matrix = tf_util.pairwise_distance(net3_1)
	net3_2 = pan_util.point_atrous_conv(net3_1, adj_matrix, dist_matrix3_2, k, de3_2, r3_21, r3_22, 256,
		scope='page3_2', bn=True, bn_decay=bn_decay, is_training=is_training, activation_fn=activation_fn)

##################################################################################################
	# Embeded Features

	net = tf.concat([net1_1, net1_2, net2_1, net2_2, net3_1, net3_2], axis=-1)
	# net = tf.concat([net1_2, net2_2, net3_2], axis=-1)
	net = tf_util.conv2d(net, 1024, [1, 1],
					   padding='VALID', stride=[1,1], activation_fn=activation_fn,
					   bn=True, is_training=is_training,
					   scope='agg', bn_decay=bn_decay)

	net = tf.reduce_max(net, axis=1, keepdims=True)

	# MLP on global point cloud vector
	net = tf.reshape(net, [batch_size, -1])

	end_points['embedding'] = net

	net = tf_util.fully_connected(net, 512, bn=True, is_training=is_training,
								scope='fc1', bn_decay=bn_decay)
	net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp1')
	net = tf_util.fully_connected(net, 256, bn=True, is_training=is_training,
								scope='fc2', bn_decay=bn_decay)
	net = tf_util.dropout(net, keep_prob=0.5, is_training=is_training, scope='dp2')
	net = tf_util.fully_connected(net, 40, activation_fn=None, scope='fc3')

	return net, end_points

def get_loss(pred, label, end_points, mmd_flag):
	""" pred: B*NUM_CLASSES,
	  label: B, """
	labels = tf.one_hot(indices=label, depth=40)
	loss = tf.losses.softmax_cross_entropy(onehot_labels=labels, logits=pred, label_smoothing=0.2)
	classify_loss = tf.reduce_mean(loss)
	tf.summary.scalar('cls loss', classify_loss)

	loss = classify_loss

	if mmd_flag > 0:
		batch_size = end_points['embedding'].get_shape()[0].value
		feature_size = end_points['embedding'].get_shape()[1].value

		true_samples = tf.random_normal(tf.stack([batch_size, feature_size]))
		mmd_loss = extra_loss.compute_mmd(end_points['embedding'], true_samples)
		mmd_loss = mmd_loss * mmd_flag
		tf.summary.scalar('mmd loss', mmd_loss)

		loss = loss + mmd_loss

	tf.add_to_collection('losses', loss)
	return loss

if __name__=='__main__':
	batch_size = 2
	num_pt = 124
	pos_dim = 3

	input_feed = np.random.rand(batch_size, num_pt, pos_dim)
	label_feed = np.random.rand(batch_size)
	label_feed[label_feed>=0.5] = 1
	label_feed[label_feed<0.5] = 0
	label_feed = label_feed.astype(np.int32)

	# # np.save('./debug/input_feed.npy', input_feed)
	# input_feed = np.load('./debug/input_feed.npy')
	# print input_feed

	with tf.Graph().as_default():
		input_pl, label_pl = placeholder_inputs(batch_size, num_pt)
		pos, ftr = get_model(input_pl, tf.constant(True))
		# loss = get_loss(logits, label_pl, None)

	with tf.Session() as sess:
		sess.run(tf.global_variables_initializer())
		feed_dict = {input_pl: input_feed, label_pl: label_feed}
		res1, res2 = sess.run([pos, ftr], feed_dict=feed_dict)
		print (res1.shape)
		print (res1)

		print (res2.shape)
		print (res2)

max_issues_repo_name: pldeqiushui/Point-Atrous-Convolution

max_forks_repo_name: pldeqiushui/Point-Atrous-Convolution


text:
import cv2
import numpy as np

class ImageAnalyzer :

    def __init__(self):
        self.img = np.zeros([1, 1, 3])
        self.img_processed = np.zeros([1, 1, 3])
        pass

    def getImage(self, img):
        self.img = img

    def img(self):
        print('retuning img')
        return self.img_processed

    def findContours(self):

        img_gray = cv2.cvtColor(self.img,cv2.COLOR_BGR2GRAY)
        # ret, thresh = cv2.threshold(img_gray, 127, 255,0)
        thresh = cv2.adaptiveThreshold(img_gray,255,cv2.ADAPTIVE_THRESH_MEAN_C,
                                       cv2.THRESH_BINARY,51,2)
        contours,hierarchy = cv2.findContours(thresh,2,1)

        contours_large = [x for x in contours if cv2.contourArea(x)> 1000]
        print('# of contours : ', len(contours_large))
        # contours_2 = [x for x in contours_large if cv2.contourAspect]
        for j in range(len(contours_large)):
            cnt = contours_large[j]

            cv2.drawContours(self.img, cnt, -1, (0, 255, 0), 3)
            print(cv2.contourArea(cnt))
        self.img_processed = np.asarray(self.img)
        self.img_inline = thresh

    def doThresholding(self):
        pass

    def doCannyEdgeDetection(self):
        pass