text:
import re
import copy
import random
import itertools

from faker import Faker

from . import filth as filth_module
from .filth import Filth
from .detectors.tagged import KnownFilthItem

from typing import List, Dict, Union, Optional, Tuple, Callable, Iterable, Type, Set
import numpy as np
import pandas as pd
import sklearn.metrics

# I was originally thinking of building this into the Filth system, but they serve subtlly different purposes:
#   * Filths need to be merged by text location so that replacements can be made
#   * TextPostions need to be merged by text location, but seperated by type so that we an correclty count them
from .utils import ToStringMixin

Grouping = Dict[str, str]
GroupingFunction = Callable[[Filth], Grouping]

class TextPosition(ToStringMixin):
    def __init__(self, filth: Filth, grouping_function: GroupingFunction):
        self.beg = filth.beg
        self.end = filth.end
        self.detected = set()  # type: Set[Tuple[str, ...]]
        self.tagged = set()  # type: Set[Tuple[str, ...]]
        self.document_name = str(filth.document_name or '')  # type: str

        if isinstance(filth, filth_module.TaggedEvaluationFilth):
            self.tagged.add(tuple(grouping_function(filth).values()))
        else:
            self.detected.add(tuple(grouping_function(filth).values()))

    @staticmethod
    def sort_key(position: 'TextPosition') -> Tuple[str, int, int]:
        return (position.document_name, position.beg, -position.end)

    def merge(self, other: 'TextPosition') -> 'TextPosition':
        if self.document_name != other.document_name:
            raise ValueError("Positions are in different documents")
        if self.beg <= other.end and self.end > other.beg:
            self.beg = min(self.beg, other.beg)
            self.end = max(self.end, other.end)
            self.tagged = self.tagged | other.tagged
            self.detected = self.detected | other.detected
            return self
        raise ValueError(f"Positions do not overlap {self.beg} to {self.end} and {other.beg} to {other.end}")

    def __repr__(self) -> str:
        return self._to_string(['beg', 'end', 'tagged', 'detected', 'document_name', ])

    def __eq__(self, other):
        return self.__dict__ == other.__dict__

class FilthTypePositions(ToStringMixin, object):
    def __init__(self, grouping_function: GroupingFunction, filth_type: str):
        self.positions = []  # type: List[TextPosition]
        self.filth_type = filth_type
        self.grouping_function = grouping_function
        self.column_names = None  # type: Optional[List[str]]

    def __repr__(self) -> str:
        return self._to_string(['filth_type', 'positions', ])

    def __eq__(self, other):
        return self.__dict__ == other.__dict__

    def add_filth(self, filth: Filth):
        self.positions.append(TextPosition(filth, grouping_function=self.grouping_function))
        if self.column_names is None:
            self.column_names = list(self.grouping_function(filth).keys())

    @staticmethod
    def _merge_position_list(position_list: List[TextPosition]) -> List[TextPosition]:
        position_list.sort(key=TextPosition.sort_key)
        merged_positions = []  # type: List[TextPosition]

        current_position = position_list[0]
        for next_position in position_list[1:]:
            if current_position.document_name != next_position.document_name or \
                    current_position.end <= next_position.beg:
                merged_positions.append(current_position)
                current_position = next_position
            else:
                current_position = current_position.merge(next_position)
        merged_positions.append(current_position)

        return merged_positions

    def merge_positions(self):
        self.positions = self._merge_position_list(self.positions)

    def get_counts(self) -> pd.DataFrame:
        self.merge_positions()

        data_list = []  # type: List[Dict[Tuple[str, ...], int]]
        for position in self.positions:
            row = {
                detected_name: 1
                for detected_name in position.detected
            }
            row.update({
                detected_name: 1
                for detected_name in position.tagged
            })
            data_list.append(row)

        dataframe = pd.DataFrame(data_list).fillna(0).astype(int)
        dataframe.columns = pd.MultiIndex.from_tuples(
            dataframe.columns.values.tolist(),
            names=self.column_names,
        )

        return dataframe

class FilthGrouper(ToStringMixin, object):
    def __init__(self, filth_types: Optional[List[str]] = None, grouping_function: Optional[GroupingFunction] = None,
                 combine_detectors: bool = False, groupby_documents: bool = False):
        self.types = {}  # type: Dict[str, FilthTypePositions]
        self.combine_detectors = combine_detectors
        self.groupby_documents = groupby_documents
        self.filth_types = filth_types

        if grouping_function is None:
            if self.combine_detectors and self.groupby_documents:
                self.grouping_function = FilthGrouper.grouping_combined_bydoc  # type: GroupingFunction
            elif self.combine_detectors and not self.groupby_documents:
                self.grouping_function = FilthGrouper.grouping_combined
            elif not self.combine_detectors and self.groupby_documents:
                self.grouping_function = FilthGrouper.grouping_default_bydoc
            else:
                self.grouping_function = FilthGrouper.grouping_default
        else:
            self.grouping_function = grouping_function

    def __repr__(self) -> str:
        return self._to_string(['combine_detectors', 'filth_types', 'types', ])

    def __eq__(self, other):
        return self.__dict__ == other.__dict__

    @staticmethod
    def grouping_default(filth: Filth) -> Grouping:
        detector_name = (
            (filth.detector_name or 'None')
            if not isinstance(filth, filth_module.TaggedEvaluationFilth) else
            filth_module.TaggedEvaluationFilth.type
        )
        result = {
            'filth': getattr(filth, 'comparison_type', filth.type),
            'detector': detector_name,
            'locale': filth.locale or 'None',
        }
        return result

    @staticmethod
    def grouping_default_bydoc(filth: Filth) -> Grouping:
        detector_name = (
            (filth.detector_name or 'None')
            if not isinstance(filth, filth_module.TaggedEvaluationFilth) else
            filth_module.TaggedEvaluationFilth.type
        )
        result = {
            'filth': getattr(filth, 'comparison_type', filth.type),
            'document_name': filth.document_name or 'None',
            'detector': detector_name,
            'locale': filth.locale or 'None',
        }

        return result

    @staticmethod
    def grouping_combined(filth: Filth) -> Grouping:
        detector_name = (
            'combined'
            if not isinstance(filth, filth_module.TaggedEvaluationFilth) else
            filth_module.TaggedEvaluationFilth.type
        )
        return {
            'filth': getattr(filth, 'comparison_type', filth.type),
            'detector': detector_name,
            'locale': filth.locale or 'None',
        }

    @staticmethod
    def grouping_combined_bydoc(filth: Filth) -> Grouping:
        detector_name = (
            'combined'
            if not isinstance(filth, filth_module.TaggedEvaluationFilth) else
            filth_module.TaggedEvaluationFilth.type
        )
        result = {
            'filth': getattr(filth, 'comparison_type', filth.type),
            'document_name': filth.document_name or 'None',
            'detector': detector_name,
            'locale': filth.locale or 'None',
        }

        return result

    def merge_positions(self):
        for positioniser in self.types.values():
            positioniser.merge_positions()

    def add_filths(self, filth_list: List[Filth]):
        for filth_item in filth_list:
            sub_filths = [filth_item]
            if isinstance(filth_item, filth_module.base.MergedFilth):
                sub_filths = filth_item.filths

            for filth in sub_filths:
                filth_type = (
                    filth.comparison_type if isinstance(filth, filth_module.TaggedEvaluationFilth) else filth.type
                )

                if filth_type is None or (self.filth_types is not None and filth_type not in self.filth_types):
                    continue

                if filth_type not in self.types:
                    self.types[filth_type] = FilthTypePositions(
                        filth_type=filth_type,
                        grouping_function=self.grouping_function
                    )

                self.types[filth_type].add_filth(filth)

    @classmethod
    def from_filth_list(
            cls, filth_list: List[Filth], filth_types: Optional[List[str]] = None,
            combine_detectors: bool = False, groupby_documents: bool = False,
            grouping_function: Optional[GroupingFunction] = None
    ) -> 'FilthGrouper':
        grouper = cls(filth_types=filth_types, combine_detectors=combine_detectors, groupby_documents=groupby_documents,
                      grouping_function=grouping_function)
        grouper.add_filths(filth_list)
        grouper.merge_positions()
        return grouper

    def expand_missing(self, df: pd.DataFrame) -> pd.DataFrame:
        set_list = [set(s) for s in zip(*df.columns.values.tolist())]
        for column in itertools.product(*set_list):
            if column not in df.columns:
                df.loc[:, column] = 0
        return df

    def get_counts(self, expand_missing: bool = False) -> pd.DataFrame:
        if len(self.types) == 0:
            return pd.DataFrame()
        df_list = []  # type: List[pd.DataFrame]
        running_rows = 0
        for positioniser in self.types.values():
            pos_df = positioniser.get_counts()
            if expand_missing:
                pos_df = self.expand_missing(pos_df)
            df_list.append(pos_df)
            df_list[-1].index += running_rows
            running_rows += max(df_list[-1].index) + 1
        return pd.concat(df_list).fillna(0).astype(int)

def get_filth_classification_report(
        filth_list: List[Filth],
        combine_detectors: bool = False,
        groupby_documents: bool = False,
        output_dict: bool = False,
) -> Optional[Union[str, Dict[str, float]]]:
    """Evaluates the performance of detectors using KnownFilth.

    An example of using this is shown below:

    .. code:: pycon

        >>> import scrubadub, scrubadub.comparison, scrubadub.detectors.text_blob
        >>> scrubber = scrubadub.Scrubber(detector_list=[
        ...     scrubadub.detectors.text_blob.TextBlobNameDetector(name='name_detector'),
        ...     scrubadub.detectors.TaggedEvaluationFilthDetector([
        ...         {'match': 'Tom', 'filth_type': 'name'},
        ...         {'match': 'tom@example.com', 'filth_type': 'email'},
        ...     ]),
        ... ])
        >>> filth_list = list(scrubber.iter_filth("Hello I am Tom"))
        >>> print(scrubadub.comparison.get_filth_classification_report(filth_list))
        filth    detector         locale      precision    recall  f1-score   support
        <BLANKLINE>
        name     name_detector    en_US            1.00      1.00      1.00         1
        <BLANKLINE>
                                    accuracy                           1.00         1
                                   macro avg       1.00      1.00      1.00         1
                                weighted avg       1.00      1.00      1.00         1
        <BLANKLINE>

    :param filth_list: The list of detected filth
    :type filth_list: A list of `Filth` objects
    :param combine_detectors: Combine performance of all detectors for the same filth/locale
    :type combine_detectors: bool, optional
    :param groupby_documents: Show performance for each file individually
    :type groupby_documents: bool, optional
    :param output_dict: Return the report in JSON format, defautls to False
    :type output_dict: bool, optional
    :return: The report in JSON (a `dict`) or in plain text
    :rtype: `str` or `dict`
    """
    if len(filth_list) == 0:
        return None

    grouper = FilthGrouper.from_filth_list(filth_list, combine_detectors=combine_detectors,
                                           groupby_documents=groupby_documents)
    results_df = grouper.get_counts(expand_missing=True)

    filth_index = results_df.columns.names.index('filth')
    detector_index = results_df.columns.names.index('detector')
    tagged_column_mask = np.array(
        [x[detector_index] == filth_module.TaggedEvaluationFilth.type for x in results_df.columns]
    )

    # Find filth types that have some tagged filth
    tagged_types = [x[filth_index] for x in results_df.columns[tagged_column_mask]]

    # Select the columns that have some related tagged filth, but are not tagged filth themselves
    detected_columns = [
        x for x in results_df.columns[~tagged_column_mask]
        if x[filth_index] in tagged_types
    ]
    detected_classes = results_df.loc[:, detected_columns].values

    # Take the detected_columns above and find their tagged counterparts
    tagged_columns = [
        (*x[:detector_index], filth_module.TaggedEvaluationFilth.type, *x[detector_index + 1:])
        for x in detected_columns
    ]
    # If they don't have any tagged counterpart, set the column to zero
    for column in tagged_columns:
        if column not in results_df.columns:
            results_df.loc[:, column] = 0
            tagged_column_mask = np.append(tagged_column_mask, [True])

    true_classes = results_df.loc[:, tagged_columns].values

    # Then no true classes were found
    if detected_classes.shape[1] == 0:
        return None

    report_prefix = None  # type: Optional[str]
    if not output_dict:
        report_prefix = ''
        class_labels = [''] * len(detected_columns)
        for i, name in enumerate(results_df.columns.names):
            max_length = max([len(str(columns[i])) for columns in results_df.columns] + [len(name)]) + 4
            class_labels = [
                name + columns[i].ljust(max_length)
                for name, columns in zip(class_labels, detected_columns)
            ]
            report_prefix += name.ljust(max_length)
        class_labels = [
            name
            for name in class_labels
        ]
        if report_prefix is not None:
            report_prefix += '  '
    else:
        base_name = ("{}:" * len(results_df.columns.names)).rstrip(':')
        class_labels = [base_name.format(*x) for x in detected_columns]

    # If there is only one label reshape the data so that
    # the classification_report interprets it less ambiguously
    report_labels = []  # type: List[int]
    if detected_classes.shape[1] == 1:
        detected_classes = detected_classes.T[0]
        true_classes = true_classes.T[0]
        report_labels = [1]
    else:
        report_labels = [class_labels.index(x) for x in sorted(class_labels)]
        class_labels = sorted(class_labels)

    report = sklearn.metrics.classification_report(
        true_classes,
        detected_classes,
        output_dict=output_dict,
        zero_division=0,
        target_names=class_labels,
        labels=report_labels,
        # **extra_args
    )

    if report_prefix is not None:
        report = report_prefix + report.lstrip(' ')

    return report

def get_filth_dataframe(filth_list: List[Filth]) -> pd.DataFrame:
    """Produces a pandas `DataFrame` to allow debugging and improving detectors.

    An example of using this is shown below:

    .. code:: pycon

        >>> import scrubadub, scrubadub.comparison, scrubadub.detectors.text_blob
        >>> scrubber = scrubadub.Scrubber(detector_list=[
        ...     scrubadub.detectors.text_blob.TextBlobNameDetector(name='name_detector'),
        ...     scrubadub.detectors.TaggedEvaluationFilthDetector([
        ...         {'match': 'Tom', 'filth_type': 'name'},
        ...         {'match': 'tom@example.com', 'filth_type': 'email'},
        ...     ]),
        ... ])
        >>> filth_list = list(scrubber.iter_filth("Hello I am Tom"))
        >>> with pd.option_context("display.max_columns", 20):
        ...     print(scrubadub.comparison.get_filth_dataframe(filth_list))  # doctest: +NORMALIZE_WHITESPACE
           group_id  filth_id filth_type  detector_name document_name text  beg  end  \\
        0         0         0       name  name_detector          None  Tom   11   14
        <BLANKLINE>
          locale  known_filth comparison_type known_text  known_beg  known_end  \\
        0  en_US         True             NaN        Tom         11         14
        <BLANKLINE>
          known_comparison_type  exact_match  partial_match  true_positive  \\
        0                  name         True           True           True
        <BLANKLINE>
           false_positive  false_negative
        0           False           False

    :param filth_list: The list of detected filth
    :type filth_list: A list of `Filth` objects
    :return: A `pd.DataFrame` containing infomatoin about the detected `Filth`
    :rtype: `pd.DataFrame`

    """
    results = []
    for group_id, filth_item in enumerate(filth_list):
        sub_filths = [filth_item]
        if isinstance(filth_item, filth_module.base.MergedFilth):
            sub_filths = filth_item.filths
        for filth_id, sub_filth in enumerate(sub_filths):
            results.append({
                'group_id': group_id,
                'filth_id': filth_id,
                'filth_type': sub_filth.type,
                'detector_name': getattr(sub_filth, 'detector_name', float('nan')),
                'document_name': getattr(sub_filth, 'document_name', float('nan')),
                'text': sub_filth.text,
                'beg': sub_filth.beg,
                'end': sub_filth.end,
                'locale': sub_filth.locale,
                'known_filth': isinstance(sub_filth, filth_module.TaggedEvaluationFilth),
                'comparison_type': getattr(sub_filth, 'comparison_type', float('nan')),
            })

    results_df = pd.DataFrame(results, columns=['group_id', 'filth_id', 'filth_type', 'detector_name', 'document_name',
                                                'text', 'beg', 'end', 'locale', 'known_filth', 'comparison_type'])
    suffix_label = '_y_suffix'

    return (
        pd.merge(
            results_df.loc[~results_df['known_filth']],
            results_df.loc[results_df['known_filth'], ['group_id', 'text', 'beg', 'end', 'comparison_type']],
            how='outer',
            left_on=('group_id', 'filth_type'),
            right_on=('group_id', 'comparison_type'),
            suffixes=('', suffix_label)
        )
        .rename(columns=lambda x: x if not x.endswith(suffix_label) else 'known_' + x[:-len(suffix_label)])
        .assign(
            known_filth=lambda df: ~pd.isnull(df['known_text']),
            exact_match=lambda df: (df['text'] == df['known_text']).fillna(False),
            partial_match=lambda df: ((df['beg'] < df['known_end']) & (df['end'] > df['known_beg']).fillna(False)),
            true_positive=lambda df: (~pd.isnull(df['known_text'])) & (~pd.isnull(df['text'])),
            false_positive=lambda df: (pd.isnull(df['known_text'])) & (~pd.isnull(df['text'])),
            false_negative=lambda df: (~pd.isnull(df['known_text'])) & (pd.isnull(df['text'])),
        )
    )

def make_fake_document(
        paragraphs: int = 20, locale: str = 'en_US', seed: Optional[int] = None, faker: Optional[Faker] = None,
        filth_types: Optional[List[str]] = None, fake_text_function: Optional[Callable[..., str]] = None,
        additional_filth_types: Optional[Iterable[Type[Filth]]] = None,
) -> Tuple[str, List[KnownFilthItem]]:
    """Creates a fake document containing `Filth` that needs to be removed. Also returns the list of known filth
    items that are needed by the `TaggedEvaluationFilthDetector`\\ .

    An example of using this is shown below:

    .. code:: pycon

        >>> import scrubadub, scrubadub.comparison
        >>> document, known_filth_items = scrubadub.comparison.make_fake_document(paragraphs=1, seed=1)
        >>> scrubber = scrubadub.Scrubber()
        >>> scrubber.add_detector(scrubadub.detectors.TaggedEvaluationFilthDetector(
        ...     known_filth_items=known_filth_items
        ... ))
        >>> filth_list = list(scrubber.iter_filth(document))
        >>> print(scrubadub.comparison.get_filth_classification_report(filth_list))
        filth    detector    locale      precision    recall  f1-score   support
        <BLANKLINE>
        email    email       en_US            1.00      1.00      1.00         2
        url      url         en_US            1.00      1.00      1.00         1
        <BLANKLINE>
                              micro avg       1.00      1.00      1.00         3
                              macro avg       1.00      1.00      1.00         3
                           weighted avg       1.00      1.00      1.00         3
                            samples avg       1.00      1.00      1.00         3
        <BLANKLINE>

    :param paragraphs: The list of detected filth
    :type paragraphs: int
    :param locale: The locale of the documents in the format: 2 letter lower-case language code followed by an
                   underscore and the two letter upper-case country code, eg "en_GB" or "de_CH"
    :type locale: str
    :param seed: The random seed used to generate the document
    :type seed: int, optional
    :param faker: A Faker object that is used to generate the text
    :type faker: int
    :param filth_types: A list of the ``Filth.type`` to generate
    :type filth_types: List[str]
    :param fake_text_function: A function that will generate a 1-3 sentances of text
    :type fake_text_function: Callable, optional
    :return: The document and a list of `KnownFilthItem`\\ s
    :rtype: Tuple[str, List[KnownFilthItem]]

    """
    if faker is None:
        faker = Faker(locale=locale)

    if fake_text_function is None:
        fake_text_function = faker.text

    # TODO: register filth types to build up a dict that can be read from, like the detectors
    possible_filth = [
        filth_module.AddressFilth,
        filth_module.EmailFilth,
        filth_module.NameFilth,
        filth_module.PhoneFilth,
        filth_module.PostalCodeFilth,
        filth_module.SocialSecurityNumberFilth,
        filth_module.TwitterFilth,
        filth_module.UrlFilth,
    ]
    if additional_filth_types is not None:
        possible_filth += list(additional_filth_types)

    if filth_types is not None:
        possible_filth = [filth for filth in possible_filth if filth.type in filth_types]

    if seed is not None:
        Faker.seed(seed)
        random.seed(seed)

    doc = ""
    known_items = []  # type: List[KnownFilthItem]
    for _ in range(paragraphs):
        for _ in range(random.randint(1, 10)):
            text = fake_text_function() + " "
            matches = list(re.finditer(r'[\s.]', text))
            position = random.choice(matches)
            chosen_filth = random.choice(possible_filth)
            pii_text = chosen_filth.generate(faker=faker)
            known_items.append({
                'match': copy.copy(pii_text),
                'filth_type': copy.copy(chosen_filth.type),
            })
            separator = position.group()
            if '\n' in pii_text:
                separator = '\n'
            doc += (
                text[:position.start()] +
                (separator if separator != '.' else separator + ' ') +
                pii_text +
                separator +
                text[position.end():]
            )
        doc += "\n\n"
    return (doc.strip(), known_items)

max_stars_repo_name: datascopeanalytics/scrubadub

max_issues_repo_name: vishalbelsare/scrubadub

max_forks_repo_name: datascopeanalytics/scrubadub


text:
import numpy as np
import math

""" Implementation of the ORCA algorithm
    Resources: van den Berg, Reciprocal n-body Collision Avoidance,
                RVO2 library (C++) https://github.com/snape/RVO2/blob/master/src/Agent.cpp
                Pyorca library to see another python implementation https://github.com/Muon/pyorca
    Other resources included in the code"""

class Line:
    def __init__(self, point=np.array([0, 0]), direction=np.array([0, 0])):
        self.point = point
        self.direction = direction

class ORCA:
    def __init__(self):
        self.max_distance_to_neighbors = 5000  # m
        self.time_horizon = 250  # in seconds, the range to query/sense neighbors is 5 km, the max speed is 20 m/s => time_horizon = range/max_speed (250 seconds ~ 4.2 minutes)
        self.inv_time_horizon = 1 / self.time_horizon
        self.epsilon = 0.00001
        self.k_neighbors = 10

    def compute_new_velocity(self, agent, dt):
        # Compute the new velocity for the agent
        neighbors = agent.get_nearest_neighbors(self.k_neighbors, self.max_distance_to_neighbors)
        orca_lines =[]
        R = agent.radius * 1.1  # Slight increase to take care of floating point errors
        # Compute the agent desired velocity
        direction_to_goal = agent.goal - agent.position
        distance_to_goal = np.linalg.norm(direction_to_goal)
        pref_vel = min(agent.maxSpeed, distance_to_goal / dt) * direction_to_goal / distance_to_goal
        for neighbor in neighbors:
            # Construct ORCA lines
            # Project on the half plane
            rel_position = neighbor.position-agent.position
            d = np.linalg.norm(rel_position)
            # Opposite for some reason
            # Relative velocity computed using current velocity
            rel_velocity = agent.velocity - neighbor.velocity
            # Is there a collision
            if d > R:
                # No collision right now
                # Vector from cutoff center to relative velocity
                w = rel_velocity - self.inv_time_horizon * rel_position
                dot_product1 = np.dot(w, rel_position)
                w_norm = np.linalg.norm(w)
                unit_w = w / w_norm
                # dot_product1<0 is a necessary but not sufficient condition for the projection having to be done on the cutoff circle
                # The second condition compares the angle between w and - rel_postion (lambda) to the angle between rel_position and the radius tangent to the line (alpha)
                # Project on the circle if lambda < alpha => cos^2 (lambda) > cos^2(alpha) because alpha and lambda are between 0 and pi/2
                # cos^2 lambda = dot_product1**2/(|w|^2*|rel_position|^2)
                # cos^2 alpha = R^2 / |rel_position|^2
                if dot_product1 < 0 and dot_product1**2 > R**2 * np.dot(w, w):
                    # Should project on cut-off circle
                    # U is in the direction of w and the remaining distance to exit the cutoff circle
                    u = (R * self.inv_time_horizon - w_norm) * unit_w
                    direction = np.array([unit_w[1], -unit_w[0]])
                else:
                    # Need to project on legs
                    leg = math.sqrt(d**2 - R**2)
                    if np.linalg.det([rel_position,w]) > 0:
                        # On left leg, find direction by multiplying by rotation matrix (cone haf angle theta such that sin theta = R / d and cos theta = leg / d
                        # Unit vector
                        direction = np.array([rel_position[0]*leg - rel_position[1] * R,
                                             rel_position[0]*R + rel_position[1] * leg ]) / d**2
                    else:
                        # On right leg
                        direction = - np.array([rel_position[0] * leg + rel_position[1] * R,
                                              - rel_position[0] * R + rel_position[1] * leg]) / d**2
                    # Project the relative velocity on the leg
                    dot_product2 = rel_velocity * direction
                    # u is the point from the rel_velocity to the boundary
                    u = dot_product2 * direction - rel_velocity
            else:
                # Already colliding with neighbor, pick the velocity that will get us out within the time step
                w = rel_velocity - rel_position / dt
                w_norm = np.linalg.norm(w)
                unit_w = w / w_norm
                u = (R / dt - w_norm) * unit_w
                direction = np.array([unit_w[1], -unit_w[0]])
            line = Line(agent.velocity + u / 2, direction)
            orca_lines.append(line)
        # Try to solve the linear program, optimize for the preferred velocity of the agent
        line_fail, new_vel = self.linear_program2(orca_lines, agent.maxSpeed, pref_vel, False)

        if line_fail < len(orca_lines):
            # The feasible region is empty
            new_vel = self.linear_program3(orca_lines, line_fail, agent.maxSpeed, pref_vel)

        return new_vel

    # de Berg, Cheong, van Kreveld and Overmars, Computational Geometry: Algorithms and Applications, Third edition
    # Chapter 4: linear programming
    # 4.3 Incremental Linear Programming

    def linear_program1(self, lines, line_no, max_speed, opt_v, opt_dir):
        # Called when current velocity violates constraint line_no and we are looking for a solution on the constraint line_no
        # This is a 1D linear problem
        # returns False and the original velocity if the program is unfeasible
        # returns True and the new velocity which lays on constraint line_no

        # Initialize the bounds of the solution (scalar value indicating how far from point along direction is the solution)
        # This solver does not work for unbounded solutions but the solution is bounded by the max speed anyway
        # We are looking for the intersection of the line with the circle: |P + t* d|= R, this results in a quadratic equation
        # Where P is the point and d the direction of the constraint, |d|=1
        dot_product = np.dot(lines[line_no].point, lines[line_no].direction)
        discriminant = dot_product ** 2 - np.dot(lines[line_no].point, lines[line_no].point) + max_speed**2
        if discriminant < 0:
            # The constraint is outside the max speed limit
            return False, opt_v
        sqrt_discriminant = math.sqrt(discriminant)
        t_left = - dot_product - sqrt_discriminant
        t_right = - dot_product + sqrt_discriminant

        for i in range(0, line_no):
            # Find intersection of both lines https://en.wikipedia.org/wiki/Line%E2%80%93line_intersection
            denom = np.linalg.det([lines[line_no].direction, lines[i].direction])
            numer = np.linalg.det([lines[i].direction,lines[line_no].point - lines[i].point])
            if abs(denom)<= self.epsilon:
                # lines are almost parallel
                if numer < 0:
                    # if line_no is in the forbidden area of i and hence there are no solution
                    # I think it should not happens given how the solution is computed
                    return False, opt_v
                else:
                    continue
            t = numer / denom
            if denom >= 0:
                # Line i bounds line_no on the right
                t_right = min(t_right, t)
            else:
                # Line i bounds line_no on the right
                t_left = max(t_left, t)

            if t_left > t_right:
                # There is no solution on the constraint
                return False, opt_v

        if opt_dir:
            # We are just looking for a feasible solution
            if np.dot(opt_v, lines[line_no].direction) > 0:
                # Take right extreme
                new_vel = lines[line_no].point + t_right * lines[line_no].direction
            else:
                new_vel = lines[line_no].point + t_left * lines[line_no].direction
        else:
            # Should we use t_left or t_right as the new solution?
            # Project the optimal velocity on the constraint
            t = np.dot (lines[line_no].direction, opt_v - lines[line_no].point)
            if t < t_left:
                # projected to the left of the limit => the left limit is the new optimal
                new_vel = lines[line_no].point + t_left * lines[line_no].direction
            elif t > t_right:
                # projected to the right of the limit => the left limit is the new optimal
                new_vel = lines[line_no].point + t_right * lines[line_no].direction
            else:
                # the projection is in the middle of the limits
                new_vel = lines[line_no].point + t * lines[line_no].direction

        return True, new_vel

    def linear_program2(self, lines, max_speed, opt_v, opt_dir):
        """Solves the 2D linear program defined by the lines
        If there is a solution returns the number of constraint
        If there is no solution returns the constraint number that made the solution space empty
        If opt_dir is True then opt_v must be unit length
        """
        # Start by setting the desired solution
        # Not quite sure what opt_dir does, opt_dir is set to True when the solution space is empty
        if opt_dir:
            # opt_v is unit length
            result = max_speed * opt_v
        elif np.linalg.norm(opt_v) > max_speed:
            result = max_speed * opt_v / np.linalg.norm(opt_v)
        else:
            result = opt_v

        # Solve the linear program
        for i in range(0, len(lines)):
            line = lines[i]
            # Does the current result satisfy the constraint?
            if np.linalg.det([line.direction, line.point - result]) > 0:
                # The current result does not satisfy the new constraint, hence the new result will be on the new constraint unless the program is unfeasible
                feasible, new_result = self.linear_program1(lines, i, max_speed, opt_v, opt_dir)
                if not feasible:
                    # Constraint i makes the solution space empty, the result velocity will be ignored
                    return i, opt_v
                else:
                    result = new_result
        return len(lines), result

    def linear_program3(self, lines, line_fail, max_speed, opt_v, num_obstacles=0):
        # Called if the original 2D linear program is unfeasible see section 5.3 of ORCA paper
        # Basically the constraints are moved away until one velocity becomes feasible (it's a 3D linear programming problem)
        # The 2D linear program to solve is how much to relax
        # For now there are no non-participating or static obstacles (num obstacles  is 0)
        # Distance is the distance that constraints have to be relaxed by to open the solution space
        distance = 0
        result = opt_v
        # Start at the line that failed
        for i in range(line_fail,len(lines)):
            if np.linalg.det([lines[i].direction, lines[i].point - result])> distance:
                # the result does not satisfy line i constraint (even accounting for relaxation)
                # Since result is the only point in the relaxed solution space this means the solution space is empty

                # Treat constraints coming from non-participating traffic differently
                # Copy all lines that are not going to be relaxed
                if num_obstacles !=0:
                    # TODO Copy lines created by non-participating obstacles
                    projected_lines = []
                    pass
                else:
                    projected_lines=[]
                # Go through all constraints generated by participating traffic that are before the current constraint
                for j in range(num_obstacles,i):
                    new_line = Line()
                    # Find intersection of both lines https://en.wikipedia.org/wiki/Line%E2%80%93line_intersection
                    denom = np.linalg.det([lines[i].direction, lines[j].direction])
                    if abs(denom) <= self.epsilon:
                        # lines are basically parallel
                        if np.dot(lines[i].direction,lines[j].direction)>0:
                            # lines are in the same direction
                            continue
                        else:
                            # lines are in opposite directions
                            # Set a point in the middle to find how much you have to relax the constraint to open the solution space
                            new_line.point = 0.5*(lines[i].point+lines[j].point)
                    else:
                        # Find the intersection of those two lines
                        numer = np.linalg.det([lines[j].direction, lines[i].point - lines[j].point])
                        new_line.point = lines[i].point + (numer / denom) * lines[i].direction
                    new_line.direction = (lines[j].direction-lines[i].direction)/ np.linalg.norm(lines[j].direction-lines[i].direction)
                    projected_lines.append(new_line)

                # We know that all the previous constraints left a non-empty solution space (since line_fail is the first constraint that emptied the solution space)
                # The new lines' direction is opposite to the directions along which the intersection of line_fail and constraint i move when the constraints are being relaxed
                projected_vel = np.array([-lines[i].direction[1],
                                          lines[i].direction[0]])
                # Solve linear program to figure out how much to relax the constraints
                # Projected_vel gives the direction along which to optimize
                line_fail, solution = self.linear_program2(projected_lines, max_speed, projected_vel, True)

                if line_fail < len(projected_lines):
                    # Failed to relax (this should not happen) if it fails it's because of floating point errors
                    pass
                result = solution
                # Set the relaxation distance based on the result of the linear program (the following formula is the signed distance from result to line i because direction is a unit vector)
                distance = np.linalg.det([lines[i].direction, lines[i].point - result])

        return result

ext: py

reason: import numpy


text:
import numpy as np
import scipy.stats as stats

__all__ = ['rejection_sampling']

def rejection_sampling(density, inter, K, *args):
    """ Generates ``K`` samples from ``density`` on the interval ``inter``
    using an implementation of the acceptance-rejection method.

    Parameters
    ----------
    density : function
        The expression for the probability density function (pdf).
    inter : tuple
        Interval on which the samples will be generated.
    K : int
        Number of desired samples
    args :
        Any parameters, if needed, of the pdf.

    Return
    ------
    rvs : numpy.ndarray
        1-D array containing ``K`` samples from the distribution ``density``.
    """

    x = np.linspace(inter[0], inter[1], K)
    pdf = density(x, *args)
    pdfmax = np.max(pdf)

    if np.isnan(pdfmax) or np.isinf(pdfmax):
    	raise ValueError("density is not well defined, pdfmax has either" +
                          "NaN of inf values.")

    K_min = 0
    rvs = np.zeros(1)
    while K_min < K+1:
    	    u1 = stats.uniform.rvs(loc=inter[0],
                                   scale=inter[1] - inter[0],
                                   size=K)
    	    u2 = stats.uniform.rvs(size=K)
    	    idx = np.where(u2<=density(u1,*args)/pdfmax)[0]
    	    rvs = np.concatenate([rvs,u1[idx]])
    	    K_min = np.size(rvs)

    rvs = np.reshape(rvs,[K_min,1])
    return rvs[1:K+1]

max_stars_repo_name: mirca/fuzzy-umbrella

max_issues_repo_name: mirca/fuzzy-umbrella

max_forks_repo_name: mirca/fuzzy-umbrella

reason: import numpy,import scipy


text:
# Load the library with the CountVectorizer method
import os

from sklearn.feature_extraction.text import CountVectorizer
import numpy as np
import json
import matplotlib.pyplot as plt
import seaborn as sns

sns.set_style('whitegrid')

with open('../test_corpus/corpus') as f:
    contents = json.loads(f.read())

print(contents)

list_of_string = [' '.join(x) for x in contents]

# Helper function
def plot_10_most_common_words(count_data, count_vectorizer):
    import matplotlib.pyplot as plt
    words = count_vectorizer.get_feature_names()
    total_counts = np.zeros(len(words))
    for t in count_data:
        total_counts += t.toarray()[0]

    count_dict = (zip(words, total_counts))
    count_dict = sorted(count_dict, key=lambda x: x[1], reverse=True)[0:10]
    words = [w[0] for w in count_dict]
    counts = [w[1] for w in count_dict]
    x_pos = np.arange(len(words))

    plt.figure(2, figsize=(15, 15 / 1.6180))
    plt.subplot(title='10 most common words')
    sns.set_context("notebook", font_scale=1.25, rc={"lines.linewidth": 2.5})
    sns.barplot(x_pos, counts, palette='husl')
    plt.xticks(x_pos, words, rotation=90)
    plt.xlabel('words')
    plt.ylabel('counts')
    plt.show()

# Initialise the count vectorizer with the English stop words
count_vectorizer = CountVectorizer(stop_words='english')
# Fit and transform the processed titles
count_data = count_vectorizer.fit_transform(list_of_string)
# Visualise the 10 most common words
# plot_10_most_common_words(count_data, count_vectorizer)

import warnings

warnings.simplefilter("ignore", DeprecationWarning)
# Load the LDA model from sk-learn
from sklearn.decomposition import LatentDirichletAllocation as LDA

# Helper function
def print_topics(model, count_vectorizer, n_top_words):
    words = count_vectorizer.get_feature_names()
    for topic_idx, topic in enumerate(model.components_):
        print("\nTopic #%d:" % topic_idx)
        print(" ".join([words[i]
                        for i in topic.argsort()[:-n_top_words - 1:-1]]))

# Tweak the two parameters below
number_topics = 4
number_words = 50
# Create and fit the LDA model
lda = LDA(n_components=number_topics, n_jobs=-1)
lda.fit(count_data)
# Print the topics found by the LDA model
print("Topics found via LDA:")
print_topics(lda, count_vectorizer, number_words)

from pyLDAvis import sklearn as sklearn_lda
import pickle
import pyLDAvis

LDAvis_data_filepath = os.path.join('./ldavis_prepared_' + str(number_topics))
# # this is a bit time consuming - make the if statement True
# # if you want to execute visualization prep yourself
if 1 == 1:
    LDAvis_prepared = sklearn_lda.prepare(lda, count_data, count_vectorizer)
    pyLDAvis.save_html(LDAvis_prepared, './ldavis_prepared_' + str(number_topics) + '.html')

vectorizer = CountVectorizer(analyzer='word',
                             stop_words='english',
                             lowercase=True,
                             token_pattern='[a-zA-Z0-9]{3,}',)

topic_probability_scores = lda.transform(['what', 'reaction', 'contains', 'ch4', 'as', 'a', 'reactant'])
print(topic_probability_scores)

max_stars_repo_name: mdhillmancmcl/TheWorldAvatar-CMCL-Fork

max_issues_repo_name: mdhillmancmcl/TheWorldAvatar-CMCL-Fork

max_forks_repo_name: mdhillmancmcl/TheWorldAvatar-CMCL-Fork


text:
import numpy as np
from scipy.stats import f

def statistic(x, y):
    r"""
    Calulates the Hotelling :math:`T^2` test statistic.

    Parameters
    ----------
    x,y : ndarray
        Input data matrices. ``x`` and ``y`` must have the same number of
        dimensions. That is, the shapes must be ``(n, p)`` and ``(m, p)`` where
        `n` is the number of samples and `p` and `q` are the number of
        dimensions.

    Returns
    -------
    stat : float
        The computed Hotelling :math:`T^2` statistic.
    """
    # ported from Hotteling packge in R
    nx, p = x.shape
    ny = y.shape[0]

    meanx = np.mean(x, axis=0)
    meany = np.mean(y, axis=0)

    covx = np.cov(x, rowvar=False)
    covy = np.cov(y, rowvar=False)

    covs = ((nx - 1) * covx + (ny - 1) * covy) / (nx + ny - 2)
    m = (nx + ny - p - 1) / (p * (nx + ny - 2))
    if p > 1:
        inv_covs = np.linalg.pinv(covs)
        stat = m * (meanx - meany).T @ inv_covs @ (meanx - meany) * nx * ny / (nx + ny)
    else:
        inv_covs = 1 / p
        stat = m * (meanx - meany) ** 2 * inv_covs * nx * ny / (nx + ny)

    return stat

def test(x, y):
    r"""
    Calculates the Hotelling :math:`T^2` test statistic and p-value.

    Parameters
    ----------
    x,y : ndarray
        Input data matrices. ``x`` and ``y`` must have the same number of
        dimensions. That is, the shapes must be ``(n, p)`` and ``(m, p)`` where
        `n` is the number of samples and `p` and `q` are the number of
        dimensions.

    Returns
    -------
    stat : float
        The computed Hotelling :math:`T^2` statistic.
    pvalue : float
        The computed Hotelling :math:`T^2` p-value.

    Examples
    --------
    >>> import numpy as np
    >>> from hyppo.ksample import Hotelling
    >>> x = np.arange(7)
    >>> y = x
    >>> stat, pvalue = Hotelling().test(x, y)
    >>> '%.3f, %.1f' % (stat, pvalue)
    '0.000, 1.0'
    """
    stat = statistic(x, y)
    nx, p = x.shape
    ny = y.shape[0]
    pvalue = f.sf(stat, p, nx + ny - p - 1)

    return stat, pvalue

max_stars_repo_name: neurodata/MCC

max_issues_repo_name: neurodata/MCC

max_forks_repo_name: neurodata/MCC

reason: import numpy,from scipy


text:
import logging
import sys

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

import pickle

def plot_feat_imp(imp):
    imp = imp.sort_values('mean', ascending=True)
    ax = imp['mean'].plot(kind='barh', color='b', alpha=0.25, xerr=imp['std'],
                          error_kw={'ecolor': 'r'}, width=1)
    ax.get_yaxis().set_visible(False)
    plt.title('Mean decrease in negative log loss.')
    plt.grid()
    plt.show()

filename = '../datasets/Bagging_Optim_importance.pickle'
optimizer = pickle.load(open(filename, 'rb'))
filename = '../datasets/Bagging_Optim.pickle'
optimizer_optim = pickle.load(open(filename, 'rb'))
# print('Importance loss per feature')
# print(optimizer.importance)

# print('Most important features table')
imp_loss_mean = optimizer.importance['mean'].mean()
# print(optimizer.importance.loc[optimizer.importance['mean'] > imp_loss_mean],)

# most_imp_features = optimizer.importance.loc[optimizer.importance['mean'] > imp_loss_mean].index
# print('Most important features list: {}'.format(most_imp_features))

plot_feat_imp(optimizer.importance)

most_imp_features = optimizer.importance.loc[optimizer.importance['mean'] > imp_loss_mean].index
print(optimizer_optim.strategies[0].Xy.loc[:,most_imp_features])

most_imp_features = most_imp_features.to_numpy()
most_imp_features = most_imp_features[most_imp_features != 'trgt']
print(most_imp_features)

correlation_mat = optimizer.df[most_imp_features].corr()
ax = sns.heatmap(correlation_mat, annot = True)
plt.title("Correlation matrix of selected features")

bottom, top = ax.get_ylim()
ax.set_ylim(bottom + 0.5, top - 0.5)

plt.show()


text:
'''
Created by Wang Qiu Li
7/3/2018

get dicom info according to malignancy.csv and ld_scan.txt
results are npy files for malignant / benign nodules
'''

import csvTools
import os
import pandas as pd
import pydicom
import scipy.misc
import cv2
import numpy as np

basedir = '/data0/LIDC/DOI/'
resdir = '303010/'

noduleinfo = csvTools.readCSV('files/malignancy.csv')
idscaninfo = csvTools.readCSV('files/id_scan.txt')

print('normal')

# LUNA2016 data prepare ,first step: truncate HU to -1000 to 400
def truncate_hu(image_array):
    image_array[image_array > 400] = 0
    image_array[image_array <-1000] = 0
    return image_array

# LUNA2016 data prepare ,second step: normalzation the HU
def normalazation(image_array):
    max = image_array.max()
    min = image_array.min()
    image_array = (image_array-min)/(max-min)  # float cannot apply the compute,or array error will occur
    avg = image_array.mean()
    image_array = image_array-avg
    return image_array   # a bug here, a array must be returned,directly appling function did't work

def cutTheImage(x, y, pix):
    temp = 15
    x1 = x - temp
    x2 = x + temp
    y1 = y - temp
    y2 = y + temp
    img_cut = pix[x1:x2, y1:y2]
    return img_cut

def caseid_to_scanid(caseid):
    returnstr = ''
    if caseid < 10:
        returnstr = '000' + str(caseid)
    elif caseid < 100:
        returnstr = '00' + str(caseid)
    elif caseid < 1000:
        returnstr = '0' + str(caseid)
    else:
        returnstr = str(caseid)
    return 'LIDC-IDRI-' + returnstr

f = open('errlist.txt', 'w')
for onenodule in noduleinfo:
    try:
        scanid = onenodule[1]
        scanid = caseid_to_scanid(int(scanid))
        print(scanid)
        scanpath = ''
        for idscan in idscaninfo:
            if scanid in idscan[0]:
                scanpath = idscan[0]
                break

        filelist1 = os.listdir(basedir + scanpath)
        filelist2 = []
        print(len(filelist1))
        for onefile in filelist1:
            if '.dcm' in onefile:
                filelist2.append(onefile)

        slices = [pydicom.dcmread(basedir + scanpath + '/' + s) for s in filelist2]
        slices.sort(key = lambda x : float(x.ImagePositionPatient[2]),reverse=True)
        x_loc = int(onenodule[6])
        y_loc = int(onenodule[7])
        z_loc = int(onenodule[8])
        pix = slices[z_loc - 1].pixel_array
        cut_img = []

        zstart = z_loc - 1 - 5
        zend = z_loc - 1 + 5

        tempsign = 0
        for zslice in slices[zstart : zend]:
            pix = zslice.pixel_array
            pix.flags.writeable = True
            pix = truncate_hu(pix)
            cutpix = cutTheImage(y_loc, x_loc, pix)

            # scipy.misc.imsave(str(tempsign) + '.jpeg', cutpix)
            tempsign += 1
            cut_img.append(cutpix)
        if float(onenodule[29]) >= 3.5:
            np.save(resdir + onenodule[0] + '_high' + '.npy', cut_img)
        elif float(onenodule[29]) < 2.5:
            np.save(resdir + onenodule[0] + '_low' + '.npy', cut_img)
    except BaseException:
        print(onenodule)

max_stars_repo_name: qiuliwang/LIDC-IDRI-Toolbox-python

max_forks_repo_name: qiuliwang/LIDC-IDRI-Toolbox-python


text:
import json
import numpy as np
import os

from argparse import Namespace
from pathlib import Path
from tqdm import tqdm

from tpp.processes.multi_class_dataset import generate_points
from tpp.utils.cli import parse_args
from tpp.utils.run import make_deterministic
from tpp.utils.record import hawkes_seq_to_record

def main(args: Namespace):
    if args.verbose:
        print(args)

    # Create paths for plots
    data_dir = os.path.expanduser(args.data_dir)
    hawkes_data_dir = os.path.join(data_dir, "hawkes")
    Path(hawkes_data_dir).mkdir(parents=True, exist_ok=True)

    seeds = {
        "train": [args.train_size, args.seed],
        "val": [args.val_size, args.seed + args.train_size],
        "test": [args.test_size, args.seed + args.train_size + args.val_size]}

    for name, [size, seed] in seeds.items():
        range_size = range(size)
        if args.verbose:
            range_size = tqdm(range_size)

        times_marked = [
            generate_points(
                n_processes=args.marks,
                mu=args.mu.astype(np.float64),
                alpha=args.alpha.astype(np.float64),
                decay=args.beta.astype(np.float64),
                window=args.window,
                seed=seed + i
            ) for i in range_size]  # D x M x Li

        records = [hawkes_seq_to_record(r) for r in times_marked]

        with open(os.path.join(hawkes_data_dir, name + ".json"), "w") as h:
            h.write(
                '[' + ',\n'.join(json.dumps(i) for i in records) + ']\n')

    codes_to_names = {str(i): str(i) for i in range(args.marks)}

    args_dict = vars(args)
    args_dict['alpha'] = args_dict['alpha'].tolist()
    args_dict['beta'] = args_dict['beta'].tolist()
    args_dict['mu'] = args_dict['mu'].tolist()
    keys_to_keep = ["seed", "mu", "alpha", "beta", "marks", "hawkes_seed",
                    "window", "train_size", "val_size", "test_size"]
    args_dict = {k: args_dict[k] for k in keys_to_keep}
    with open(os.path.join(hawkes_data_dir, 'args.json'), 'w') as fp:
        json.dump(args_dict, fp)
    with open(os.path.join(hawkes_data_dir, 'int_to_codes.json'), 'w') as fp:
        json.dump(codes_to_names, fp)
    with open(os.path.join(hawkes_data_dir, 'codes_to_int.json'), 'w') as fp:
        json.dump(codes_to_names, fp)
    with open(os.path.join(hawkes_data_dir, 'codes_to_names.json'), 'w') as fp:
        json.dump(codes_to_names, fp)
    with open(os.path.join(hawkes_data_dir, 'names_to_codes.json'), 'w') as fp:
        json.dump(codes_to_names, fp)

if __name__ == "__main__":
    parsed_args = parse_args()

    # check_repo(allow_uncommitted=not parsed_args.use_mlflow)
    make_deterministic(seed=parsed_args.seed)

    main(args=parsed_args)


text:
from keras import models
import retrieval
import replace
import numpy as np
import generate_cnn_data as gcd
from argparse import ArgumentParser

parser = ArgumentParser()
parser.add_argument("-i", "--query-map-id", dest="query_map_id", required=True)
args = parser.parse_args()

query_path = gcd.current_path() + '/data_flp/queries/query_' + args.query_map_id + '.map'
query_triples_path = gcd.current_path() + '/data_flp/queries/query_triples_' + args.query_map_id + '.map'

classificator_model = models.load_model(gcd.current_path() + '/data_flp/classificator/saves/classificator.h5')

query_triples = gcd.get_triples(query_triples_path)
room_count = len(query_triples[0])
query_data = gcd.dataset(num_classes=1, convnet_type=None, dataset_type='queries', shape=room_count,
                         query_id=args.query_map_id)
assert len(query_data) == 1
query_connmap = query_data[0]

query_data.clear()
query_data.append(gcd.get_extened_connmap(query_connmap, room_count))

max_room_count = gcd.max_room_count
assert len(query_data[0]) == max_room_count * max_room_count

pred_classes_classificator = []
for e in query_data:
    pred_classes_classificator.append(
        classificator_model.predict(np.array(e).reshape(1, max_room_count, max_room_count, 1)).argmax())
assert len(pred_classes_classificator) == 1

all_sims = retrieval.get_sims(query_path, room_count)
assert len(all_sims) == 1

query_data_file = open(query_path).readlines()
case_data_file = open(gcd.current_path() + '/data_flp/case_data/cases/0.txt').readlines()
case_triples_file = open(gcd.current_path() + '/data_flp/case_data/cases/0_uuids.txt').readlines()
# save adapted maps
file_adapted = open(gcd.current_path() + '/data_flp/results/adapted_' + args.query_map_id + '.map', 'a+')
file_adapted_extended = open(gcd.current_path() + '/data_flp/results/adapted_extended_' + args.query_map_id + '.map',
                             'a+')
file_adapted_triples = open(gcd.current_path() + '/data_flp/results/adapted_triples_' + args.query_map_id + '.map', 'a+')
query_connmap_1d = query_data_file[0]
mode = pred_classes_classificator[0]  # e.g. 0, 1, or 2
for case_num, sim in all_sims[0]:
    case_connmap_1d = case_data_file[int(case_num)]
    query_map = replace.mrx(query_connmap_1d, room_count)
    case_triples = gcd.get_triples(case_triples_file[int(case_num)])
    case_room_count = len(case_triples[0])
    case_map = replace.mrx(case_connmap_1d, case_room_count)
    try:
        query_map_replaced, query_triples_replaced = replace.replace(query_map,
                                                                     query_triples[0],
                                                                     case_map,
                                                                     case_triples[0],
                                                                     mode=mode)
        np_replaced = np.array(query_map_replaced).flatten()
        file_adapted.write(str(np_replaced).replace('\n', '') + '\n')
        query_map_replaced_extended = gcd.get_extened_connmap(query_map_replaced, len(query_map_replaced))
        file_adapted_extended.write(str(query_map_replaced_extended).replace(',', '') + '\n')
        file_adapted_triples.write(str(query_triples_replaced) + '\n')
    except TypeError:
        # TODO check why it sometimes gives 'TypeError: cannot unpack non-iterable NoneType object'
        pass

max_stars_repo_name: cenetp/roomconf-adaptation

max_issues_repo_name: cenetp/roomconf-adaptation

max_forks_repo_name: cenetp/roomconf-adaptation


text:
"""Generate train/test split"""
from collections import defaultdict
import os
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split

import random
import re

BOTH_GENDERS = False
SAVE_IDS = True
ID_SAVE_PATH = os.path.join("data", "id_train_test_split_males.csv")
IDS_FROM_CSV = os.path.join("data", "id_train_test_split_males.csv")

def get_id(filename):
    name = filename.split("_")[-2]
    if name.endswith("A"):
        return name[:-1]
    if len(name.split(".")) > 1:
        return name.split(".")[0]
    pattern = re.compile("^[A-Z]{3}")
    regex_match = pattern.search(name)
    if regex_match:
        return regex_match[0]
    return name

def get_task(filename):
    return filename.split("_")[1]

def get_trial(filename):
    return filename.split("_")[-1][:-4]

def sample_ids(df, diagnosis, gender, k):
    return random.choices(df[(df["Diagnosis"] == diagnosis) & (df["Gender"] == gender)]["ID"].tolist()[0], k=k)

def train_test_split_to_csv(test_ids, train_ids):
    test_ids = {"ID" : test_ids,
                "split" : "test"}
    train_ids = {"ID" : train_ids,
                "split" : "train"}
    test_ids = pd.DataFrame(test_ids)
    train_ids = pd.DataFrame(train_ids)
    id_split = test_ids.append(train_ids)
    id_split.to_csv(ID_SAVE_PATH, index=False)

def get_train_test_from_csv():
        train_test_df = pd.read_csv(IDS_FROM_CSV)
        train_ids = train_test_df[train_test_df["split"] == "train"]["ID"].astype(str).tolist()
        test_ids = train_test_df[train_test_df["split"] == "test"]["ID"].astype(str).tolist()
        return train_ids, test_ids

if __name__ == "__main__":
    data = os.path.join(os.getcwd(), "data", "autism_data")
    key = os.path.join(os.getcwd(), "data", "autism_data_key", "FullDataByTrial.csv")
    filepaths = os.listdir(data)
    file_dict = [{"file": os.path.join(data, path)} for path in filepaths if path.endswith(".wav")]

    df_key = pd.read_csv(key)
    df = pd.DataFrame(file_dict)
    df["name"] = df.file.apply(lambda x: x.split("/")[-1])
    df["ID"] = df.name.apply(lambda x: get_id(x))
    unique_wav_ids = set(df["ID"])
    unique_key_ids = set(df_key["ID"])

    diff_ids = unique_wav_ids - unique_key_ids
    # print(diff_ids)

    # make new key rows for the known missing
    key_cols = df_key.columns
    new_rows = defaultdict()
    for col in key_cols:
        new_rows[col] = [np.nan] * 4
    new_rows["ID"] = ["7320", "7231", "7029", "7094"]
    new_rows["Diagnosis"] = ["ASD"] * 4
    new_rows["Language"] = ["us"] * 4
    new_rows = pd.DataFrame(new_rows)
    df_key = df_key.append(new_rows)

    keep_cols = [
        "ID",
        "Language",
        "Diagnosis",
        "Gender",
        "Age",
        "AdosCommunication",
        "AdosSocial",
        "AdosCreativity",
        "AdosStereotyped",
        "VIQ",
        "PIQ",
        "TIQ",
        "ParentalEducation",
        "SRS",
        "CARS",
        "PPVT",
        "Leiter",
        "language",
        "AgeS"]
    df_merge_key = df_key[keep_cols]
    df_merge_key = df_merge_key.drop_duplicates()
    df = df.merge(df_merge_key, on="ID")

    df["Trial"] = df.name.apply(lambda x: get_trial(x))
    df["Task"] = df.name.apply(lambda x: get_task(x))

    # only keeping the Danish data
    df = df[df["language"] == "dk"]

    if BOTH_GENDERS:
    # stratifying - sampling 2 females and 4 males from each group
        unique_ids = pd.DataFrame(df.groupby(["Diagnosis", "Gender"])["ID"].unique()).reset_index()
        asd_male = sample_ids(unique_ids, "ASD", "Male", k=4)
        asd_female = sample_ids(unique_ids, "ASD", "Female", k=2)
        td_male = sample_ids(unique_ids, "TD", "Male", k=4)
        td_female = sample_ids(unique_ids, "TD", "Female", k=2)

        test_ids = asd_male + asd_female + td_male + td_female
        all_ids = np.concatenate(unique_ids["ID"].tolist())

    else:
        df = df[df["Gender"] == "Male"]
        test_ids = ["109", "121", "126", "127", "135", "211", "212", "219", "228"]
        all_ids = df["ID"].unique()

    train_ids = [x for x in list(all_ids) if x not in test_ids]

    if SAVE_IDS:
        train_test_split_to_csv(test_ids, train_ids)

    if IDS_FROM_CSV:
        train_ids, test_ids = get_train_test_from_csv()

    train = df[df["ID"].isin(train_ids)]
    test = df[df["ID"].isin(test_ids)]

    stories_train = train[train["Task"] == "stories"]
    stories_test = test[test["Task"] == "stories"]
    triangles_train = train[train["Task"] == "triangles"]
    triangles_test = test[test["Task"] == "triangles"]

    stories_train.to_csv(os.path.join("data", "splits", f"stories_train_data_gender_{str(BOTH_GENDERS)}.csv"), index=False)
    stories_test.to_csv(os.path.join("data", "splits", f"stories_test_data_gender_{str(BOTH_GENDERS)}.csv"), index=False)
    triangles_train.to_csv(os.path.join("data", "splits", f"triangles_train_data_gender_{str(BOTH_GENDERS)}.csv"), index=False)
    triangles_test.to_csv(os.path.join("data", "splits", f"triangles_test_data_gender_{str(BOTH_GENDERS)}.csv"), index=False)


text:
from collections import deque
from copy import deepcopy
import imgaug.augmenters as iaa
import numpy as np
import os
from os import path
from PIL import Image
from random import shuffle
import skimage
from warnings import warn

from train import datautils

__all__ = ['Davis2017Dataset',
           'get_trainval', 'get_test_dev', 'get_test_challenge']

_subset_param_desc = "which subset of DAVIS 2017 to get (either 'trainval', " \
                      "'test-dev', or 'test-challenge')"

_quality_param_desc = "quality of the video (either '480p', '1080p', or 'fullres'); " \
                       "'1080p' and 'fullres' refer to the same image quality"

_selection_param_desc = "which objects to retrieve (either 'images', 'labels', or " \
                         "'videos')"

class Davis2017Dataset(datautils.Dataset):
    """
    Class for representing a dataset of images from the DAVIS 2017 Dataset.

    Methods for Loading Images into Dataset:
     * load_subset - loads a particular subset of the data, as defined by the DAVIS 2017 Challenge
     * load_video - loads a particular video from a particular subset of the data
     * load_frame - loads a particular frame from a particular video from a particular subset of the data

    After loading images, you must call prepare() in order for the dataset to be
    use properly. Once you call prepare(), you can load images/masks (into memory).

    Methods for Actually Loading Images:
     * load_image - returns the image tensor
     * load_mask - returns the mask tensor and instance array of classes
        * has_mask - returns whether the image_id has a mask in the dataset
          (call this before load_mask() )

    Generators:
     * paired_generator - generates image pairs (with masks) for a Keras model
     * sequential_generator - generates single images in sequence
    """

    name = 'DAVIS2017'
    size = (480, 854)

    def __init__(self, subset: str, quality: str, data_dir='./'):
        super(self.__class__, self).__init__(self)

        if not path.exists(data_dir):
            warn('Creating a Dataset with a root directory that does not exist on the current machine. Most '
                 'load methods will fail.')

        self.subset = subset
        self.quality = quality
        self.data_dir = data_dir

    def __len__(self):
        return len(self.image_ids)

    @property
    def all_masked(self) -> bool:
        """Whether all images in this dataset have a mask. This method is O(n),
        although the coefficients are relatively small.

        It is not safe to use the current dataset as a training dataset if this
        property is not true.
        """
        for img_dict in self.image_info:
            if 'mask_path' not in img_dict:
                return False

        return True

    def load_subset(self, *videos):
        """
        Loads all the videos within this subset and video quality for the videos
        given. If a list of videos is given, only those videos will be loaded;
        otherwise, all available videos will be loaded.

        Args:
            *videos: specific videos to load
        """

        if not len(videos):
            _, videos, _ = next(os.walk(self.build_absolute_path_to('images', '')))
            # FIXME there should be an error in logic here

        for vid in videos:
            self.load_video(vid)

    def load_video(self, video: str):
        """
        Loads all the frames from a specific video.

        Args:
            video: name of the video to load into dataset
        """

        _, _, frame_filenames = next(os.walk(self.build_absolute_path_to('images', video)))

        for fname in frame_filenames:
            self.load_frame(video, fname)

    def load_frame(self, video, img_filename):
        """
        Loads a single frame (as specified by the filename) from a specific video
        into this dataset.

        Args:
            video: name of the video to load frame from
            img_filename: full filename (with no path) of the frame to load
        """

        img_id = img_filename[:-4]
        img_path_to_store = path.join(video, img_filename)
        mask_path_to_store = path.join(video, f'{img_id}.png')

        if path.exists(self.build_absolute_path_to('labels', mask_path_to_store)):
            self.add_image(self.name, img_id, img_path_to_store, mask_path=mask_path_to_store, video=video)
        else:
            self.add_image(self.name, img_id, img_path_to_store, video=video)

    def load_image(self, image_id: int):
        """
        Load a specific image into memory.

        Args:
            image_id: id of the frame to load

        Returns:
            image tensor
        """
        info = self.image_info[image_id]

        if info['source'] != self.name:
            return super(self.__class__, self).load_image(image_id)

        img_path = self.build_absolute_path_to('images', info['path'])

        # Load image
        image = skimage.io.imread(img_path)

        # If has an alpha channel, remove it for consistency
        if image.shape[-1] == 4:
            image = image[..., :3]

        return image

    def has_mask(self, image_id: int):
        """
        Check whether the specified frame has a mask associated with it.

        Args:
            image_id: id of the frame

        Returns:
            True if there's a mask; False otherwise
        """

        return 'mask_path' in self.image_info[image_id]

    def __getitem__(self, item: int):
        return self.load_float_mask(item)

    def load_mask(self, image_id: int):
        """
        Loads the mask and class information for a specified frame into memory.

        Args:
            image_id: id of the frame

        Returns:
            (mask, labels) pair, where mask and labels are tensors

        The returned mask has shape [h, w, n], where n is the number of instances
        in this frame.

        The returned labels tensor has shape [n], where n is the same as above.
        """

        info = self.image_info[image_id]

        if info['source'] != self.name:
            return super(self.__class__, self).load_mask(image_id)

        if not self.has_mask(image_id):
            raise ValueError('this image does not have a mask')

        mask_path = self.build_absolute_path_to('labels', info['mask_path'])

        mask = np.expand_dims(np.atleast_3d(Image.open(mask_path))[..., 0], axis=2)
        uniqs = np.delete(np.expand_dims(np.expand_dims(np.unique(mask), axis=0), axis=0), 0)

        # Return mask and class of mask
        return mask == uniqs, uniqs

    def load_int_mask(self, image_id: int):
        """
        Loads the mask for a frame as an integer-based image.

        Args:
            image_id: id of the frame

        Returns:
            TODO continue from here
        """
        mask, ids = self.load_mask(image_id)

        return 255 * mask.astype(int), ids

    def load_float_mask(self, image_id: int):
        mask, ids = self.load_mask(image_id)

        return mask.astype(np.float32), ids

    def image_reference(self, image_id):
        return 'https://davischallenge.org/davis2017/code.html'

    def __str__(self):
        try:
            return f'<Davis 2017 Dataset: {len(self.image_ids)} images prepared >'
        except AttributeError:
            return '<Davis 2017 Dataset (unprepared)>'

    def paired_generator(self, augmentation=iaa.Noop(), training=True, max_pair_dist=3):
        """
        Creates a generator that returns pairs of consecutive images (as input)
        and the mask for the second image (as ground truth).
        Args:
            augmentation: sequence of imgaug augmentations to perform on each
                          mask to transform it into an input mask (ignored if
                          mask_as_input is False)
            training: whether to return an augmented mask as an input and
                      normalize inputs
            max_pair_dist: the maximum number of frames apart between two images
                           in the same pair

        Returns:
            inputs as requested (see details below)

        If mask_as_input is True, then the following are returned in this order:
        previous image      [1, h, w, 3]
        current image       [1, h, w, 3]
        augmented mask      [1, h, w, 1]   (excluded if mask_as_input is False)
        ground-truth mask   [1, h, w, 1]
        All returned arrays are of type np.float32. Values are normalized to
        [0, 1] if training is False and mean-centered if training is True.
        """

        def make_batch_dim(tensor):
            return np.expand_dims(tensor, axis=0)

        ordered_ids = deepcopy(self.image_ids[1:])

        id_pairs = []

        i = 0
        while i < len(ordered_ids) - 1:
            # at the change point between videos
            if self.image_info[i]['video'] != self.image_info[i + 1]['video']:
                i += 1
                continue

            # add image pairs until we reach the max distance (in frame) away
            j = i + 1
            while j < len(ordered_ids) - 1 \
                    and self.image_info[i]['video'] == self.image_info[j]['video'] \
                    and j - i < max_pair_dist:
                id_pairs.append((i, j))

                j += 1

            i += 1

        print(f'Created paired generator with {len(id_pairs)} image pairs.')
        sentinel = (-1, -1)

        shuffle(id_pairs)
        id_pair_queue = deque(id_pairs)
        id_pair_queue.appendleft(sentinel)

        while True:
            # process the next image
            prev_id, curr_id = id_pair_queue.pop()

            # reshuffle if reached the sentinel
            if prev_id == curr_id:
                shuffle(id_pairs)
                id_pair_queue = deque(id_pairs)
                id_pair_queue.appendleft(sentinel)
                continue

            # load originals (and transform to [0, 1] range)
            prev_image = self.load_image(prev_id).astype(np.float32) / 255
            curr_image = self.load_image(curr_id).astype(np.float32) / 255
            gt_masks, _ = self.load_float_mask(curr_id)

            pre_aug_masks, _ = self.load_int_mask(curr_id)
            pre_aug_masks = pre_aug_masks.astype(np.uint8)

            # generate a pair for each mask instance
            for i in range(gt_masks.shape[-1]):
                gt_mask = np.expand_dims(gt_masks[..., i], axis=2)

                if training:
                    aug_for_this = augmentation.to_deterministic()

                    aug_mask = np.expand_dims(pre_aug_masks[..., i], axis=2)
                    aug_mask = aug_for_this.augment_image(aug_mask).astype(np.float32) / 255

                    yield tuple(map(make_batch_dim, (prev_image, curr_image, aug_mask, gt_mask)))
                else:
                    yield tuple(map(make_batch_dim, (prev_image, curr_image, gt_mask)))

            # add the image to the back of the queue
            id_pair_queue.appendleft(curr_id)

    def sequential_generator(self, random=None):
        # TODO implement

        pass

    def build_absolute_path_to(self, selection: str, video_and_filename: str) -> str:
        """
        Builds and returns an absolute path to a specified file, given a selection.

        Args:
            selection: either 'images', 'labels', or 'videos'
            video_and_filename: name of the file

        Returns:
            full absolute path to the file specified
        """

        return path.join(self.data_dir,
                         Davis2017Dataset.build_relative_path(self.subset, self.quality, selection),
                         video_and_filename)

    @staticmethod
    def build_relative_path(subset: str, quality: str, selection: str) -> str:
        f"""
        Builds a relative path to the directory containing the dataset with the
        parameters given.
        :param subset: {_subset_param_desc}
        :param quality: {_quality_param_desc}
        :param selection: {_selection_param_desc}
        :return: path to the desired selection
        """
        if subset not in ['trainval', 'test-dev', 'test-challenge']:
            raise ValueError(f'{subset} not a subset for DAVIS 2017')
        if quality not in ['480p', 'fullres', '1080p']:
            raise ValueError(f'{quality} not one of the available qualities')
        if selection not in ['images', 'labels', 'videos']:
            raise ValueError('selection must be either \'images\' or \'labels\'')

        if quality == 'fullres' or quality == '1080p':
            quality = 'Full-Resolution'

        base_path = f'DAVIS-2017-{subset}-{quality}/DAVIS/'

        if selection == 'images':
            selection = 'JPEGImages'
        elif selection == 'labels':
            selection = 'Annotations'

        if selection == 'videos':
            rel_path = 'ImageSets/2017/'
        else:
            rel_path = f'{selection}/{quality}/'

        return path.join(base_path, rel_path)

def _load_predefined(root_dir, subset, quality) -> Davis2017Dataset:
    """
    Load one of the DAVIS-predefined subsets of the DAVIS 2017 dataset.

    Args:
        root_dir: root directory of the dataset
        subset: name of predefined subset (e.g. 'trainval')
        quality: either '480p' or 'fullres'

    Returns:
        prepared dataset containing that subset of DAVIS
    """
    dataset = Davis2017Dataset(subset, quality, data_dir=root_dir)
    dataset.load_subset()
    dataset.prepare()

    return dataset

def get_trainval(root_dir, quality='480p') -> Davis2017Dataset:
    """
    Returns the prepared trainval dataset as defined by DAVIS 2017.
    :param root_dir: root directory of the dataset
    :param quality: {_quality_param_desc}
    :return: trainval dataset for quality requested
    """
    # TODO produce validation set separately

    return _load_predefined(root_dir, 'trainval', quality)

def get_test_dev(root_dir, quality='480p') -> Davis2017Dataset:
    """
    Returns the prepared test-dev dataset as defined by DAVIS 2017.
    :param root_dir: root directory of the dataset
    :param quality: {_quality_param_desc}
    :return: test-dev dataset for quality requested
    """

    return _load_predefined(root_dir, 'test-dev', quality)

def get_test_challenge(root_dir, quality='480p') -> Davis2017Dataset:
    """
    Returns the prepared test-challenge dataset as defined by DAVIS 2017.
    :param root_dir: root directory of the dataset
    :param quality: {_quality_param_desc}
    :return: test-challenge dataset for quality requested
    """

    return _load_predefined(root_dir, 'test-challenge', quality)

max_stars_repo_name: umd-fire-coml/MultiSeg

max_issues_repo_name: umd-fire-coml/MultiSeg

max_forks_repo_name: umd-fire-coml/MultiSeg


text:
# we generated the training pair for Part Refiner is based on the shapes with res 32

import numpy as np
import scipy.io as sio
import os
import sys
import argparse
import itertools

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(BASE_DIR)
sys.path.append(os.path.dirname(BASE_DIR))

parser = argparse.ArgumentParser()
parser.add_argument('--input_obj', default='chair', help='The input data path for seperating [default: None]')
parser.add_argument('--num_semantic_parts', type=int, default=4, help='The amount of semantic parts on shape')
FLAGS = parser.parse_args()

label_num = FLAGS.num_semantic_parts
obj = FLAGS.input_obj

def load_mat(matFile, cube):
    data = sio.loadmat(matFile)
    volume_size = (cube, cube, cube)
    array = np.ndarray(volume_size, np.int32)
    array = data['instance']
    return array

def load_real_data_part(real_part_path, label_num):
    real_parts = {}

    for i in xrange(label_num):
        real_part_path = os.path.join(real_part_path, 'part'+str(i))
        real_parts[i] = np.load(real_part_path)

    return real_part

def load_np_array(path):
    volume = np.load(path)
    return volume

def l1_distance(chair, data, part_num):
    dist = np.zeros(len(data))
    for i in xrange(len(data)):
        dist[i] = np.sum(np.abs(chair - data[i]))
    return np.argpartition(dist, part_num)[:part_num]

def intersection_over_union(chair, data, part_num):

    data = np.squeeze(data)
    dist = np.zeros(len(data))

    for i in xrange(len(data)):
        intersection = np.count_nonzero((data[i] - 0.5 * chair) == 0.5)
        union = np.count_nonzero(data[i] + chair)
        dist[i] = intersection / float(union)
    return np.argpartition(dist, -part_num)[-part_num:]

def load_real_part(input_path, part_num):

    print 'Loading data'
    loaded_real_parts = {}

    for ID in xrange(part_num):
    	loaded_real_parts[ID] = np.load(input_path + '/part'+str(ID)+'.npy'),

    return loaded_real_parts

def separate_parts(data, dim, part_num):

    separate = {}

    for ID in xrange(part_num):
    	separate[ID] = np.empty([0, dim, dim, dim, 2])

    for i in xrange(len(data)):
        tmp = np.zeros([part_num, dim, dim, dim, 2])
        print 'Separate', i
        for j in xrange(data.shape[1]):
            for k in xrange(data.shape[2]):
                for l in xrange(data.shape[3]):
                    index = np.argmax(data[i, j, k, l])
                    for n in xrange(len(separate)):
                        if (index - 1) == n:
                            tmp[n, j, k, l, 1] = 1
                        else:
                            tmp[n, j, k, l, 0] = 1
        for n in xrange(len(separate)):
            separate[n] = np.concatenate((separate[n], np.reshape(tmp[n], (np.append([1], tmp[n].shape)))), axis=0)

    return separate

def prepare_pair_for_PR(input_data_path, real_parts, dim, part_num):

    gen_parts = {}
    pairs = {}

    for ID in xrange(part_num):
    	gen_parts[ID] = np.empty([0, dim, dim, dim, 2])
        pairs[ID] = np.empty([0, 2])

    gen_data_filelist =  os.listdir(input_data_path)
    len_gen_data_filelist = len(gen_data_filelist)

    dataset_files_32 = []

    for i in xrange(len_gen_data_filelist):
    	print 'File:', i
        #we assume the initial generated shapes are .npy files
    	gen_shapes = load_np_array(input_data_path + '/' + gen_data_filelist[i])

    	# numpy convert to one-hot
    	print 'Converts to One-Hot'
    	max_idx = np.argmax(gen_shapes, axis=4)
    	max_idx = np.reshape(max_idx, (gen_shapes.shape[0] * gen_shapes.shape[1] * gen_shapes.shape[2] * gen_shapes.shape[3]))
    	tmp = np.zeros((gen_shapes.shape[0] * gen_shapes.shape[1] * gen_shapes.shape[2] * gen_shapes.shape[3], part_num+1))
    	tmp[np.arange(gen_shapes.shape[0] * gen_shapes.shape[1] * gen_shapes.shape[2] * gen_shapes.shape[3]), max_idx] = 1
    	gen_shapes = np.reshape(tmp, gen_shapes.shape)

    	# separates the generated parts
    	gen_shapes_parts = separate_parts(gen_shapes, 32, part_num)

        for j in xrange(label_num):
            print 'Category:', j
            for gen_part in gen_shapes_parts[j]:
                num_of_arm_rest_voxels = np.count_nonzero(gen_part)
                print(j)

                if num_of_arm_rest_voxels == 0:
                    continue
                elif num_of_arm_rest_voxels < 10:
                    match_idx = l1_distance(gen_part, real_parts[j], part_num)
                else:
                    match_idx = intersection_over_union(gen_part, real_parts[j], part_num)

                reshaped_part = np.reshape(gen_part, np.concatenate(([1], gen_part.shape)))
                gen_parts[j] = np.concatenate((gen_parts[j], reshaped_part), axis=0)

                g = len(gen_parts[j]) - 1

                for m in match_idx:
                    pair = np.array([g, m])
                    reshaped_pair = np.reshape(pair, np.concatenate(([1], pair.shape)))
                    pairs[j] = np.concatenate((pairs[j], reshaped_pair), axis=0)

    return pairs, gen_parts

def main():

    INITIAL_GEN_DATA_PATH = os.path.join(BASE_DIR, 'input_data/'+obj+'/gen_32')
    REAL_32_PART_DATA_PATH = os.path.join(BASE_DIR, 'seperate_result/'+obj+'/real_32')

    TRAINING_DATA_FOR_PR_SAVE_PATH = os.path.join(BASE_DIR, 'pair_index/'+obj)
    GEN_SHAPE_PART_SAVE_PATH = os.path.join(BASE_DIR, 'seperate_result/'+obj+'/gen_32')

    if not os.path.exists(TRAINING_DATA_FOR_PR_SAVE_PATH):
        os.makedirs(TRAINING_DATA_FOR_PR_SAVE_PATH)

    if not os.path.exists(GEN_SHAPE_PART_SAVE_PATH):
        os.makedirs(GEN_SHAPE_PART_SAVE_PATH)

    real_data_parts = load_real_part(REAL_32_PART_DATA_PATH, label_num)

    pair_index, gen_data_parts = prepare_pair_for_PR(INITIAL_GEN_DATA_PATH, real_data_parts, 32, label_num)

    for i in xrange(label_num):
        print 'Saving training data for PR', i

        np.save(TRAINING_DATA_FOR_PR_SAVE_PATH + '/pair_index_part_' + str(i), pair_index[i])
        np.save(GEN_SHAPE_PART_SAVE_PATH + '/part' + str(i), gen_data_parts[i])

if __name__ == "__main__":
    main()

max_stars_repo_path: Part Refiner/Data Preparation for PR/gen_paris.py


text:
import sys
sys.path.append("../bin/")
import pyHiChi as hichi
import numpy as np

field_size = hichi.Vector3d(144, 30, 1)
pml_size = hichi.Vector3d(8, 0, 0)
min_coords = hichi.Vector3d(0.0, 0.0, 0.0)
max_coords = hichi.Vector3d(field_size.x * hichi.c, field_size.y * hichi.c, field_size.z * hichi.c)

field_step = (max_coords - min_coords) / field_size
time_step = 0.1

pml_left_end = min_coords.x + pml_size.x*field_step.x
pml_right_end = max_coords.x - pml_size.x*field_step.x
internal_width = pml_right_end - pml_left_end

def value_Ex(x, y, z):
    Ex = 0
    return Ex

def value_Ey(x, y, z):
    if (x < pml_left_end or x >= pml_right_end):
        Ey=0
    else:
        Ey = np.sin(2*np.pi/internal_width*(x - pml_left_end))
    return Ey

def value_Ez(x, y, z):
    Ez = 0
    return Ez

def value_Bx(x, y, z):
    Bx = 0
    return Bx

def value_By(x, y, z):
    By = 0
    return By

def value_Bz(x, y, z):
    if (x < pml_left_end or x >= pml_right_end):
        Bz=0
    else:
        Bz = np.sin(2*np.pi/internal_width*(x - pml_left_end))
    return Bz

field = hichi.PSTDField(field_size, min_coords, field_step, time_step)
# field = hichi.PSATDField(field_size, min_coords, field_step, time_step)
# field = hichi.PSATDTimeStraggeredField(field_size, min_coords, field_step, time_step)
# field = hichi.YeeField(field_size, min_coords, field_step, time_step)
field.set_E(value_Ex, value_Ey, value_Ez)
field.set_B(value_Bx, value_By, value_Bz)
field.set_PML(int(pml_size.x), int(pml_size.y), int(pml_size.z))

# ------------------- show -------------------

import matplotlib.pyplot as plt
import matplotlib.animation as animation

Nx = int(field_size.x)
Ny = int(field_size.y)
x = np.arange(min_coords.x, max_coords.x, (max_coords.x - min_coords.x)/Nx)
y = np.arange(min_coords.y, max_coords.y, (max_coords.y - min_coords.y)/Ny)

def get_fields():
    global field, x, y, Nx, Ny
    Ey = np.zeros(shape=(Ny, Nx))
    Bz = np.zeros(shape=(Ny, Nx))
    for ix in range(Nx):
        for iy in range(Ny):
            coord_xy = hichi.Vector3d(x[ix], y[iy], 0.0)
            E = field.get_E(coord_xy)
            Ey[iy, ix] = E.y
            B = field.get_B(coord_xy)
            Bz[iy, ix] = B.z
    return Ey, Bz

def update_data():
    for i in range(3):
        field.update_fields()

def compute_energy():
    (Ey, Bz) = get_fields()
    energy=0
    for ix in range(Nx):
        for iy in range(Ny):
            energy += Ey[iy, ix]**2 + Bz[iy, ix]**2
    return energy

(Ey, Bz) = get_fields()

fig, axes = plt.subplots(ncols=2, nrows=1)

im11 = axes[0].imshow(Ey, cmap='RdBu', interpolation='none', extent=(0, 2, 0, 1), animated = True)
fig.colorbar(im11, ax=axes[0])
axes[0].set_title("Ey")
axes[0].set_xlabel("x")
axes[0].set_ylabel("y")

im12 = axes[1].imshow(Bz, cmap='RdBu', interpolation='none', extent=(0, 2, 0, 1), animated = True)
fig.colorbar(im12, ax=axes[1])
axes[1].set_title("Bz")
axes[1].set_xlabel("x")
axes[1].set_ylabel("y")

iter = 0
def update_fig(*args):
    global iter
    update_data()
    (Ey, Bz) = get_fields()
    im11.set_array(Ey)
    im12.set_array(Bz)
    if (iter % 50 == 0):
        print("Energy = " + str(compute_energy()))
    iter += 1
    return im11, im12

ani = animation.FuncAnimation(fig, update_fig, interval=10, blit=True)

plt.tight_layout()

plt.show()

max_stars_repo_name: AleksandrPanov/pyHiChi

max_issues_repo_name: AleksandrPanov/pyHiChi

max_forks_repo_name: AleksandrPanov/pyHiChi


text:
from sympy.liealgebras.cartan_type import CartanType
from sympy.matrices import Matrix

def test_type_G():
    c = CartanType("G2")
    m = Matrix(2, 2, [2, -1, -3, 2])
    assert c.cartan_matrix() == m
    assert c.simple_root(2) == [1, -2, 1]
    assert c.basis() == 14
    assert c.roots() == 12
    assert c.dimension() == 3

max_stars_repo_name: torsknod/sympy-torsknod

max_issues_repo_name: torsknod/sympy-torsknod

max_forks_repo_name: torsknod/sympy-torsknod

reason: from sympy


text:
import timeit
import numpy as np

def load():
    import conllu
    load, read, write, text, relchain, save = [], [], [], [], [], []
    for _ in range(30):
        start = timeit.default_timer()
        data_file = open("cs-ud-train-l.conllu", "r", encoding="utf-8")
        # parse_incr is recommended for large files (more than 1 MB)
        # since it returns a generator, which is why a conversion to list
        tokenlist = list(conllu.parse_incr(data_file))
        end = timeit.default_timer()
        load.append(end - start)

        start = timeit.default_timer()
        for sentence in tokenlist:
            for token in sentence:
                form_lemma = token['form'] + token['lemma']
        end = timeit.default_timer()
        read.append(end - start)

        start = timeit.default_timer()
        for sentence in tokenlist:
            chain = []
            for token in sentence:
                if token['head']:
                    parent = sentence[token['head'] - 1]
                    if token['deprel'] == "case" and parent['deprel'] == "nmod":
                        chain.append(token)
        end = timeit.default_timer()
        relchain.append(end - start)

        start = timeit.default_timer()
        for sentence in tokenlist:
            for token in sentence:
                token['deprel'] = 'dep'
        end = timeit.default_timer()
        write.append(end - start)

        text.append(-1)

        start = timeit.default_timer()
        with open('out.conllu', 'w', encoding='utf8') as f:
            f.writelines([sentence.serialize() + "\n" for sentence in tokenlist])
        end = timeit.default_timer()
        save.append(end - start)

    for x, y in [('load', load), ('read', read), ('write', write), ('text', text), ('relchain', relchain), ('save', save)]:
        print("{}\t{} +/- {}".format(x, round(np.mean(y), 2), round(np.std(y), 2)))

if __name__ == '__main__':
    load()


text:
from itertools import cycle

import numpy as np

from ..categorical_colormap_utils import ColorCycle, compare_colormap_dicts

def test_color_cycle():
    color_values = np.array([[1, 0, 0, 1], [0, 0, 1, 1]])
    color_cycle = cycle(color_values)
    cc_1 = ColorCycle(values=color_values, cycle=color_cycle)
    cc_2 = ColorCycle(values=color_values, cycle=color_cycle)

    np.testing.assert_allclose(cc_1.values, color_values)
    assert isinstance(cc_1.cycle, cycle)
    assert cc_1 == cc_2

    other_color_values = np.array([[1, 0, 0, 1], [1, 1, 1, 1]])
    other_color_cycle = cycle(other_color_values)
    cc_3 = ColorCycle(values=other_color_values, cycle=other_color_cycle)
    assert cc_1 != cc_3

    # verify that checking equality against another type works
    assert cc_1 != color_values

def test_compare_colormap_dicts():
    cmap_dict_1 = {
        0: np.array([0, 0, 0, 1]),
        1: np.array([1, 1, 1, 1]),
        2: np.array([1, 0, 0, 1]),
    }
    cmap_dict_2 = {
        0: np.array([0, 0, 0, 1]),
        1: np.array([1, 1, 1, 1]),
        2: np.array([1, 0, 0, 1]),
    }
    assert compare_colormap_dicts(cmap_dict_1, cmap_dict_2)

    # same keys different values
    cmap_dict_3 = {
        0: np.array([1, 1, 1, 1]),
        1: np.array([1, 1, 1, 1]),
        2: np.array([1, 0, 0, 1]),
    }
    assert not compare_colormap_dicts(cmap_dict_1, cmap_dict_3)

    # different number of keys
    cmap_dict_4 = {
        0: np.array([1, 1, 1, 1]),
        1: np.array([1, 1, 1, 1]),
    }
    assert not compare_colormap_dicts(cmap_dict_1, cmap_dict_4)
    assert not compare_colormap_dicts(cmap_dict_3, cmap_dict_4)

    # same number of keys, but different keys
    cmap_dict_5 = {
        'hi': np.array([1, 1, 1, 1]),
        'hello': np.array([1, 1, 1, 1]),
        'hallo': np.array([1, 0, 0, 1]),
    }
    assert not compare_colormap_dicts(cmap_dict_1, cmap_dict_5)
    assert not compare_colormap_dicts(cmap_dict_3, cmap_dict_5)

max_stars_repo_name: MaksHess/napari

max_issues_repo_name: MaksHess/napari

max_forks_repo_name: MaksHess/napari


text:
import tensorflow as tf

import argparse
import numpy as np
from keras.callbacks import CSVLogger, ModelCheckpoint

from models.vaes import ARVAE
from utils.io import load_gzdata
from utils.data_loaders import one_hot_generator

# Define training parameters
batch_size = 32
seed = 0
n_epochs = 50
verbose = 1
save_all_epochs = False

seed = np.random.seed(seed)

# Load aligned sequences
_, raw_seqs = load_gzdata('data/training_data/ll_train.fa.gz', one_hot=False)
_, val_raw_seqs = load_gzdata('data/training_data/ll_val.fa.gz', one_hot=False)

# Define data generators
train_gen = one_hot_generator(raw_seqs, padding=504)
val_gen = one_hot_generator(val_raw_seqs, padding=504)

# Define model
print('Building model')
model = ARVAE()

# (Optionally) define callbacks
callbacks=[CSVLogger('output/logs/arvae.csv')]

if save_all_epochs:
    callbacks.append(ModelCheckpoint('output/weights/arvae'+'.{epoch:02d}-{luxa_errors_mean:.2f}.hdf5',
                                     save_best_only=False, verbose=1))

# Train model https://github.com/keras-team/keras/issues/8595
model.VAE.fit_generator(generator=train_gen,
                        steps_per_epoch=len(raw_seqs) // batch_size,
                        verbose=verbose,
                        epochs=n_epochs,
                        validation_data=val_gen,
                        validation_steps=len(val_raw_seqs) // batch_size,
                        callbacks=callbacks)

if not save_all_epochs:
  model.save_weights('output/weights/arvae.h5')

max_stars_repo_name: kellisfm/deep-protein-generation

max_issues_repo_name: kellisfm/deep-protein-generation

max_forks_repo_name: kellisfm/deep-protein-generation


text:
"""Script to run stacking scripts on the DESY cluster.

Through use of argparse, a given configuration for the code can be selected.
This can be given from the command line, in the form:

python RunCluster.py -c Desired_Configuration_Name -n Number_Of_Tasks -s

Each available configuration must be listed in "config.ini", and controls
options for fitting, such as which catalogue is to be used, and which seasons
of data should be included. If -x is included, then a new job is submitted
to the cluster. Having submitted the job to the cluster it will be run in
parallel Number_of_Tasks times. The shell script SubmitOne.sh is called for
each task, which in turn calls RunLocal.py with the given configuration setting.

The Wait function will periodically query the cluster
to check on the status of the job, and will output the job status occasionally.

Once all sub-tasks are completed, the script will proceed to call
MergeFiles.run() for the given configuration, combining results.

"""
import subprocess
import time
import os
import os.path
import logging
import argparse
import numpy as np
from flarestack.shared import log_dir, fs_dir
from flarestack.cluster.submitter import Submitter
from flarestack.cluster.make_desy_cluster_script import make_desy_submit_file, submit_file

logger = logging.getLogger(__name__)

username = os.path.basename(os.environ['HOME'])

cmd = 'qstat -u ' + username

def wait_for_cluster(job_ids=None):
    logger.warning('The wait_for_cluster function is deprecated! '
                   'Use the Submitter class instead.')
    Submitter.wait_for_cluster(job_ids)

    # if not job_ids:
    #     wait_for_job()
    # else:
    #     try:
    #         for i, job_id in enumerate(job_ids):
    #
    #             logger.debug(f'waiting for job {job_id}')
    #             prog_str = f'{i}/{len(job_ids)}'
    #             wait_for_job(job_id, prog_str)
    #
    #     except TypeError:
    #         logger.debug('Only waiting for one job')
    #         wait_for_job(job_ids)

def wait_for_job(job_id=None, progress_str=None):
    """
    Runs the command cmd, which queries the status of the job on the
    cluster, and reads the output. While the output is not an empty
    string (indicating job completion), the cluster is re-queried
    every 30 seconds. Occasionally outputs the number of remaining sub-tasks
    on cluster, and outputs full table result every ~ 8 minutes. On
    completion of job, terminates function process and allows the script to
    continue.
    """

    if not job_id:
        job_id_str = 's'
    else:
        if progress_str:
            job_id_str = f' {progress_str} {job_id}'
        else:
            job_id_str = ' ' + str(job_id)

    time.sleep(10)

    cmd = f'qstat -u {username}'
    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
    tmp = process.stdout.read().decode()
    n_total = n_tasks(tmp, job_id)
    i = 31
    j = 6
    while n_total != 0:
        if i > 3:

            running_process = subprocess.Popen(
                cmd + " -s r", stdout=subprocess.PIPE, shell=True)
            running_tmp = running_process.stdout.read().decode()

            if running_tmp != '':
                n_running = n_tasks(running_tmp, job_id)
            else:
                n_running = 0

            logger.info(f'{time.asctime(time.localtime())} - Job{job_id_str}:'
                         f' {n_total} entries in queue. '
                         f'Of these, {n_running} are running tasks, and '
                         f'{n_total-n_running} are tasks still waiting to be executed.')
            i = 0
            j += 1

        if j > 7:
            logger.info(str(tmp))
            j = 0

        time.sleep(30)
        i += 1
        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
        tmp = process.stdout.read().decode()
        n_total = n_tasks(tmp, job_id)

def submit_to_cluster(path, n_cpu=2, n_jobs=10, ram_per_core=None, **kwargs):

    for file in os.listdir(log_dir):
        os.remove(log_dir + file)

    # Submits job to the cluster

    submit_cmd = "qsub "

    if n_cpu > 1:
        submit_cmd += " -pe multicore {0} -R y ".format(n_cpu)

    ram_per_core = "{0:.1f}G".format(6./float(n_cpu) + 2.) if not ram_per_core else ram_per_core
    print("Ram per core:", ram_per_core)

    submit_cmd += "-t 1-{0}:1 {1} {2} {3}".format(
        n_jobs, submit_file, path, n_cpu
    )

    make_desy_submit_file(ram_per_core, **kwargs)

    print(time.asctime(time.localtime()), submit_cmd, "\n")

    process = subprocess.Popen(submit_cmd, stdout=subprocess.PIPE, shell=True)
    msg = process.stdout.read().decode()
    print(msg)
    job_id = int(str(msg).split('job-array')[1].split('.')[0])

    return job_id

def n_tasks(tmp, job_id):
    """
    Returns the number of tasks given the output of qsub
    :param tmp: output of qsub
    :param job_id: int, optional, if given only tasks belonging to this job will we counted
    :return: int
    """
    st = str(tmp)
    ids = np.array([int(s.split(' ')[2]) for s in st.split('\n')[2:-1]])

    if job_id:
        return len(ids[ids == job_id])
    else:
        return len(ids)

if not os.path.isfile(submit_file):
    make_desy_submit_file()

max_stars_repo_name: grburgess/flarestack

max_issues_repo_name: Raimer/flarestack

max_forks_repo_name: Raimer/flarestack


text:
import os

import numpy as np

from scipy.linalg import svd
from scipy.sparse.linalg import svds

from sklearn.utils.extmath import svd_flip

from glob import glob
from tqdm import trange, tqdm
import h5py

from nekio import *

def load_stability_results():
    # -->
    fname = "./data/STABILITY/RevCYL0.f00001"
    ωr = readnek(fname)["data"][:, :, -1].ravel()

    # -->
    fname = "./data/STABILITY/ImvCYL0.f00001"
    ωi = readnek(fname)["data"][:, :, -1].ravel()

    # -->
    fname = "./data/STABILITY/Spectre_NS_conv.dat"
    λr, λi = np.loadtxt(fname, unpack=True)

    return ωr + 1j * ωi, λr[0] + 1j * λi[0]

def load_base_flow():
    # --> Name of the file.
    fname = "./data/STABILITY/BF_CYL0.f00001"

    # --> Base flow vorticity field.
    ω = readnek(fname)["data"][:, :, -1].ravel()

    return ω

def main():

    # --> Get the Reynolds number to process.
    Re = sys.argv[1]

    # --> Load the base flow.
    X̄ = load_base_flow().reshape(-1, 1)

    # --> Load the stability results.
    X, λ = load_stability_results()

    # --> Store the data in the HDF5 archive.
    with h5py.File("cylinder_dataset.hdf5", "a") as f:

        # -->
        grp = require_group("RE{0}".format(Re))
        grp.attrs["Eigenvalue"] = λ

        # --> Store the base flow and stability mode.
        grp.create_dataset("Base flow", data=X̄)
        grp.create_dataset("Stability mode", data=X)

if __name__ == "__main__":
    main()