text:
import numpy as np
import pydicom
import os
import argparse
from DicomData import DicomData

def get_filepaths(directory):
    """
        This function will generate the file names in a directory
        tree by walking the tree either top-down or bottom-up. For each
        directory in the tree rooted at directory top (including top itself),
        it yields a 3-tuple (dirpath, dirnames, filenames).
    """
    file_paths = []  # List which will store all of the full filepaths.

    # Walk the tree.
    for root, directories, files in os.walk(directory):
        for filename in files:
            # Join the two strings in order to form the full filepath.
            filepath = os.path.join(root, filename)
            file_paths.append(filepath)  # Add it to the list.

    return file_paths

def get_volume(vol_dir):
    """
        Get phase or magnitude image volume
    """
    volume = []
    # Retrieve all the dicom filepaths
    files  = get_filepaths(vol_dir)

    for slice_nr, dicom_path in enumerate(files):
        ds = pydicom.dcmread(dicom_path)
        img = ds.pixel_array

        if slice_nr == 0:
            # Get this on the first slice only
            spacing = ds.PixelSpacing
            spacing.append(ds.SliceThickness)
            spacing = np.asarray(spacing)

            # Note: In our case, sequence name contains venc and direction info
            sequence_name = ds.SequenceName
            # print(sequence_name)

        volume.append(img)
    volume = np.asarray(volume)
    return volume, spacing, sequence_name

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--input-dir', type=str, required=True, help='Input directory contains the Dicom files with [Phase/Magnitude series desc]/[triggerTime] structure')
    parser.add_argument('--output-dir', type=str, default='../../Dataset', help='Output directory where the dataset will be saved')
    parser.add_argument('--output-filename', type=str, default='mri_data.h5', help='Output filename (HDF5)')
    parser.add_argument('--phase-pattern', type=str, default='_P_', help='Pattern of phase series description')
    parser.add_argument('--mag-pattern', type=str, default='_M_', help='Pattern of magnitude series description')
    parser.add_argument('--fh-mul', type=int, default= -1, help='Velocity multiplier for Foot-Head direction')
    parser.add_argument('--rl-mul', type=int, default=  1, help='Velocity multiplier for Right-Left direction')
    parser.add_argument('--in-mul', type=int, default=  1, help='Velocity multiplier for Inplane direction')
    args = parser.parse_args()

    case_dir      = args.input_dir
    phase_pattern = args.phase_pattern
    mag_pattern   = args.mag_pattern
    output_path   = args.output_dir
    output_filename = args.output_filename
    in_multiplier = args.in_mul
    fh_multiplier = args.fh_mul
    rl_multiplier = args.rl_mul

    output_filepath = f'{output_path}/{output_filename}'

    # 1. Get the phase and magnitude directories
    directories = os.listdir(case_dir)
    phase_dirs = [item for item in directories if phase_pattern in item]
    mag_dirs   = [item for item in directories if mag_pattern   in item]
    print("Phase dirs:\n", "\n".join(phase_dirs))
    print("Mag dirs:\n"  , "\n".join(mag_dirs))

    assert len(phase_dirs) == 3, f"There must be exactly 3 Phase directories with {phase_pattern} pattern"
    assert len(mag_dirs)   == 3, f"There must be exactly 3 Magnitude directories with {mag_pattern} pattern"

    # 2. Get and sort the trigger times
    dirpath = f'{case_dir}/{phase_dirs[0]}'
    timeFrames = os.listdir(dirpath)
    timeFrames = sorted(timeFrames, key=float)
    # print('All frames sorted:', timeFrames)

    # Create the output dir
    if not os.path.isdir(output_path):
        os.makedirs(output_path)

    # 3. Looping through the time frames
    for j in range(0, len(timeFrames)):
        triggerTime = timeFrames[j]
        print(f"\rProcessing {j+1}/{len(timeFrames)} (frame {triggerTime})          ", end="\r")

        # Wrap it as DicomData instance
        dicom_data = DicomData()
        # Collect the 3 phase and magnitude volumes for 1 time frame
        for mag_dir, p_dir in zip(mag_dirs, phase_dirs):
            magnitude_path = f'{case_dir}/{mag_dir}/{triggerTime}'
            phase_path = f'{case_dir}/{p_dir}/{triggerTime}'

            # Get the magnitude and phase images
            mag_images, _, _                = get_volume(magnitude_path)
            phase_images, spacing, sequence = get_volume(phase_path)

            dicom_data._phaseImages.append(phase_images)
            dicom_data._magImages.append(mag_images)
            dicom_data.sequenceNames.append(sequence)
            dicom_data.spacing = spacing
        # Save per row
        dicom_data.determine_velocity_components(in_multiplier, fh_multiplier, rl_multiplier)
        dicom_data.save_dataset(output_filepath, triggerTime)

    # End of trigger time loop
    print(f'\nDone! saved at {output_filepath}')

if __name__ == '__main__':
    main()


text:
#!/usr/bin/env python
# coding=utf-8
from pycocotools.coco import COCO
import numpy as np
annfile='/home/dataset/coco/coco2017/annotations/instances_train2017.json'
catNames=['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush']

catSuperNames=['appliance', 'food', 'indoor', 'accessory', 'electronic', 'furniture', 'vehicle', 'sports', 'animal', 'kitchen', 'person', 'outdoor']

coco = COCO(annfile)

### go through all category id and names
catIds = coco.getCatIds(catNms=catNames)
for index in range(len(catIds)):
    print('%d %s'%(catIds[index],catNames[index]))

### get one image info
catIds = coco.getCatIds(catNms='dog')
imgIds = coco.getImgIds(catIds=catIds)
img = coco.loadImgs(imgIds[np.random.randint(0, len(imgIds))])[0]
print(img)

### load Ann info
annIds = coco.getAnnIds(imgIds=img['id'], catIds=catIds, iscrowd=None)
anns = coco.loadAnns(annIds)
coco.showAnns(anns)

ext: py

max_stars_repo_name: smallcraft/cocoapi

max_issues_repo_name: smallcraft/cocoapi

max_forks_repo_name: smallcraft/cocoapi

reason: import numpy


text:
# Copyright 2018 Deepmind Technologies Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Defines some `predicates` for the predicate_task."""
import abc
import colorsys
import numpy as np

HSV_SATURATION = 0.5
HSV_ACTIVATED_SATURATION = 0.75
HSV_VALUE = 1.0

WALKER_GOAL_RGBA = [0, 0, 0]
WALKER_GOAL_PRESSED_RGBA = [128, 128, 128]
INACTIVE_OBSERVATION_VALUE = [-1] * 5

# Define globals for the special encoding.
MOVABLE_TYPES = {'walker': 0, 'box': 1}
TARGET_TYPES = {'box': 0, 'target': 1}
PREDICATE_TYPES = {'on': 0, 'close_to': 1, 'far_from': 2}

class BasePredicate(object, metaclass=abc.ABCMeta):
  """Base class for all predicates."""

  def __init__(self, walker):
    self._walker = walker

  @abc.abstractmethod
  def reinitialize(self, random_state):
    """Reinitializes a new, potentially random, predicate state.

    The reinitialize method should reset to a new predicate state which could
    update the `objects_in_use` by the `Predicate`. This method could be called
    multiple times before a finally binding predicate set has been found.
    Therefore no changes to the model that are not reversible should be made
    here (setting colors etc). Any changes affecting the Mujoco model should
    instead be made in the `activate_predicate` method call.

    Args:
      random_state: An instance of `np.RandomState` which may be seeded to
        ensure a deterministic environment.
    """
    pass

  @abc.abstractmethod
  def activate_predicate(self):
    """Activates the current predicate configuration.

    Any changes that are non-reversible like setting object properties or
    affinities *must* only be done in this method. At this point, the
    `predicate_task` logic has confirmed that a valid predicate configuration
    has been found.
    """
    pass

  @property
  def objects_in_use(self):
    """Returns the `set` of objects used for this episode."""
    return set()

  @abc.abstractproperty
  def observation_value(self):
    """Returns a `dict` to be used as the predicate observable."""
    pass

  @abc.abstractmethod
  def is_active(self, physics):
    """Boolean method indicating whether the predicate has been activated.

    If `True`, it implies the condition for the predicate has been satisfied
    and the walker can be rewarded.

    Args:
      physics: An instance of `control.Physics`.
    """
    pass

  @property
  def inactive_observation_value(self):
    """observation_value indicating a `Predicate` is inactive.

    The `PredicateTask` randomly samples the number of active predicates to be
    used on each episode. For a consistent `observation_spec`, the predicates
    that are not active need a special observation_value that cannot be used
    anywhere else.

    Returns:
      A special value indicating that the predicate is inactive and is not used
      by any other predicate in the task.
    """
    return INACTIVE_OBSERVATION_VALUE

class MoveWalkerToTarget(BasePredicate):
  """Predicate to move a walker to a specific target."""

  def __init__(self, walker, target, target_index=0):
    """Predicate to move a walker or box to a target.

    Args:
      walker: An locomotion `Walker` instance to use for this predicate.
      target: `locomotion.prop` instance containing an `activated` property.
      target_index: An 'int' argument to add to the observable to indicate the
        index of the target.
    """
    super(MoveWalkerToTarget, self).__init__(walker)
    self._target = target
    self._target_id = target_index

  def reinitialize(self, random_state):
    self._target.deregister_entities()

  def activate_predicate(self):
    self._target.register_entities(self._walker)
    self._target.set_colors(WALKER_GOAL_RGBA, WALKER_GOAL_PRESSED_RGBA)

  @property
  def objects_in_use(self):
    return set([self._walker, self._target])

  @property
  def observation_value(self):
    return np.array([
        MOVABLE_TYPES['walker'], 0, TARGET_TYPES['target'], self._target_id,
        PREDICATE_TYPES['close_to']
    ])

  def is_active(self, physics):
    return self._target.activated

class MoveWalkerToRandomTarget(BasePredicate):
  """Predicate to move a walker to a random target."""

  def __init__(self, walker, targets=None):
    """Predicate to move a walker or box to a target.

    Args:
      walker: An locomotion `Walker` instance to use for this predicate.
      targets: An optional list of `locomotion.prop` instances each of which
        contains an `activated` property.
    """
    super(MoveWalkerToRandomTarget, self).__init__(walker)
    self._targets = targets
    self._target_to_move_to = None

  def reinitialize(self, random_state):
    if self._target_to_move_to is not None:
      self._target_to_move_to.deregister_entities()
    self._target_to_move_to = random_state.choice(self._targets)
    self._target_idx = self._targets.index(self._target_to_move_to)

  def activate_predicate(self):
    self._target_to_move_to.register_entities(self._walker)
    self._target_to_move_to.set_colors(WALKER_GOAL_RGBA,
                                       WALKER_GOAL_PRESSED_RGBA)

  @property
  def objects_in_use(self):
    return set([self._walker, self._target_to_move_to])

  @property
  def observation_value(self):
    return np.array([
        MOVABLE_TYPES['walker'], 0, TARGET_TYPES['target'], self._target_idx,
        PREDICATE_TYPES['close_to']
    ])

  def is_active(self, physics):
    return self._target_to_move_to.activated

class MoveWalkerToBox(BasePredicate):
  """Predicate to move a walker to a specific box."""

  def __init__(self, walker, box, box_index=0, detection_region=None):
    """Predicate to move a walker to a specific box.

    Args:
      walker: An locomotion `Walker` instance to use for this predicate.
      box: A `manipulation.prop` instance to move.
      box_index: An integer index to use for the observable to identify the
        `box`.
      detection_region: A 2-tuple indicating the tolerances in x and y for the
        walker to be deemed `close_to` the box. If `None`, contact based
        detection is used.
    """
    super(MoveWalkerToBox, self).__init__(walker)
    self._box = box
    self._detection_region = detection_region
    self._box_index = box_index
    self._walker_geoms = None

  def reinitialize(self, random_state):
    if self._walker_geoms is None:
      # pylint: disable=protected-access
      self._walker_geoms = set(self._walker._mjcf_root.find_all('geom'))

  def activate_predicate(self):
    self._box.geom.rgba[:3] = WALKER_GOAL_RGBA

  @property
  def objects_in_use(self):
    return set([self._walker, self._box])

  @property
  def observation_value(self):
    return np.array([
        MOVABLE_TYPES['walker'], 0, TARGET_TYPES['box'], self._box_index,
        PREDICATE_TYPES['close_to']
    ])

  def is_active(self, physics):
    if self._detection_region is None:
      return self._is_walker_contacting_box(physics)
    else:
      return np.all(
          np.abs(
              physics.bind(self._walker.root_body).xpos -
              physics.bind(self._box.geom).xpos)[:2] < self._detection_region)

  def _is_walker_contacting_box(self, physics):
    walker_geom_ids = [
        physics.bind(geom).element_id for geom in self._walker_geoms
    ]
    for contact in physics.data.contact:
      contact_geoms = set([contact.geom1, contact.geom2])
      if (physics.bind(self._box.geom).element_id in contact_geoms and
          contact_geoms.intersection(walker_geom_ids)):
        return True
    return False

class MoveBoxToBox(BasePredicate):
  """Predicate to move a walker to a specific box."""

  def __init__(self,
               walker,
               first_box,
               second_box,
               first_box_index=0,
               second_box_index=1,
               detection_region=None):
    """Predicate to move a walker to a specific box.

    Args:
      walker: An locomotion `Walker` instance to use for this predicate.
      first_box: A `manipulation.prop` instance to move.
      second_box: A `manipulation.prop` instance to move.
      first_box_index: An integer index to use for the observable to identify
        the  `box`.
      second_box_index: An integer index to use for the observable to identify
        the  `box`.
      detection_region: A 2-tuple indicating the tolerances in x and y for the
        walker to be deemed `close_to` the box. If `None`, contact based
        detection is used.
    """
    super(MoveBoxToBox, self).__init__(walker)
    self._first_box = first_box
    self._second_box = second_box
    self._detection_region = detection_region
    self._first_box_index = first_box_index
    self._second_box_index = second_box_index
    self._walker_geoms = None

  def reinitialize(self, random_state):
    if self._walker_geoms is None:
      # pylint: disable=protected-access
      self._walker_geoms = set(self._walker._mjcf_root.find_all('geom'))

  def activate_predicate(self):
    self._first_box.geom.rgba[:3] = WALKER_GOAL_RGBA

  @property
  def objects_in_use(self):
    return set([self._first_box, self._second_box])

  @property
  def observation_value(self):
    return np.array([
        MOVABLE_TYPES['box'], self._first_box_index, TARGET_TYPES['box'],
        self._second_box_index, PREDICATE_TYPES['close_to']
    ])

  def is_active(self, physics):
    if self._detection_region is None:
      return self._are_boxes_in_contact(physics)
    else:
      return np.all(
          np.abs(
              physics.bind(self._first_box.geom).xpos -
              physics.bind(self._second_box.geom).xpos)[:2] <
          self._detection_region)

  def _are_boxes_in_contact(self, physics):
    for contact in physics.data.contact:
      contact_geoms = set([contact.geom1, contact.geom2])
      if (physics.bind(self._first_box.geom).element_id in contact_geoms and
          physics.bind(self._second_box.geom).element_id in contact_geoms):
        return True
    return False

class MoveBoxToTarget(BasePredicate):
  """Predicate to move a walker to a specific target."""

  def __init__(self, walker, box, target, box_index=0, target_index=0):
    """Predicate to move a walker or box to a target.

    Args:
      walker: An locomotion `Walker` instance to use for this predicate.
      box: A `manipulation.prop` to move to the target.
      target: `locomotion.prop` instance containing an `activated` property.
      box_index: An 'int' argument to add to the observable to indicate the
        index of the box.
      target_index: An 'int' argument to add to the observable to indicate the
        index of the target.
    """
    super(MoveBoxToTarget, self).__init__(walker)
    self._box = box
    self._target = target
    self._box_id = box_index
    self._target_id = target_index
    self._original_box_size = np.copy(box.geom.size)
    self._rgb = None
    self._activated_rgb = None

  def reinitialize(self, random_state):
    self._target.deregister_entities()
    self._get_box_properties(random_state)

  def _get_box_properties(self, random_state):
    hue0 = random_state.uniform()
    hue = (hue0 + self._target_id) % 1.0
    self._rgb = colorsys.hsv_to_rgb(hue, HSV_SATURATION, HSV_VALUE)
    self._activated_rgb = colorsys.hsv_to_rgb(hue, HSV_ACTIVATED_SATURATION,
                                              HSV_VALUE)

  def activate_predicate(self):
    self._target.set_colors(self._rgb, self._activated_rgb)
    self._box.geom.rgba[:3] = self._rgb
    self._target.register_entities(self._box)

  @property
  def objects_in_use(self):
    return set([self._box, self._target])

  @property
  def observation_value(self):
    return np.array([
        MOVABLE_TYPES['box'], self._box_id, TARGET_TYPES['target'],
        self._target_id, PREDICATE_TYPES['close_to']
    ])

  def is_active(self, physics):
    return self._target.activated

class MoveBoxToRandomTarget(BasePredicate):
  """Predicate to move a walker to a random target."""

  def __init__(self, walker, box, box_index=0, targets=None):
    """Predicate to move a walker or box to a target.

    Args:
      walker: An locomotion `Walker` instance to use for this predicate.
      box: A `manipulation.prop` to move to the target.
      box_index: An optional 'int' argument to add to the observable to indicate
        the index of the box.
      targets: An optional list of `locomotion.prop` instances each of which
        contains an `activated` property.
    """
    super(MoveBoxToRandomTarget, self).__init__(walker)
    self._targets = targets
    self._box_to_move = box
    self._box_index = box_index
    self._target_to_move_to = None
    self._original_box_size = np.copy(box.geom.size)
    self._rgb = None
    self._activated_rgb = None

  def reinitialize(self, random_state):
    if self._target_to_move_to is not None:
      self._target_to_move_to.deregister_entities()
    self._target_to_move_to = random_state.choice(self._targets)
    self._target_idx = self._targets.index(self._target_to_move_to)
    self._get_box_properties(random_state)

  def _get_box_properties(self, random_state):
    hue0 = random_state.uniform()
    hue = (hue0 + (self._target_idx / len(self._targets))) % 1.0
    self._rgb = colorsys.hsv_to_rgb(hue, HSV_SATURATION, HSV_VALUE)
    self._activated_rgb = colorsys.hsv_to_rgb(hue, HSV_ACTIVATED_SATURATION,
                                              HSV_VALUE)

  def activate_predicate(self):
    self._target_to_move_to.set_colors(self._rgb, self._activated_rgb)
    self._box_to_move.geom.rgba[:3] = self._rgb
    self._target_to_move_to.register_entities(self._box_to_move)

  @property
  def objects_in_use(self):
    return set([self._box_to_move, self._target_to_move_to])

  @property
  def observation_value(self):
    return np.array([
        MOVABLE_TYPES['box'], self._box_index,
        TARGET_TYPES['target'], self._target_idx,
        PREDICATE_TYPES['close_to']
    ])

  def is_active(self, physics):
    return self._target_to_move_to.activated

max_stars_repo_name: kawa-work/deepmind-research

max_issues_repo_name: subhayuroy/deepmind-research

max_forks_repo_name: subhayuroy/deepmind-research


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Jun  5 16:22:00 2018

@author: lijie
"""
from numpy import *
from adaboosting import *

'''simple classification test'''
def adaClassifier(dataToClass,classifierArr):
    dataMatrix = mat(dataToClass)
    m = shape(dataMatrix)[0]
    aggClassEst = mat(zeros((m,1)))
    for i in range(len(classifierArr)):
        classEst = stumpClassify(dataMatrix,classifierArr[i]['dim'],classifierArr[i]['thresh'],classifierArr[i]['ineq'])
        #stumpClassify(dataMatrix,dimen,threshVal,threshIneq)
        aggClassEst += classifierArr[i]['alpha']*classEst
        print(aggClassEst)
    return sign(aggClassEst)

if __name__=='__main__' :
#test
    dataArr,labelArr=testData()
    classifier=adaBoostTrainDS(dataArr,labelArr,numIt=30)
    classifierArr=classifier[0]
    adaClassifier([0,0],classifierArr)

reason: from numpy


text:
import math
import re
import networkx as nx
from parse import *
import copy
from collections import defaultdict
import itertools
import numpy as np
from math import cos, sin, pi
import contextlib
from unittest import TestCase

def read_input(filename="input.txt"):
    with open(filename) as f:
        blocks = f.read().split("\n\n")
    print(blocks)
    tiles = dict()
    for t in blocks:
        head, body = t.split(":\n")
        _, num = head.split("Tile ")
        tiles[num] = body
    inp = tiles
    return inp

def c_to_num(c):
    if c == "#":
        return 1
    elif c == ".":
        return 0
    else:
        raise ValueError

def get_rot_and_flip_variations(img):
    return [np.rot90(img, k=n) for n in range(4)] + [
        np.rot90(np.fliplr(img), k=n) for n in range(4)
    ]

class Tile:
    def __init__(self, num, body):
        self.num = num
        self._raw = body
        self.b = np.array([[c_to_num(c) for c in s] for s in body], int)
        self.rotations = get_rot_and_flip_variations(self.b)
        assert len(self.rotations) == 8

def get_next_empty(grid, size):
    """Grid will be filled left to right and top to bottom to simplify neighbour checks."""
    for r in range(size):
        for c in range(size):
            if grid[r, c] is None:
                return r, c

def get_rotated_tile(pos, tiles_dict):
    """Return the variation given by the pos tuple.

    pos: (tile_id, rotation_id)
    """
    return tiles_dict[pos[0]].rotations[pos[1]]

def lr_fit(left, right, tiles_dict):
    """Check whether both tiles fit next to one another."""
    left_rc = get_rotated_tile(left, tiles_dict)[:, -1]
    right_lc = get_rotated_tile(right, tiles_dict)[:, 0]
    if all(left_rc == right_lc):
        return True
    return False

def tb_fit(top, bottom, tiles_dict):
    """Check whether both tiles fit on top of each other."""
    top_br = get_rotated_tile(top, tiles_dict)[-1, :]
    bottom_tr = get_rotated_tile(bottom, tiles_dict)[0, :]
    if all(top_br == bottom_tr):
        return True
    return False

def get_candidates(grid, tiles_left, location, tiles_dict):
    """Return a list with all candidates that fit in the specified location.

    Because we fill the grid from left to right and top to bottom, we only
    need to check whether the tiles fit with the left and top neighbours (unless
    they are on the first row or column."""
    # print(f"looking for candidates at location {location}")
    all_positions_left = itertools.product(tiles_left, range(8))
    candidates = []
    r, c = location
    for pos in all_positions_left:
        if r != 0:
            if not tb_fit(grid[(location[0] - 1, location[1])], pos, tiles_dict):
                continue
        if c != 0:
            if not lr_fit(grid[(location[0], location[1] - 1)], pos, tiles_dict):
                continue
        candidates.append(pos)
    return candidates

def solve(grid, tiles_left, size, tiles_dict):
    """Run an iteration to solve the grid.

    As long as there are tiles left to place, it tries each of them in any
    possible position (rotation and flip) until the remaining tiles in the new
    grid can also be solved."""
    if not tiles_left:
        return True
    location = get_next_empty(grid, size)
    potential_positions = get_candidates(grid, tiles_left, location, tiles_dict)
    while potential_positions:
        t = potential_positions.pop()
        grid[location] = t
        new_tiles_left = [ti for ti in tiles_left if t[0] != ti]
        result = solve(grid, new_tiles_left, size, tiles_dict)
        if result:
            return True
        else:
            grid[location] = None  # remove the tile that did not fit from the grid
    return False

def create_image(grid, side_length, tiles_dict):
    """Part 2 image from a given grid."""
    blocks = []
    for r in range(side_length):
        r_blocks = []
        for c in range(side_length):
            pos = grid[(r, c)]
            t = get_rotated_tile(pos, tiles_dict)
            t_strip = t[1:-1, 1:-1]
            r_blocks.append(t_strip)
        blocks.append(np.concatenate(r_blocks, axis=1))
    return np.concatenate(blocks, axis=0)

def count_sea(variation, sea_monster):
    """Find monsters and return empty sea water count."""
    monster_shape = sea_monster.shape
    variation_shape = variation.shape
    monster_space = np.sum(sea_monster)
    monsters_found = 0
    for r in range(variation_shape[0] - monster_shape[0]):
        for c in range(variation_shape[1] - monster_shape[1]):
            search_space = variation[r: r + monster_shape[0], c: c + monster_shape[1]]
            assert search_space.shape == monster_shape
            if np.sum(np.multiply(search_space, sea_monster)) == monster_space:
                print("found a monster! coords:", r, c)
                monsters_found += 1
    return np.sum(variation) - monsters_found * monster_space

def main(input_file):
    """Solve puzzle and connect part 1 with part 2 if needed."""
    # part 1
    inp = read_input(input_file)
    tiles = list()
    for t, b in inp.items():
        body = [s.strip() for s in b.splitlines() if s.strip()]
        tiles.append(Tile(t, body))

    side_length = int(round(math.sqrt(len(tiles))))
    print(side_length)
    tiles_dict = {t.num: t for t in tiles}
    grid = {(r, c): None for r in range(side_length) for c in range(side_length)}
    tiles_left = [t.num for t in tiles]
    solution = solve(grid, tiles_left, side_length, tiles_dict)
    print(grid)
    p1 = (
            int(grid[0, 0][0])
            * int(grid[0, side_length - 1][0])
            * int(grid[side_length - 1, 0][0])
            * int(grid[side_length - 1, side_length - 1][0])
    )

    # part 2
    img = create_image(grid, side_length, tiles_dict)
    sea_monster = np.array(
        [
            [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
            [1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1],
            [0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0],
        ]
    )
    print(img)
    print(img.shape)
    non_monsters = []
    for variation in get_rot_and_flip_variations(img):
        non_monsters.append(count_sea(variation, sea_monster))

    print("Non monsters", non_monsters)
    p2 = min(non_monsters)
    print(f"Solution to part 1: {p1}")
    print(f"Solution to part 2: {p2}")
    return p1, p2

def test_samples(self):
    input_file = "sample_1.txt"
    p1, p2 = main(input_file)
    self.assertEqual(20899048083289, p1)
    self.assertEqual(273, p2)
    print("***Tests passed so far***")

if __name__ == "__main__":
    test_samples(TestCase())
    p1, p2 = main("full.txt")
    assert p1 == 107399567124539
    assert p2 == 1555

max_stars_repo_name: dandiez/AdventOfCode

max_issues_repo_name: dandiez/AdventOfCode

max_forks_repo_name: dandiez/AdventOfCode

reason: import numpy,import networkx


text:
import os
import sys
import argparse
import numpy as np
import pandas as pd
import datetime
import bisect
import re
from clean import simple_clean
from clean import complex_clean
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import accuracy_score
from sklearn.svm import LinearSVC
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB
import matplotlib.pyplot as plt
import csv
from modelutils import modelutils
from sklearn.metrics import precision_recall_fscore_support

# this version allows for multiple fit methods (SVM, Logistic, Logistic Lasso)
# this method also computes the mean and std deviation of the accuracy by
# this version also allows for ngrams
# allows the text preprocessing algorithm to be selected
# running each model Niter Times

# python ./model2_0.py --ngram 1,1 2,2 3,3 1,3 2,3 --Niter 3 --pctTrain .8 --data minutes speeches statements

def runModels(models, model_data_set, Nitr, pctTrain, ngram):
    results, prec,recall,f1, trainPos, testPos=[],[],[],[], np.zeros(Nitr), np.zeros(Nitr)
    for iter in range(Nitr):
        train_data, test_data = modelutils.splitTrainTest(model_data_set, pctTrain)
        training_features, test_features = modelutils.getFeatures(train_data, test_data, ngram)
        for i, m in enumerate(models):
            model=m[1]
            model.fit(training_features, train_data["ActionFlag"])
            y_pred = model.predict(test_features)
            acc = accuracy_score(test_data["ActionFlag"], y_pred)
            if iter == 0:
                results.append(np.zeros(Nitr))
                prec.append(np.zeros(Nitr))
                recall.append(np.zeros(Nitr))
                f1.append(np.zeros(Nitr))
            results[i][iter]=acc
            prec_recall = precision_recall_fscore_support(test_data['ActionFlag'].tolist(), y_pred, average='binary')
            prec[i][iter] = prec_recall[0]
            recall[i][iter] = prec_recall[1]
            f1[i][iter] = prec_recall[2]
        trainPos[iter] = train_data["ActionFlag"].sum()/ len(train_data)
        testPos[iter] = test_data["ActionFlag"].sum()/ len(test_data)
    return results, trainPos, testPos, prec, recall, f1

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='SocialNetworks CSV to HDF5')
    parser.add_argument('--decision', default="../text/history/RatesDecision.csv")
    parser.add_argument('--minutes', default="../text/minutes")
    parser.add_argument('--speeches', default="../text/speeches")
    parser.add_argument('--statements', default="../text/statements")
    parser.add_argument('--pctTrain', default=0.75, type=float)
    parser.add_argument('--outputFileName', default='output_data.csv')
    parser.add_argument('--Niter', help="number of times to refit data", default=10, type=int)
    parser.add_argument('--cleanAlgo', default="complex")
    parser.add_argument('--ngram', nargs='+', default=['1,1'])
    parser.add_argument('--max_iter', help="max iterations for sklearn solver", default=25000, type=int)
    parser.add_argument('--solver', help="solver for sklearn algo", default='liblinear')
    parser.add_argument('--data', nargs="+", default=["minutes", "speeches", "statements"])
    args = parser.parse_args()

    ngrams = [(int(x.split(",")[0]),int(x.split(",")[1])) for x in args.ngram]
    clean_algo = complex_clean if args.cleanAlgo == "complex" else simple_clean
    pctTrain, cleanA, Niter, ngram = args.pctTrain, args.cleanAlgo, args.Niter, args.ngram
    solver, max_iter, datasetlist = args.solver, args.max_iter, args.data
    outputFileName = args.outputFileName

    assert len(datasetlist) > 0, "no data sets specified"
    datasetlabel=":".join(d for d in datasetlist)

    # fed rate decison matrix
    df = modelutils.decisionDF(args.decision)

    # datsets
    data_set, N, start, end = [], 0, datetime.datetime.now(), datetime.datetime(1970, 1, 1)
    if "minutes" in datasetlist:
        data_set.append(modelutils.getMinutes(args.minutes, df, clean_algo))
        (N, start, end) = modelutils.getBounds(data_set[-1])
    if "speeches" in datasetlist:
        data_set.append(modelutils.getSpeeches(args.speeches, df, clean_algo))
        (N1, start1, end1) = modelutils.getBounds(data_set[-1])
        N , start, end = N + N1, min(start, start1), max(end, end1)
    if "statements" in datasetlist:
        data_set.append(modelutils.getStatements(args.statements, df, clean_algo))
        (N1, start1, end1) = modelutils.getBounds(data_set[-1])
        N , start, end = N + N1, min(start, start1), max(end, end1)

    assert N  > 0, "no data in data_set"
    start, end = start.strftime("%m/%d/%Y"), end.strftime("%m/%d/%Y")

    # Train on current minutes
    models=[("svm",LinearSVC(max_iter=max_iter)),
            ("logistic",LogisticRegression(solver=solver,max_iter=max_iter)),
            ("logistic_lasso",LogisticRegression(penalty='l1',solver=solver,max_iter=max_iter)),
            ("Naive Bayes",MultinomialNB())]

    print("Determining Fed Action from minutes")
    print("%-20s %5s %5s %10s %10s %5s %8s %7s %10s %10s %-27s %6s %6s %6s %6s %6s" %
          ("Model Name", "NGram", "Niter", "mean(acc)", "std(acc)","N","PctTrain", "clean", "start", "end", "Data Sets", "TrainP", "TestP", "Prec", "Recall", "F1"))

    outputDF = []

    for ngram in ngrams:
        results, trainPos, testPos, prec, recall, f1 = runModels(models, data_set, Niter, pctTrain, ngram)
        ngramstr = str(ngram[0]) + ":" + str(ngram[1])
        for m, r, t, u, v in zip(models, results, prec, recall, f1):
            name, mu, s, precMu, recallMu, f1Mu = m[0], np.mean(r), np.std(r), np.mean(t), np.mean(u), np.mean(v)
            print("%-20s %5s %5s %10.4f %10.4f %5d %8.3f %7s %10s %10s %-27s %6.3f %6.3f  %6.3f %6.3f  %6.3f " %
                  (name, ngramstr, Niter, mu, s, N, pctTrain, cleanA, start, end, datasetlabel, np.mean(trainPos), np.mean(testPos), precMu, recallMu, f1Mu))

            outputDF.append([name, ngramstr, Niter, mu, s, N, pctTrain, cleanA, start, end, datasetlabel, np.mean(trainPos), np.mean(testPos), precMu, recallMu, f1Mu])
        print("")

    outputDF = pd.DataFrame(outputDF)
    outputDF.to_csv('../text/data_for_graphs/'+outputFileName, index=False, header=["Model Name", "NGram", "Niter", "mean(acc)", "std(acc)","N","PctTrain", "clean", "start", "end", "Data Sets", "TrainP", "TestP", "Prec", "Recall", "F1"])


text:
import pytest
import numpy.testing as npt

from lenstronomy.Cosmo.background import Background

class TestCosmoProp(object):

    def setup(self):
        self.z_L = 0.8
        self.z_S = 3.0
        from astropy.cosmology import FlatLambdaCDM
        cosmo = FlatLambdaCDM(H0=70, Om0=0.3, Ob0=0.05)
        self.bkg = Background(cosmo=cosmo)

    def test_scale_factor(self):
        z = 0.7
        assert self.bkg.a_z(z) == 1./(1+z)

    def test_rho_crit(self):
        assert self.bkg.rho_crit == 135955133951.10692

    def test_interpol(self):
        from astropy.cosmology import FlatLambdaCDM
        cosmo = FlatLambdaCDM(H0=70, Om0=0.3, Ob0=0.05)
        bkg = Background(cosmo=cosmo)

        bkg_interp = Background(cosmo=cosmo, interp=True, num_interp=100, z_stop=10)
        d_xy = bkg.d_xy(z_observer=0.1, z_source=0.8)
        d_xy_interp = bkg_interp.d_xy(z_observer=0.1, z_source=0.8)
        npt.assert_almost_equal(d_xy_interp / d_xy, 1, decimal=5)

if __name__ == '__main__':
    pytest.main()

reason: import numpy,from astropy


text:
## Generic Plotting Outline for rhovsenergy.out data
##
## Author:  Zander Mausolff
## Usage:  python plt_cmdline-args.py
## Usage help: python plt_cmdline-args.py help
import os
import numpy as np
import matplotlib.pyplot as plt
import sys
import re

##-------Change font to 'Palatino' throughout the plot---##
##-------This will match latex documents exactly---------##
from matplotlib import rcParams

current_dir = os.getcwd()
##-------Check if user wants help w/cmd line options
arglist = str(sys.argv)

f=plt.figure()

##------Shape functions------##
def shape_fcn(fcn,x,i):
    if (i == 0) :
        fcn = -0.5*x*(1.0 - x)
    if (i == 1) :
        fcn = (1 + x)*(1 - x)
    if (i == 2) :
        fcn = 0.5*x*(1 + x)
    return fcn

data_file_names="data_file_names.txt"
file_names = [line.rstrip('\n') for line in open(data_file_names)]
tick_marks = ['--',':','-','.']

i=0
color = ['.', 'o', '*', '^']

#set number of precursor groups
labels = ['Lagged','Instant']
skp=1
while i < len(file_names):
    print file_names[i]

    data1 = np.loadtxt(current_dir + "/" + file_names[i] , skiprows=skp)
    time = data1[:,0]
    mass_flow = data1[:,5]
    beta   = data1[:,4]

    plt.plot(time,beta,tick_marks[i],label=labels[i] )
    plt.ylabel('Beta',size=14)

    #plt.xlabel('Mass Flow [g/s]',size=14)
    plt.xlabel('Time [s]',size=14)
    plt.ylim(6.00E-3, 7.0E-3)
    i=i+1

## Name of saved image
name = 'beta_variation'

##------Configure the legend --- ##
plt.legend( loc='upper left',  shadow=True)

if re.search(r'\bpng\b',arglist[1:]):
	f.savefig("./" + name + ".png", bbox_inches = "tight")

if re.search(r'\bpdf\b',arglist[1:]):
        f.savefig("./" + name + ".pdf", bbox_inches = "tight")

if re.search(r'\bs\b',arglist[1:]):
	plt.show()


text:
"""
Utilities for benchmarking calibration methods.
"""

from kbbq import compare_reads
import pysam
import numpy as np

def get_ref_dict(reffilename):
    fasta = pysam.FastaFile(reffilename)
    ref = {chrom : np.array(list(fasta.fetch(reference = chrom)), dtype = np.unicode) for chrom in fasta.references}
    return ref

def get_var_sites(vcf):
    vcf = pysam.VariantFile(vcf)
    d = dict()
    for record in vcf:
        for i in range(record.start, record.stop, 1):
            d.setdefault(record.chrom, list()).append(i)
    return d

def get_bed_dict(refdict, bedfh):
    beddict = {chrom: np.zeros(len(refdict[chrom]), dtype = np.bool) for chrom in refdict.keys()}
    for bedline in pysam.tabix_iterator(bedfh, parser = pysam.asBed()):
        beddict[bedline.contig][bedline.start:bedline.end] = True #end is 1 past the actual end, so this slice should work properly
    return beddict

def get_full_skips(refdict, var_sites, bedfh = None):
    skips = {chrom: np.zeros(len(refdict[chrom]), dtype = np.bool) for chrom in refdict.keys()}
    for chrom in skips.keys():
        variable_positions = np.array(var_sites[chrom], dtype = np.int)
        skips[chrom][variable_positions] = True

    if bedfh is not None:
        beddict = get_bed_dict(refdict, bedfh)
        for chrom in skips.keys():
            skips[chrom][~beddict[chrom]] = True #skip anything not in the BED

    return skips

def get_bam_readname(read):
    """
    Given a bam read, get a canonicalized name.
    The name is the query name + a suffix
    based on whether it is first or second in pair.
    """
    suffix = ("/2" if read.is_read2 else "/1")
    return read.query_name + suffix

def get_fastq_readname(read):
    """
    Given a fastq read, get a canonicalized name.
    This is based on whether the read is first in pair.
    """
    return read.name.split(sep = '_')[0]

def get_error_dict(bamfile, refdict, fullskips):
    """
    Finds errors given an authoritative BAM, ref, and sites to skip.

    The returned dict has readname keys and tuple(np.array, np.array) of bools as values.
    """
    edict = dict()
    for read in bamfile:
        name = get_bam_readname(read)
        e, s = compare_reads.find_read_errors(read, refdict, fullskips)
        #we have to reverse e and s because the samtools fastq command will
        #automatically reverse them, causing the values from the fastq to
        #not match up.
        if read.is_reverse:
            e = np.flip(e)
            s = np.flip(s)
        edict[name] = (e,s)
    return edict

def calculate_q(errors, quals):
    """
    Calculate Actual Q and Predicted Q given a flat
    array of errors and flat array of quality scores.

    The index of the returned array represents the predicted quality score,
    the value represents the actual quality score or the number of bases.
    """
    numtotal = np.bincount(quals.reshape(-1))
    numerrs = np.bincount(quals[errors].reshape(-1), minlength = len(numtotal))
    nonzero = (numtotal != 0)
    p = np.true_divide(numerrs[nonzero], numtotal[nonzero])
    q = compare_reads.p_to_q(p)
    actual_q = np.zeros(len(numtotal), dtype = np.int)
    actual_q[nonzero] = q
    return actual_q, numtotal

def benchmark_fastq(fqfile, bamfile, ref, var_sites, bedfh = None):
    fullskips = get_full_skips(ref, var_sites, bedfh)
    edict = get_error_dict(bamfile, ref, fullskips)
    errors, skips, quals = zip(*(edict[get_fastq_readname(r)] + (np.array(r.get_quality_array()),) for r in pysam.FastxFile(fqfile)))
    #turn list of small arrays into 2d arrays
    errors = np.concatenate(errors)
    skips = np.concatenate(skips)
    quals = np.concatenate(quals)
    #get rid of skipped sites (we can't tell if these are errors or not)
    e = errors[~skips]
    q = quals[~skips]
    return calculate_q(e, q) #actual_q, ntotal

def get_bamread_quals(read, use_oq = False):
    """
    Return the qualities of the read as an np.array.

    Use the OQ tag for read quals if use_oq = True.
    """
    if use_oq:
        return compare_reads.bamread_get_oq(read)
    else:
        return np.array(read.query_qualities, dtype = np.int)

def benchmark_bam(bamfile, ref, var_sites, use_oq = False, bedfh = None):
    fullskips = get_full_skips(ref, var_sites, bedfh)
    errors, skips, quals = zip(*(compare_reads.find_read_errors(read, ref, fullskips) + (get_bamread_quals(read, use_oq),) for read in bamfile)) #generator
    #turn list of 1d arrays into 2d arrays
    errors = np.concatenate(errors)
    skips = np.concatenate(skips)
    quals = np.concatenate(quals)
    #get rid of skipped sites
    e = errors[~skips]
    q = quals[~skips]
    return calculate_q(e, q)

def print_benchmark(actual_q, label, nbases):
    """
    Print benchmark data to a tab separated table.

    Label should be a string and is used for all values
    in the table. No headers are printed. This means
    the benchmark command can be called many times and each
    table can be concatenated together.
    """
    nonzero = (nbases != 0)
    nbases = nbases[nonzero]
    predicted_q = np.arange(len(actual_q))[nonzero]
    actual_q = actual_q[nonzero]
    for pq, aq, nb in zip(predicted_q, actual_q, nbases):
        print(pq, aq, label, nb, sep = "\t")

def benchmark(bamfile, fafile, vcffile, fastqfile = None, label = None, use_oq = False, bedfh = None):
    """
    Perform the benchmark and print the results to stdout.

    If a fastqfile is not given, the benchmark will be taken from
    the reads in the BAM file. If a fastqfile is given, it should
    have the same read names as the reads in the BAM file. The
    names will then be used to look up where errors occur in the
    reads.
    """
    bam = pysam.AlignmentFile(bamfile, 'r')
    ref = get_ref_dict(fafile)
    var_sites = get_var_sites(vcffile)
    if fastqfile is not None:
        actual_q, nbases = benchmark_fastq(fastqfile, bam, ref, var_sites, bedfh)
        label = (fastqfile if label is None else label)
    else:
        actual_q, nbases = benchmark_bam(bam, ref, var_sites, use_oq, bedfh)
        label = (bamfile if label is None else label)
    print_benchmark(actual_q, label, nbases)

max_stars_repo_name: adamjorr/kbbq-py

max_issues_repo_name: adamjorr/kbbq-py

max_forks_repo_name: adamjorr/kbbq-py


text:
"""Compile BBox position from meta file into training vector.

The training output `y` is a feature map with 5 features: label, BBox centre
relative to anchor, and BBox absolute width/height.

The label values, ie the entries in y[0, :, :], are non-negative integers. A
label of zero always means background.
"""
import os
import bz2
import sys
import glob
import tqdm
import json
import pickle
import argparse
import multiprocessing
import inspect_feature

import numpy as np
import PIL.Image as Image
import PIL.ImageFilter as ImageFilter
from feature_utils import ft2im, downsampleMatrix

def parseCmdline():
    """Parse the command line arguments."""
    # Create a parser and program description.
    parser = argparse.ArgumentParser(description='Compile training data')
    parser.add_argument(
        'path', nargs='?', type=str,
        metavar='path',
        help='File or Path with training images and *-meta.json.bz2')
    parser.add_argument(
        '--debug', action='store_true', default=False,
        help='Create debug plots for instant inspection')

    param = parser.parse_args()
    if param.path is None:
        cur_dir = os.path.dirname(os.path.abspath(__file__))
        param.path = os.path.join(cur_dir, 'data', '3dflight')

    if not os.path.exists(param.path):
        print(f'Error: cannot open <{param.path}>')
        sys.exit(1)

    if os.path.isdir(param.path):
        fnames = glob.glob(os.path.join(param.path, '*.jpg'))
    else:
        fnames = [param.path]

    param.fnames = [_[:-4] for _ in sorted(fnames)]
    return param

def _computeBBoxes(bb_rects, objID_at_pixel_ft, im_dim):
    # Find all feature map locations that show anything but background.
    fg_idx = np.nonzero(objID_at_pixel_ft)
    ft_dim = objID_at_pixel_ft.shape

    # Convert the absolute BBox corners to relative values with respect to
    # the anchor point (all in image coordinates).
    bboxes = np.zeros((4, *ft_dim), np.float32)
    for y, x in zip(*fg_idx):
        objID = objID_at_pixel_ft[y, x]
        anchor_x = ft2im(x, ft_dim[1], im_dim[1])
        anchor_y = ft2im(y, ft_dim[0], im_dim[0])
        x0, y0, x1, y1 = bb_rects[objID]
        x0 = float(x0 - anchor_x)
        x1 = float(x1 - anchor_x)
        y0 = float(y0 - anchor_y)
        y1 = float(y1 - anchor_y)
        bboxes[:, y, x] = (x0, y0, x1, y1)
    return bboxes

def _maskForeground(objID_at_pixel_ft):
    """Return the "this-is-not-a-background-pixel" mask."""
    mask = np.zeros(objID_at_pixel_ft.shape, np.uint8)

    # Activate all feature map locations that show anything but background.
    fg_idx = np.nonzero(objID_at_pixel_ft)
    mask[fg_idx] = 1
    return mask

def _maskBBox(objID_at_pixel_ft, obj_pixels_ft):
    """Return the "you-can-estimate-bbox-size-at-this-anchor" mask.

    To estimating the BBox it often suffices to see only a small portion of the
    object. In this case, all objects that have more than 10% of its pixels
    visible are considered "visible enough for BBox estimation". NOTE: just
    because it is possible to estimate the BBox size does *not* mean it is also
    possible to estimate its label (ie the number on the cube), as considerably
    more pixels may have to be visible for that (see `_maskFgLabel`).
    """
    # Iterate over each object and determine (mostly guess) if it is possible
    # to recognise the object.
    mask = np.zeros(objID_at_pixel_ft.shape, np.uint8)
    for objID, pixels in obj_pixels_ft.items():
        # Skip background locations.
        if objID == 0:
            continue

        # Find out which pixels belong to the current object AND are visible in
        # the scene right now.
        idx_visible = np.nonzero(objID_at_pixel_ft == objID)

        # We decide that BBox estimation is possible if at least 10% of all
        # pixels for the object are visible.
        num_visible = len(idx_visible[0])
        num_tot = np.count_nonzero(pixels)
        if num_visible >= 9 and num_visible >= 0.1 * num_tot:
            mask[idx_visible] = 1
    return mask

def _maskValid(objID_at_pixels_ft):
    """Return the "only-train-on-these-pixels" mask.

    The main purpose of this mask is to remove all pixels close to
    foreground/background boundaries. This will avoid anchor positions where
    the foreground/background label is ambiguous.
    """
    src = np.array(objID_at_pixels_ft, np.uint8)
    out = np.array(Image.fromarray(src).filter(ImageFilter.FIND_EDGES))

    mask = np.zeros(src.shape, np.uint8)
    mask[np.nonzero(out == 0)] = 1
    return mask

def _maskFgLabel(img, objID_at_pixel_ft, obj_pixels_ft):
    """Return the "it is possible to estimate the label at that pixel" mask.

    To estimate the label the object must be
      1. large enough to see the number
      2. bright enough to see the number
      3. unobstructed enough to see the number
      4. not too so close to the screen edge that the number is clipped

    For Condition 1, objects are large enough if it occupies least 9 pixels in
    feature space. 9 pixels (ie a 3x3 patch) corresponds to a 24x24 receptive
    field for our default feature and image sizes of 64x64 and 512x512,
    respectively.

    Objects are bright enough if their average pixel value is at least 40. I
    determine this threshold with an empirical study of one image :)

    We consider the object unobstructed if at least 50% of its pixels are
    actually visible in the scene (Condition 3).

    I do not know how to recognise Condition 4 with the given data.

    This is not foolproof but works well enough for now. A notable case where
    this fails is a cube close to the camera but clipped by the image boundary.
    These cubes may occupy a lot of screen real estate, yet its only visible
    portion is the red frame and parts of the white surface.
    """
    # Compute the average pixel intensity.
    img = np.mean(np.array(img, np.int64), axis=2)
    assert img.shape == objID_at_pixel_ft.shape

    # Iterate over each object and determine (mostly guess) if it is possible
    # to recognise the object.
    mask = np.zeros(objID_at_pixel_ft.shape, np.uint8)
    for objID, pixels in obj_pixels_ft.items():
        idx_visible = np.nonzero(objID_at_pixel_ft == objID)
        if len(idx_visible[0]) == 0:
            continue

        # Is it bright enough.
        avg_brightness = np.mean(img[idx_visible])
        if avg_brightness < 40:
            continue

        # Are enough pixels visible in the scene.
        num_visible = len(idx_visible[0])
        num_tot = np.count_nonzero(pixels)
        if num_visible >= 9 and num_visible >= 0.5 * num_tot:
            mask[idx_visible] = 1
    return mask

def generate(fname, img, ft_dims):
    assert img.ndim == 3 and img.shape[2] == 3 and img.dtype == np.uint8
    im_dim = img.shape[:2]

    # Load the True output and verify that all files use the same
    # int->label mapping.
    img_meta = bz2.open(fname + '-meta.json.bz2', 'rb').read()
    img_meta = json.loads(img_meta.decode('utf8'))

    # Undo JSON's int->str conversion for dict keys.
    int2name = {int(k): v for k, v in img_meta['int2name'].items()}
    bb_rects = {int(k): v for k, v in img_meta['bb_rects'].items()}
    obj_pixels = {int(k): v for k, v in img_meta['obj-pixels'].items()}
    objID2label = {int(k): v for k, v in img_meta['objID2label'].items()}
    objID_at_pixel = np.array(img_meta['objID-at-pixel'], np.int32)
    del img_meta

    # The label map *must* contain a None labels -> these are the background
    # pixels.
    assert int2name[0] == 'None'
    name2int = {v: k for k, v in int2name.items()}

    # For each non-zero pixel, map the object ID to its label. This
    # will produce an image where each pixel corresponds to a label
    # that can be looked up with `int2name`.
    label_at_pixel = np.zeros_like(objID_at_pixel)
    for idx in zip(*np.nonzero(objID_at_pixel)):
        label_name = objID2label[objID_at_pixel[idx]]
        assert label_name != 'None'
        label_at_pixel[idx] = name2int[label_name]

    # Add the int2name map to the function output.
    out = {}
    out['int2name'] = int2name

    # Compile dictionary with feature size specific data. This includes the
    # BBox data relative to the anchor point.
    for ft_dim in ft_dims:
        img_ft = Image.fromarray(img).resize((ft_dim[1], ft_dim[0]))
        img_ft = np.array(img_ft)

        # Downsample the label/objID maps to the feature size.
        label_at_pixel_ft = downsampleMatrix(label_at_pixel, ft_dim)
        objID_at_pixel_ft = downsampleMatrix(objID_at_pixel, ft_dim)
        obj_pixels_ft = {k: downsampleMatrix(v, ft_dim) for k, v in obj_pixels.items()}

        bboxes = _computeBBoxes(bb_rects, objID_at_pixel_ft, im_dim)
        mask_fg = _maskForeground(objID_at_pixel_ft)
        mask_bbox = _maskBBox(objID_at_pixel_ft, obj_pixels_ft)
        mask_valid = _maskValid(objID_at_pixel_ft)
        mask_cls = _maskFgLabel(img_ft, objID_at_pixel_ft, obj_pixels_ft)

        # Compile all the information into the output dictionary.
        out[ft_dim] = {
            'bboxes': np.array(bboxes, np.float32),
            'objID_at_pixel': objID_at_pixel_ft,
            'label_at_pixel': label_at_pixel_ft,
            'mask_fg': mask_fg,
            'mask_bbox': mask_bbox,
            'mask_cls': mask_cls,
            'mask_valid': mask_valid,
        }
    return out

def compileSingle(args):
    fname, ft_dims = args
    img = np.array(Image.open(fname + '.jpg').convert('RGB'))
    features = generate(fname, img, ft_dims)
    pickle.dump(features, open(fname + '-compiled.pickle', 'wb'))

def main():
    param = parseCmdline()
    ft_dims = [(64, 64)]

    args = [(_, ft_dims) for _ in param.fnames]

    if len(args) == 1:
        compileSingle(args[0])
    else:
        with multiprocessing.Pool() as pool:
            # Setup parallel execution and wrap it into a TQDM progress bar. Then
            # consume the iterator.
            progbar = tqdm.tqdm(
                pool.imap_unordered(compileSingle, args),
                total=len(args), desc='Compiling Features', leave=False
            )
            [_ for _ in progbar]

    # Show debug plots for the first file in the list.
    if param.debug:
        inspect_feature.main(param.fnames[0] + '.jpg')

if __name__ == '__main__':
    main()


text:
__author__ = "Simon Bonnaud, Arthur Chevallier, Anaïs Pignet, Eliott Vincent"
__license__ = "MIT"
__version__ = "0.1"

#================================================================================
# modules
#================================================================================
import pandas as pd
import numpy as np
import plotly
from plotly.graph_objs import Scatter, Layout
import plotly.graph_objs as go
from collections import Counter

#================================================================================
# properties
#================================================================================
continuousFeatures = ['Elevation','Aspect','Slope','Horizontal_Distance_To_Hydrology','Vertical_Distance_To_Hydrology','Horizontal_Distance_To_Roadways','Hillshade_9am','Hillshade_Noon','Hillshade_3pm','Horizontal_Distance_To_Fire_Points']
categoricalFeatures = ['Cover_Type', 'Wilderness_Area', 'Soil_Type']
continuousStatistics = ['FEATURENAME', 'count', 'miss_percentage', 'card', 'minimum', 'first_quartile', 'mean', 'median', 'third_quartile', 'maximum', 'std_dev']
categoricalStatistics = ['FEATURENAME', 'count', 'miss_percentage', 'card', 'mode', 'mode_frequency', 'mode_percentage', 'second_mode', 'second_mode_frequency', 'second_mode_percentage']
dataPath = './data/'
teamName = 'A'

def main():

	# loading the dataset as a Pandas DataFrame
	df = load_dataframe('DataSet-cleaned-string.csv')

	# splitting the main DataFrame in two sub-dataframes (continuous and categorical features)
	continuousDf = getContinuousDf(df)
	categoricalDf = getCategoricalDf(df)

	# generating the reports for bot continuous and categorical features
	continuousReport = generateContinuousReport(continuousDf)
	categoricalReport = generateCategoricalReport(categoricalDf)

	# saving the reports as .csv files
	write_dataframe(continuousReport, 'dqr/' + teamName + '-DQR-ContinuousFeatures.csv')
	write_dataframe(categoricalReport, 'dqr/' + teamName + '-DQR-CategoricalFeatures.csv')

	# generating the graphs for continuous and categorical features
	generateContinuousGraphs(continuousDf, continuousReport)
	generateCategoricalGraphs(categoricalDf)

#  ██╗███╗   ██╗██████╗ ██╗   ██╗████████╗    ██████╗ ██╗   ██╗████████╗██████╗ ██╗   ██╗████████╗
#  ██║████╗  ██║██╔══██╗██║   ██║╚══██╔══╝   ██╔═══██╗██║   ██║╚══██╔══╝██╔══██╗██║   ██║╚══██╔══╝
#  ██║██╔██╗ ██║██████╔╝██║   ██║   ██║█████╗██║   ██║██║   ██║   ██║   ██████╔╝██║   ██║   ██║
#  ██║██║╚██╗██║██╔═══╝ ██║   ██║   ██║╚════╝██║   ██║██║   ██║   ██║   ██╔═══╝ ██║   ██║   ██║
#  ██║██║ ╚████║██║     ╚██████╔╝   ██║      ╚██████╔╝╚██████╔╝   ██║   ██║     ╚██████╔╝   ██║
#  ╚═╝╚═╝  ╚═══╝╚═╝      ╚═════╝    ╚═╝       ╚═════╝  ╚═════╝    ╚═╝   ╚═╝      ╚═════╝    ╚═╝

def load_dataframe(fileName):
	"""load_dataframe
	Loads a DataFrame from a .csv file.

    Input:
    fileName -- the name of the file (should be located in ./data/)

	Output:
	pd.read_csv() -- the DataFrame loaded by Pandas
    """
	path = dataPath + fileName
	return pd.read_csv(path, header=0)

def write_dataframe(df, fileName):
	"""write_dataframe
	Writes a DataFrame into a .csv file.

    Input:
    df -- the DataFrame to write
    fileName -- the name of the file (will be saved in ./data/)
    """
	path = dataPath + fileName
	df.to_csv(path)

def getContinuousDf(df):
	"""getContinuousDf
	Extract continuous features from the main DataFrame.

    Input:
    df -- the main DataFrame

	Output:
	df.drop() -- the main DataFrame with categoricalFeatures omitted
    """
	return df.drop(categoricalFeatures, axis=1)

def getCategoricalDf(df):
	"""getCategoricalDf
	Extract categorical features from the main DataFrame.

    Input:
    df -- the main DataFrame

	Output:
	df.drop() -- the main DataFrame with continuousFeatures omitted
    """
	return df.drop(continuousFeatures, axis=1)

#  ███████╗████████╗ █████╗ ████████╗██╗███████╗████████╗██╗ ██████╗███████╗
#  ██╔════╝╚══██╔══╝██╔══██╗╚══██╔══╝██║██╔════╝╚══██╔══╝██║██╔════╝██╔════╝
#  ███████╗   ██║   ███████║   ██║   ██║███████╗   ██║   ██║██║     ███████╗
#  ╚════██║   ██║   ██╔══██║   ██║   ██║╚════██║   ██║   ██║██║     ╚════██║
#  ███████║   ██║   ██║  ██║   ██║   ██║███████║   ██║   ██║╚██████╗███████║
#  ╚══════╝   ╚═╝   ╚═╝  ╚═╝   ╚═╝   ╚═╝╚══════╝   ╚═╝   ╚═╝ ╚═════╝╚══════╝

def computeContinuousStatistics(df):
	"""computeContinuousStatistics
	Computes statistics for continuous features.

    Input:
    df -- DataFrame to use

	Output:
	{} -- a dictionary containing the statistics
    """

	# returning the final statistics
	return {
		'count': len(df),
		'miss_percentage': ((len(df)-df.count())*100)/len(df),
		'card': df.nunique(),
		'minimum': df.min(),
		'first_quartile': df.quantile([0.25][0]),
		'mean': df.mean(),
		'median': df.median(),
		'third_quartile': df.quantile([0.75][0]),
		'maximum': df.max(),
		'std_dev': df.std()
	}

def computeCategoricalStatistics(df):
	"""computeCategoricalStatistics
	Computes statistics for categorical features.

    Input:
    df -- DataFrame containing to use

	Output:
	{} -- a dictionary containing the statistics
    """

	# computing first mode
	tmpDf = df.copy()
	firstMode = tmpDf.mode()[0]
	firstModeFrequency = tmpDf.describe(include='all')['freq']
	firstModePercentage = (firstModeFrequency/len(df))*100

	# computing second mode
	tmpDf = tmpDf[tmpDf != firstMode]	# removing rows with first mode values
	secondMode = tmpDf.mode()[0]
	secondModeFrequency = tmpDf.describe(include='all')['freq']
	secondModePercentage = (secondModeFrequency/len(df))*100

	# returning the final statistics
	return {
		'count': len(df),
		'miss_percentage': ((len(df)-df.count())*100)/len(df),
		'card': df.nunique(),
		'mode': firstMode,
		'mode_frequency': firstModeFrequency,
		'mode_percentage': firstModePercentage,
		'second_mode': secondMode,
		'second_mode_frequency': secondModeFrequency,
		'second_mode_percentage': secondModePercentage
	}

#  ██████╗ ███████╗██████╗  ██████╗ ██████╗ ████████╗███████╗
#  ██╔══██╗██╔════╝██╔══██╗██╔═══██╗██╔══██╗╚══██╔══╝██╔════╝
#  ██████╔╝█████╗  ██████╔╝██║   ██║██████╔╝   ██║   ███████╗
#  ██╔══██╗██╔══╝  ██╔═══╝ ██║   ██║██╔══██╗   ██║   ╚════██║
#  ██║  ██║███████╗██║     ╚██████╔╝██║  ██║   ██║   ███████║
#  ╚═╝  ╚═╝╚══════╝╚═╝      ╚═════╝ ╚═╝  ╚═╝   ╚═╝   ╚══════╝

def generateContinuousReport(continuousDf):
	"""generateContinuousReport
	Generates a report of continuous features statistics from a dataframe.

    Input:
    continuousDf -- DataFrame to use

	Output:
	generateReport() -- output of the generateReport() method
    """
	report = generateReport(continuousDf, continuousStatistics, computeContinuousStatistics)
	print('Report for continuous features: OK')
	return report

def generateCategoricalReport(categoricalDf):
	"""generateCategoricalReport
	Generates a report of categorical features statistics from a dataframe.

    Input:
    categoricalDf -- DataFrame to use

	Output:
	generateReport() -- output of the generateReport() method
    """
	report = generateReport(categoricalDf, categoricalStatistics, computeCategoricalStatistics)
	print('Report for categorical features: OK')
	return report

def generateReport(dataFrame, statisticsNames, computeFunction):
	"""generateReport
	Generates a report from a dataframe.

    Input:
    dataFrame -- DataFrame to use
    statisticsNames -- statistics to compute
    computeFunction -- method to use to compute the mesures

	Output:
	statisticsDf -- DataFrame containing the statistics
    """

	# 1st step: create an empty dataframe with the statistics names as columns
	statisticsDf = pd.DataFrame(columns=statisticsNames)
	statisticsDf.set_index('FEATURENAME', inplace=True)

	# 2nd step: loop over each feature
	for featureName in dataFrame:

		# gathering all values for the current feature
		featureValues = dataFrame[featureName]

		# computing statistics
		statisticsDict = computeFunction(featureValues)

		# creating a serie with those statistics
		statisticsSeries = pd.Series(statisticsDict, name=featureName)

		# adding the serie to the final dataframe
		statisticsDf = statisticsDf.append(statisticsSeries)

	# 3rd step: return the completed dataframe
	return statisticsDf

#   ██████╗ ██████╗  █████╗ ██████╗ ██╗  ██╗███████╗
#  ██╔════╝ ██╔══██╗██╔══██╗██╔══██╗██║  ██║██╔════╝
#  ██║  ███╗██████╔╝███████║██████╔╝███████║███████╗
#  ██║   ██║██╔══██╗██╔══██║██╔═══╝ ██╔══██║╚════██║
#  ╚██████╔╝██║  ██║██║  ██║██║     ██║  ██║███████║
#   ╚═════╝ ╚═╝  ╚═╝╚═╝  ╚═╝╚═╝     ╚═╝  ╚═╝╚══════╝

def drawHistogramFromFeature(featureName, featureValues):
	"""drawHistogramFromFeature
	Draws an histogram graph from a feature.
	The graph is then saved in ./data/

    Input:
    featureName -- the name of the feature
    featureValues -- the values of the feature
    """

	data = [go.Histogram(x=featureValues)]
	path = dataPath + 'dqr/' + featureName + '-histogram.html'
	plotly.offline.plot(data, filename=path, auto_open=False)

def drawBarPlotFromFeature(featureName, featureValues):
	"""drawBarPlotFromFeature
	Draws a bar plot graph from a feature.
	The graph is then saved in ./data/

    Input:
    featureName -- the name of the feature
    featureValues -- the values of the feature
    """

    # we need to count the number of each occurrence
	occurrences = Counter(featureValues)
	x_axis = []
	y_axis = []
	for occurrence in occurrences:
		x_axis.append(occurrence)
		y_axis.append(occurrences[occurrence])

	data = [go.Bar(x=x_axis, y=y_axis)]
	path = dataPath + 'dqr/' + featureName + '-bar-plot.html'
	plotly.offline.plot(data, filename=path, auto_open=False)

def generateContinuousGraphs(continuousDf, continuousReport):
	"""generateContinuousGraphs
	Draws graphs for each continuous feature of a DataFrame.
	According to the cardinality of the feature, the generated graph will either be an histogram or a bar plot.

    Input:
    continuousDf -- the DataFrame containing all the continuous features
    continuousReport -- the report associated with the continuous features
    """

	for featureName in continuousDf:

		cardinality = continuousReport.loc[featureName]['card']	# getting the cardinality from the report

		# if the continuous feature has a low cardinality (< 10), we draw a bar plot
		if cardinality < 10:
			drawBarPlotFromFeature(featureName, continuousDf[featureName])
		# else we draw an histogram
		else:
			drawHistogramFromFeature(featureName, continuousDf[featureName])

	print('Graphs for continuous features: OK')

def generateCategoricalGraphs(categoricalDf):
	"""generateCategoricalGraphs
	Draws graphs for each categorical feature of a DataFrame.

    Input:
    categoricalDf -- the DataFrame containing all the categorical features
    """

	for featureName in categoricalDf:
		drawBarPlotFromFeature(featureName, categoricalDf[featureName])

	print('Graphs for categorical features: OK')

# program launch
if __name__ == '__main__':
	main()


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Tests for `wafflemaker` package."""

import tempfile

import numpy as np
import matplotlib.font_manager as fm
import matplotlib as mpl
import matplotlib.pyplot as plt
import pandas as pd
import pytest
import requests

from wafflemaker import waffle, CellFillDirection

@pytest.fixture
def df():
    return pd.DataFrame(dict(
        values=[4, 3, 2],
        categories=['category #1', 'category #2', 'category #3'],
        hues=['blue', 'orange', 'white']
    ))

# DATA
@pytest.mark.mpl_image_compare
def test_data_list(df):
    return waffle(
        nrows=3, ncols=3,
        values=[4, 3, 2],
        title='data: list'
    ).figure

@pytest.mark.mpl_image_compare
def test_data_nparray(df):
    return waffle(
        nrows=3, ncols=3,
        values=np.array([4, 3, 2]),
        title='data: np.ndarray'
    ).figure

@pytest.mark.mpl_image_compare
def test_data_dataframe(df):
    return waffle(
        nrows=3, ncols=3,
        values='values',
        data=df,
        title='data: pd.DataFrame'
    ).figure

# UNSCALED DATA
def test_unscaled_validation():
    fail_kwargs = [
        dict(values=[10, 20]),
        dict(nrows=3, values=[10, 20]),
        dict(ncols=3, values=[10, 20]),
        dict(scale_to_dims=False, values=[10, 20]),
        dict(nrows=3, ncols=3, scale_to_dims=False, values=[10, 20])
    ]
    for kwargs in fail_kwargs:
        with pytest.raises(ValueError):
            waffle(**kwargs)

@pytest.mark.mpl_image_compare
def test_unscaled_data_column(df):
    return waffle(
        ncols=8,
        scale_to_dims=False,
        values=[25, 15, 10],
        labels=['A', 'B', 'C'],
        title='data: unscaled, column specified',
        fill_direction=CellFillDirection.ByRow,
    ).figure

@pytest.mark.mpl_image_compare
def test_unscaled_data_row(df):
    return waffle(
        nrows=4,
        scale_to_dims=False,
        values=[25, 15, 10],
        labels=['A', 'B', 'C'],
        title='data: unscaled, row specified',
        fill_direction=CellFillDirection.ByColumn,
    ).figure

# EMBEDDING
@pytest.mark.mpl_image_compare
def test_subplot(df):
    np.random.seed(123)
    fig, axes = plt.subplots(1, 3, figsize=(15, 5))
    for i, ax in enumerate(axes):
        data = np.random.normal(100, 50, 4)
        data.sort()
        waffle(nrows=7, ncols=5,
               values=list(reversed(data)),
               title='subplot #%d' % (i + 1),
               ax=ax)
    return fig

# LAYOUT
@pytest.mark.mpl_image_compare
def test_order_row(df):
    return waffle(
        nrows=3, ncols=3,
        values='values', hue='hues', data=df,
        fill_direction=CellFillDirection.ByRow,
        title='layout: row'
    ).figure

@pytest.mark.mpl_image_compare
def test_order_column(df):
    return waffle(
        nrows=3, ncols=3,
        values='values', hue='hues', data=df,
        fill_direction=CellFillDirection.ByColumn,
        title='layout: column'
    ).figure

# LEGEND
@pytest.mark.mpl_image_compare
def test_legend_dataframe(df):
    return waffle(
        nrows=3, ncols=3,
        values='values', labels='categories', data=df,
        title='legend: pd.DataFrame'
    ).figure

@pytest.mark.mpl_image_compare
def test_legend_list(df):
    return waffle(
        nrows=3, ncols=3,
        values='values',
        labels=['a', 'b', 'c'], data=df,
        title='legend: list'
    ).figure

@pytest.mark.mpl_image_compare
def test_legend_long(df):
    return waffle(
        nrows=3, ncols=3,
        values='values',
        labels=[
            'Supercalifragilisticexpialidocious',
            'Even though the sound of it\nIs something quite atrocious',
            'If you say it loud enough\nYou\'ll always sound precocious',
        ], data=df,
        title='legend: long label'
    ).figure

# FIGURE_OPTIONS
def test_figure_options(df):
    # custom figsize
    fig = waffle(
        nrows=3, ncols=3,
        values='values', data=df,
        figure_options={'figsize': (10, 10)}
    ).figure
    figsize = fig.get_size_inches()
    assert np.isclose(figsize, (10, 10), atol=0.1).all()

# ICON
@pytest.mark.mpl_image_compare
def test_icon(df):
    # custom figsize
    return waffle(
        nrows=3, ncols=3,
        values='values', labels='categories', data=df,
        icon='\u26AB',
        background_color='#efefef',
        icon_options=dict(fontsize=130),
        icon_legend_options=dict(fontsize=30),
        grid_options=dict(color='#eaeaea', linewidth=10),
        title='icon: custom icon'
    ).figure

@pytest.mark.mpl_image_compare
def test_unscaled_icon(df):
    # custom figsize
    return waffle(
        nrows=5,
        values='values', labels='categories', hue=['r', 'g', 'b'], data=df,
        icon='\u2764',
        scale_to_dims=False,
        background_color='grey',
        icon_options=dict(fontsize=75),
        icon_legend_options=dict(fontsize=30),
        grid_options=dict(color='black', linewidth=10),
        title='icon: custom icon'
    ).figure

# COLOR
@pytest.mark.mpl_image_compare
def test_color_base(df):
    return waffle(
        nrows=3, ncols=3,
        values='values', data=df,
        title='color: default'
    ).figure

@pytest.mark.mpl_image_compare
def test_color_dataframe(df):
    return waffle(
        nrows=3, ncols=3,
        values='values', hue='hues', data=df,
        title='color: pd.DataFrame'
    ).figure

@pytest.mark.mpl_image_compare
def test_color_colormap(df):
    return waffle(
       nrows=3, ncols=3,
       values='values', data=df, colormap=mpl.cm.Purples_r,
       title='color: colormap'
    ).figure

@pytest.mark.mpl_image_compare
def test_color_list(df):
    return waffle(
        nrows=3, ncols=3,
        values='values', hue=['#c1d3a2', '#1d4dff', '#cecece'], data=df,
        title='color: list'
    ).figure

# BACKGROUND COLOR
@pytest.mark.mpl_image_compare
def test_background_color(df):
    return waffle(
        nrows=3, ncols=3,
        values='values', data=df,
        background_color='black',
        title='background color: black'
    ).figure

# GRID STYLES
@pytest.mark.mpl_image_compare
def test_grid_color_base(df):
    return waffle(
        nrows=3, ncols=3,
        values='values', data=df,
        title='grid: base'
    ).figure

@pytest.mark.mpl_image_compare
def test_grid_color_altered(df):
    return waffle(
        nrows=3, ncols=3,
        values='values', data=df,
        grid_options=dict(color='black', linewidth=25),
        title='grid: custom'
    ).figure

# DOCUMENTATION
@pytest.mark.mpl_image_compare
def test_docs_household_debt(df):
    # https://github.com/hrbrmstr/waffle
    # http://www.nytimes.com/2008/07/20/business/20debt.html
    df = pd.DataFrame(dict(
        values=[84911, 14414, 10062, 8565],
        categories=['Mortgage ($85k)', 'Auto and tuition loans ($14k)', 'Home equity loans ($10k)', 'Credit cards ($9k)'],
        hues=["#c7d4b6", "#a3aabd", "#a0d0de", "#97b5cf"]
    ))

    df['scaled_values'] = df['values'] / 500.

    return waffle(
        nrows=7,
        scale_to_dims=False,

        data=df,
        values='scaled_values',
        labels='categories',
        hue='hues',

        title='Average Household Debt',
        grid_options=dict(linewidth=2),
        figure_options=dict(figsize=(14, 5)),

    ).figure

@pytest.mark.mpl_image_compare
def test_docs_icon(df):
    font_awesome_url = 'https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/fonts/fontawesome-webfont.ttf'
    tf = tempfile.NamedTemporaryFile()
    r = requests.get(font_awesome_url)
    with open(tf.name, 'wb') as f:
        f.write(r.content)

    # use font in waffle plot
    prop = fm.FontProperties(fname=tf.name)

    # build plot
    figure = waffle(
            nrows=3, ncols=3,
            values='values', labels='categories', data=df,
            colormap=mpl.cm.cool,
            icon='\uf015',
            background_color='#efefef',
            icon_options=dict(fontproperties=prop, size=85),
            icon_legend_options=dict(size=20),
            grid_options=dict(color='#eaeaea', linewidth=15),
            title='icon: custom icon'
        ).figure

    # delete temporary font
    tf.close()

    return figure

max_stars_repo_name: shapiromatron/wafflemaker

max_issues_repo_name: shapiromatron/wafflemaker

max_forks_repo_name: shapiromatron/wafflemaker


text:
import sys
import cv2
import json
import numpy as np
import tensorflow as tf
from imagenet40 import get_db
from architecture import shufflenet
from input_pipeline import resize_keeping_aspect_ratio, central_crop
from evaluation import evaluate_csv

def benchmark_accuracy(DEPTH_MULTIPLIER):
    if DEPTH_MULTIPLIER == '1.0':
        SAVE_PATH = 'run01/model.ckpt-1661328'
    elif DEPTH_MULTIPLIER == '0.5':
        SAVE_PATH = 'run00/model.ckpt-1331064'
    else:
        raise NotImplementedError()

    import os
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

    # Prepare benchmark information
    with open('data/integer_encoding.json', 'r') as f:
        encoding = json.load(f)

    with open('data/wordnet_decoder.json', 'r') as f:
        wordnet_decoder = json.load(f)

    decoder = {i: wordnet_decoder[n] for n, i in encoding.items()}

    DB_PATH = '/data/imagenet40/val'
    im_paths, labels, classes = get_db(DB_PATH)

    classes_idx = [encoding[c] for c in classes]

    # Prepare Network
    tf.reset_default_graph()

    raw_images = tf.placeholder(tf.uint8, [None, None, 3])
    images = tf.to_float(raw_images)/255.0
    MIN_DIMENSION = 256
    IMAGE_SIZE = 224
    images = (1.0 / 255.0) * tf.to_float(raw_images)
    images = resize_keeping_aspect_ratio(images, MIN_DIMENSION)
    images = central_crop(images, crop_height=IMAGE_SIZE, crop_width=IMAGE_SIZE)
    images = tf.expand_dims(images, 0)

    logits = shufflenet(images, is_training=False, depth_multiplier=DEPTH_MULTIPLIER)

    ema = tf.train.ExponentialMovingAverage(decay=0.995)
    variables_to_restore = ema.variables_to_restore()
    saver = tf.train.Saver(variables_to_restore)

    print "Running accuracy benchmark...",
    sys.stdout.flush()
    pred = []
    conf = []
    with tf.Session() as sess:
        saver.restore(sess, SAVE_PATH)

        # Read Image
        for p in im_paths:
            image = cv2.imread(p)
            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

            feed_dict = {raw_images: image}
            result = sess.run(logits, feed_dict)[0]
            prob = result[classes_idx]
            prob = np.exp(prob)
            prob = prob / np.sum(prob)
            pred.append(np.argmax(prob))
            conf.append(np.max(prob))

    csv_path = "results.csv"
    with open(csv_path, "w") as f:
        for im_path, p, c in zip(im_paths, pred, conf):
            f.writelines("%s,%d,%f\n" % (im_path, p, c))

    print "Done"
    sys.stdout.flush()

    evaluate_csv(csv_path)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('model_width', type=str, help='model_width (1.0 / 0.5)')
    args = parser.parse_args()

    benchmark_accuracy(args.model_width)


text:
import numpy as np
from sklearn.cluster import KMeans
from sklearn.datasets import make_blobs

import tkinter
import pandas as pd
import matplotlib.pyplot as plt

'''
# define number of samples
n_samples = 100

# define random state value to initialize the center
random_state = 20

# define number of feature as 5
X,y = make_blobs(n_samples=n_samples, n_features=5, random_state=None)
# print(X,y)

# The goal of K-means algorithm is to find groups in the data, with the number of groups represented by the variable K.
# mean distance to the centroid as a function of K is plotted and the "elbow point," where the rate of decrease sharply shifts, can be used to roughly determine K.

# define number of cluster to be formed as 3 and
# in random state and fit features into the model
predict_y = KMeans(n_clusters=3, random_state=random_state)

print(predict_y)

predict_y_fit = predict_y.fit_predict(X)
# estimator function
# print(predict_y_fit)
print(type(predict_y_fit))
# print(predict_y_fit.labels_)
print(predict_y_fit.shape)

df = pd.DataFrame(predict_y_fit)
# print(df)
print(df.describe())

print(df[0].value_counts())
'''
# rule
'''
df[0].value_counts().plot.bar()
plt.show()

df[0].value_counts().plot.pie()
plt.gca().set_aspect("equal")
plt.show()

labels = []
for i, dfi in enumerate(df.groupby(df[0])):
    labels.append(dfi[0])
    plt.bar(i, dfi[1].count(), label=dfi[0])
plt.xticks(range(len(labels)), labels)
plt.legend()
plt.show()
'''

data = pd.read_csv('data_1024.csv', sep='\t')
print(data.describe())
print(data.head())

# plt.scatter(data['Distance_Feature'], data['Speeding_Feature'])
# plt.show()

X = np.matrix(list(zip(data['Distance_Feature'].values,data['Speeding_Feature'].values)))
kmeans = KMeans(n_clusters=2)
print(kmeans)

kmeans_fit = kmeans.fit(X)
print(kmeans_fit)

kmeans_predict = kmeans_fit.predict(X)
print(kmeans_predict)
print(kmeans_predict.shape)

dfx = pd.DataFrame(X)

plt.scatter(dfx[0], dfx[1], c=kmeans_predict, s=50, cmap='viridis')

centers = kmeans.cluster_centers_
print(centers)
plt.scatter(centers[:, 0], centers[:, 1], c='black', s=200, alpha=0.5)

plt.show()

kmeans = KMeans(n_clusters=4)
kmeans_fit = kmeans.fit(X)
kmeans_predict = kmeans_fit.predict(X)
plt.scatter(dfx[0], dfx[1], c=kmeans_predict, s=50, cmap='viridis')
centers = kmeans.cluster_centers_
plt.scatter(centers[:, 0], centers[:, 1], c='black', s=200, alpha=0.5)
plt.show()


text:
# Copyright 2021 The NetKet Authors - All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import abc
import numbers
from functools import partial

from tqdm import tqdm

import jax
from jax.tree_util import tree_map

from netket.logging import JsonLog
from netket.utils import mpi

def _to_iterable(maybe_iterable):
    """
    _to_iterable(maybe_iterable)

    Ensure the result is iterable. If the input is not iterable, it is wrapped into a tuple.
    """
    if hasattr(maybe_iterable, "__iter__"):
        surely_iterable = maybe_iterable
    else:
        surely_iterable = (maybe_iterable,)

    return surely_iterable

# Note: to implement a new Driver (see also _vmc.py for an example)
# If you want to inherit the nice interface of AbstractMCDriver, you should
# subclass it, defining the following methods:
# - Either _forward_and_backward or individually _forward, _backward, that should
#   compute the loss function and the gradient. If the driver is minimizing or
#   maximising some loss function, this quantity should be assigned to self._stats
#   in order to monitor it.
# - _estimate_stats should return the MC estimate of a single operator
# - reset should reset the driver (usually the sampler).
# - info should return a string with an overview of the driver.
# - The __init__ method shouldbe called with the machine and the optimizer. If this
#   driver is minimising a loss function and you want it's name to show up automatically
#   in the progress bar/ouput files you should pass the optional keyword argument
#   minimized_quantity_name.
class AbstractVariationalDriver(abc.ABC):
    """Abstract base class for NetKet Variational Monte Carlo drivers"""

    def __init__(self, variational_state, optimizer, minimized_quantity_name=""):
        self._mynode = mpi.node_number
        self._mpi_nodes = mpi.n_nodes
        self._loss_stats = None
        self._loss_name = minimized_quantity_name
        self._step_count = 0

        self._variational_state = variational_state
        self.optimizer = optimizer

    def _forward_and_backward(self):
        """
        Performs the forward and backward pass at the same time.
        Concrete drivers should either override this method, or override individually
        _forward and _backward.

        Returns:
            the update for the weights.
        """
        self._forward()
        dp = self._backward()
        return dp

    def _forward(self):
        """
        Performs the forward pass, computing the loss function.
        Concrete should either implement _forward and _backward or the joint method
        _forward_and_backward.
        """
        raise NotImplementedError()

    def _backward(self):
        """
        Performs the backward pass, computing the update for the parameters.
        Concrete should either implement _forward and _backward or the joint method
        _forward_and_backward.
        """
        raise NotImplementedError()

    def _estimate_stats(self, observable):
        """
        Returns the MCMC statistics for the expectation value of an observable.
        Must be implemented by super-classes of AbstractVMC.

        :param observable: A quantum operator (netket observable)
        :return:
        """
        return self.state.expect(observable)

    def reset(self):
        """
        Resets the driver.
        Concrete drivers should also call super().reset() to ensure that the step
        count is set to 0.
        """
        self.state.reset()
        self.step_count = 0
        pass

    @abc.abstractmethod
    def info(self, depth=0):
        """
        Returns an info string used to print information to screen about this driver.
        """
        pass

    @property
    def state(self):
        """
        Returns the machine that is optimized by this driver.
        """
        return self._variational_state

    @property
    def optimizer(self):
        """
        The optimizer used to update the parameters at every iteration.
        """
        return self._optimizer

    @optimizer.setter
    def optimizer(self, optimizer):
        self._optimizer = optimizer
        self._optimizer_state = optimizer.init(self.state.parameters)

    @property
    def step_count(self):
        """
        Returns a monotonic integer labelling all the steps performed by this driver.
        This can be used, for example, to identify the line in a log file.
        """
        return self._step_count

    def iter(self, n_steps: int, step: int = 1):
        """
        Returns a generator which advances the VMC optimization, yielding
        after every `step_size` steps.

        Args:
            n_iter: The total number of steps to perform.
            step_size: The number of internal steps the simulation
                is advanced every turn.

        Yields:
            int: The current step.
        """
        for _ in range(0, n_steps, step):
            for i in range(0, step):
                dp = self._forward_and_backward()
                if i == 0:
                    yield self.step_count

                self._step_count += 1
                self.update_parameters(dp)

    def advance(self, steps: int = 1):
        """
        Performs `steps` optimization steps.

        steps: (Default=1) number of steps
        """
        for _ in self.iter(steps):
            pass

    def run(
        self,
        n_iter,
        out=None,
        obs=None,
        show_progress=True,
        save_params_every=50,  # for default logger
        write_every=50,  # for default logger
        step_size=1,  # for default logger
        callback=lambda *x: True,
    ):
        """
        Executes the Monte Carlo Variational optimization, updating the weights of the network
        stored in this driver for `n_iter` steps and dumping values of the observables `obs`
        in the output `logger`. If no logger is specified, creates a json file at `out`,
        overwriting files with the same prefix.

        By default uses :ref:`netket.logging.JsonLog`. To know about the output format
        check it's documentation. The logger object is also returned at the end of this function
        so that you can inspect the results without reading the json output.

        Args:
            n_iter: the total number of iterations
            out: A logger object, or an iterable of loggers, to be used to store simulation log and data.
                If this argument is a string, it will be used as output prefix for the standard JSON logger.
            obs: An iterable containing all observables that should be computed
            save_params_every: Every how many steps the parameters of the network should be
                serialized to disk (ignored if logger is provided)
            write_every: Every how many steps the json data should be flushed to disk (ignored if
                logger is provided)
            step_size: Every how many steps should observables be logged to disk (default=1)
            show_progress: If true displays a progress bar (default=True)
            callback: Callable or list of callable callback functions to stop training given a condition
        """

        if not isinstance(n_iter, numbers.Number):
            raise ValueError(
                "n_iter, the first positional argument to `run`, must be a number!"
            )

        if obs is None:
            obs = {}

        if out is None:
            out = tuple()
            print(
                "No output specified (out=[apath|nk.logging.JsonLogger(...)])."
                "Running the optimization but not saving the output."
            )

        # Log only non-root nodes
        if self._mynode == 0:
            # if out is a path, create an overwriting Json Log for output
            if isinstance(out, str):
                loggers = (JsonLog(out, "w", save_params_every, write_every),)
            else:
                loggers = _to_iterable(out)
        else:
            loggers = tuple()
            show_progress = False

        callbacks = _to_iterable(callback)
        callback_stop = False

        with tqdm(total=n_iter, disable=not show_progress) as pbar:
            old_step = self.step_count
            first_step = True

            for step in self.iter(n_iter, step_size):

                log_data = self.estimate(obs)

                # if the cost-function is defined then report it in the progress bar
                if self._loss_stats is not None:
                    pbar.set_postfix_str(self._loss_name + "=" + str(self._loss_stats))
                    log_data[self._loss_name] = self._loss_stats

                # Execute callbacks before loggers because they can append to log_data
                for callback in callbacks:
                    if not callback(step, log_data, self):
                        callback_stop = True

                for logger in loggers:
                    logger(self.step_count, log_data, self.state)

                if len(callbacks) > 0:
                    if mpi.mpi_any(callback_stop):
                        break

                # Reset the timing of tqdm after the first step, to ignore compilation time
                if first_step:
                    first_step = False
                    pbar.unpause()

                # Update the progress bar
                pbar.update(self.step_count - old_step)
                old_step = self.step_count

            # Final update so that it shows up filled.
            pbar.update(self.step_count - old_step)

        # flush at the end of the evolution so that final values are saved to
        # file
        for logger in loggers:
            logger.flush(self.state)

        return loggers

    def estimate(self, observables):
        """
        Return MCMC statistics for the expectation value of observables in the
        current state of the driver.

        Args:
            observables: A pytree of operators for which statistics should be computed.

        Returns:
            A pytree of the same structure as the input, containing MCMC statistics
            for the corresponding operators as leaves.
        """
        return tree_map(self._estimate_stats, observables)

    def update_parameters(self, dp):
        """
        Updates the parameters of the machine using the optimizer in this driver

        Args:
            dp: the pytree containing the updates to the parameters
        """
        self._optimizer_state, self.state.parameters = apply_gradient(
            self._optimizer.update, self._optimizer_state, dp, self.state.parameters
        )

@partial(jax.jit, static_argnums=0)
def apply_gradient(optimizer_fun, optimizer_state, dp, params):
    import optax

    updates, new_optimizer_state = optimizer_fun(dp, optimizer_state, params)

    new_params = optax.apply_updates(params, updates)
    return new_optimizer_state, new_params

max_stars_repo_name: PhilipVinc/netket

max_issues_repo_name: PhilipVinc/netket

max_forks_repo_name: PhilipVinc/netket

reason: import jax,from jax


text:
import sympy as sp
from sympy import lambdify, nsolve

#Python functions to calculate the enthalpy & adiabatic flame temp of a species based on the a_n & b_n constants

#calculate enthalpy (absolute value) using sympy
def enthalpy(input:dict, T:int=298.15):
    x = sp.Symbol("x")

    #define the polynomial
    def f(x):
        return input["a1"]*x**-2 + input["a2"]*x**-1 + input["a3"] + input["a4"]*x + input["a5"]*x**2 + input["a6"]*x**3 + input["a7"]*x**4

    #print(f(x))
    #integrate the polynomial
    int_f = sp.integrate(f(x),x)
    b1 = input["b1"]

    #evaluate the polynomial
    return 8.314*float(int_f.subs(x, T) + b1)

#Symbolically represent the enthalpy as a function of T
#this does the same as the above function, but returns a function of T rather than evaluating the definite integral
def enthalpy_function(input:dict, T=298.15):
    x = sp.Symbol("x")

    def f(x):
        return input["a1"]*x**-2 + input["a2"]*x**-1 + input["a3"] + input["a4"]*x + input["a5"]*x**2 + input["a6"]*x**3 + input["a7"]*x**4

    int_f = sp.integrate(f(x),x)
    b1 = input["b1"]

    return 8.314*(int_f.subs(x, T) + b1)

#this function uses the enthalpy as a function of T (above) to evaluate the AFT for question 1
#each instance of an aft will have water, nitrogen, and an excess gas (h2 or o2)
def aft_t1(water:dict, nitrogen:dict, extra:dict,  er:float=1.0):
    er_2 = er
    if er >=1:
        er_2 = 1

    T = sp.Symbol("T")

    #sub the enthalpy as a function of T
    def f(T):
        return 2*er_2*enthalpy_function(water, T) + 3.76*enthalpy_function(nitrogen, T) + abs((er - 1))*enthalpy_function(extra, T)

    eq = f(T)

    #numerically solve the function of T
    #this is done numerically because it's a nonlinear function, sympy can only symbolically solve linear functions
    T_aft = nsolve(eq, 1)

    return T_aft

reason: import sympy,from sympy