text:
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

"""ETCI 2021 dataset."""

import glob
import os
from typing import Callable, Dict, List, Optional

import matplotlib.pyplot as plt
import numpy as np
import torch
from PIL import Image
from torch import Tensor

from .geo import VisionDataset
from .utils import download_and_extract_archive

class ETCI2021(VisionDataset):
    """ETCI 2021 Flood Detection dataset.

    The `ETCI2021 <https://nasa-impact.github.io/etci2021/>`_
    dataset is a dataset for flood detection

    Dataset features:

    * 33,405 VV & VH Sentinel-1 Synthetic Aperture Radar (SAR) images
    * 2 binary masks per image representing water body & flood, respectively
    * 2 polarization band images (VV, VH) of 3 RGB channels per band
    * 3 RGB channels per band generated by the Hybrid Pluggable Processing
      Pipeline (hyp3)
    * Images with 5x20m per pixel resolution (256x256) px) taken in
      Interferometric Wide Swath acquisition mode
    * Flood events from 5 different regions

    Dataset format:

    * VV band three-channel png
    * VH band three-channel png
    * water body mask single-channel png where no water body = 0, water body = 255
    * flood mask single-channel png where no flood = 0, flood = 255

    Dataset classes:

    1. no flood/water
    2. flood/water

    If you use this dataset in your research, please add the following to your
    acknowledgements section::

        The authors would like to thank the NASA Earth Science Data Systems Program,
        NASA Digital Transformation AI/ML thrust, and IEEE GRSS for organizing
        the ETCI competition.
    """

    bands = ["VV", "VH"]
    masks = ["flood", "water_body"]
    metadata = {
        "train": {
            "filename": "train.zip",
            "md5": "1e95792fe0f6e3c9000abdeab2a8ab0f",
            "directory": "train",
            "url": "https://drive.google.com/file/d/14HqNW5uWLS92n7KrxKgDwUTsSEST6LCr",
        },
        "val": {
            "filename": "val_with_ref_labels.zip",
            "md5": "fd18cecb318efc69f8319f90c3771bdf",
            "directory": "test",
            "url": "https://drive.google.com/file/d/19sriKPHCZLfJn_Jmk3Z_0b3VaCBVRVyn",
        },
        "test": {
            "filename": "test_without_ref_labels.zip",
            "md5": "da9fa69e1498bd49d5c766338c6dac3d",
            "directory": "test_internal",
            "url": "https://drive.google.com/file/d/1rpMVluASnSHBfm2FhpPDio0GyCPOqg7E",
        },
    }

    def __init__(
        self,
        root: str = "data",
        split: str = "train",
        transforms: Optional[Callable[[Dict[str, Tensor]], Dict[str, Tensor]]] = None,
        download: bool = False,
        checksum: bool = False,
    ) -> None:
        """Initialize a new ETCI 2021 dataset instance.

        Args:
            root: root directory where dataset can be found
            split: one of "train", "val", or "test"
            transforms: a function/transform that takes input sample and its target as
                entry and returns a transformed version
            download: if True, download dataset and store it in the root directory
            checksum: if True, check the MD5 of the downloaded files (may be slow)

        Raises:
            AssertionError: if ``split`` argument is invalid
            RuntimeError: if ``download=False`` and data is not found, or checksums
                don't match
        """
        assert split in self.metadata.keys()

        self.root = root
        self.split = split
        self.transforms = transforms
        self.checksum = checksum

        if download:
            self._download()

        if not self._check_integrity():
            raise RuntimeError(
                "Dataset not found or corrupted. "
                + "You can use download=True to download it"
            )

        self.files = self._load_files(self.root, self.split)

    def __getitem__(self, index: int) -> Dict[str, Tensor]:
        """Return an index within the dataset.

        Args:
            index: index to return

        Returns:
            data and label at that index
        """
        files = self.files[index]
        vv = self._load_image(files["vv"])
        vh = self._load_image(files["vh"])
        water_mask = self._load_target(files["water_mask"])

        if self.split != "test":
            flood_mask = self._load_target(files["flood_mask"])
            mask = torch.stack(tensors=[water_mask, flood_mask], dim=0)
        else:
            mask = water_mask.unsqueeze(0)

        image = torch.cat(tensors=[vv, vh], dim=0)  # type: ignore[attr-defined]
        sample = {"image": image, "mask": mask}

        if self.transforms is not None:
            sample = self.transforms(sample)

        return sample

    def __len__(self) -> int:
        """Return the number of data points in the dataset.

        Returns:
            length of the dataset
        """
        return len(self.files)

    def _load_files(self, root: str, split: str) -> List[Dict[str, str]]:
        """Return the paths of the files in the dataset.

        Args:
            root: root dir of dataset
            split: subset of dataset, one of [train, val, test]

        Returns:
            list of dicts containing paths for each pair of vv, vh,
            water body mask, flood mask (train/val only)
        """
        files = []
        directory = self.metadata[split]["directory"]
        folders = sorted(glob.glob(os.path.join(root, directory, "*")))
        folders = [os.path.join(folder, "tiles") for folder in folders]
        for folder in folders:
            vvs = sorted(glob.glob(os.path.join(folder, "vv", "*.png")))
            vhs = sorted(glob.glob(os.path.join(folder, "vh", "*.png")))
            water_masks = sorted(
                glob.glob(os.path.join(folder, "water_body_label", "*.png"))
            )

            if split != "test":
                flood_masks = sorted(
                    glob.glob(os.path.join(folder, "flood_label", "*.png"))
                )

                for vv, vh, flood_mask, water_mask in zip(
                    vvs, vhs, flood_masks, water_masks
                ):
                    files.append(
                        dict(vv=vv, vh=vh, flood_mask=flood_mask, water_mask=water_mask)
                    )
            else:
                for vv, vh, water_mask in zip(vvs, vhs, water_masks):
                    files.append(dict(vv=vv, vh=vh, water_mask=water_mask))

        return files

    def _load_image(self, path: str) -> Tensor:
        """Load a single image.

        Args:
            path: path to the image

        Returns:
            the image
        """
        filename = os.path.join(path)
        with Image.open(filename) as img:
            array = np.array(img.convert("RGB"))
            tensor: Tensor = torch.from_numpy(array)  # type: ignore[attr-defined]
            # Convert from HxWxC to CxHxW
            tensor = tensor.permute((2, 0, 1))
            return tensor

    def _load_target(self, path: str) -> Tensor:
        """Load the target mask for a single image.

        Args:
            path: path to the image

        Returns:
            the target mask
        """
        filename = os.path.join(path)
        with Image.open(filename) as img:
            array = np.array(img.convert("L"))
            tensor: Tensor = torch.from_numpy(array)  # type: ignore[attr-defined]
            tensor = torch.clamp(tensor, min=0, max=1)  # type: ignore[attr-defined]
            tensor = tensor.to(torch.long)  # type: ignore[attr-defined]
            return tensor

    def _check_integrity(self) -> bool:
        """Checks the integrity of the dataset structure.

        Returns:
            True if the dataset directories and split files are found, else False
        """
        directory = self.metadata[self.split]["directory"]
        dirpath = os.path.join(self.root, directory)
        if not os.path.exists(dirpath):
            return False
        return True

    def _download(self) -> None:
        """Download the dataset and extract it.

        Raises:
            AssertionError: if the checksum of split.py does not match
        """
        if self._check_integrity():
            print("Files already downloaded and verified")
            return

        download_and_extract_archive(
            self.metadata[self.split]["url"],
            self.root,
            filename=self.metadata[self.split]["filename"],
            md5=self.metadata[self.split]["md5"] if self.checksum else None,
        )

    def plot(
        self,
        sample: Dict[str, Tensor],
        show_titles: bool = True,
        suptitle: Optional[str] = None,
    ) -> plt.Figure:
        """Plot a sample from the dataset.

        Args:
            sample: a sample returned by :meth:`__getitem__`
            show_titles: flag indicating whether to show titles above each panel
            suptitle: optional string to use as a suptitle

        Returns:
            a matplotlib Figure with the rendered sample
        """
        vv = np.rollaxis(sample["image"][:3].numpy(), 0, 3)
        vh = np.rollaxis(sample["image"][3:].numpy(), 0, 3)
        water_mask = sample["mask"][0].numpy()

        showing_flood_mask = sample["mask"].shape[0] > 1
        showing_predictions = "prediction" in sample
        num_panels = 3
        if showing_flood_mask:
            flood_mask = sample["mask"][1].numpy()
            num_panels += 1

        if showing_predictions:
            predictions = sample["prediction"].numpy()
            num_panels += 1

        fig, axs = plt.subplots(1, num_panels, figsize=(num_panels * 4, 3))
        axs[0].imshow(vv)
        axs[0].axis("off")
        axs[1].imshow(vh)
        axs[1].axis("off")
        axs[2].imshow(water_mask)
        axs[2].axis("off")
        if show_titles:
            axs[0].set_title("VV")
            axs[1].set_title("VH")
            axs[2].set_title("Water mask")

        idx = 0
        if showing_flood_mask:
            axs[3 + idx].imshow(flood_mask)
            axs[3 + idx].axis("off")
            if show_titles:
                axs[3 + idx].set_title("Flood mask")
            idx += 1

        if showing_predictions:
            axs[3 + idx].imshow(predictions)
            axs[3 + idx].axis("off")
            if show_titles:
                axs[3 + idx].set_title("Predictions")
            idx += 1

        if suptitle is not None:
            plt.suptitle(suptitle)
        return fig

max_stars_repo_name: khlaifiabilel/torchgeo

max_issues_repo_name: khlaifiabilel/torchgeo

max_forks_repo_name: khlaifiabilel/torchgeo


text:
'''Decorator for the compiled Fortran library tish
'''

from dsmpy.flib import tish
import numpy as np

parameters = dict(maxnzone=tish.parameters.maxnzone.item(),
                  maxnr=tish.parameters.maxnr.item(),
                  maxlmax=tish.parameters.maxlmax.item(),
                  maxnlay=tish.parameters.maxnlay.item(),
                  flattening=tish.parameters.flattening.item())

def _pinput(parameter_file):
    """DSM input from DSM input parameter file.

    Args:
        parameter_file (str): path to DSM parameter file
            (e.g. 'PREM_SH.inf')

    Returns:
        inputs (tuple): tupe of input parameters
            (in correct order for _tish)

    """
    inputs = tish.pinput_tish(parameter_file)
    return inputs

def _tish(
        re, ratc, ratl, tlen, nspc, omegai,
        imin, imax, nzone, vrmin, vrmax, rho,
        vsv, vsh, qmu, r0, eqlat, eqlon, mt, nr,
        theta, phi, lat, lon, output, write_to_file):
    """Compute spectra for SH wavefield using DSM.

    """
    spcs = tish.tish(
        re, ratc, ratl, tlen, nspc, omegai,
        imin, imax, nzone, vrmin, vrmax, rho,
        vsv, vsh, qmu, r0, eqlat, eqlon, mt, nr,
        theta, phi, lat, lon, output, write_to_file)
    return spcs

def _translat(lat_geodetic):
    """Return geocentric latitude from geodetic latitude.

    Notes:
        flattening = 1/298.25
    """
    return tish.translat(lat_geodetic)

def _calthetaphi(stalat, stalon, eqlat, eqlon):
    """Compute theta and phi for spherical coordinate system.

    Args:
        eqlat (float): event geodetic latitude
        eqlon (float): event longitude
        stalat (float): station geodetic latitude
        stalon (float): station longitude

    Returns:
        theta (float): theta spherical coordinate
        phi (float): phi spherical coordinate

    """
    eqlat_geocentric = _translat(eqlat)
    stalat_geocentric = _translat(stalat)
    theta, phi = tish.calthetaphi(eqlat_geocentric, eqlon,
                            stalat_geocentric, stalon)
    # the conversion to radians is now done in pinput
    # so it must be reproduced here
    theta *= np.pi / 180.
    phi *= np.pi / 180.
    return theta, phi

ext: py

max_stars_repo_name: seismobassoon/dsmpy

max_issues_repo_name: seismobassoon/dsmpy

max_forks_repo_name: seismobassoon/dsmpy

reason: import numpy


text:
# Train a new network on a data set with train.py
#
#    Basic usage: python train.py data_directory
#    Prints out training loss, validation loss, and validation accuracy as the network trains
#    Options:
#        Set directory to save checkpoints: python train.py data_dir --save_dir save_directory
#        Choose architecture: python train.py data_dir --arch "vgg13"
#        Set hyperparameters: python train.py data_dir --learning_rate 0.01 --hidden_units 512 --epochs 20
#        Use GPU for training: python train.py data_dir --gpu
#

# Imports here
import argparse
import os
import numpy as np
import torch
import random
from util import datestr
from torch import nn, optim
from torchvision import datasets, transforms, models
from collections import OrderedDict

# Parse input arguments
parser = argparse.ArgumentParser(
    description='Train a new network on a data set of images.'
)

parser.add_argument('data_directory', type=str, help='path to directory of image training data')
parser.add_argument('--seed', type=int, default=16, help='manual random seed (default=%(default)s)')
parser.add_argument('--gpu', action='store_const', const='use_gpu', default=False, help='use gpu (default=%(default)s)')
parser.add_argument('--arch', type=str, choices=['vgg13', 'vgg16', 'densenet161'], default='vgg16', help='deep neural network architecture for image features (default=%(default)s)') # , 'resnet50', 'inception_v3'
parser.add_argument('--learning_rate', type=float, default='0.0001', help='learning rate (default=%(default)s)')
parser.add_argument('--hidden_units', type=int, nargs='?', default=[], help='hidden layer sizes (default=%(default)s)')
parser.add_argument('--prob_dropout', type=float, default=0.25, help='dropout probability (fraction) (default=%(default)s)')
parser.add_argument('--epochs', type=int, default=5, help='number of training epochs (default=%(default)s)')
parser.add_argument('--warm_start', action='store_const', const=True, default=False, help='warm start with checkpoint.pth')
parser.add_argument('--print_every', type=int, default=5, help='decimation factor for printing per-batch performance (default=%(default)s)')
parser.add_argument('--save_dir', type=str, nargs='?', default='.', help='directory for saving checkpoints (default=\'%(default)s\')')
args = parser.parse_args()
print(args)

# Note: I evolved my idea of what the classifier structure should be over time, failing to reach the
# 70% threshold (possibly partially owing to other bugs) with hidden_units of [4096, 1024, 1024] and
# [1024, 512, 256, 256, 256, 256]. While i did not copy any code, I do attribute the /idea/ of
# running with hidden_units = [] to https://www.freecodecamp.org/news/how-to-build-the-best-image-classifier-3c72010b3d55/,
# found via a Google search for "flowers vgg 102 hidden classifier".

# Set randomizer behavior
manual_seed = args.seed
attempt_determinism = True # Ran into some problems that /seemed/ influenced by this
# https://forums.fast.ai/t/solved-reproducibility-where-is-the-randomness-coming-in/31628/4
# https://pytorch.org/docs/stable/notes/randomness.html
if attempt_determinism:
    np.random.seed(manual_seed)
    random.seed(manual_seed)
    torch.manual_seed(manual_seed)

    # if you are using GPU
    torch.cuda.manual_seed(manual_seed)
    torch.cuda.manual_seed_all(manual_seed)
    #torch.backends.cudnn.enabled = False
    #torch.backends.cudnn.benchmark = False
    #torch.backends.cudnn.deterministic = True

# Load the data
data_dir = args.data_directory
train_dir = data_dir + '/train'
valid_dir = data_dir + '/valid'
test_dir = data_dir + '/test'

## DONE: Define your transforms for the training, validation, and testing sets
## https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html
data_transforms = {
    'train': transforms.Compose([transforms.Resize(255),
                                 transforms.RandomRotation(30),
                                 transforms.RandomResizedCrop(224),
                                 transforms.RandomHorizontalFlip(),
                                 transforms.ToTensor(),
                                 transforms.Normalize([0.485, 0.456, 0.406],
                                                      [0.229, 0.224, 0.225])
                                ]),
    'valid': transforms.Compose([transforms.Resize(255),
                                 transforms.CenterCrop(224),
                                 transforms.ToTensor(),
                                 transforms.Normalize([0.485, 0.456, 0.406],
                                                      [0.229, 0.224, 0.225])
                                ]),
    'test': transforms.Compose([transforms.Resize(255),
                                transforms.CenterCrop(224),
                                transforms.ToTensor(),
                                transforms.Normalize([0.485, 0.456, 0.406],
                                                     [0.229, 0.224, 0.225])])
}

## DONE: Load the datasets with ImageFolder
image_datasets = {
    x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x])
    for x in ['train', 'valid', 'test']
}

## DONE: Using the image datasets and the transforms, define the dataloaders
dataloaders = {
    x: torch.utils.data.DataLoader(image_datasets[x], batch_size=32, shuffle=True)
    for x in ['train', 'valid', 'test']
}

dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'valid', 'test']}
n_batches = {x: len(dataloaders[x]) for x in ['train', 'valid', 'test']}

## Use GPU if it's available and specified
if args.gpu and torch.cuda.is_available():
    device = torch.device("cuda")
elif args.gpu:
    print("Warning: --gpu specified but cuda is not available.")
    device = torch.device("cpu")
else:
    device = torch.device("cpu")

print("Training with {}...".format(device))

# DONE: Build and train your network

## https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html
if args.arch == 'vgg13':
    model = models.vgg13(pretrained=True)
    n_input = model.classifier[0].in_features
elif args.arch == 'vgg16':
    model = models.vgg16(pretrained=True)
    n_input = model.classifier[0].in_features
elif args.arch == 'densenet161':
    model = models.densenet161(pretrained=True)
    n_input = model.classifier.in_features
#elif args.arch == 'inception_v3':
    # RuntimeError: Expected tensor for argument #1 'input' to have the same dimension as tensor for 'result'; but 4 does not equal 2 (while checking arguments for cudnn_convolution)
#    model = models.inception_v3(pretrained=True)
#    n_input = model.fc.in_features
#elif args.arch == 'resnet50':
     # https://discuss.pytorch.org/t/runtimeerror-element-0-of-variables-does-not-require-grad-and-does-not-have-a-grad-fn/11074/2
#    model = models.resnet50(pretrained=True)
#    n_input = model.fc.in_features
else:
    # https://stackoverflow.com/questions/2052390/manually-raising-throwing-an-exception-in-python
    raise ValueError('Encountered unexpected --arch argument')

## Remember the number of input features (size of input to the classifier)

## Freeze parameters so that we don't backpropagate through them
model.to(device)
for param in model.parameters():
    param.requires_grad = False

## Create classifier for our flower classification task
n_hidden = args.hidden_units
#print(args.hidden_units)
#print(n_hidden)
#print(type(n_hidden))
#print(len(n_hidden))
n_output = 102
n_sequence = n_hidden.copy()
# https://stackoverflow.com/questions/8537916/whats-the-idiomatic-syntax-for-prepending-to-a-short-python-list
n_sequence.insert(0, n_input)
n_sequence.append(n_output)

## Build nn.Sequential()
# https://pymotw.com/3/collections/ordereddict.html
new_classifier = OrderedDict()

print("n_hidden = {}".format(n_hidden))
print("n_sequence = {}".format(n_sequence))
for i in range(len(n_sequence) - 1):
    ordstr = str(i + 1)
    new_classifier['fc' + ordstr] = nn.Linear(n_sequence[i], n_sequence[i + 1])
    # Don't do ReLU or dropout on output layer
    if i < len(n_sequence) - 2:
        new_classifier['relu' + ordstr] = nn.ReLU()
        new_classifier['drop' + ordstr] = nn.Dropout(args.prob_dropout)

new_classifier['output'] = nn.LogSoftmax(dim=1)

## Replace classifier in deep neural network with my own
model.classifier = nn.Sequential(new_classifier)
print(model)

## Warm start with existing checkpoint
if args.warm_start:
    filepath = os.path.join(args.save_dir, 'checkpoint.pth')
    checkpoint = torch.load(filepath)
    model.classifier.load_state_dict(checkpoint['model_classifier_state_dict'])
    model.class_to_idx = checkpoint['class_to_idx']
    print('Loaded checkpoint for warm start with args: {}'.format(checkpoint['args']))
else:
    checkpoint = dict()
    checkpoint['cum_epochs'] = 0

# Initialize dictionary keys if null (appended to at end for checkpoint)
for key in ['epoch_loss_train', 'epoch_loss_valid', 'epoch_acc_train', 'epoch_acc_valid']:
    if checkpoint.get(key) is None:
        checkpoint[key] = []

print('Validation accuracy history after {} cumulative epochs is {}.'.format(
    checkpoint['cum_epochs'], checkpoint['epoch_acc_valid']))

## Initialize the weights
# https://stackoverflow.com/questions/49433936/how-to-initialize-weights-in-pytorch
def init_weights(m):
    if type(m) == nn.Linear:
        # https://pytorch.org/cppdocs/api/function_namespacetorch_1_1nn_1_1init_1ace282f75916a862c9678343dfd4d5ffe.html
        torch.nn.init.xavier_uniform_(m.weight)
        m.bias.data.fill_(0.01)

manually_init_weights = False
if manually_init_weights:
    model.classifier.apply(init_weights)

## Use negative log likelihood loss (corresponding to log-softmax)
criterion = nn.NLLLoss()

## Only train the classifier parameters (feature parameters are frozen)
optimizer = optim.Adam(model.classifier.parameters(), lr=args.learning_rate)

# Train the network
# https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html
num_epochs = args.epochs
print_every = args.print_every
epoch_losses = {x: [] for x in ['train', 'test', 'valid']}
epoch_accuracies = {x: [] for x in ['train', 'test', 'valid']}

model.to(device) # Use GPU if available

for epoch in range(num_epochs):

    for phase in ['train', 'valid']:
        if phase == 'train':
            model.train() # Training mode
            print(datestr() + "(epoch {}/{}) Model is set to training mode for '{}' phase.".format(
                epoch + 1, num_epochs, phase))
        else:
            model.eval() # Evaluation mode
            print(datestr() + "(epoch {}/{}) Model is set to evaluation mode for '{}' phase.".format(
                epoch + 1, num_epochs, phase))

        batch_counter = 0
        running_loss = 0.0
        running_n_correct = 0

        for inputs, labels in dataloaders[phase]:
            batch_counter += 1

            # Move inputs and label tensors to the target device (CPU or GPU)
            inputs = inputs.to(device)
            labels = labels.to(device)

            # Zero the parameter gradients
            optimizer.zero_grad()

            with torch.set_grad_enabled(phase == 'train'):

                # Forward pass
                outputs = model.forward(inputs)
                _, preds = torch.max(outputs, dim=1)
                loss = criterion(outputs, labels)

                # Backward pass and step optimizer only in training phase
                if phase == 'train':
                    loss.backward()
                    optimizer.step()

                # Calculate and record statistics
                batch_loss = loss.item()
                #print(outputs.shape)
                #print(preds)
                #print(labels.data)
                batch_n_correct = torch.sum(preds == labels.data)
                batch_acc = float(batch_n_correct) / len(labels)
                running_loss += batch_loss
                running_n_correct += batch_n_correct

                # Subsample training batch output but print results for every validation batch
                if (((batch_counter - 1) % print_every) == 0) or (phase == 'valid'):
                    print(datestr()
                        + "    (epoch {}/{} batch {}/{}) {} loss = {:.4f}, acc = {:.2f}% ({}/{})".format(
                            epoch + 1, num_epochs, batch_counter, n_batches[phase], phase,
                            batch_loss, 100.0*batch_acc, batch_n_correct, len(labels)))

        # Batches complete
        epoch_loss = running_loss / dataset_sizes[phase]
        epoch_acc = running_n_correct.float() / dataset_sizes[phase]

        epoch_losses[phase].append(epoch_loss)
        epoch_accuracies[phase].append(epoch_acc)

        print(datestr() + "(epoch {}/{}) {} loss = {:.4f}, acc = {:.2f}% ({}/{})".format(
            epoch + 1, num_epochs, phase, epoch_losses[phase][-1], 100.0*epoch_accuracies[phase][-1],
            running_n_correct, dataset_sizes[phase]))

# DONE: Save the checkpoint
# https://pytorch.org/tutorials/beginner/saving_loading_models.html
model.class_to_idx = image_datasets['train'].class_to_idx

# Update checkpoint data (these are set to null above when not warm start)
cum_epochs = checkpoint['cum_epochs'] + num_epochs
checkpoint['epoch_loss_train'].append(epoch_losses['train'])
checkpoint['epoch_loss_valid'].append(epoch_losses['valid'])
checkpoint['epoch_acc_train'].append(epoch_accuracies['train'])
checkpoint['epoch_acc_valid'].append(epoch_accuracies['valid'])

checkpoint['epoch'] = epoch
checkpoint['model_classifier_state_dict'] = model.classifier.state_dict()
checkpoint['optimizer_state_dict'] = optimizer.state_dict()
checkpoint['class_to_idx'] = model.class_to_idx
checkpoint['args'] = args
checkpoint['n_input'] = n_input
checkpoint['cum_epochs'] = cum_epochs

torch.save(checkpoint, os.path.join(args.save_dir, 'checkpoint.pth'))
print("Saved checkpoint. For this training session:")

# Print out summary performance
print("    epoch_losses['train'] = {}".format(epoch_losses['train']))
print("    epoch_losses['valid'] = {}".format(epoch_losses['valid']))
print("    epoch_accuracies['train'] = {}".format(epoch_accuracies['train']))
print("    epoch_accuracies['valid'] = {}".format(epoch_accuracies['valid']))
print("    cum_epochs = {}".format(cum_epochs))
print("    args = {}".format(args))

max_stars_repo_name: gitlars/aipnd-project

max_issues_repo_name: gitlars/aipnd-project

max_forks_repo_name: gitlars/aipnd-project


text:
import numpy as np

from mushroom.algorithms.value import QLearning
from mushroom.core import Core
from mushroom.environments import *
from mushroom.policy import EpsGreedy
from mushroom.utils.parameters import Parameter

"""
Simple script to solve a simple chain with Q-Learning.

"""

def experiment():
    np.random.seed()

    # MDP
    mdp = generate_simple_chain(state_n=5, goal_states=[2], prob=.8, rew=1,
                                gamma=.9)

    # Policy
    epsilon = Parameter(value=.15)
    pi = EpsGreedy(epsilon=epsilon)

    # Agent
    learning_rate = Parameter(value=.2)
    algorithm_params = dict(learning_rate=learning_rate)
    agent = QLearning(pi, mdp.info, **algorithm_params)

    # Algorithm
    core = Core(agent, mdp)

    # Train
    core.learn(n_steps=10000, n_steps_per_fit=1)

if __name__ == '__main__':
    experiment()

max_stars_repo_name: doroK/mushroom

max_issues_repo_name: doroK/mushroom

max_forks_repo_name: doroK/mushroom


text:
import sys
import numpy as np
import torch

sys.path.append("src/models")
from train_model import _main

x = torch.Tensor(np.ones((10, 1, 28, 28)))
y = torch.tensor(np.arange((10)), dtype=torch.long)

try:
    _main([[x, y]], [x, y], "models/model.pth", 1, True)
    val = True
except RuntimeError as err:
    print(err)
    val = False

assert val, "training failed on 1 epoch."


text:
# Copyright (c) Facebook, Inc. and its affiliates.
import inspect
import logging
import numpy as np
import os
from pathlib import Path
from typing import Dict, List, Optional, Tuple, Union
import cv2
import torch
from torch import nn

from detectron2.config import configurable
from detectron2.layers import ShapeSpec
from detectron2.modeling.meta_arch import build_sem_seg_head
from detectron2.modeling.poolers import ROIPooler, convert_boxes_to_pooler_format
from detectron2.modeling.roi_heads import ROI_HEADS_REGISTRY
from detectron2.modeling.roi_heads.box_head import build_box_head
from detectron2.modeling.roi_heads.keypoint_head import build_keypoint_head
from detectron2.modeling.roi_heads.mask_head import build_mask_head
from detectron2.structures import Boxes, ImageList, Instances

from wsl.layers import CSC
from wsl.modeling.roi_heads.fast_rcnn import WSDDNOutputLayers
from wsl.modeling.roi_heads.roi_heads import (
    ROIHeads,
    get_image_level_gt,
    select_foreground_proposals,
    select_proposals_with_visible_keypoints,
)
from wsl.modeling.roi_heads.third_party.cpg_stats import Statistic

logger = logging.getLogger(__name__)

@ROI_HEADS_REGISTRY.register()
class WSJDSROIHeads(ROIHeads):
    """
    It's "standard" in a sense that there is no ROI transform sharing
    or feature sharing between tasks.
    Each head independently processes the input features by each head's
    own pooler and head.

    This class is used by most models, such as FPN and C5.
    To implement more models, you can subclass it and implement a different
    :meth:`forward()` or a head.
    """

    @configurable
    def __init__(
        self,
        *,
        box_in_features: List[str],
        box_pooler: ROIPooler,
        box_head: nn.Module,
        box_predictor: nn.Module,
        mask_in_features: Optional[List[str]] = None,
        mask_pooler: Optional[ROIPooler] = None,
        mask_head: Optional[nn.Module] = None,
        keypoint_in_features: Optional[List[str]] = None,
        keypoint_pooler: Optional[ROIPooler] = None,
        keypoint_head: Optional[nn.Module] = None,
        train_on_pred_boxes: bool = False,
        output_dir: str = None,
        csc_max_iter: int = None,
        sem_seg_in_features: List[str] = None,
        constraint: bool = False,
        sem_seg_head: nn.Module = None,
        device: torch.device = None,
        pixel_mean: torch.Tensor = None,
        pixel_std: torch.Tensor = None,
        **kwargs
    ):
        """
        NOTE: this interface is experimental.

        Args:
            box_in_features (list[str]): list of feature names to use for the box head.
            box_pooler (ROIPooler): pooler to extra region features for box head
            box_head (nn.Module): transform features to make box predictions
            box_predictor (nn.Module): make box predictions from the feature.
                Should have the same interface as :class:`FastRCNNOutputLayers`.
            mask_in_features (list[str]): list of feature names to use for the mask
                pooler or mask head. None if not using mask head.
            mask_pooler (ROIPooler): pooler to extract region features from image features.
                The mask head will then take region features to make predictions.
                If None, the mask head will directly take the dict of image features
                defined by `mask_in_features`
            mask_head (nn.Module): transform features to make mask predictions
            keypoint_in_features, keypoint_pooler, keypoint_head: similar to ``mask_*``.
            train_on_pred_boxes (bool): whether to use proposal boxes or
                predicted boxes from the box head to train other heads.
        """
        super().__init__(**kwargs)
        # keep self.in_features for backward compatibility
        self.in_features = self.box_in_features = box_in_features
        self.box_pooler = box_pooler
        self.box_head = box_head
        self.box_predictor = box_predictor

        self.mask_on = mask_in_features is not None
        if self.mask_on:
            self.mask_in_features = mask_in_features
            self.mask_pooler = mask_pooler
            self.mask_head = mask_head

        self.keypoint_on = keypoint_in_features is not None
        if self.keypoint_on:
            self.keypoint_in_features = keypoint_in_features
            self.keypoint_pooler = keypoint_pooler
            self.keypoint_head = keypoint_head

        self.train_on_pred_boxes = train_on_pred_boxes

        # CSC
        self.output_dir = output_dir
        self.csc_max_iter = csc_max_iter
        self.csc_cur_iter = 0
        self.tau = 0.7
        self.fg_threshold = 0.1
        self.bg_threshold = 0.005
        self.csc = CSC(
            tau=self.tau,
            debug_info=False,
            fg_threshold=self.fg_threshold,
            mass_threshold=0.2,
            density_threshold=0.0,
            area_sqrt=True,
            context_scale=1.8,
        )

        self.csc_stats = Statistic(
            self.csc_max_iter, self.tau, 4, self.num_classes, self.output_dir, ""
        )

        self.mask_csc_stats = Statistic(
            self.csc_max_iter, self.tau, 4, self.num_classes, self.output_dir, "mask_"
        )

        self.sem_seg_in_features = sem_seg_in_features
        self.constraint = constraint
        self.sem_seg_head = sem_seg_head

        self.device = device
        self.img_normalizer = lambda x: (x * pixel_std + pixel_mean)

    @classmethod
    def from_config(cls, cfg, input_shape):
        ret = super().from_config(cfg)
        ret["train_on_pred_boxes"] = cfg.MODEL.ROI_BOX_HEAD.TRAIN_ON_PRED_BOXES
        # Subclasses that have not been updated to use from_config style construction
        # may have overridden _init_*_head methods. In this case, those overridden methods
        # will not be classmethods and we need to avoid trying to call them here.
        # We test for this with ismethod which only returns True for bound methods of cls.
        # Such subclasses will need to handle calling their overridden _init_*_head methods.
        if inspect.ismethod(cls._init_box_head):
            ret.update(cls._init_box_head(cfg, input_shape))
        if inspect.ismethod(cls._init_mask_head):
            ret.update(cls._init_mask_head(cfg, input_shape))
        if inspect.ismethod(cls._init_keypoint_head):
            ret.update(cls._init_keypoint_head(cfg, input_shape))
        if inspect.ismethod(cls._init_sem_seg_head):
            ret.update(cls._init_sem_seg_head(cfg, input_shape))

        device = torch.device(cfg.MODEL.DEVICE)
        ret["device"] = device
        # TODO(YH): batch size=1
        assert len(cfg.MODEL.PIXEL_MEAN) == len(cfg.MODEL.PIXEL_STD)
        ret["pixel_mean"] = torch.Tensor(cfg.MODEL.PIXEL_MEAN).to(device).view(1, -1, 1, 1)
        ret["pixel_std"] = torch.Tensor(cfg.MODEL.PIXEL_STD).to(device).view(1, -1, 1, 1)

        return ret

    @classmethod
    def _init_box_head(cls, cfg, input_shape):
        # fmt: off
        in_features       = cfg.MODEL.ROI_HEADS.IN_FEATURES
        pooler_resolution = cfg.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION
        pooler_scales     = tuple(1.0 / input_shape[k].stride for k in in_features)
        sampling_ratio    = cfg.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO
        pooler_type       = cfg.MODEL.ROI_BOX_HEAD.POOLER_TYPE
        # fmt: on

        # If StandardROIHeads is applied on multiple feature maps (as in FPN),
        # then we share the same predictors and therefore the channel counts must be the same
        in_channels = [input_shape[f].channels for f in in_features]
        # Check all channel counts are equal
        assert len(set(in_channels)) == 1, in_channels
        in_channels = in_channels[0]

        box_pooler = ROIPooler(
            output_size=pooler_resolution,
            scales=pooler_scales,
            sampling_ratio=sampling_ratio,
            pooler_type=pooler_type,
        )
        # Here we split "box head" and "box predictor", which is mainly due to historical reasons.
        # They are used together so the "box predictor" layers should be part of the "box head".
        # New subclasses of ROIHeads do not need "box predictor"s.
        box_head = build_box_head(
            cfg, ShapeSpec(channels=in_channels, height=pooler_resolution, width=pooler_resolution)
        )
        box_predictor = WSDDNOutputLayers(cfg, box_head.output_shape)

        # CSC
        output_dir = cfg.OUTPUT_DIR
        csc_max_iter = cfg.WSL.CSC_MAX_ITER

        return {
            "box_in_features": in_features,
            "box_pooler": box_pooler,
            "box_head": box_head,
            "box_predictor": box_predictor,
            "output_dir": output_dir,
            "csc_max_iter": csc_max_iter,
        }

    @classmethod
    def _init_sem_seg_head(cls, cfg, input_shape):
        # fmt: off
        in_features       = cfg.MODEL.ROI_HEADS.IN_FEATURES
        constraint   = cfg.MODEL.SEM_SEG_HEAD.CONSTRAINT
        # fmt: on

        # in_channels = [input_shape[f].channels for f in in_features][0]
        in_channels = [input_shape[f].channels for f in in_features]

        sem_seg_head = build_sem_seg_head(cfg, ShapeSpec(channels=in_channels))

        return {
            "sem_seg_in_features": in_features,
            "constraint": constraint,
            "sem_seg_head": sem_seg_head,
        }

    @classmethod
    def _init_mask_head(cls, cfg, input_shape):
        if not cfg.MODEL.MASK_ON:
            return {}
        return {}
        # fmt: off
        in_features       = cfg.MODEL.ROI_HEADS.IN_FEATURES
        pooler_resolution = cfg.MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION
        pooler_scales     = tuple(1.0 / input_shape[k].stride for k in in_features)
        sampling_ratio    = cfg.MODEL.ROI_MASK_HEAD.POOLER_SAMPLING_RATIO
        pooler_type       = cfg.MODEL.ROI_MASK_HEAD.POOLER_TYPE
        # fmt: on

        in_channels = [input_shape[f].channels for f in in_features][0]

        ret = {"mask_in_features": in_features}
        ret["mask_pooler"] = (
            ROIPooler(
                output_size=pooler_resolution,
                scales=pooler_scales,
                sampling_ratio=sampling_ratio,
                pooler_type=pooler_type,
            )
            if pooler_type
            else None
        )
        if pooler_type:
            shape = ShapeSpec(
                channels=in_channels, width=pooler_resolution, height=pooler_resolution
            )
        else:
            shape = {f: input_shape[f] for f in in_features}
        ret["mask_head"] = build_mask_head(cfg, shape)
        return ret

    @classmethod
    def _init_keypoint_head(cls, cfg, input_shape):
        if not cfg.MODEL.KEYPOINT_ON:
            return {}
        # fmt: off
        in_features       = cfg.MODEL.ROI_HEADS.IN_FEATURES
        pooler_resolution = cfg.MODEL.ROI_KEYPOINT_HEAD.POOLER_RESOLUTION
        pooler_scales     = tuple(1.0 / input_shape[k].stride for k in in_features)  # noqa
        sampling_ratio    = cfg.MODEL.ROI_KEYPOINT_HEAD.POOLER_SAMPLING_RATIO
        pooler_type       = cfg.MODEL.ROI_KEYPOINT_HEAD.POOLER_TYPE
        # fmt: on

        in_channels = [input_shape[f].channels for f in in_features][0]

        ret = {"keypoint_in_features": in_features}
        ret["keypoint_pooler"] = (
            ROIPooler(
                output_size=pooler_resolution,
                scales=pooler_scales,
                sampling_ratio=sampling_ratio,
                pooler_type=pooler_type,
            )
            if pooler_type
            else None
        )
        if pooler_type:
            shape = ShapeSpec(
                channels=in_channels, width=pooler_resolution, height=pooler_resolution
            )
        else:
            shape = {f: input_shape[f] for f in in_features}
        ret["keypoint_head"] = build_keypoint_head(cfg, shape)
        return ret

    def forward(
        self,
        images: ImageList,
        features: Dict[str, torch.Tensor],
        proposals: List[Instances],
        targets: Optional[List[Instances]] = None,
    ) -> Tuple[List[Instances], Dict[str, torch.Tensor]]:
        """
        See :class:`ROIHeads.forward`.
        """
        self.gt_classes_img, self.gt_classes_img_int, self.gt_classes_img_oh = get_image_level_gt(
            targets, self.num_classes
        )

        # del images
        self.images = images
        if self.training:
            assert targets, "'targets' argument is required during training"
            proposals = self.label_and_sample_proposals(proposals, targets)
        del targets

        if self.training:
            losses = self._forward_box(features, proposals)
            # Usually the original proposals used by the box head are used by the mask, keypoint
            # heads. But when `self.train_on_pred_boxes is True`, proposals will contain boxes
            # predicted by the box head.
            losses.update(self._forward_mask(features, proposals))
            losses.update(self._forward_keypoint(features, proposals))
            return proposals, losses
        else:
            pred_instances, all_scores, all_boxes = self._forward_box(features, proposals)
            # During inference cascaded prediction is used: the mask and keypoints heads are only
            # applied to the top scoring box detections.
            pred_instances = self.forward_with_given_boxes(features, pred_instances)
            return pred_instances, {}, all_scores, all_boxes

    def forward_with_given_boxes(
        self, features: Dict[str, torch.Tensor], instances: List[Instances]
    ) -> List[Instances]:
        """
        Use the given boxes in `instances` to produce other (non-box) per-ROI outputs.

        This is useful for downstream tasks where a box is known, but need to obtain
        other attributes (outputs of other heads).
        Test-time augmentation also uses this.

        Args:
            features: same as in `forward()`
            instances (list[Instances]): instances to predict other outputs. Expect the keys
                "pred_boxes" and "pred_classes" to exist.

        Returns:
            instances (list[Instances]):
                the same `Instances` objects, with extra
                fields such as `pred_masks` or `pred_keypoints`.
        """
        assert not self.training
        assert instances[0].has("pred_boxes") and instances[0].has("pred_classes")

        instances = self._forward_mask(features, instances)
        instances = self._forward_keypoint(features, instances)
        return instances

    def _forward_box(self, features: Dict[str, torch.Tensor], proposals: List[Instances]):
        """
        Forward logic of the box prediction branch. If `self.train_on_pred_boxes is True`,
            the function puts predicted boxes in the `proposal_boxes` field of `proposals` argument.

        Args:
            features (dict[str, Tensor]): mapping from feature map names to tensor.
                Same as in :meth:`ROIHeads.forward`.
            proposals (list[Instances]): the per-image object proposals with
                their matching ground truth.
                Each has fields "proposal_boxes", and "objectness_logits",
                "gt_classes", "gt_boxes".

        Returns:
            In training, a dict of losses.
            In inference, a list of `Instances`, the predicted instances.
        """
        features = [features[f] for f in self.box_in_features]
        box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])

        objectness_logits = torch.cat([x.objectness_logits + 1 for x in proposals], dim=0)
        # objectness_logits = torch.cat([x.objectness_logits for x in proposals], dim=0)
        # objectness_logits[objectness_logits > 1] = 0
        # objectness_logits[objectness_logits < -1] = 0
        # objectness_logits = objectness_logits + 1
        box_features = box_features * objectness_logits.view(-1, 1, 1, 1)

        # torch.cuda.empty_cache()

        box_features = self.box_head(box_features)
        predictions = self.box_predictor(box_features)
        pred_class_logits, _ = predictions
        del box_features

        # det
        cpgs = self._forward_cpg(pred_class_logits, proposals)
        W_pos, W_neg, PL, NL = self._forward_csc(cpgs, pred_class_logits, proposals)
        self._save_mask(cpgs, "cpgs", "cpg")

        if self.training:
            losses = self.box_predictor.losses_csc(
                predictions, proposals, W_pos, W_neg, PL, NL, self.csc_stats
            )
            if self.csc_cur_iter > self.csc_max_iter:
                self.csc_cur_iter = self.csc_cur_iter + 1
                return losses

            # det -->> seg
            targets, weights = self._get_sem_seg_target(pred_class_logits, cpgs)
            self._save_mask(cpgs, "masks", "cpg")
            self._save_mask(targets, "masks", "target")
            self._save_mask(weights, "masks", "weight")

            # seg
            masks, losses_mask = self.sem_seg_head(
                self.images, features, targets=targets, weights=weights
            )
            losses.update(losses_mask)
            if self.constraint:
                masks, masks_crf = masks
                self._save_mask(masks_crf, "masks", "masks_crf")
            self._save_mask(masks, "masks", "mask")

            # det <<-- seg
            masks = torch.sigmoid(masks)
            masks = masks / masks.view(1, self.num_classes, -1).max(dim=2, keepdim=True)[0].view(
                1, self.num_classes, 1, 1
            ).expand_as(masks)
            self._save_mask(masks, "masks", "mask_norm")
            W_pos, W_neg, PL, NL = self._forward_csc(masks, pred_class_logits, proposals)

            losses_refine = self.box_predictor.losses_csc(
                predictions,
                proposals,
                W_pos,
                W_neg,
                PL,
                NL,
                self.csc_stats,
                loss_weight=0.1,
                prefix="mask_",
            )
            losses.update(losses_refine)

            # proposals is modified in-place below, so losses must be computed first.
            if self.train_on_pred_boxes:
                with torch.no_grad():
                    pred_boxes = self.box_predictor.predict_boxes_for_gt_classes(
                        predictions, proposals
                    )
                    for proposals_per_image, pred_boxes_per_image in zip(proposals, pred_boxes):
                        proposals_per_image.proposal_boxes = Boxes(pred_boxes_per_image)

            self.csc_cur_iter = self.csc_cur_iter + 1
            return losses
        else:
            pred_instances, _, all_scores, all_boxes = self.box_predictor.inference(
                predictions, proposals
            )

            # seg
            masks, _ = self.sem_seg_head(self.images, features)
            pred_instances = self._get_ins_seg_result(pred_instances, masks)

            return pred_instances, all_scores, all_boxes

    def _forward_mask(self, features: Dict[str, torch.Tensor], instances: List[Instances]):
        """
        Forward logic of the mask prediction branch.

        Args:
            features (dict[str, Tensor]): mapping from feature map names to tensor.
                Same as in :meth:`ROIHeads.forward`.
            instances (list[Instances]): the per-image instances to train/predict masks.
                In training, they can be the proposals.
                In inference, they can be the boxes predicted by R-CNN box head.

        Returns:
            In training, a dict of losses.
            In inference, update `instances` with new fields "pred_masks" and return it.
        """
        if not self.mask_on:
            # https://github.com/pytorch/pytorch/issues/43942
            if self.training:
                return {}
            else:
                return instances

        if self.training:
            # head is only trained on positive proposals.
            instances, _ = select_foreground_proposals(instances, self.num_classes)

        if self.mask_pooler is not None:
            features = [features[f] for f in self.mask_in_features]
            boxes = [x.proposal_boxes if self.training else x.pred_boxes for x in instances]
            features = self.mask_pooler(features, boxes)
        else:
            # https://github.com/pytorch/pytorch/issues/41448
            features = dict([(f, features[f]) for f in self.mask_in_features])
        return self.mask_head(features, instances)

    def _forward_keypoint(self, features: Dict[str, torch.Tensor], instances: List[Instances]):
        """
        Forward logic of the keypoint prediction branch.

        Args:
            features (dict[str, Tensor]): mapping from feature map names to tensor.
                Same as in :meth:`ROIHeads.forward`.
            instances (list[Instances]): the per-image instances to train/predict keypoints.
                In training, they can be the proposals.
                In inference, they can be the boxes predicted by R-CNN box head.

        Returns:
            In training, a dict of losses.
            In inference, update `instances` with new fields "pred_keypoints" and return it.
        """
        if not self.keypoint_on:
            if self.training:
                return {}
            else:
                return instances

        if self.training:
            # head is only trained on positive proposals with >=1 visible keypoints.
            instances, _ = select_foreground_proposals(instances, self.num_classes)
            instances = select_proposals_with_visible_keypoints(instances)

        if self.keypoint_pooler is not None:
            features = [features[f] for f in self.keypoint_in_features]
            boxes = [x.proposal_boxes if self.training else x.pred_boxes for x in instances]
            features = self.keypoint_pooler(features, boxes)
        else:
            features = dict([(f, features[f]) for f in self.keypoint_in_features])
        return self.keypoint_head(features, instances)

    @torch.no_grad()
    def _forward_cpg(self, pred_class_logits, proposals):
        if not self.training:
            return None

        if self.csc_cur_iter > self.csc_max_iter:
            return None

        pred_class_img_logits = torch.sum(pred_class_logits, dim=0, keepdim=True)
        # pred_class_img_logits = torch.clamp(pred_class_img_logits, min=1e-6, max=1.0 - 1e-6)

        image_sizes = self.images.image_sizes[0]
        cpgs = torch.zeros(
            (1, self.num_classes, image_sizes[0], image_sizes[1]),
            dtype=pred_class_img_logits.dtype,
            device=pred_class_img_logits.device,
        )
        for c in range(self.num_classes):
            if self.gt_classes_img_oh[0, c] < 0.5:
                continue
            if pred_class_img_logits[0, c] < self.tau:
                continue

            grad_outputs = torch.zeros(
                pred_class_logits.size(),
                dtype=pred_class_logits.dtype,
                device=pred_class_logits.device,
            )
            grad_outputs[:, c] = 1.0
            (cpg,) = torch.autograd.grad(  # grad_outputs[0, c] = self.pred_class_img_logits[0, c]
                outputs=pred_class_logits,
                inputs=self.images.tensor,
                grad_outputs=grad_outputs,
                retain_graph=True,
                create_graph=False,
            )
            cpg.abs_()
            cpg, _ = torch.max(cpg, dim=1)

            # cpg_scale_op
            max_value = torch.max(cpg)
            cpg = cpg / max_value

            cpgs[0, c, :, :] = cpg[0, :, :]
            del cpg
            del grad_outputs
            torch.cuda.empty_cache()

        self.images.tensor.requires_grad = False
        self.images.tensor.detach()

        return cpgs

    @torch.no_grad()
    def _forward_csc(self, masks, pred_class_logits, proposals):
        if not self.training:
            return None, None, None, None

        if self.csc_cur_iter > self.csc_max_iter:
            PL = self.gt_classes_img_oh
            NL = torch.zeros(
                self.gt_classes_img_oh.size(),
                dtype=self.gt_classes_img_oh.dtype,
                device=self.gt_classes_img_oh.device,
            )
            W_pos = torch.ones(
                pred_class_logits.size(),
                dtype=pred_class_logits.dtype,
                device=pred_class_logits.device,
            )
            W_neg = torch.zeros(
                pred_class_logits.size(),
                dtype=pred_class_logits.dtype,
                device=pred_class_logits.device,
            )
            return W_pos, W_neg, PL, NL

        pred_class_img_logits = torch.sum(pred_class_logits, dim=0, keepdim=True)

        pooler_fmt_boxes = convert_boxes_to_pooler_format([x.proposal_boxes for x in proposals])

        W, PL, NL = self.csc(masks, self.gt_classes_img_oh, pred_class_img_logits, pooler_fmt_boxes)

        W_pos = torch.clamp(W, min=0.0)
        W_neg = torch.clamp(W, max=0.0)

        W_pos.abs_()
        W_neg.abs_()

        return W_pos, W_neg, PL, NL

    @torch.no_grad()
    def _save_mask(self, masks, prefix, suffix):
        if masks is None:
            return
        if self.csc_cur_iter % 128 > 0:
            return

        output_dir = os.path.join(self.output_dir, prefix)
        Path(output_dir).mkdir(parents=True, exist_ok=True)

        id_str = "iter" + str(self.csc_cur_iter) + "_gpu" + str(masks.device.index)

        for c in range(self.num_classes):
            if self.gt_classes_img_oh[0, c] < 0.5:
                continue
            # if self.pred_class_img_logits[0, c] < self.tau:
            # continue

            mask = masks[0, c, ...].clone().detach().cpu().numpy()
            max_value = np.max(mask)
            if max_value > 0:
                max_value = max_value * 0.1
                mask = np.clip(mask, 0, max_value)
                mask = mask / max_value * 255
            mask = mask.astype(np.uint8)
            im_color = cv2.applyColorMap(mask, cv2.COLORMAP_JET)

            file_name = os.path.join(output_dir, id_str + "_c" + str(c) + "_" + suffix + ".png")
            cv2.imwrite(file_name, im_color)

        imgs = self.img_normalizer(self.images.tensor)
        imgs = imgs.clone().detach().cpu().numpy().transpose((0, 2, 3, 1))
        imgs = imgs.astype(np.uint8)
        file_name = os.path.join(output_dir, id_str + ".png")
        cv2.imwrite(file_name, imgs[0, ...])

    @torch.no_grad()
    def _get_sem_seg_target(self, pred_class_logits, cpgs):
        pred_class_img_logits = torch.sum(pred_class_logits, dim=0, keepdim=True)

        # 1   : pos
        # 0   : neg
        # 255 : ignore
        targets = torch.ones_like(cpgs)
        targets[cpgs < self.fg_threshold] = 255
        targets[cpgs < self.bg_threshold] = 0
        targets[pred_class_img_logits < self.tau, :, :] = 255
        targets[self.gt_classes_img_oh == 0.5, :, :] = 255
        targets[self.gt_classes_img_oh == 0, :, :] = 0

        pos = (targets == 1).sum(dim=[2, 3], keepdim=True).expand_as(cpgs).type_as(cpgs)
        neg = (targets == 0).sum(dim=[2, 3], keepdim=True).expand_as(cpgs).type_as(cpgs)

        weights = torch.ones_like(cpgs)
        # spatial_size = cpgs.size(1) * cpgs.size(2) * cpgs.size(3)
        weights[targets == 1] = pos[targets == 1].reciprocal()
        weights[targets == 0] = neg[targets == 0].reciprocal()
        weights[targets == 255] = 0

        targets[targets == 255] = 0

        return targets, weights

    @torch.no_grad()
    def _get_ins_seg_result(self, pred_instances, masks):
        for i, pred_instance in enumerate(pred_instances):
            img_h, img_w = pred_instance.image_size

            pred_classes = pred_instance.pred_classes
            pred_boxes = pred_instance.pred_boxes

            pred_classes_int = pred_classes.to(dtype=torch.int64)
            boxes = pred_boxes.tensor

            N = boxes.shape[0]
            # C = masks.shape[1]
            mask_h, mask_w = masks.shape[2:]

            img_masks = torch.zeros(N, 1, mask_h, mask_w, device=masks.device, dtype=masks.dtype)

            x0, y0, x1, y1 = torch.split(boxes.to(dtype=torch.int64), 1, dim=1)  # each is Nx1
            # print(x0.shape, x1.shape, y0.shape, y1.shape, pred_classes_int.shape)

            # x0_int, y0_int = torch.clamp(boxes.min(dim=0).values.floor()[:2] - 1, min=0).to(dtype=torch.int32)
            # x1_int = torch.clamp(boxes[:, 2].max().ceil() + 1, max=img_w).to(dtype=torch.int32)
            # y1_int = torch.clamp(boxes[:, 3].max().ceil() + 1, max=img_h).to(dtype=torch.int32)

            # indices_0 = torch.full((N,), 0, dtype=torch.int64, device=masks.device)
            # indices_i = torch.full((N,), i, dtype=torch.int64, device=masks.device)
            # indices_N = torch.arange(N, device=masks.device)

            # img_masks[(indices_N, indices_0, slice(y0_int, y1_int), slice(x0_int, x1_int))] = masks[(indices_i, pred_classes_int, slice(y0_int, y1_int), slice(x0_int, x1_int))]
            for j in range(N):
                img_masks[j, 0, y0[j][0] : y1[j][0], x0[j][0] : x1[j][0]] = masks[
                    i, pred_classes_int[j], y0[j][0] : y1[j][0], x0[j][0] : x1[j][0]
                ]

            # print(img_masks.size(), img_masks.device)

            pred_instance.pred_masks = img_masks

            pred_instance.no_paste = (pred_classes_int[:] >= 0).to(dtype=torch.bool)

        return pred_instances

max_stars_repo_name: shenyunhang/JTSM

max_issues_repo_name: shenyunhang/JTSM

max_forks_repo_name: shenyunhang/JTSM


text:
# --------------
#Importing header files

import pandas as pd
from sklearn.model_selection import train_test_split

# Code starts here
data = pd.read_csv(path)
X = data.drop(['customer.id','paid.back.loan'], axis=1)
y = data['paid.back.loan']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
# Code ends here

# --------------
#Importing header files
import matplotlib.pyplot as plt

# Code starts here
fully_paid = y_train.value_counts()
print (fully_paid)
plt.bar(y_train.unique(),fully_paid)
plt.show()
# Code ends here

# --------------
#Importing header files
import numpy as np
from sklearn.preprocessing import LabelEncoder

# Code starts here
X_train['int.rate'] = X_train['int.rate'].apply(lambda x: (float(x.replace("%","")))/100)
X_test['int.rate'] = X_test['int.rate'].apply(lambda x: (float(x.replace("%","")))/100)
num_df = X_train.select_dtypes(exclude='object')
cat_df = X_train.select_dtypes(include='object')
# Code ends here

# --------------
#Importing header files
import seaborn as sns

# Code starts here
cols = num_df.columns
fig, axes = plt.subplots(nrows=9, ncols=1, figsize=(5,25))
for i in range(0,9):
    sns.boxplot(x=y_train, y=num_df[cols[i]], ax=axes[i])
plt.show()

# Code ends here

# --------------
# Code starts here
cols = cat_df.columns
fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(25,10))
for i in range(0,2):
    for j in range(0,2):
        sns.countplot(x=X_train[cols[i*2+j]], hue=y_train, ax=axes[i,j])
plt.show()
# Code ends here

# --------------
#Importing header files
from sklearn.tree import DecisionTreeClassifier

#Code starts here

#Looping through categorical columns
for col in cat_df.columns:

    #Filling null values with 'NA'
    X_train[col].fillna('NA',inplace=True)

    #Initalising a label encoder object
    le=LabelEncoder()

    #Fitting and transforming the column in X_train with 'le'
    X_train[col]=le.fit_transform(X_train[col])

    #Filling null values with 'NA'
    X_test[col].fillna('NA',inplace=True)

    #Fitting the column in X_test with 'le'
    X_test[col]=le.transform(X_test[col])

# Replacing the values of y_train
y_train.replace({'No':0,'Yes':1},inplace=True)

# Replacing the values of y_test
y_test.replace({'No':0,'Yes':1},inplace=True)

#Initialising 'Decision Tree' model
model=DecisionTreeClassifier(random_state=0)

#Training the 'Decision Tree' model
model.fit(X_train, y_train)

#Finding the accuracy of 'Decision Tree' model
acc=model.score(X_test, y_test)

#Printing the accuracy
print(acc)

#Code ends here

# --------------
#Importing header files
from sklearn.model_selection import GridSearchCV

#Parameter grid
parameter_grid = {'max_depth': np.arange(3,10), 'min_samples_leaf': range(10,50,10)}
model_2 = DecisionTreeClassifier(random_state=0)
# Code starts here
p_tree = GridSearchCV(estimator=model_2, param_grid=parameter_grid, cv=5)
p_tree.fit(X_train, y_train)
acc_2 = p_tree.score(X_test, y_test)
print (acc_2)
# Code ends here

# --------------
#Importing header files

from io import StringIO
from sklearn.tree import export_graphviz
from sklearn import tree
from sklearn import metrics
from IPython.display import Image
import pydotplus

# Code starts here
dot_data = export_graphviz(decision_tree=p_tree.best_estimator_, out_file=None, feature_names=X.columns, filled = True, class_names=['loan_paid_back_yes','loan_paid_back_no'])
graph_big = pydotplus.graph_from_dot_data(dot_data)

# show graph - do not delete/modify the code below this line
img_path = user_data_dir+'/file.png'
graph_big.write_png(img_path)

plt.figure(figsize=(20,15))
plt.imshow(plt.imread(img_path))
plt.axis('off')
plt.show()

# Code ends here

max_stars_repo_name: girishvankudre/ga-learner-dsmp-repo

max_issues_repo_name: girishvankudre/ga-learner-dsmp-repo

max_forks_repo_name: girishvankudre/ga-learner-dsmp-repo


text:
import numpy as np
import numpy.linalg as la
import scipy.linalg as sla
from pdb import set_trace
from sklearn.linear_model import  Lasso

from .model import Model

# Simulates 2-state system
# x1[k+1] = x1[k] + x2[k]**3
# x2[k+1] = x2[k] + u

class DummyNonlinear(Model):
    def __init__(self, system):
        super().__init__(system)

    def state_dim(self):
        return 2

    def train(self, trajs):
        pass

    def traj_to_state(self, traj):
        state = np.zeros((2,))
        state[:] = traj[-1].obs[:]
        return state[:]

    def update_state(state, new_obs, new_ctrl):
        return state[:]

    def pred(self, state, ctrl):
        u = ctrl[0]
        x1, x2 = state[0], state[1]
        xpred = np.array([x1 + x2**3, x2 + u])

        return xpred

    def pred_diff(self, state, ctrl):
        u = ctrl[0]
        x1, x2 = state[0], state[1]
        xpred = np.array([x1 + x2**3, x2 + u])
        grad1 = np.array([[1.0, 3 * x2 ** 2], [0., 1.]])
        grad2 = np.array([[0.], [1.]])
        return xpred, grad1, grad2

    @staticmethod
    def get_configuration_space(system):
        """
        Returns the model configuration space.
        """
        return None

max_stars_repo_name: StochLab/autompc

max_issues_repo_name: StochLab/autompc

max_forks_repo_name: StochLab/autompc

reason: import numpy,import scipy


text:
"""Visualization module."""
import gc
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
import warnings

from eda_viz.exceptions import InvalidDataError, \
    NonNumericTypeNotSupportedError, TypeNotSupportedError
from eda_viz.warnings import DifferentLengthWarning
# from tests.test_data import *

sns.set(style='whitegrid', palette='muted', font_scale=1.5)
FIG_SIZE = (14, 8)
ACCEPTED_GENERIC_TYPES = (pd.Series, np.ndarray, list)
DEFAULT_COLOR = '#4878d0'

def column_distribution(column, title='Class Distribution', xlabel='Class',
                        ylabel='Count', sort=True):
    """
    Displays distribution bar chart.

    :param column: pandas Series, numpy ndarray or list
    :param title: title of plot (str)
    :param xlabel: label of x-axis (str)
    :param ylabel: label of y-axis (str)
    :param sort: boolean, to sort the column distribution bars from highest to lowest
    :return: None
    :raise TypeNotSupportedError: if the column passed is one of pandas Series, numpy ndarray or list
    :raise InvalidDataError: if the data cannot be displayed
    """
    if not isinstance(column, ACCEPTED_GENERIC_TYPES):
        raise TypeNotSupportedError(
            'Column of type {} is not supported by '
            'this method. Make sure the type is one '
            'of {}'.format(type(column), ', '.join(
                [str(t) for t in ACCEPTED_GENERIC_TYPES])))

    try:
        plt.subplots(figsize=FIG_SIZE)
        if sort:
            count_classes = pd.value_counts(column, sort=True)
        else:
            count_classes = pd.value_counts(column)
        count_classes.plot(kind='bar', rot=0)
        plt.title(title)
        plt.xlabel(xlabel)
        plt.ylabel(ylabel)
        plt.show()
        del count_classes
        gc.collect()
    except TypeError:
        raise InvalidDataError('Something is wrong with the data provided. '
                               'Make sure there are some valid data values to '
                               'plot.')

def histogram(column, title='Histogram', xlabel='Class Bins',
              ylabel='Count'):
    """
    Displays histogram.

    :param column: pandas Series, numpy ndarray or list
    :param title: title of plot (str)
    :param xlabel: label of x-axis (str)
    :param ylabel: label of y-axis (str)
    :return: None
    :raise TypeNotSupportedError: if the column passed is one of pandas Series, numpy ndarray or list
    :raise NonNumericTypeNotSupportedError: if the column passed is non-numeric
    :raise InvalidDataError: if the data cannot be displayed
    """
    if not isinstance(column, ACCEPTED_GENERIC_TYPES):
        raise TypeNotSupportedError(
            'Column of type {} is not supported by '
            'this method. Make sure the type is one '
            'of {}'.format(type(column), ', '.join(
                [str(t) for t in ACCEPTED_GENERIC_TYPES])))

    pd_column = pd.Series(column)
    if not pd.api.types.is_numeric_dtype(pd_column):
        raise NonNumericTypeNotSupportedError('Column other than type numeric '
                                              'is not supported by this '
                                              'method. Make sure the type is '
                                              'numeric.')

    try:
        plt.subplots(figsize=FIG_SIZE)
        pd_column.plot(kind='hist', rot=0)
        plt.title(title)
        plt.xlabel(xlabel)
        plt.ylabel(ylabel)
        plt.show()
        del pd_column
        gc.collect()
    except TypeError:
        raise InvalidDataError('Something is wrong with the data provided. '
                               'Make sure there are some valid data values to '
                               'plot.')

def bar_plot(x, y, title='Bar Plot', xlabel='x-axis',
              ylabel='y-axis'):
    """
    Displays Bar plot.

    :param x: pandas Series, numpy ndarray or list for x-axis
    :param y: pandas Series, numpy ndarray or list for y-axis
    :param title: title of plot (str)
    :param xlabel: label of x-axis (str)
    :param ylabel: label of y-axis (str)
    :return: None
    :raise TypeNotSupportedError: if the column passed is one of pandas Series, numpy ndarray or list
    :raise NonNumericTypeNotSupportedError: if the column passed is non-numeric
    :raise InvalidDataError: if the data cannot be displayed
    :raise DifferentLengthWarning: when x and y data are of different lengths
    """
    if not isinstance(x, ACCEPTED_GENERIC_TYPES):
        raise TypeNotSupportedError(
            'Column of type {} is not supported by '
            'this method. Make sure the type is one '
            'of {}'.format(type(x), ', '.join(
                [str(t) for t in ACCEPTED_GENERIC_TYPES])))

    if not isinstance(y, ACCEPTED_GENERIC_TYPES):
        raise TypeNotSupportedError(
            'Column of type {} is not supported by '
            'this method. Make sure the type is one '
            'of {}'.format(type(y), ', '.join(
                [str(t) for t in ACCEPTED_GENERIC_TYPES])))

    x_pd_column = pd.Series(x)
    y_pd_column = pd.Series(y)
    if not pd.api.types.is_numeric_dtype(y_pd_column):
        raise NonNumericTypeNotSupportedError('Column other than type numeric '
                                              'is not supported for y-axis by '
                                              'this method. Make sure the type '
                                              'is numeric.')

    # Check if x and y have equal length
    if len(x_pd_column) != len(y_pd_column):
        warnings.warn('x and y have different lengths.', DifferentLengthWarning)

    try:
        plt.subplots(figsize=FIG_SIZE)
        sns.barplot(x_pd_column, y_pd_column, color=DEFAULT_COLOR)
        plt.title(title)
        plt.xlabel(xlabel)
        plt.ylabel(ylabel)
        plt.show()
        del x_pd_column
        del y_pd_column
        gc.collect()
    except TypeError:
        raise InvalidDataError('Something is wrong with the data provided. '
                               'Make sure there are some valid data values to '
                               'plot.')
    except ValueError:
        raise InvalidDataError('Something is wrong with the data provided. '
                               'Make sure there are some valid data values to '
                               'plot.')

# if __name__ == '__main__':
#     bar_plot(CATEGORICAL_LIST, DICTIONARY_LIST)

max_stars_repo_name: ajaymaity/eda-viz

max_issues_repo_name: ajaymaity/eda-viz

max_forks_repo_name: ajaymaity/eda-viz


text:
# This file has resusable funtions in Fbprophet and Arima model for forecasting purpose

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

def load_data(name):
    '''
    name: string, name of the csv file, file has to be in the current directory

    return
    ______
    df: dataframe

    '''
    df = pd.read_csv(name)
    return df

def prepare_data(df, ds_col, y_col):
    '''
    rename the data and target column into 'ds' and 'y'

    ds_col: string, column name
    y_col: string, column name
    '''
    df = df.copy()
    df = df.rename(columns = {ds_col:'ds', y_col:'y'})
    df['ds']= pd.to_datetime(df['ds'])

    return df

def group_by_date(df, date_type, start_date, end_date):
    '''
    group dataframe by date

    df: has column 'ds'

    '''
    df = df.copy()
    df['ds']= pd.to_datetime(df['ds']) # make sure 'ds' is in datetime datatype

    df = df[(df['ds'] >= start_date ) & (df['ds'] <= end_date)]

    if date_type == "Y":
        df = df.groupby(df['ds'].dt.to_period('Y')).sum()
    elif date_type == "M":
        df = df.groupby(df['ds'].dt.to_period('M')).sum()
    else:
        df = df.groupby(df['ds'].dt.to_period('D')).sum()

    df = df.reset_index()
    df['ds']= pd.to_datetime(df.ds.astype(str))

    return df

def slice_df(df, start_date, end_date, selected_col=[]):
    '''
    selects the desired date frame and columns from the dataframe

    selected_col: list, list of column names
    '''
    df = df.copy()
    if len(selected_col) != 0:
        df = df[selected_col]

    mask = (df['ds'] >= start_date) & (df['ds'] <= end_date)
    return df[mask]

def train_test_split(df, train_end_date,test_end_date):
    '''
    df has a column of 'ds' which stores datetime variable
    '''
    train = (df['ds'] <= train_end_date)
    test = (df['ds'] > train_end_date) & (df['ds'] <= test_end_date )

    df_tr = df.loc[train]
    df_tst = df.loc[test]

    print("train shape",df_tr.shape)
    print("test shape",df_tst.shape)

    return df_tr, df_tst

def calculate_monthly_revenue(df, start_date, end_date):
    '''
    calculate monthly revenue from start_date to end_date

    '''
    df = df[(df['ds'] >= start_date ) & (df['ds'] <= end_date)]
    df = df.groupby(df['ds'].dt.to_period('m')).sum() # group by 'YYYY-MM'
    return df

def mean_absolute_percentage_error(y_pred,y_true):
    y_true, y_pred = np.array(y_true), np.array(y_pred)
    return np.mean(np.abs((y_true - y_pred) / y_true))

def calc_mape(df_pred, df_true, date_type, start_date, end_date):
    '''
    This function can first group by data and then calc mape

    df_pred: dataframe, has a column of 'yhat' that stores predicted
             monthly revenue a column of 'ds' (datetime)
    df_true: dataframe, has a column of 'y' that stores true monthly
             revenue, a column of 'ds' (datetime)
    '''

    df_pred = group_by_date(df_pred, date_type, start_date, end_date)
    df_true = group_by_date(df_true, date_type, start_date, end_date)
    return  mean_absolute_percentage_error(df_true['y'], df_pred['yhat'])

def make_dataframe(arr, df_true):
    '''
    convert prediction array to a dataframe

    this function will be useful do convert the prediction data from arima model

    arr: numpy array
    df_true: pandas dataframe, which has the desired data frame as 'ds'
    '''
    if type(arr) is np.ndarray:
        df = pd.DataFrame({'ds':df_true.ds, 'yhat': arr})

    return df

def plot_train_and_test(df_tst, df_tr):
    '''
    plot line graphs of test data and training data into a graph

    df_pred: dataframe, has column 'ds' (Datetime variable), 'y'
    df_tr: dataframe, training data, has columns 'ds', 'y'
    '''
    f, ax = plt.subplots(figsize=(16,5)) #make empty graph
    df_tst.plot(kind='line',x='ds', y='y', color='green', label='Test', ax=ax)
    df_tr.plot(kind='line',x='ds',y='y', color='blue',label='Train', ax=ax)

    plt.legend()
    plt.title(' y Train and Test')
    plt.show()

def plot_predict_and_actual(df_pred, df_tr):
    '''
    plot line graphs of prediction value and actual value into a graph

    df_pred: dataframe, has column 'ds' (Datetime variable), 'yhat'
    df_tr: dataframe, training data, has columns 'ds', 'y'
    '''

    f, ax = plt.subplots(figsize=(16,5)) #make empty graph
    df_pred.plot(kind='line',x='ds', y='yhat', color='orange', label='Prediction', ax=ax)
    df_tr.plot(kind='line',x='ds',y='y', color='blue',label='Actual', ax=ax)

    plt.legend()
    plt.title(' y Forecast and Actual')
    plt.show()

def plot_predict_vs_actual(df_pred, df_true, date_type, start_date, end_date):
    '''
    plot line graphs of prediction value and actual value into a graph

    df_pred: has column 'ds' (Datetime variable), 'yhat'
    df_true: has columns 'ds', 'y'

    Note: Graph for yearly data is weird
    '''

    df_pred = group_by_date(df_pred, date_type, start_date, end_date)
    df_true = group_by_date(df_true, date_type, start_date, end_date)
    print(df_pred.shape, df_true.shape)

    f, ax = plt.subplots(figsize=(15,10)) #make empty graph
    df_pred.plot(kind='line',x='ds', y='yhat', color='orange', label='Prediction', ax=ax)
    df_true.plot(kind='line',x='ds',y='y', color='blue',label='Actual', ax=ax)

    plt.legend()
    plt.title( start_date[:4] + ' y Forecast vs Actual')
    plt.show()

def save_to_file(df,col_names,  filename, groupby = False, date_type = None, start_date = None, end_date = None ):
    '''
    save data to csv file with columns:year, month, yhat

    df: the dataframe with predicted data
    col_names: list of strings, desired columns in the dataframe to save to output
    '''

    df = df.copy()
    df = df[col_names]
    if groupby:
        df = group_by_date(df, date_type, start_date, end_date)

    df['year'] = df.ds.dt.year # create a separate column for 'year'
    df['month'] = df.ds.dt.month # create a separate column for 'month'

    # choose the desired columns and save to file
    df[['year','month','yhat']].to_csv(filename, index = False)

max_stars_repo_name: naomiyjchen/sales-forecast-time-series

max_issues_repo_name: naomiyjchen/sales-forecast-time-series

max_forks_repo_name: naomiyjchen/sales-forecast-time-series


text:
import json
from pathlib import Path
from types import SimpleNamespace
from typing import NamedTuple, Any, Optional

from flask.testing import FlaskClient
from numpy.testing import assert_almost_equal

from osgeo_utils.auxiliary.base import PathLikeOrStr

test_env = SimpleNamespace()
test_env.root = Path('.')
test_env.talos = 'http://localhost:5000'
test_env.tasc = test_env.talos
test_env.talos_wps = test_env.talos + '/wps/'
test_env.tasc_api1 = test_env.tasc + '/api/1.0'
test_env.talos_api1 = test_env.talos + '/api/1.0'

class TalosTest(NamedTuple):
    c: FlaskClient
    url: str
    request: Any
    response: Any
    dump_response: Optional[PathLikeOrStr] = None

    def assert_response(self, other):
        assert self.response == other

    def run_test(self, is_json=True):
        if is_json:
            rv = self.c.post(self.url, json=self.request)
        else:
            rv = self.c.post(self.url, data=self.request)
        assert rv.status_code == 200
        json_data = rv.get_json()
        if self.dump_response:
            with open(self.dump_response, 'w') as f:
                json.dump(json_data, f, indent=4)
        self.assert_response(json_data)

class ProfileTest(TalosTest):
    def assert_response(self, other):
        c1 = self.response['features'][0]['geometry']['coordinates']
        c2 = other['features'][0]['geometry']['coordinates']
        assert_almost_equal(c1, c2)
        self.response['features'][0]['geometry']['coordinates'] = None
        other['features'][0]['geometry']['coordinates'] = None
        super().assert_response(other)

class CzmlTest(TalosTest):
    def assert_response(self, other):
        self.response[1]['name'] = None
        other[1]['name'] = None
        super().assert_response(other)

def read_file(filename):
    with open(filename) as f:
        data = f.read()
    return data

reason: from numpy


text:
"""This module handles image input and output
"""

import cv2
from PIL import Image
import numpy as np

def imread(img, mode='O'):
    """read image file

    Arguments:
        img {path} -- image path

    Keyword Arguments:
        mode {str} -- image format ['np', 'pil'] (default: {'O'})

    Returns:
        image -- numpy ndarray or PIL data
    """

    if isinstance(img, str):
        img = cv2.imread(img)

    if mode == 'O':
        if not isinstance(img, np.ndarray):
            img = np.array(img)
    else:
        if isinstance(img, np.ndarray):
            img = Image.fromarray(img)

    return img

max_stars_repo_name: tkianai/tk-cv

max_issues_repo_name: tkianai/tk-cv

max_forks_repo_name: tkianai/tk-cv


text:
import os,sys
from itertools import product
from collections import Counter,OrderedDict
from Bio import SeqIO
import numpy as np
import math

######################## USAGE ##########################
#
# python create_kmer_freq_vectors.py <FASTA> <k-mer size>
#
#########################################################

fasta_fn =     sys.argv[1]
k        = int(sys.argv[2])

def kmer_freq ( ref_str, k ):
	"""
	Walk through sequence and return k-mer counts plus
	a pseudocount of 1.
	"""
	ref_str = ref_str.upper()
	kmers = []
	for seq in product("ATGC",repeat=k):
		kmers.append( "".join(seq) )

	kmer_counts = Counter()
	for j in range( len(ref_str)-(k-1) ):
		motif    = ref_str[j:j+k]
		kmer_counts[motif] += 1

	# Combine forward and reverse complement motifs into one count
	combined_kmer = Counter()
	for kmer in kmers:
		kmer_rc = rev_comp_motif(kmer)
		if not combined_kmer.get(kmer_rc):
			combined_kmer[kmer] = kmer_counts[kmer] + kmer_counts[kmer_rc] + 1

	return combined_kmer

def rev_comp_motif( motif ):
	"""
	Return the reverse complement of the input motif.
	"""
	COMP = {"A":"T", \
			"T":"A", \
			"C":"G", \
			"G":"C", \
			"W":"S", \
			"S":"W", \
			"M":"K", \
			"K":"M", \
			"R":"Y", \
			"Y":"R", \
			"B":"V", \
			"D":"H", \
			"H":"D", \
			"V":"B", \
			"N":"N", \
			"X":"X", \
			"*":"*"}
	rc_motif = []
	for char in motif[::-1]:
		rc_motif.append( COMP[char] )

	return "".join(rc_motif)

for i,entry in enumerate(SeqIO.parse(fasta_fn, "fasta")):
	# Return k-mer counts for each entry in the fasta
	kmers     = []
	seq_comp  = []
	seq_kmers = kmer_freq( str(entry.seq), k )

	seq_kmers_sort = OrderedDict(sorted(seq_kmers.items()))
	for kmer,count in seq_kmers_sort.iteritems():

		# Normalize each count by total k-mers and log2 transform
		kmer_normed_comp = math.log(float(count) / sum(seq_kmers_sort.values()))
		# kmer_normed_comp = count
		kmers.append(kmer)
		seq_comp.append(kmer_normed_comp)

	kmer_vector = np.array(kmers)
	comp_vector = np.array(seq_comp)

	if i==0:
		# print header
		print "seq_name\t%s" % "\t".join(kmer_vector)

	print "%s\t%s" % (entry.id, "\t".join(map(lambda x: str(round(x,4)), comp_vector)))

max_stars_repo_name: NeolithEra/mbin

max_issues_repo_name: NeolithEra/mbin

max_forks_repo_name: NeolithEra/mbin