text:
# coding:utf-8
# 2019-1-10
# tf-record tool

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from datetime import datetime
import os
import random
import sys
import threading

import numpy as np
import tensorflow as tf
import logging

__all__ = ['main']

def check_and_set_default_args(input_para):
    assert not (input_para['train_shards'] % input_para['num_threads'] and input_para['validation_shards'] % input_para['num_threads']), ("train_shards 和 validation_shards 必须是 num_threads 的公约数")

def _find_image_files(data_dir,
                    labels_file,
                    class_label_base):
    """
    return：
        filenames : list, data_dir下所有图片名
        texts : list, 父文件名
        labels : list, 每张图片对应的分类索引，下标从class_label_base开始
    """
    logging.info('处理的数据来源于 %s.' % data_dir)
    unique_labels = [l.strip() for l in tf.gfile.FastGFile(
        labels_file, 'r').readlines()] # 获得需要处理的文件列表

    labels, filenames, texts = [], [], []

    label_index = class_label_base

    # 各个文件夹下的图片
    for text in unique_labels:
        pic_file_pattern = '%s/%s/*' % (data_dir, text) # 匹配格式 `*` 通配符
        matching_files = []
        try:
            matching_files = tf.gfile.Glob(pic_file_pattern)
        except:
            pass
        labels.extend([label_index] * len(matching_files)) # 分类索引
        texts.extend([text] * len(matching_files)) # 父文件夹名
        filenames.extend(matching_files) # 图片名

        logging.info("第 %s 个分类，共处理文件 %s 个" % (
                label_index, len(labels)))
        label_index += 1

    # 打乱列表
    shuffled_index = [i for i in range(len(filenames))]
    random.seed(12345)
    random.shuffle(shuffled_index)

    filenames = [filenames[i] for i in shuffled_index]
    texts = [texts[i] for i in shuffled_index]
    labels = [labels[i] for i in shuffled_index]

    logging.info('在 %s 内, 共找到 %d 张图片,  %d 个标签' %
                 (data_dir, len(filenames), len(unique_labels)))

    return filenames, texts, labels

class ImageCoder(object):
    """
    将TensorFlow图像读取转换
    """
    def __init__(self):
        self._sess = tf.Session() # 创建Session处理图像

        # 将PNG转换为JPEG
        self._png_data = tf.placeholder(dtype=tf.string) # 读取图片未经过解码为字符串
        image = tf.image.decode_png(self._png_data, channels=3)
        self._png_to_jpeg = tf.image.encode_jpeg(image, format='rgb', quality=100)

        # JPEG解码
        self._decode_jpeg_data = tf.placeholder(dtype=tf.string)
        self._decode_jpeg = tf.image.decode_jpeg(self._decode_jpeg_data, channels=3)

    def png_to_jpeg(self, image_data):
        return self._sess.run(self._png_to_jpeg,
                              feed_dict={self._png_data: image_data})

    def decode_jpeg(self, image_data):
        image = self._sess.run(self._decode_jpeg,
                               feed_dict={self._decode_jpeg_data: image_data})
        assert len(image.shape) == 3
        assert image.shape[2] == 3, '图片不是三通道'
        return image

def _is_png(filename):
    """
    return bool
    """
    return '.png' in filename

def _process_image(filename, coder):
    """
    读取图片
    return:
        image_buffer: string, JPEG
        height: int
        width: int
    """
    with open(filename, 'rb') as f:
        image_data = f.read()

    if _is_png(filename):
        logging.info('%s 转换为JPEG格式' % filename)
        image_data = coder.png_to_jpeg(image_data)

    image = coder.decode_jpeg(image_data) # JPEG解码,解码后可获得图像属性
    assert len(image.shape) == 3
    assert image.shape[2] == 3, '图像不是三通道'

    height, width = image.shape[0], image.shape[1]

    return image_data, height, width

def _int64_feature(value):
    """
    转换为int64格式
    """
    if not isinstance(value, list):
        value = [value]
    return tf.train.Feature(int64_list=tf.train.Int64List(value=value))

def _bytes_feature(value):
    """
    转换为bytes
    """
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def _convert_to_example(filename,
                        image_buffer,
                        label,
                        text,
                        height,
                        width):
    """
    使用tf.train.Example生成 Example proto
    """
    colorspace = b'RGB'
    channels = 3
    image_format = b'JPEG'

    example = tf.train.Example(features=tf.train.Features(feature={
        'image/height': _int64_feature(height),
        'image/width': _int64_feature(width),
        'image/colorspace': _bytes_feature(colorspace),
        'image/channels': _int64_feature(channels),
        'image/class/label': _int64_feature(label),
        'image/class/text': _bytes_feature(str.encode(text)),
        'image/format': _bytes_feature(image_format),
        'image/filename': _bytes_feature(os.path.basename(str.encode(filename))),
        'image/encoded': _bytes_feature(image_buffer)}))

    return example

def _process_image_files_batch(coder,
                                thread_index,
                                ranges,
                                name,
                                filenames,
                                texts,
                                labels,
                                num_shards,
                                output_dir,
                                dataset_name):
    """
    每个线程的call函数
    """
    num_threads = len(ranges) # 计算线程数
    assert not num_shards % num_threads, 'num_shars必须能整除num_threads'
    num_shards_per_batch = int(num_shards / num_threads) # 每个线程需要处理的数据

    shard_ranges = np.linspace(
        ranges[thread_index][0],
        ranges[thread_index][1],
        num_shards_per_batch + 1).astype(int)
    num_files_in_thread = ranges[thread_index][1] - ranges[thread_index][0] # 计算当前线程需要处理的图像数量

    counter = 0
    for s in range(num_shards_per_batch):
        shard = thread_index * num_shards_per_batch + s # 计算当前图像处理批次
        output_filename = '%s_%s_%.5d-of-%.5d.tfrecord' % (dataset_name, name, shard, num_shards)

        output_file = os.path.join(output_dir, output_filename)

        writer = tf.python_io.TFRecordWriter(output_file)

        shard_counter = 0
        files_in_shard = np.arange(shard_ranges[s], shard_ranges[s + 1], dtype=int) # 获得当前处理的文件索引

        for i in files_in_shard:
            filename = filenames[i]
            label = labels[i]
            text = texts[i]

            image_buffer, height, width = _process_image(filename, coder)
            example = _convert_to_example(
                filename,
                image_buffer,
                label,
                text,
                height,
                width) # tf.train.Features 格式化数据
            writer.write(example.SerializeToString())
            shard_counter += 1
            counter += 1

            if not counter % 300:
                logging.info('%s [thread %d]: Processed %d of %d images in thread batch.' %
                             (datetime.now(), thread_index, counter, num_files_in_thread))
                sys.stdout.flush()
        writer.close()
        logging.info('%s [thread %d]: Wrote %d images to %s' %
                     (datetime.now(), thread_index, shard_counter, output_file))
        sys.stdout.flush()
        shard_counter = 0
    logging.info('%s [thread %d]: Wrote %d images to %d shards.' %
                 (datetime.now(), thread_index, counter, num_files_in_thread))
    sys.stdout.flush()

def _process_image_files(name,
                        filenames,
                        texts,
                        labels,
                        num_shards,
                        num_threads,
                        output_dir,
                        dataset_name):

    # 判断图片是否对应标签
    assert len(filenames) == len(texts)
    assert len(filenames) == len(labels)

    # 将列表分割为num_threads份
    # np.linspace(0, 20, 4 + 1).astype(np.int)
    # [ 0  5 10 15 20]
    spacing = np.linspace(0, len(filenames), num_threads + 1).astype(np.int)

    # 组合索引
    ranges = []
    for i in range(len(spacing) - 1):
        ranges.append([spacing[i], spacing[i + 1]])

    logging.info('开启 %s 个线程' % num_threads)
    sys.stdout.flush() # 刷新输出

    coord = tf.train.Coordinator() # 监控线程

    coder = ImageCoder() # 初始化图像处理类

    # 线程入队
    threads = []
    for thread_index in range(len(ranges)): # range(num_threads)
        args = (coder, thread_index, ranges, name, filenames, texts, labels, num_shards, output_dir, dataset_name)
        t = threading.Thread(target=_process_image_files_batch, args=args)
        t.start()
        threads.append(t)

    coord.join(threads)
    logging.info('%s : 完成 %d 图像的数据转换.' %
                (datetime.now(), len(filenames)))
    sys.stdout.flush()

def process_dataset(name,
                    directory,
                    num_shards,
                    labels_file,
                    num_threads,
                    output_dir,
                    dataset_name,
                    class_label_base):
    filenames, texts, labels = _find_image_files(directory, labels_file, class_label_base)
    _process_image_files(name, filenames, texts, labels, num_shards, num_threads, output_dir, dataset_name)

def main(input_para):
    logging.info('Saving results to %s' % input_para['output_dir'])
    process_dataset(
        'validation',
        input_para['validation_dir'],
        input_para['validation_shards'],
        input_para['labels_file'],
        input_para['num_threads'],
        input_para['output_dir'],
        input_para['dataset_name'],
        input_para['class_label_base'])

    process_dataset(
        'train',
        input_para['train_dir'],
        input_para['train_shards'],
        input_para['labels_file'],
        input_para['output_dir'],
        input_para['num_threads'],
        input_para['output_dir'],
        input_para['dataset_name'],
        input_para['class_label_base'])

    logging.info('%s : Finish!' % datetime.now())

if __name__ == '__main__':
    input_para = {

    }
    main(input_para)

max_stars_repo_name: YangXiaoo/NoteBook

max_stars_repo_licenses: CNRI-Python RSA-MD CECILL-B

max_issues_repo_name: YangXiaoo/NoteBook

max_issues_repo_licenses: CNRI-Python RSA-MD CECILL-B

max_forks_repo_name: YangXiaoo/NoteBook

max_forks_repo_licenses: CNRI-Python RSA-MD CECILL-B


text:
# Copyright (c) 2019 Mohammad Nadji-Tehrani <m.nadji.tehrani@gmail.com>

import os
import struct
import numpy as np

def read_mnist_labels(dataset="training", path="../../Fashion-MNIST/"):
    """
    For importing the MNIST data set.  It returns an iterator
    of 2-tuples with the first element being the label and the second element
    being a numpy.uint8 2D array of pixel data for the given image.
    """
    if dataset is "training":
        fname_lbl = os.path.join(path, 'train-labels.idx1-ubyte')
    elif dataset is "testing":
        fname_lbl = os.path.join(path, 't10k-labels.idx1-ubyte')
    else:
        raise Exception(ValueError, "data set must be 'testing' or 'training'")

    # Load everything in some numpy arrays
    with open(fname_lbl, 'rb') as flbl:
        magic, num = struct.unpack(">II", flbl.read(8))
        lbl = np.fromfile(flbl, dtype=np.int8)

    get_img_lbl = lambda idx: (lbl[idx])

    # Create an iterator which returns each image in turn
    for i in range(len(lbl)):
        yield get_img_lbl(i)

mnist_labels = read_mnist_labels()
for label in mnist_labels:
    print(label)

ext: py

max_stars_repo_name: feagi/feagi-core-deprecated

max_issues_repo_name: feagi/feagi-core-deprecated

max_forks_repo_name: feagi/feagi-core-deprecated

reason: import numpy


text:
import numpy as np
import torch
from torch import nn
from torch.optim.lr_scheduler import ReduceLROnPlateau
import pytorch_lightning as pl
from torchmetrics.functional import accuracy, f1

from transformers import BertForTokenClassification, AdamW

# lists for classification report
eval_labels = list()
eval_preds = list()

class NerBertModel(pl.LightningModule):
    def __init__(self, unique_tags: int):
        super(NerBertModel, self).__init__()
        self.num_labels = len(unique_tags)
        self.bert = BertForTokenClassification.from_pretrained('bert-base-german-cased',
                                                               num_labels=self.num_labels,
                                                               output_attentions=True,
                                                               return_dict=True)

    def forward(self, ids, mask, labels):
        outputs = self.bert(input_ids=ids, attention_mask=mask, labels=labels)
        # outputs.loss   outputs.logits   outputs.attentions
        return outputs.loss, outputs.logits

    def training_step(self, batch, batch_idx):
        ids = batch["ids"]
        mask = batch["mask"]
        labels = batch["tags"]

        loss, logits = self(ids=ids, mask=mask, labels=labels)

        # compute accuracy
        flattened_targets = labels.view(-1) # shape (batch_size * seq_len,)
        active_logits = logits.view(-1, self.bert.num_labels) # shape (batch_size * seq_len, num_labels)
        flattened_predictions = torch.argmax(active_logits, axis=1) # shape (batch_size * seq_len,)

        # only compute accuracy at active labels
        active_accuracy = labels.view(-1) != -100 # shape (batch_size, seq_len)
        labels = torch.masked_select(flattened_targets, active_accuracy)
        predictions = torch.masked_select(flattened_predictions, active_accuracy)

        return {"loss": loss, "logits": logits, "predictions": predictions, "labels": labels, "train_acc": accuracy(predictions, labels)}

    def training_epoch_end(self, outputs):
        avg_train_loss = torch.stack([x["loss"] for x in outputs]).mean()
        avg_train_acc = torch.stack([x["train_acc"] for x in outputs]).mean()

        self.log("train_loss", avg_train_loss, prog_bar=True, logger=True, on_step=False, on_epoch=True)
        self.log("train_acc", avg_train_acc, prog_bar=True, logger=True, on_step=False, on_epoch=True)

    def validation_step(self, batch, batch_idx):
        ids = batch["ids"]
        mask = batch["mask"]
        labels = batch["tags"]

        loss, logits = self(ids=ids, mask=mask, labels=labels)

        # compute accuracy
        flattened_targets = labels.view(-1)
        active_logits = logits.view(-1, self.bert.num_labels)
        flattened_predictions = torch.argmax(active_logits, axis=1)

        # only compute accuracy at active labels
        active_accuracy = labels.view(-1) != -100
        labels = torch.masked_select(flattened_targets, active_accuracy)
        predictions = torch.masked_select(flattened_predictions, active_accuracy)

        return {"loss": loss, "logits": logits, "predictions": predictions, "labels": labels, "val_acc": accuracy(predictions, labels), "val_f1": f1(predictions, labels, num_classes=self.bert.num_labels)}

    def validation_epoch_end(self, outputs):
        avg_val_loss = torch.stack([x["loss"] for x in outputs]).mean()
        avg_val_acc = torch.stack([x["val_acc"] for x in outputs]).mean()
        avg_val_f1 = torch.stack([x["val_f1"] for x in outputs]).mean()

        self.log("val_loss", avg_val_loss, prog_bar=True, logger=True, on_step=False, on_epoch=True)
        self.log("val_acc", avg_val_acc, prog_bar=True, logger=True, on_step=False, on_epoch=True)
        self.log("val_f1", avg_val_f1, prog_bar=True, logger=True, on_step=False, on_epoch=True)

    def test_step(self, batch, batch_idx):
        ids = batch["ids"]
        mask = batch["mask"]
        labels = batch["tags"]

        loss, logits = self(ids=ids, mask=mask, labels=labels)

        # compute accuracy
        flattened_targets = labels.view(-1)
        active_logits = logits.view(-1, self.bert.num_labels)
        flattened_predictions = torch.argmax(active_logits, axis=1)

        # only compute accuracy at active labels
        active_accuracy = labels.view(-1) != -100
        labels = torch.masked_select(flattened_targets, active_accuracy)
        predictions = torch.masked_select(flattened_predictions, active_accuracy)

        # add for classification report
        eval_labels.extend(labels)
        eval_preds.extend(predictions)

        return {"loss": loss, "logits": logits, "predictions": predictions, "labels": labels, "test_acc": accuracy(predictions, labels), "test_f1": f1(predictions, labels, num_classes=self.bert.num_labels)}

    def test_epoch_end(self, outputs):
        avg_test_acc = torch.stack([x["test_acc"] for x in outputs]).mean()
        avg_test_f1 = torch.stack([x["test_f1"] for x in outputs]).mean()

        self.log("test_acc", avg_test_acc, prog_bar=True, logger=True, on_step=False, on_epoch=True)
        self.log("test_f1", avg_test_f1, prog_bar=True, logger=True, on_step=False, on_epoch=True)

    def configure_optimizers(self):
        optimizer = AdamW(self.parameters(), lr=0.0001)
        scheduler = ReduceLROnPlateau(optimizer)
        self.log("learning_rate", optimizer.param_groups[0]['lr'], prog_bar=True, logger=True)
        return {
            'optimizer': optimizer,
            'lr_scheduler': scheduler,
            'monitor': 'val_loss'
        }


text:
# -*- coding: utf-8 -*-

"""UNIT TESTS FOR EXAMPLE SUBPACKAGE.

This module contains unit tests for the example subpackage.

"""

from unittest import TestCase

import numpy as np
from numpy import testing as npt

from configure_package_name.example import classes, hello, math

class ClassesTestCase(TestCase):
    """Test case for ``classes`` module."""

    def setUp(self):
        """Set test parameter values."""
        r_sun = 7e8
        t_sun = 5800
        self._sb_inst = classes.StefBoltz(r_sun, t_sun)
        self._expected_out = 3.951223664081994e26

    def tearDown(self):
        """Unset test parameter values."""
        self._sb_inst = None
        self._expected_out = None

    def test_luminosity(self):
        """Test ``configure_package_name.example.classes.StefBoltz.luminosity`` method.

        See Also
        --------
        configure_package_name.example.classes.StefBoltz : Implementation of the
            ``StefBoltz`` class.

        """
        npt.assert_equal(
            self._sb_inst.luminosity(),
            self._expected_out,
            err_msg='Incorrect luminosity output.',
        )

class HelloTestCase(TestCase):
    """Test case for ``hello`` module."""

    def setUp(self):
        """Set test parameter values."""
        self._expected_out = 'Hello World!'

    def tearDown(self):
        """Unset test parameter values."""
        self._expected_out = None

    def test_hello_world(self):
        """Test ``configure_package_name.example.hello.hello_world`` function.

        See Also
        --------
        configure_package_name.example.hello.hello_world : Implementation of the
            ``hello_world`` function.

        """
        npt.assert_equal(
            hello.hello_world(),
            self._expected_out,
            err_msg='Incorrect hello_world output.',
        )

class MathTestCase(TestCase):
    """Test case for ``math`` module."""

    def setUp(self):
        """Set test parameter values."""
        self._first = 1
        self._second = 2
        self._drake_param = [1, 0.2, 1, 1, 1, 0.1, 1000]
        self._mad_data = np.arange(9).reshape(3, 3)
        self._add_res = 3
        self._drake_res = 20
        self._mad_res = 2.0

    def tearDown(self):
        """Unset test parameter values."""
        self._first = None
        self._second = None
        self._drake_param = None
        self._mad_data = None
        self._add_res = None
        self._drake_res = None
        self._mad_res = None

    def test_add_two_ints(self):
        """Test ``configure_package_name.example.math.add_two_ints`` function.

        See Also
        --------
        configure_package_name.example.math.test_add_two_ints : Implementation of the
            ``test_add_two_ints`` function.

        """
        npt.assert_equal(
            math.add_two_ints(self._first, self._second),
            self._add_res,
            err_msg='Incorrect addition result.',
        )

        npt.assert_raises(
            TypeError, math.add_two_ints, self._first, float(self._second),
        )

    def test_drake_equation(self):
        """Test ``configure_package_name.example.math.drake_equation`` function.

        See Also
        --------
        configure_package_name.example.math.drake_equation : Implementation of the
            ``drake_equation`` function.

        """
        npt.assert_equal(
            math.drake_equation(self._drake_param),
            self._drake_res,
            err_msg='Incorrect Drake eq result.',
        )

    def test_mad(self):
        """Test ``configure_package_name.example.math.mad`` function.

        See Also
        --------
        configure_package_name.example.math.mad : Implementation of the
            ``mad`` function.

        """
        npt.assert_equal(
            math.mad(self._mad_data),
            self._mad_res,
            err_msg='Incorrect median absolute deviation',
        )

max_forks_repo_name: sfarrens/pyralid-template

reason: import numpy,from numpy


text:
import abc
import numpy as np
import cv2
import Polygon as polygon3
from shapely.geometry import Polygon as shapely_poly
import math
from arg_parser import PARAMS

MAX_FIDUCIAL_POINTS = 50

def get_midpoints(p1, p2):
    return (p1[0] + p2[0])/2, (p1[1] + p2[1])/2

def point_distance(p1, p2):
    distx = math.fabs(p1[0] - p2[0])
    disty = math.fabs(p1[1] - p2[1])
    return math.sqrt(distx * distx + disty * disty)

def point_angle(p1, p2):
    return (np.arctan2(p1[0]-p2[0], p1[1]-p2[1]) * 180 / np.pi + 360) % 360

def corner_continuous_check(index1, index2, index3, index4):
    pass

class Box(metaclass=abc.ABCMeta):
    def __init__(self, points, confidence, transcription):
        self.points = points
        self.confidence = confidence
        self.transcription = transcription
        self.is_dc = transcription == "###"

    @abc.abstractmethod
    def __and__(self, other) -> float:
        """Returns intersection between two objects"""
        pass

    @abc.abstractmethod
    def subtract(self, other):
        """polygon subtraction"""
        pass

    @abc.abstractmethod
    def center(self):
        pass

    @abc.abstractmethod
    def center_distance(self, other):
        """center distance between each box"""

    @abc.abstractmethod
    def diagonal_length(self) -> float:
        """Returns diagonal length for box-level"""
        pass

    @abc.abstractmethod
    def is_inside(self, x, y) -> bool:
        """Returns point (x, y) is inside polygon."""
        pass

    @abc.abstractmethod
    def make_polygon_obj(self):
        # TODO: docstring 좀 더 자세히 적기
        """Make polygon object to calculate for future"""
        pass

    @abc.abstractmethod
    def pseudo_character_center(self) -> list:
        """get character level boxes for TedEval pseudo center"""
        pass

class QUAD(Box):
    """Points should be x1,y1,...,x4,y4 (8 points) format"""
    def __init__(self, points, confidence=0.0, transcription=""):
        super(QUAD, self).__init__(points, confidence, transcription)
        self.polygon = self.make_polygon_obj()
        if self.is_dc:
            self.transcription = "#" * self.pseudo_transcription_length()

    def __and__(self, other) -> float:
        """Get intersection between two area"""
        poly_intersect = self.polygon & other.polygon
        if len(poly_intersect) == 0:
            return 0.0
        return poly_intersect.area()

    def subtract(self, other):
        self.polygon = self.polygon - other.polygon

    def center(self):
        return self.polygon.center()

    def center_distance(self, other):
        return point_distance(self.center(), other.center())

    def area(self):
        return self.polygon.area()

    def __or__(self, other):
        return self.polygon.area() + other.polygon.area() - (self & other)

    def make_polygon_obj(self):
        point_matrix = np.empty((4, 2), np.int32)
        point_matrix[0][0] = int(self.points[0])
        point_matrix[0][1] = int(self.points[1])
        point_matrix[1][0] = int(self.points[2])
        point_matrix[1][1] = int(self.points[3])
        point_matrix[2][0] = int(self.points[4])
        point_matrix[2][1] = int(self.points[5])
        point_matrix[3][0] = int(self.points[6])
        point_matrix[3][1] = int(self.points[7])
        return polygon3.Polygon(point_matrix)

    def aspect_ratio(self):
        top_side = point_distance((self.points[0], self.points[1]), (self.points[2], self.points[3]))
        right_side = point_distance((self.points[2], self.points[3]), (self.points[4], self.points[5]))
        bottom_side = point_distance((self.points[4], self.points[5]), (self.points[6], self.points[7]))
        left_side = point_distance((self.points[0], self.points[1]), (self.points[2], self.points[3]))
        avg_hor = (top_side + bottom_side) / 2
        avg_ver = (right_side + left_side) / 2

        return (avg_hor + 1e-5) / (avg_ver + 1e-5)

    def pseudo_transcription_length(self):
        return min(round(0.5+(max(self.aspect_ratio(), 1/self.aspect_ratio()))), 10)

    def pseudo_character_center(self):
        chars = list()
        length = len(self.transcription)
        aspect_ratio = self.aspect_ratio()

        if length == 0:
            return chars

        if aspect_ratio < PARAMS.VERTICAL_ASPECT_RATIO_THRES:
            left_top = self.points[0], self.points[1]
            right_top = self.points[2], self.points[3]
            right_bottom = self.points[4], self.points[5]
            left_bottom = self.points[6], self.points[7]
        else:
            left_top = self.points[6], self.points[7]
            right_top = self.points[0], self.points[1]
            right_bottom = self.points[2], self.points[3]
            left_bottom = self.points[4], self.points[5]

        p1 = get_midpoints(left_top, left_bottom)
        p2 = get_midpoints(right_top, right_bottom)

        unit_x = (p2[0] - p1[0]) / length
        unit_y = (p2[1] - p1[1]) / length

        for i in range(length):
            x = p1[0] + unit_x / 2 + unit_x * i
            y = p1[1] + unit_y / 2 + unit_y * i
            chars.append((x, y))
        return chars

    def diagonal_length(self) -> float:
        left_top = self.points[0], self.points[1]
        right_top = self.points[2], self.points[3]
        right_bottom = self.points[4], self.points[5]
        left_bottom = self.points[6], self.points[7]
        diag1 = point_distance(left_top, right_bottom)
        diag2 = point_distance(right_top, left_bottom)
        return (diag1 + diag2) / 2

    def is_inside(self, x, y) -> bool:
        return self.polygon.isInside(x, y)

class POLY(Box):
    """Points should be x1,y1,...,xn,yn (2*n points) format"""
    def __init__(self, points, confidence=0.0, transcription=""):
        super(POLY, self).__init__(points, confidence, transcription)
        self.num_points = len(self.points) // 2
        self.polygon = self.make_polygon_obj()
        self._aspect_ratio = self.make_aspect_ratio()
        if self.is_dc:
            self.transcription = "#" * self.pseudo_transcription_length()

    def __and__(self, other):
        """Get intersection between two area"""
        poly_intersect = self.polygon.intersection(other.polygon)
        return poly_intersect.area

    def subtract(self, other):
        """get substraction"""
        self.polygon = self.polygon.difference(self.polygon.intersection(other.polygon))

    def __or__(self, other):
        return 1.0

    def area(self):
        return self.polygon.area

    def center(self):
        return self.polygon.centroid.coords[0]

    def center_distance(self, other):
        try:
            return point_distance(self.center(), other.center())
        except:
            return 0.0001

    def diagonal_length(self):
        left_top = self.points[0], self.points[1]
        right_top = self.points[self.num_points-2], self.points[self.num_points-1]
        right_bottom = self.points[self.num_points], self.points[self.num_points+1]
        left_bottom = self.points[self.num_points*2-2], self.points[self.num_points*2-1]

        diag1 = point_distance(left_top, right_bottom)
        diag2 = point_distance(right_top, left_bottom)

        return (diag1 + diag2) / 2

    def is_inside(self, x, y) -> bool:
        return self.polygon.contains(Point(x, y))

    def check_corner_points_are_continuous(self, lt, rt, rb, lb):
        count = 0
        while lt != rt:
            lt = (lt+1) % self.num_points
            count += 1

        while rb != lb:
            rb = (rb+1) % self.num_points
            count += 1

        return True

    def get_four_max_distance_from_center(self):
        center_x, center_y = self.center()
        distance_from_center = list()
        point_x = self.points[0::2]
        point_y = self.points[1::2]

        for px, py in zip(point_x, point_y):
            distance_from_center.append(point_distance((center_x, center_y), (px, py)))

        distance_idx_max_order = np.argsort(distance_from_center)[::-1]
        return distance_idx_max_order[:4]

    def make_polygon_obj(self):
        point_x = self.points[0::2]
        point_y = self.points[1::2]
        # in TotalText dataset, there there are under 4 points annotation for Polygon shape
        # so, we have to deal with it

        # if points are given 3, fill last quad points with left bottom coordinates
        if len(point_x) == len(point_y) == 3:
            point_x.append(point_x[0])
            point_y.append(point_y[2])
            self.points.append(point_x[0])
            self.points.append(point_y[2])
            self.num_points = len(self.points) // 2
        # if points are given 2, copy value 2 times
        elif len(point_x) == len(point_y) == 2:
            point_x *= 2
            point_y *= 2
            self.points.append(point_x[1])
            self.points.append(point_y[0])
            self.points.append(point_x[0])
            self.points.append(point_y[1])
            self.num_points = len(self.points) // 2
        # if points are given 1, copy value 4 times
        elif len(point_x) == len(point_y) == 1:
            point_x *= 4
            point_y *= 4
            for _ in range(3):
                self.points.append(point_x[0])
                self.points.append(point_x[0])
            self.num_points = len(self.points) // 2
        return shapely_poly(np.stack([point_x, point_y], axis=1)).buffer(0)

    def aspect_ratio(self):
        return self._aspect_ratio

    def pseudo_transcription_length(self):
        return min(round(0.5+ (max(self._aspect_ratio, 1/self._aspect_ratio))), 10)

    def make_aspect_ratio(self):
        np.array(np.reshape(self.points, [-1, 2]))
        rect = cv2.minAreaRect(np.array(np.reshape(self.points, [-1, 2]), dtype=np.float32))
        width = rect[1][0]
        height = rect[1][1]

        width += 1e-6
        height += 1e-6

        return min(10, height/width) + 1e+5

    def pseudo_character_center(self):
        chars = list()
        length = len(self.transcription)

        # Prepare polygon line estimation with interpolation
        point_x = self.points[0::2]
        point_y = self.points[1::2]
        points_x_top = point_x[:self.num_points//2]
        points_x_bottom = point_x[self.num_points//2:]
        points_y_top = point_y[:self.num_points//2]
        points_y_bottom = point_y[self.num_points//2:]

        # reverse bottom point order from left to right
        points_x_bottom = points_x_bottom[::-1]
        points_y_bottom = points_y_bottom[::-1]

        num_interpolation_section = (self.num_points//2) - 1
        num_points_to_interpolate = length

        new_point_x_top, new_point_x_bottom = list(), list()
        new_point_y_top, new_point_y_bottom = list(), list()

        for sec_idx in range(num_interpolation_section):
            start_x_top, end_x_top = points_x_top[sec_idx], points_x_top[sec_idx+1]
            start_y_top, end_y_top = points_y_top[sec_idx], points_y_top[sec_idx+1]
            start_x_bottom, end_x_bottom = points_x_bottom[sec_idx], points_x_bottom[sec_idx + 1]
            start_y_bottom, end_y_bottom = points_y_bottom[sec_idx], points_y_bottom[sec_idx + 1]

            diff_x_top = (end_x_top - start_x_top) / num_points_to_interpolate
            diff_y_top = (end_y_top - start_y_top) / num_points_to_interpolate
            diff_x_bottom = (end_x_bottom - start_x_bottom) / num_points_to_interpolate
            diff_y_bottom = (end_y_bottom - start_y_bottom) / num_points_to_interpolate

            new_point_x_top.append(start_x_top)
            new_point_x_bottom.append(start_x_bottom)
            new_point_y_top.append(start_y_top)
            new_point_y_bottom.append(start_y_bottom)

            for num_pt in range(1, num_points_to_interpolate):
                new_point_x_top.append(int(start_x_top + diff_x_top * num_pt))
                new_point_x_bottom.append(int(start_x_bottom + diff_x_bottom * num_pt))
                new_point_y_top.append(int(start_y_top + diff_y_top * num_pt))
                new_point_y_bottom.append(int(start_y_bottom + diff_y_bottom * num_pt))
        new_point_x_top.append(points_x_top[-1])
        new_point_y_top.append(points_y_top[-1])
        new_point_x_bottom.append(points_x_bottom[-1])
        new_point_y_bottom.append(points_y_bottom[-1])

        len_section_for_single_char = (len(new_point_x_top)-1) / len(self.transcription)
        print(self.num_points)
        print(len(self.transcription))

        for c in range(len(self.transcription)):
            print(len(new_point_x_top), c, len_section_for_single_char)
            center_x = (new_point_x_top[int(c*len_section_for_single_char)] +
                        new_point_x_top[int((c+1)*len_section_for_single_char)] +
                        new_point_x_bottom[int(c*len_section_for_single_char)] +
                        new_point_x_bottom[int((c+1)*len_section_for_single_char)]) / 4

            center_y = (new_point_y_top[int(c*len_section_for_single_char)] +
                        new_point_y_top[int((c+1)*len_section_for_single_char)] +
                        new_point_y_bottom[int(c*len_section_for_single_char)] +
                        new_point_y_bottom[int((c+1)*len_section_for_single_char)]) / 4

            chars.append((center_x, center_y))
        return chars

    # def pseudo_character_center(self):
    #     chars = list()

    #     # TODO: write comment more fancier(?)
    #     # Prepare polygon line estimation with interpolation

    #     point_x = self.points[0::2]
    #     point_y = self.points[1::2]
    #     points_x_top = point_x[:self.num_points//2]
    #     points_x_bottom = point_x[self.num_points//2:]
    #     points_y_top = point_y[:self.num_points//2]
    #     points_y_bottom = point_y[self.num_points//2:]

    #     points_x_bottom = points_x_bottom[::-1]
    #     points_y_bottom = points_y_bottom[::-1]
    #     # print(points_x_top, points_x_bottom, points_y_top, points_y_bottom)

    #     num_interpolation_section = (self.num_points//2) - 1
    #     num_points_to_interpolate = MAX_FIDUCIAL_POINTS // num_interpolation_section

    #     # TODO: using computer vision?
    #     new_point_x_top, new_point_x_bottom = list(), list()
    #     new_point_y_top, new_point_y_bottom = list(), list()

    #     for sec_idx in range(num_interpolation_section):
    #         start_x_top, end_x_top = points_x_top[sec_idx], points_x_top[sec_idx+1]
    #         start_y_top, end_y_top = points_y_top[sec_idx], points_y_top[sec_idx+1]
    #         start_x_bottom, end_x_bottom = points_x_bottom[sec_idx], points_x_bottom[sec_idx + 1]
    #         start_y_bottom, end_y_bottom = points_y_bottom[sec_idx], points_y_bottom[sec_idx + 1]

    #         diff_x_top = (end_x_top - start_x_top) / num_points_to_interpolate
    #         diff_y_top = (end_y_top - start_y_top) / num_points_to_interpolate
    #         diff_x_bottom = (end_x_bottom - start_x_bottom) / num_points_to_interpolate
    #         diff_y_bottom = (end_y_bottom - start_y_bottom) / num_points_to_interpolate

    #         new_point_x_top.append(start_x_top)
    #         new_point_x_bottom.append(start_x_bottom)
    #         new_point_y_top.append(start_y_top)
    #         new_point_y_bottom.append(start_y_bottom)

    #         for num_pt in range(1, num_points_to_interpolate):
    #             new_point_x_top.append(int(start_x_top + diff_x_top * num_pt))
    #             new_point_x_bottom.append(int(start_x_bottom + diff_x_bottom * num_pt))
    #             new_point_y_top.append(int(start_y_top + diff_y_top * num_pt))
    #             new_point_y_bottom.append(int(start_y_bottom + diff_y_bottom * num_pt))

    #     len_section_for_single_char = len(new_point_x_top) / len(self.transcription)

    #     for c in range(len(self.transcription)):
    #         center_x = (new_point_x_top[int(c*len_section_for_single_char)] +
    #                     new_point_x_top[min(int((c+1)*len_section_for_single_char), len(new_point_x_top)-1)] +
    #                     new_point_x_bottom[int(c*len_section_for_single_char)] +
    #                     new_point_x_bottom[min(int((c+1)*len_section_for_single_char), len(new_point_x_top)-1)]) / 4

    #         center_y = (new_point_y_top[int(c*len_section_for_single_char)] +
    #                     new_point_y_top[min(int((c+1)*len_section_for_single_char), len(new_point_x_top)-1)] +
    #                     new_point_y_bottom[int(c*len_section_for_single_char)] +
    #                     new_point_y_bottom[min(int((c+1)*len_section_for_single_char), len(new_point_x_top)-1)]) / 4

    #         chars.append((center_x, center_y))
    #     return chars

    # def pseudo_character_center_using_curve_fitting(self):
    #     if self.num_points == 4:
    #         return self.pseudo_character_center_prev()
    #
    #     chars = list()
    #
    #     point_x = self.points[0::2]
    #     point_y = self.points[1::2]
    #
    #     center_x, center_y = self.center()
    #
    #     # calculate distance between each point & center
    #     distance = []
    #     for px, py in zip(point_x, point_y):
    #         distance.append(point_distance((center_x, center_y), (px, py)))
    #
    #     # filter 4 points with maximum distance (to find each corner points)
    #     distance_max_idx_order = np.argsort(distance)[::-1][:4]
    #
    #     # distinguish left/right point by x coordinate difference with center point
    #     left_candidate_indices = []
    #     right_candidate_indices = []
    #
    #     for idx in distance_max_idx_order:
    #         if point_x[idx] < center_x:
    #             left_candidate_indices.append(idx)
    #         else:
    #             right_candidate_indices.append(idx)
    #
    #     # there should be 2: 2 points for each left/right location
    #     print(self.transcription, left_candidate_indices, right_candidate_indices)
    #
    #     # case
    #     # find minimum distance point with single left/right points, and append it to left/right candidate
    #     if len(left_candidate_indices) == 0 or len(right_candidate_indices) == 0:
    #         left_candidate_indices = [0, self.num_points-1]
    #         right_candidate_indices = [self.num_points//2-1, self.num_points//2]
    #
    #     if not len(left_candidate_indices) == len(right_candidate_indices):
    #         distance_between_corner = list()
    #         if len(left_candidate_indices) == 1:
    #             for right_idx in right_candidate_indices:
    #                 distance_between_corner.append(point_distance((point_x[right_idx], point_y[right_idx]),
    #                                                               (point_x[left_candidate_indices[0]],
    #                                                                point_y[left_candidate_indices[0]])))
    #         elif len(right_candidate_indices) == 1:
    #             for left_idx in left_candidate_indices:
    #                 distance_between_corner.append(point_distance((point_x[left_idx], point_y[left_idx]),
    #                                                               (point_x[right_candidate_indices[0]],
    #                                                                point_y[right_candidate_indices[0]])))
    #
    #         min_distance_point_idx = np.argsort(distance_between_corner)[0]
    #         if len(left_candidate_indices) == 1:
    #             left_candidate_indices.append(right_candidate_indices[min_distance_point_idx])
    #             right_candidate_indices.pop(min_distance_point_idx)
    #         else:
    #             right_candidate_indices.append(left_candidate_indices[min_distance_point_idx])
    #             left_candidate_indices.pop(min_distance_point_idx)
    #
    #     # distinguish top / bottom points by y coordinate difference compared with each left/right points
    #     if point_angle((point_x[left_candidate_indices[0]], point_y[left_candidate_indices[0]]), (center_x, center_y)) <\
    #        point_angle((point_x[left_candidate_indices[1]], point_y[left_candidate_indices[1]]), (center_x, center_y)):
    #         left_top_idx = left_candidate_indices[0]
    #         left_bottom_idx = left_candidate_indices[1]
    #     else:
    #         left_top_idx = left_candidate_indices[1]
    #         left_bottom_idx = left_candidate_indices[0]
    #
    #     if point_angle((point_x[right_candidate_indices[0]], point_y[right_candidate_indices[0]]), (center_x, center_y)) > \
    #        point_angle((point_x[right_candidate_indices[1]], point_y[right_candidate_indices[1]]), (center_x, center_y)):
    #         right_top_idx = right_candidate_indices[0]
    #         right_bottom_idx = right_candidate_indices[1]
    #     else:
    #         right_top_idx = right_candidate_indices[1]
    #         right_bottom_idx = right_candidate_indices[0]
    #
    #     # if point_y[left_candidate_indices[0]] < point_y[left_candidate_indices[1]]:
    #     #     left_top_idx = left_candidate_indices[0]
    #     #     left_bottom_idx = left_candidate_indices[1]
    #     # else:
    #     #     left_top_idx = left_candidate_indices[1]
    #     #     left_bottom_idx = left_candidate_indices[0]
    #     #
    #     # if point_y[right_candidate_indices[0]] < point_y[right_candidate_indices[1]]:
    #     #     right_top_idx = right_candidate_indices[0]
    #     #     right_bottom_idx = right_candidate_indices[1]
    #     # else:
    #     #     right_top_idx = right_candidate_indices[1]
    #     #     right_bottom_idx = right_candidate_indices[0]
    #
    #     top_x, top_y = list(), list()
    #     bottom_x, bottom_y = list(), list()
    #     top_start = left_top_idx
    #     top_end = right_top_idx
    #     bottom_start = right_bottom_idx
    #     bottom_end = left_bottom_idx
    #
    #     self.check_corner_points_are_continuous(left_top_idx, right_top_idx, right_bottom_idx, left_bottom_idx)
    #
    #     while top_start != top_end:
    #         top_x.append(point_x[top_start])
    #         top_y.append(point_y[top_start])
    #         top_start = (top_start+1) % self.num_points
    #     top_x.append(point_x[top_end])
    #     top_y.append(point_y[top_end])
    #
    #     while bottom_start != bottom_end:
    #         bottom_x.append(point_x[bottom_start])
    #         bottom_y.append(point_y[bottom_start])
    #         bottom_start = (bottom_start+1) % self.num_points
    #
    #     bottom_x.append(point_x[bottom_end])
    #     bottom_y.append(point_y[bottom_end])
    #
    #     # reverse bottom points to make it left -> right order
    #     bottom_x = bottom_x[::-1]
    #     bottom_y = bottom_y[::-1]
    #
    #     # pseudo interpolation for more accurate polynomial estimation
    #     new_point_x_top, new_point_x_bottom = list(), list()
    #     new_point_y_top, new_point_y_bottom = list(), list()
    #
    #     NUM_POINTS_TO_FILL = 3
    #
    #     new_point_x_top.append(top_x[0])
    #     new_point_y_top.append(top_y[0])
    #     for sec_idx in range(len(top_x)-1):
    #         start_top_x, end_top_x = top_x[sec_idx], top_x[sec_idx+1]
    #         start_top_y, end_top_y = top_y[sec_idx], top_y[sec_idx+1]
    #
    #         diff_x_top = (end_top_x - start_top_x) / NUM_POINTS_TO_FILL
    #         diff_y_top = (end_top_y - start_top_y) / NUM_POINTS_TO_FILL
    #
    #         new_point_x_top.append(start_top_x)
    #         new_point_y_top.append(start_top_y)
    #
    #         for num_pt in range(1, NUM_POINTS_TO_FILL+1):
    #             new_point_x_top.append(int(start_top_x + diff_x_top * num_pt))
    #             new_point_y_top.append(int(start_top_y + diff_y_top * num_pt))
    #
    #     new_point_x_bottom.append(bottom_x[0])
    #     new_point_y_bottom.append(bottom_y[0])
    #     for sec_idx in range(len(bottom_x) - 1):
    #         start_bottom_x, end_bottom_x = bottom_x[sec_idx], bottom_x[sec_idx + 1]
    #         start_bottom_y, end_bottom_y = bottom_y[sec_idx], bottom_y[sec_idx + 1]
    #
    #         diff_x_top = (end_bottom_x - start_bottom_x) / NUM_POINTS_TO_FILL
    #         diff_y_top = (end_bottom_y - start_bottom_y) / NUM_POINTS_TO_FILL
    #
    #         new_point_x_bottom.append(start_bottom_x)
    #         new_point_y_bottom.append(start_bottom_y)
    #
    #         for num_pt in range(1, NUM_POINTS_TO_FILL + 1):
    #             new_point_x_bottom.append(int(start_bottom_x + diff_x_top * num_pt))
    #             new_point_y_bottom.append(int(start_bottom_y + diff_y_top * num_pt))
    #
    #     # do polynomial curve fitting
    #     top_curve = np.poly1d(np.polyfit(np.array(new_point_x_top), np.array(new_point_y_top), 2))
    #     bottom_curve = np.poly1d(np.polyfit(np.array(new_point_x_bottom), np.array(new_point_y_bottom), 2))
    #
    #     # # do polynomial curve fitting
    #     # top_curve = np.poly1d(np.polyfit(np.array(top_x), np.array(top_y), 2))
    #     # bottom_curve = np.poly1d(np.polyfit(np.array(bottom_x), np.array(bottom_y), 2))
    #
    #     # point interpolation
    #     interpolated_top_x = np.linspace(top_x[0], top_x[-1], len(self.transcription)+1)
    #     interpolated_bottom_x = np.linspace(bottom_x[0], bottom_x[-1], len(self.transcription)+1)
    #
    #     # estimate middle point from top/bottom polynomial
    #     interpolated_top_y = top_curve(interpolated_top_x)
    #     interpolated_bottom_y = bottom_curve(interpolated_bottom_x)
    #     middle_curve_point_x = (interpolated_top_x + interpolated_bottom_x) / 2
    #     middle_curve_point_y = (interpolated_top_y + interpolated_bottom_y) / 2
    #
    #     # fill character center list from middle curve information
    #     for c in range(len(self.transcription)):
    #         chars.append(((middle_curve_point_x[c] + middle_curve_point_x[c+1])/2,
    #                       (middle_curve_point_y[c] + middle_curve_point_y[c+1])/2))
    #
    #     return chars

max_stars_repo_name: WarwickMasson/CLEval

max_issues_repo_name: WarwickMasson/CLEval

max_forks_repo_name: WarwickMasson/CLEval


text:
import tensorflow as tf
import numpy as np
import pandas as pd

#데이터 불러오기
all_data=pd.read_csv('D:\Python\MyGitHub\Project5-DL-Projet-of-Prediction\stock/SSUN.F.csv', skiprows=1)
data_arrayed=np.array(all_data)
print(data_arrayed)

#전처리
# xdata=xy[:,1:-3]
# ydata=xy[:,-3:]


text:
from sympy.physics.units.definitions.dimension_definitions import current, temperature, amount_of_substance, \
    luminous_intensity, angle, charge, voltage, impedance, conductance, capacitance, inductance, magnetic_density, \
    magnetic_flux, information

from sympy.core.numbers import (Rational, pi)
from sympy.core.singleton import S as S_singleton
from sympy.physics.units.prefixes import kilo, milli, micro, deci, centi, nano, pico, kibi, mebi, gibi, tebi, pebi, exbi
from sympy.physics.units.quantities import Quantity

One = S_singleton.One

#### UNITS ####

# Dimensionless:
percent = percents = Quantity("percent", latex_repr=r"\%")
percent.set_global_relative_scale_factor(Rational(1, 100), One)

permille = Quantity("permille")
permille.set_global_relative_scale_factor(Rational(1, 1000), One)

# Angular units (dimensionless)
rad = radian = radians = Quantity("radian", abbrev="rad")
radian.set_global_dimension(angle)
deg = degree = degrees = Quantity("degree", abbrev="deg", latex_repr=r"^\circ")
degree.set_global_relative_scale_factor(pi/180, radian)
sr = steradian = steradians = Quantity("steradian", abbrev="sr")
mil = angular_mil = angular_mils = Quantity("angular_mil", abbrev="mil")

# Base units:
m = meter = meters = Quantity("meter", abbrev="m")

# gram; used to define its prefixed units
g = gram = grams = Quantity("gram", abbrev="g")

# NOTE: the `kilogram` has scale factor 1000. In SI, kg is a base unit, but
# nonetheless we are trying to be compatible with the `kilo` prefix. In a
# similar manner, people using CGS or gaussian units could argue that the
# `centimeter` rather than `meter` is the fundamental unit for length, but the
# scale factor of `centimeter` will be kept as 1/100 to be compatible with the
# `centi` prefix.  The current state of the code assumes SI unit dimensions, in
# the future this module will be modified in order to be unit system-neutral
# (that is, support all kinds of unit systems).
kg = kilogram = kilograms = Quantity("kilogram", abbrev="kg")
kg.set_global_relative_scale_factor(kilo, gram)

s = second = seconds = Quantity("second", abbrev="s")
A = ampere = amperes = Quantity("ampere", abbrev='A')
ampere.set_global_dimension(current)
K = kelvin = kelvins = Quantity("kelvin", abbrev='K')
kelvin.set_global_dimension(temperature)
mol = mole = moles = Quantity("mole", abbrev="mol")
mole.set_global_dimension(amount_of_substance)
cd = candela = candelas = Quantity("candela", abbrev="cd")
candela.set_global_dimension(luminous_intensity)

mg = milligram = milligrams = Quantity("milligram", abbrev="mg")
mg.set_global_relative_scale_factor(milli, gram)

ug = microgram = micrograms = Quantity("microgram", abbrev="ug", latex_repr=r"\mu\text{g}")
ug.set_global_relative_scale_factor(micro, gram)

# derived units
newton = newtons = N = Quantity("newton", abbrev="N")
joule = joules = J = Quantity("joule", abbrev="J")
watt = watts = W = Quantity("watt", abbrev="W")
pascal = pascals = Pa = pa = Quantity("pascal", abbrev="Pa")
hertz = hz = Hz = Quantity("hertz", abbrev="Hz")

# CGS derived units:
dyne = Quantity("dyne")
dyne.set_global_relative_scale_factor(One/10**5, newton)
erg = Quantity("erg")
erg.set_global_relative_scale_factor(One/10**7, joule)

# MKSA extension to MKS: derived units
coulomb = coulombs = C = Quantity("coulomb", abbrev='C')
coulomb.set_global_dimension(charge)
volt = volts = v = V = Quantity("volt", abbrev='V')
volt.set_global_dimension(voltage)
ohm = ohms = Quantity("ohm", abbrev='ohm', latex_repr=r"\Omega")
ohm.set_global_dimension(impedance)
siemens = S = mho = mhos = Quantity("siemens", abbrev='S')
siemens.set_global_dimension(conductance)
farad = farads = F = Quantity("farad", abbrev='F')
farad.set_global_dimension(capacitance)
henry = henrys = H = Quantity("henry", abbrev='H')
henry.set_global_dimension(inductance)
tesla = teslas = T = Quantity("tesla", abbrev='T')
tesla.set_global_dimension(magnetic_density)
weber = webers = Wb = wb = Quantity("weber", abbrev='Wb')
weber.set_global_dimension(magnetic_flux)

# CGS units for electromagnetic quantities:
statampere = Quantity("statampere")
statcoulomb = statC = franklin = Quantity("statcoulomb", abbrev="statC")
statvolt = Quantity("statvolt")
gauss = Quantity("gauss")
maxwell = Quantity("maxwell")
debye = Quantity("debye")
oersted = Quantity("oersted")

# Other derived units:
optical_power = dioptre = diopter = D = Quantity("dioptre")
lux = lx = Quantity("lux", abbrev="lx")

# katal is the SI unit of catalytic activity
katal = kat = Quantity("katal", abbrev="kat")

# gray is the SI unit of absorbed dose
gray = Gy = Quantity("gray")

# becquerel is the SI unit of radioactivity
becquerel = Bq = Quantity("becquerel", abbrev="Bq")

# Common length units

km = kilometer = kilometers = Quantity("kilometer", abbrev="km")
km.set_global_relative_scale_factor(kilo, meter)

dm = decimeter = decimeters = Quantity("decimeter", abbrev="dm")
dm.set_global_relative_scale_factor(deci, meter)

cm = centimeter = centimeters = Quantity("centimeter", abbrev="cm")
cm.set_global_relative_scale_factor(centi, meter)

mm = millimeter = millimeters = Quantity("millimeter", abbrev="mm")
mm.set_global_relative_scale_factor(milli, meter)

um = micrometer = micrometers = micron = microns = \
    Quantity("micrometer", abbrev="um", latex_repr=r'\mu\text{m}')
um.set_global_relative_scale_factor(micro, meter)

nm = nanometer = nanometers = Quantity("nanometer", abbrev="nm")
nm.set_global_relative_scale_factor(nano, meter)

pm = picometer = picometers = Quantity("picometer", abbrev="pm")
pm.set_global_relative_scale_factor(pico, meter)

ft = foot = feet = Quantity("foot", abbrev="ft")
ft.set_global_relative_scale_factor(Rational(3048, 10000), meter)

inch = inches = Quantity("inch")
inch.set_global_relative_scale_factor(Rational(1, 12), foot)

yd = yard = yards = Quantity("yard", abbrev="yd")
yd.set_global_relative_scale_factor(3, feet)

mi = mile = miles = Quantity("mile")
mi.set_global_relative_scale_factor(5280, feet)

nmi = nautical_mile = nautical_miles = Quantity("nautical_mile")
nmi.set_global_relative_scale_factor(6076, feet)

# Common volume and area units

l = L = liter = liters = Quantity("liter")

dl = dL = deciliter = deciliters = Quantity("deciliter")
dl.set_global_relative_scale_factor(Rational(1, 10), liter)

cl = cL = centiliter = centiliters = Quantity("centiliter")
cl.set_global_relative_scale_factor(Rational(1, 100), liter)

ml = mL = milliliter = milliliters = Quantity("milliliter")
ml.set_global_relative_scale_factor(Rational(1, 1000), liter)

# Common time units

ms = millisecond = milliseconds = Quantity("millisecond", abbrev="ms")
millisecond.set_global_relative_scale_factor(milli, second)

us = microsecond = microseconds = Quantity("microsecond", abbrev="us", latex_repr=r'\mu\text{s}')
microsecond.set_global_relative_scale_factor(micro, second)

ns = nanosecond = nanoseconds = Quantity("nanosecond", abbrev="ns")
nanosecond.set_global_relative_scale_factor(nano, second)

ps = picosecond = picoseconds = Quantity("picosecond", abbrev="ps")
picosecond.set_global_relative_scale_factor(pico, second)

minute = minutes = Quantity("minute")
minute.set_global_relative_scale_factor(60, second)

h = hour = hours = Quantity("hour")
hour.set_global_relative_scale_factor(60, minute)

day = days = Quantity("day")
day.set_global_relative_scale_factor(24, hour)

anomalistic_year = anomalistic_years = Quantity("anomalistic_year")
anomalistic_year.set_global_relative_scale_factor(365.259636, day)

sidereal_year = sidereal_years = Quantity("sidereal_year")
sidereal_year.set_global_relative_scale_factor(31558149.540, seconds)

tropical_year = tropical_years = Quantity("tropical_year")
tropical_year.set_global_relative_scale_factor(365.24219, day)

common_year = common_years = Quantity("common_year")
common_year.set_global_relative_scale_factor(365, day)

julian_year = julian_years = Quantity("julian_year")
julian_year.set_global_relative_scale_factor((365 + One/4), day)

draconic_year = draconic_years = Quantity("draconic_year")
draconic_year.set_global_relative_scale_factor(346.62, day)

gaussian_year = gaussian_years = Quantity("gaussian_year")
gaussian_year.set_global_relative_scale_factor(365.2568983, day)

full_moon_cycle = full_moon_cycles = Quantity("full_moon_cycle")
full_moon_cycle.set_global_relative_scale_factor(411.78443029, day)

year = years = tropical_year

#### CONSTANTS ####

# Newton constant
G = gravitational_constant = Quantity("gravitational_constant", abbrev="G")

# speed of light
c = speed_of_light = Quantity("speed_of_light", abbrev="c")

# elementary charge
elementary_charge = Quantity("elementary_charge", abbrev="e")

# Planck constant
planck = Quantity("planck", abbrev="h")

# Reduced Planck constant
hbar = Quantity("hbar", abbrev="hbar")

# Electronvolt
eV = electronvolt = electronvolts = Quantity("electronvolt", abbrev="eV")

# Avogadro number
avogadro_number = Quantity("avogadro_number")

# Avogadro constant
avogadro = avogadro_constant = Quantity("avogadro_constant")

# Boltzmann constant
boltzmann = boltzmann_constant = Quantity("boltzmann_constant")

# Stefan-Boltzmann constant
stefan = stefan_boltzmann_constant = Quantity("stefan_boltzmann_constant")

# Atomic mass
amu = amus = atomic_mass_unit = atomic_mass_constant = Quantity("atomic_mass_constant")

# Molar gas constant
R = molar_gas_constant = Quantity("molar_gas_constant", abbrev="R")

# Faraday constant
faraday_constant = Quantity("faraday_constant")

# Josephson constant
josephson_constant = Quantity("josephson_constant", abbrev="K_j")

# Von Klitzing constant
von_klitzing_constant = Quantity("von_klitzing_constant", abbrev="R_k")

# Acceleration due to gravity (on the Earth surface)
gee = gees = acceleration_due_to_gravity = Quantity("acceleration_due_to_gravity", abbrev="g")

# magnetic constant:
u0 = magnetic_constant = vacuum_permeability = Quantity("magnetic_constant")

# electric constat:
e0 = electric_constant = vacuum_permittivity = Quantity("vacuum_permittivity")

# vacuum impedance:
Z0 = vacuum_impedance = Quantity("vacuum_impedance", abbrev='Z_0', latex_repr=r'Z_{0}')

# Coulomb's constant:
coulomb_constant = coulombs_constant = electric_force_constant = \
    Quantity("coulomb_constant", abbrev="k_e")

atmosphere = atmospheres = atm = Quantity("atmosphere", abbrev="atm")

kPa = kilopascal = Quantity("kilopascal", abbrev="kPa")
kilopascal.set_global_relative_scale_factor(kilo, Pa)

bar = bars = Quantity("bar", abbrev="bar")

pound = pounds = Quantity("pound")  # exact

psi = Quantity("psi")

dHg0 = 13.5951  # approx value at 0 C
mmHg = torr = Quantity("mmHg")

atmosphere.set_global_relative_scale_factor(101325, pascal)
bar.set_global_relative_scale_factor(100, kPa)
pound.set_global_relative_scale_factor(Rational(45359237, 100000000), kg)

mmu = mmus = milli_mass_unit = Quantity("milli_mass_unit")

quart = quarts = Quantity("quart")

# Other convenient units and magnitudes

ly = lightyear = lightyears = Quantity("lightyear", abbrev="ly")

au = astronomical_unit = astronomical_units = Quantity("astronomical_unit", abbrev="AU")

# Fundamental Planck units:
planck_mass = Quantity("planck_mass", abbrev="m_P", latex_repr=r'm_\text{P}')

planck_time = Quantity("planck_time", abbrev="t_P", latex_repr=r't_\text{P}')

planck_temperature = Quantity("planck_temperature", abbrev="T_P",
                              latex_repr=r'T_\text{P}')

planck_length = Quantity("planck_length", abbrev="l_P", latex_repr=r'l_\text{P}')

planck_charge = Quantity("planck_charge", abbrev="q_P", latex_repr=r'q_\text{P}')

# Derived Planck units:
planck_area = Quantity("planck_area")

planck_volume = Quantity("planck_volume")

planck_momentum = Quantity("planck_momentum")

planck_energy = Quantity("planck_energy", abbrev="E_P", latex_repr=r'E_\text{P}')

planck_force = Quantity("planck_force", abbrev="F_P", latex_repr=r'F_\text{P}')

planck_power = Quantity("planck_power", abbrev="P_P", latex_repr=r'P_\text{P}')

planck_density = Quantity("planck_density", abbrev="rho_P", latex_repr=r'\rho_\text{P}')

planck_energy_density = Quantity("planck_energy_density", abbrev="rho^E_P")

planck_intensity = Quantity("planck_intensity", abbrev="I_P", latex_repr=r'I_\text{P}')

planck_angular_frequency = Quantity("planck_angular_frequency", abbrev="omega_P",
                                    latex_repr=r'\omega_\text{P}')

planck_pressure = Quantity("planck_pressure", abbrev="p_P", latex_repr=r'p_\text{P}')

planck_current = Quantity("planck_current", abbrev="I_P", latex_repr=r'I_\text{P}')

planck_voltage = Quantity("planck_voltage", abbrev="V_P", latex_repr=r'V_\text{P}')

planck_impedance = Quantity("planck_impedance", abbrev="Z_P", latex_repr=r'Z_\text{P}')

planck_acceleration = Quantity("planck_acceleration", abbrev="a_P",
                               latex_repr=r'a_\text{P}')

# Information theory units:
bit = bits = Quantity("bit")
bit.set_global_dimension(information)

byte = bytes = Quantity("byte")

kibibyte = kibibytes = Quantity("kibibyte")
mebibyte = mebibytes = Quantity("mebibyte")
gibibyte = gibibytes = Quantity("gibibyte")
tebibyte = tebibytes = Quantity("tebibyte")
pebibyte = pebibytes = Quantity("pebibyte")
exbibyte = exbibytes = Quantity("exbibyte")

byte.set_global_relative_scale_factor(8, bit)
kibibyte.set_global_relative_scale_factor(kibi, byte)
mebibyte.set_global_relative_scale_factor(mebi, byte)
gibibyte.set_global_relative_scale_factor(gibi, byte)
tebibyte.set_global_relative_scale_factor(tebi, byte)
pebibyte.set_global_relative_scale_factor(pebi, byte)
exbibyte.set_global_relative_scale_factor(exbi, byte)

# Older units for radioactivity
curie = Ci = Quantity("curie", abbrev="Ci")

rutherford = Rd = Quantity("rutherford", abbrev="Rd")

max_stars_repo_name: Michal-Gagala/sympy

max_issues_repo_name: Michal-Gagala/sympy

max_forks_repo_name: Michal-Gagala/sympy

reason: from sympy


text:
import pandas as pd
import numpy as np
import seaborn as sns
import seaborn as sn
from training.config import Config
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
from scipy.optimize import curve_fit

def get_heatmaps(data, no_pred=False):
    """
    get_heatmaps(data, no_pred) -> Array, int
    Gets upper and lower comperison heatmaps.
    Args:
        data (DataFrame): Frame with values and predictions
        no_pred (bool): One of True or False. If True, then plots observed values on both sides.
    """

    st = int(data["i"].min())
    data["i"] = data["i"] - st
    data["j"] = data["j"] - st
    nr = int(data["j"].max()) + 1
    rows = np.array(data["i"]).astype(int)
    cols = np.array(data["j"]).astype(int)

    "initialize"
    hic_mat = np.zeros((nr, nr))
    hic_mat[rows, cols] = np.array(data["v"])
    hic_upper = np.triu(hic_mat)

    "check for pred"
    if no_pred:
        hic_mat[cols, rows] = np.array(data["v"])
    else:
        hic_mat[cols, rows] = np.array(data["pred"])

    hic_lower = np.tril(hic_mat)
    hic_mat = hic_upper + hic_lower
    hic_mat[np.diag_indices_from(hic_mat)] /= 2
    return hic_mat, st

def plot_foxg1(cfg, data):
    """
    plot_foxg1(cfg, data) -> No return object
    Plots window around foxg1 ko site.
    Args:
        cfg (Config): configuration to use
        data (DataFrame): Frame with values and predictions
    """

    site = cfg.foxg1_indices
    data["i"] = data["i"] - site
    data["j"] = data["j"] - site

    "window"
    data = data.loc[(data["i"] >= -100) & (data["i"] <= 100) &
                    (data["j"] >= -100) & (data["j"] <= 100)]
    data["i"] = data["i"] + 100
    data["j"] = data["j"] + 100

    "form matrix"
    nr = 201
    rows = np.array(data["i"]).astype(int)
    cols = np.array(data["j"]).astype(int)
    hic_mat = np.zeros((nr, nr))
    hic_mat[rows, cols] = np.array(data["v"])
    hic_upper = np.triu(hic_mat)
    hic_mat[cols, rows] = np.array(data["pred"])
    hic_lower = np.tril(hic_mat)
    hic_mat = hic_upper + hic_lower
    hic_mat[np.diag_indices_from(hic_mat)] /= 2

    "plot"
    simple_plot(hic_mat, mode="reds")

def simple_plot(hic_win, mode):
    """
    simple_plot(hic_win, mode) -> No return object
    plots heatmaps of reds or differences.
    Args:
        hic_win (Array): Matrix of Hi-C values
        mode (string): one of reds or diff
    """

    if mode == "reds":
        plt.figure()
        sns.set_theme()
        ax = sns.heatmap(hic_win, cmap="Reds", vmin=0, vmax=1)
        ax.set_yticks([])
        ax.set_xticks([])
        plt.savefig("/home/kevindsouza/Downloads/heatmap.svg", format="svg")
        plt.show()

    if mode == "diff":
        plt.figure()
        sns.set_theme()
        rdgn = sns.diverging_palette(h_neg=220, h_pos=14, s=79, l=55, sep=3, as_cmap=True)
        sns.heatmap(hic_win, cmap=rdgn, center=0.00, cbar=True)
        plt.yticks([])
        plt.xticks([])
        # plt.savefig("/home/kevindsouza/Downloads/ctcf_ko.png")
        plt.show()

def indices_diff_mat(indice, st, hic_mat, mode="ctcf"):
    """
    indices_diff_mat(indice, st, hic_mat, mode) -> Array
    gets window matrices given indices
    Args:
        indice (Array): Matrix of Hi-C values
        st (int): Starting indice
        hic_mat (Array): Matrix of Hi-C values
        mode (string): tadbs or others
    """

    nrows = len(hic_mat)

    if mode == "tadbs":
        i = indice[0] - st
        j = indice[1] - st
        if i - 98 >= 0:
            win_start = i - 98
        else:
            win_start = 0
        if j + 98 <= (nrows - 1):
            win_stop = i + 98
        else:
            win_stop = nrows - 1
    else:
        i = indice - st
        if i - 100 >= 0:
            win_start = i - 100
        else:
            win_start = 0
        if i + 100 <= (nrows - 1):
            win_stop = i + 100
        else:
            win_stop = nrows - 1

    hic_win = hic_mat[win_start:win_stop, win_start:win_stop]
    return hic_win

def plot_frame_error(error_list):
    """
    plot_frame_error(error_list) -> No return object
    Plot frame error given error list
    Args:
        error_list (List): List of errors
    """

    pos_list = np.arange(0, 150)
    plt.figure()
    plt.xlabel("Position in Frame", fontsize=14)
    plt.ylabel("Average Error", fontsize=14)
    plt.plot(pos_list, error_list)
    plt.grid(False)
    plt.show()

def plot_smoothness(representations):
    """
    plot_smoothness(representations) -> No return object
    Plot smoothness of representations.
    Args:
        representations (Array): representation matrix
    """

    window = 2000
    nrows = len(representations)
    diff_list = np.arange(-window, window + 1)
    diff_list = np.delete(diff_list, [window])
    diff_vals = np.zeros((nrows, 2 * window))
    for r in range(nrows):
        for i, d in enumerate(diff_list):
            if (r + d) >= 0 and (r + d) <= nrows - 1:
                diff_vals[r, i] = np.linalg.norm(representations[r, :] - representations[r + d, :], ord=1)
            else:
                continue

    diff_reduce = diff_vals.mean(axis=0)
    plt.title("Average L2 Norm of Embeddings with Distance")
    plt.xlabel("Distance in 10 Kbp", fontsize=14)
    plt.ylabel("Average L2 Norm", fontsize=14)
    plt.plot(diff_list, diff_reduce)
    plt.grid(b=None)
    plt.show()

def plot3d(representations):
    """
    plot3d(representations) -> No return object
    Plot first 3 dims of representations.
    Args:
        representations (Array): representation matrix
    """

    plt.figure()
    ax = plt.axes(projection='3d')
    ax.scatter3D(representations[:, 0], representations[:, 1], representations[:, 2], 'red')
    plt.show()

def plot_euclid_heatmap(representations):
    """
    plot_euclid_heatmap(representations) -> No return object
    Plot heatmap of euclidean distance.
    Args:
        representations (Array): representation matrix
    """

    nr = len(representations)
    euclid_heatmap = np.zeros((nr, nr))

    for r1 in range(nr):
        for r2 in range(nr):
            euclid_heatmap[r1, r2] = np.linalg.norm(representations[r1, :] - representations[r2, :])

    simple_plot(euclid_heatmap, mode="reds")

def plot_pr_curve(precision, recall):
    """
    plot_pr_curve(precision, recall) -> No return object
    Plot PR curve.
    Args:
        precision (List): List of precision values
        recall (List): List of recall values
    """

    plt.step(recall, precision, color='b', alpha=0.2, where='post')
    plt.xlabel('Recall')
    plt.ylabel('Precision')
    plt.ylim([0.0, 1.05])
    plt.xlim([0.0, 1.0])
    plt.title('2-class Precision-Recall curve')
    plt.savefig('XGBoost_PR')
    plt.show()

def plot_confusion_matrix(predictions):
    """
    plot_confusion_matrix(predictions) -> No return object
    Plot confusion matrix for subcompartments.
    Args:
        predictions (DataFrame): frame of true and predicted subcompartments
    """

    conf_matrix = confusion_matrix(predictions[:, 7], predictions[:, 6])
    conf_matrix = conf_matrix[1:, 1:]
    df_cm = pd.DataFrame(conf_matrix)
    df_cm = df_cm.div(df_cm.sum(axis=0), axis=1)

    x_axis_labels = ["A2", "A1", "B1", "B2", "B3"]
    y_axis_labels = ["A2",
                     "A1", "B1", "B2", "B3"]

    sn.set(font_scale=1.4)
    sn.heatmap(df_cm, annot=True, cmap="YlGnBu", fmt="d", xticklabels=x_axis_labels,
               yticklabels=y_axis_labels)
    plt.show()

def plot_combined(map_frame):
    """
    plot_combined(map_frame) -> No return object
    Plot map for tasks
    Args:
        map_frame (DataFrame): dataframe of map values
    """
    tasks = ["Gene Expression", "Replication Timing", "Enhancers", "TSS", "PE-Interactions", "FIREs",
             "Non-loop Domains", "Loop Domains"]

    df_main = pd.DataFrame(columns=["Tasks", "Hi-C-LSTM"])
    df_main["Tasks"] = tasks
    df_main["Hi-C-LSTM"] = [map_frame["gene_map"].mean(), map_frame["rep_map"].mean(),
                            map_frame["enhancers_map"].mean(), map_frame["tss_map"].mean(),
                            map_frame["pe_map"].mean(), map_frame["fire_map"].mean(),
                            map_frame["domains_map"].mean(), map_frame["loops_map"].mean()]

    plt.figure(figsize=(12, 10))
    plt.xticks(rotation=90, fontsize=20)
    plt.yticks(fontsize=20)
    plt.xlabel("Prediction Target", fontsize=20)
    plt.ylabel("mAP ", fontsize=20)
    plt.plot('Tasks', 'Hi-C-LSTM', data=df_main, marker='o', markersize=16, color="C3",
             linewidth=3,
             label="Hi-C-LSTM")
    plt.legend(fontsize=18)
    plt.show()

def plot_gbr(main_df):
    """
    plot_gbr(main_df) -> No return object
    Gets violin plots of Segway GBR
    Args:
        main_df (DataFrame): DF containing if values and targets
    """
    main_df["ig"] = main_df["ig"].astype(float)

    plt.figure(figsize=(16, 7))
    sns.set(font_scale=1.8)
    sns.set_style(style='white')
    plt.xticks(rotation=90, fontsize=20)
    plt.ylim(-1, 1)
    ax = sns.violinplot(x="target", y="ig", data=main_df)
    ax.set(xlabel='', ylabel='IG Importance')
    plt.show()

def plot_r2(comb_r2_df):
    """
    plot_r2(comb_r2_df) -> No return object
    plots average R2 values at a particular difference.
    Args:
        comb_r2_df (DataFrame): DF containing R2 values for various differences in positions.
    """

    max_diff = int(comb_r2_df['diff'].max())
    max_mb = 100
    num_bins_1mb = 10
    pos = np.arange(0, max_mb)
    avg_diff = pd.DataFrame(columns=["diff", "r2"])
    r2_list = []
    final_r2 = np.zeros((max_mb, ))

    "get average r2"
    for diff in range(max_diff):
        subset_diff = comb_r2_df.loc[comb_r2_df["diff"] == diff]
        r2_mean = subset_diff["r2"].mean()
        avg_diff = avg_diff.append({"diff": diff, "r2": r2_mean}, ignore_index=True)

    "mean in window"
    for i in range(int(np.ceil(max_diff/num_bins_1mb))):
        r2_sub = avg_diff.loc[(avg_diff["diff"] >= i*num_bins_1mb) & (avg_diff["diff"] < (i+1)*num_bins_1mb)]
        r2_mean = r2_sub["r2"].mean(skipna=True)
        if r2_mean < -1:
            r2_list.append(0)
        else:
            r2_list.append(r2_mean)

    num_windows = int(np.ceil(len(r2_list)/max_mb))
    if num_windows == 1:
        r2_list_pos = np.zeros((num_windows, len(r2_list)))
    else:
        r2_list_pos = np.zeros((num_windows, max_mb))
    for k in range(num_windows):
        if k == num_windows - 1:
            r2_list_pos[k, :len(r2_list[k * max_mb: ])] = r2_list[k * max_mb: ]
        else:
            r2_list_pos[k] = r2_list[k * max_mb: (k + 1) * max_mb]

    r2_list_pos = np.mean(r2_list_pos, axis=0)
    final_r2[:len(r2_list_pos)] = r2_list_pos

    "plot"
    plt.figure(figsize=(12, 10))
    plt.plot(pos, final_r2, marker='', markersize=14, color='C0', label='Hi-C-LSTM')
    plt.tick_params(axis="x", labelsize=20, length=0)
    plt.tick_params(axis="y", labelsize=20)
    plt.xlabel('Distance between positions in Mbp', fontsize=20)
    plt.ylabel('R-squared for Replicate-1', fontsize=20)
    plt.legend(loc='upper right', fontsize=20)
    plt.show()
    print("done")

def scatter_tal_lm(ko, wt):
    """
    scatter_tal_lm(ko, wt) -> No return object
    Scatter plot of TAL1 and LMO2 prediction differences.
    Args:
        ko (Array): Array containing after knockout values
        wt (Array): Array containing before knockout values
    """

    def func(x, a):
        return a * x

    diff_mat = ko - wt
    diff_mat[0,0] = 0
    og = np.triu(diff_mat)
    og = og.flatten(order='C')
    pred = np.triu(diff_mat.T)
    pred = pred.flatten(order='C')

    plt.figure(figsize=(10, 8))
    #res = sm.OLS(pred, og).fit()
    m, _ = curve_fit(func, og, pred)
    plt.scatter(og, pred, marker='o', alpha=0.5)
    plt.plot(og, m*og, "g")
    # sns.regplot(og, pred)
    plt.tick_params(axis="x", labelsize=20, length=0)
    plt.tick_params(axis="y", labelsize=20)
    plt.xlim([-1.0, 1.0])
    plt.ylim([-1.0, 1.0])
    plt.xlabel('TAL1 KO - WT (Observed)', fontsize=20)
    plt.ylabel('TAL1 KO - WT (Predicted)', fontsize=20)
    plt.tight_layout()
    plt.savefig("/home/kevindsouza/Downloads/tal1_scatter.png")

def hist_2d(og, pred):
    """
    hist_2d(og, pred) -> No return object
    2D histogram of observed and predicted differences.
    Args:
        og (Array): Array containing observed differences
        pred (Array): Array containing predicted differences
    """
    x_min = np.min(og)
    x_max = np.max(og)

    y_min = np.min(pred)
    y_max = np.max(pred)

    x_bins = np.linspace(x_min, x_max, 50)
    y_bins = np.linspace(y_min, y_max, 50)

    plt.figure(figsize=(10, 8))
    hist, _, _, _ = plt.hist2d(og, pred, bins=[x_bins, y_bins])
    plt.xticks(fontsize=18)
    plt.xlim([0, 0.1])
    plt.yticks(fontsize=18)
    plt.ylim([0.004, 0.1])
    plt.xlabel('LMO2 KO - WT (Original)', fontsize=20)
    plt.ylabel('LMO2 KO - WT (Predicted)', fontsize=20)
    plt.tight_layout()
    plt.savefig("/home/kevindsouza/Downloads/lmo2_hist.png")

if __name__ == '__main__':
    cfg = Config()
    cell = cfg.cell

    for chr in cfg.chr_test_list:
        '''
        r2_diff = pd.read_csv(cfg.output_directory + "r2frame_%s_chr%s.csv" % (cell, str(chr)), sep="\t")
        r2_diff = r2_diff.drop(['Unnamed: 0'], axis=1)
        comb_r2_df = comb_r2_df.append(r2_diff, ignore_index=True)
        plot_r2(comb_r2_df)
        '''

        pred_data = pd.read_csv(cfg.output_directory + "hiclstm_%s_predictions_chr%s.csv" % (cell, str(chr)), sep="\t")
        hic_mat, st = get_heatmaps(pred_data, no_pred=False)
        simple_plot(hic_mat[1400:1500, 1400:1500], mode="reds")

        '''
        foxg1_data = pd.read_csv(cfg.output_directory + "shuffle_%s_afko_chr%s.csv" % (cell, str(chr)), sep="\t")
        plot_foxg1(foxg1_data)
        '''

        '''
        shift_pad = np.load(cfg.output_directory + "ctcf_diff_shift_padding.npy")
        simple_plot(shift_pad, mode="diff")
        '''
        print("done")

max_stars_repo_name: kevinbdsouza/Hi-C-LSTM

max_issues_repo_name: kevinbdsouza/Hi-C-LSTM

max_forks_repo_name: kevinbdsouza/Hi-C-LSTM

reason: import numpy,from scipy


text:
import mmap
import random
import sys
from time import sleep
import numpy as np
from beamngpy import BeamNGpy, Scenario, Vehicle, setup_logging, ScenarioObject
from beamngpy.sensors import Lidar

def generate_script():
    # Create the script
    script = list()
    points = list()
    point_colors = list()
    spheres = list()
    sphere_colors = list()

    # Hardcoded traj
    traj = [[-499.55,238.29,102.01], [-504.67,232.95,102.06], [-510.00,227.36,102.09], [-515.57,221.53,102.12], [-522.3,214.45,102.17], [-526.53,207.74,102.18], [-529.22,207.19,102.24], [-531.59,208.04,102.28], [-533.72,209.82,102.31], [-535.93, 211.59, 102.31], [-537.26,212.84, 102.4]]

    t = 1
    for p in traj:
        node = {'x': p[0], 'y':p[1], 'z': p[2], 't': t}
        t += 1
        script.append(node)
        points.append([node['x'], node['y'], node['z']])
        point_colors.append([0, 1, 0, 0.1])
        spheres.append([node['x'], node['y'], node['z'], 0.25])
        sphere_colors.append([1, 0, 0, 0.8])

    # Return data
    return script, points, point_colors, spheres, sphere_colors

def load_locations(filename):
    # Open the file
    with open(filename) as f:
        # Read the line
        contents = f.read()
        lines = contents.split()
        # Holds the results
        result = []
        # Holds the individual location
        individual_location = []
        # For each line
        line_count = 0
        for line in lines:
            # Ignore the count, only worry about x,y,z
            if line_count % 4 != 0:
                # Process the file to get the x,y and z co-oridinates
                line = line[2:]
                line = line.replace(',',"")
                individual_location.append(float(line))
            else:
                # Save result and reset
                if len(individual_location) > 0:
                    result.append(individual_location)
                individual_location = []
            # Increment line count
            line_count += 1
    # Return results
    return result

def load_vehicle_types(filename):
    # Open the file
    with open(filename) as f:
        # Read the line
        contents = f.read()
        lines = contents.split()
        # Holds the results
        result = []
        # For each line
        for line in lines:
            result.append(line)
    # Return results
    return result

def create_lidar(viz="physcov"):
    lidar = Lidar()
    if viz == "physcov":
        lidar.__init__(offset=(2.4, 1, 1), direction=(-0.707, -0.707, 0), vres=2,
                       vangle=0.1, rps=10000, hz=60, angle=180, max_dist=200,
                       visualized=True)

    elif viz == "physcovthick":
        lidar.__init__(offset=(2.4, 1, 1), direction=(-0.707, -0.707, 0), vres=2,
                       vangle=0.1, rps=2200000, hz=60, angle=180, max_dist=200,
                       visualized=True)
    elif viz == "roof":
        lidar.__init__(offset=(0, 0, 1.7), direction=(-0.707, -0.707, 0), vres=32,
                       vangle=0.01, rps=2200000, hz=20, angle=360, max_dist=200,
                       visualized=True)
    elif viz == "default":
        lidar.__init__(offset=(0, 0, 1.7), direction=(0, -1, 0), vres=32,
                 vangle=26.9, rps=2200000, hz=20, angle=360, max_dist=200,
                 visualized=True)
    else: # no viz
        lidar = Lidar(offset=(0, 0, 1.6), visualized=False)
    return lidar

def main():
    number_parked_vehicles = 5

    # random.seed(1703)

    setup_logging()

    beamng = BeamNGpy('localhost', 64256, home='H:/BeamNG.research.v1.7.0.1clean')
    bng = beamng.open(launch=True)

    # Create a scenario in west_coast_usa
    scenario = Scenario("west_coast_usa", 'tech_test', description='Random driving for research')

    # https://documentation.beamng.com/vehicle_system/vehicles/index.html
    vehicle = Vehicle('ego_vehicle', model='hatch', licence='Big C', color='Red')

    # Create sensors
    lidar = create_lidar("physcovthick") # physcov, physcovthick, default, no viz
    vehicle.attach_sensor('lidar', lidar)

    # Add the vehicle
    scenario.add_vehicle(vehicle, pos=(-496.16, 241.5, 102), rot=None, rot_quat=(0, 0, 0.258819, 0.9659258))

    # Compile the scenario and place it in BeamNG's map folder
    scenario.make(bng)

    # Get the parking locations and vehicle types
    parking_locations = load_locations("all_parking_locations.txt")
    vehicle_types = load_vehicle_types("car_types.txt")
    orientations = [[0.423, -0.906], [0.940, 0.342]]

    # Add the objects
    if number_parked_vehicles > -1:
        parking_locations = random.sample(parking_locations, number_parked_vehicles)
    for i in range(len(parking_locations)):
        # Randomly select a car type
        vehicle_type = random.choice(vehicle_types)
        # Get an orientation
        orient = random.choice(orientations)
        # Get the car position
        x, y, z = parking_locations[i]
        z += 0.1
        # Create the object
        scenario_obj = ScenarioObject(oid='car' + str(i), name=vehicle_type, otype='BeamNGVehicle', pos=(x, y, z), rot=None, rot_quat=(0, 0, orient[0], orient[1]), scale=(1, 1, 1), JBeam = vehicle_type, datablock="default_vehicle")
        # Add an object
        scenario.add_object(scenario_obj)

    scenario.make(beamng)

    # Start BeamNG and enter the main loop
    try:
        bng.hide_hud()
        bng.set_deterministic()  # Set simulator to be deterministic
        bng.set_steps_per_second(60)  # With 60hz temporal resolution

        # Load and start the scenario
        bng.load_scenario(scenario)
        bng.start_scenario()
        # file1 = open("parking_locations.txt","w")

        # keep track of the number of parking spots checked
        parking_counter = 1
        was_pressed = False

        # Generate a vehicle script
        script, points, point_colors, spheres, sphere_colors = generate_script()
        # bng.add_debug_line(points, point_colors, spheres=spheres, sphere_colors=sphere_colors, cling=True, offset=0.1)
        vehicle.ai_set_script(script)
        vehicle.ai_set_speed(100, mode='limit')
        vehicle.ai_set_aggression(0.2)

        while True:

            # # Retrieve sensor data and show the camera data.
            # vehicle.poll_sensors()
            # sensors = vehicle.sensors
            #
            # # Get the current vehicle position
            # current_x, current_y, current_z = np.round(sensors["state"].data["pos"], 4)
            #
            # # print("-----------------------------")
            # # current_x, current_y, current_z = np.round(sensors["state"].data["pos"], 4)
            # # print("Position {}, {}".format(current_x, current_y))
            # # file1.write("x:{}, y:{}, z:{}\n".format(current_x, current_y, current_z))
            # # print("-----------------------------")
            vehicle.update_vehicle()
            sensors = bng.poll_sensors(vehicle)
            state = vehicle.state
            print("-----------------------------")
            print("SENSORS:\n{}".format(sensors))
            print("VEHICLE STATE:\n{}".format(state))
            print("-----------------------------")

    finally:
        bng.close()
        # file1.close()

if __name__ == '__main__':
    main()

max_stars_repo_name: MissMeriel/BeamNGpy

max_issues_repo_name: MissMeriel/BeamNGpy

max_forks_repo_name: MissMeriel/BeamNGpy


text:
#!/usr/bin/env python

import os
import numpy as np
from joblib import Memory

from ..utils.files import ensure_dir_exists

from . import ampds, ucr, pamap, uci_gas, msrc
from . import paths

from .. import compress

_memory = Memory('.', verbose=1)

# SAVE_DIR = paths.COMPRESSION_ROWMAJOR_DIR

# This file writes out the datasets used in the experiments, including:k
#
# UCR datasets (ucr/)
#   -all examples concatenated (with interp)
#   -one file per data
# MSRC (msrc/), PAMAP (pamap/)
#   -all recordings concatenated (with interp), excluding timestamps and annotations
# AMPDS
#   -{weather, electricity, gas, water}, excluding timestamps
# UCI Gas
#   -both gas levels and sensor readings (but not the timestamps)
# ECG
#   -TODO
#
# All files are just dumps of the raw binary (numpy tofile()) in
# little-endian byte order. All files are available as:
#   {row-major, column-major} x {8bit, 16bit}
# where row-major means that all variables at a given timestamp or stored
# contiguously and column-major means that all samples from a given variable
# are stored contiguously.
#
# Note that the UCR datasets are only in "colmajor" since they're univariate,
# so row-major and column-major are the same.

def _quantize(mat, dtype, axis=0):
    # print "quantize: got mat with shape: ", mat.shape
    mat -= np.min(mat, axis=axis, keepdims=True)
    mat = mat.astype(np.float32)
    mat /= max(1, np.max(mat, axis=axis, keepdims=True))
    if dtype == np.uint8:
        max_val = 255
    elif dtype in (np.uint16, np.uint32):  # NOTE: u32 -> 16 leading 0s
        max_val = 65535
    # elif dtype == np.uint32:
        # max_val == (1 << 32) - 1
    else:
        raise ValueError("Invalid dtype '{}'".format(dtype))

    return (mat * max_val).astype(dtype)

def _ensure_list_or_tuple(x):
    if not isinstance(x, (list, tuple)):
        return [x]
    return x

# want to be able to write:
#   u8 as u8
#   u16 as u16
#   u8 as u32
#   u16 as u32
#   delta [zigzag] u8 as u32
#   delta [zigzag] u16 as u32
#
#   more compactly:
#       {delta, delta+zigzag, raw} {u8, u16} as {same type, u32}

# def write_dataset(mat, name, dtypes=(np.uint8, np.uint16),
def write_dataset(mat, name, dtype, store_as_dtype=None, order='f',
                  delta_encode=False, zigzag_encode=False, subdir='', verbose=1):

    dtype_names = {np.uint8: 'uint8', np.uint16: 'uint16', np.uint32: 'uint32',
                   np.int8: 'int8', np.int16: 'int16', np.int32: 'int32'}
    utype_to_stype = {np.uint8: np.int8, np.uint16: np.int16,
                      np.uint32: np.int32}
    order_to_dir = {'c': paths.COMPRESSION_ROWMAJOR_DIR,
                    'f': paths.COMPRESSION_COLMAJOR_DIR}

    store_mat = _quantize(mat, dtype=dtype)

    if delta_encode:
        store_mat = store_mat.astype(np.int32)
        store_mat[1:, :] = store_mat[1:, :] - store_mat[:-1, :]
        if not zigzag_encode:
            dtype = utype_to_stype[dtype]
    if zigzag_encode:
        store_mat = compress.zigzag_encode(store_mat)

    # construct path at which to write data based on params
    base_savedir = order_to_dir[order]
    if store_as_dtype is None:
        child_dir = dtype_names[dtype]
    else:
        child_dir = '{}-as-{}'.format(
            dtype_names[dtype], dtype_names[store_as_dtype])
    if delta_encode:
        child_dir += '_deltas'
    if zigzag_encode:
        child_dir += '_zigzag'
    savedir = os.path.join(base_savedir, child_dir)
    if subdir:
        savedir = os.path.join(savedir, subdir)
    ensure_dir_exists(savedir)
    path = os.path.join(savedir, name + '.dat')

    # actually write out the data
    store_as_dtype = dtype if (store_as_dtype is None) else store_as_dtype
    store_mat = store_mat.astype(store_as_dtype)
    if order == 'f':
        store_mat = np.ascontiguousarray(store_mat.T)  # tofile always writes in C order
    store_mat.tofile(path)
    # out_paths.append(path)

    if verbose > 0:
        print("saved mat {} ({}) as {}".format(name, store_mat.shape, path))

    load_mat = np.fromfile(path, dtype=store_as_dtype)

    if verbose > 1:
        print("stored mat shape: ", load_mat.shape)
        print("stored mat[:10]: ", store_mat[:10])
        print("loaded mat[:10]: ", load_mat[:10])

    assert np.array_equal(store_mat.ravel(), load_mat.ravel())

    if verbose > 1:
        import matplotlib.pyplot as plt
        _, axes = plt.subplots(2, 2, figsize=(10, 7))
        if order == 'f':
            length = 5000
            axes[0, 0].plot(store_mat[0, :length], lw=.5)
            axes[0, 1].plot(store_mat[-1, -length:], lw=.5)
        else:
            length = 2000
            axes[0, 0].plot(store_mat.ravel()[:length], lw=.5)
            axes[0, 1].plot(store_mat.ravel()[-length:], lw=.5)
        axes[1, 0].plot(load_mat[:length], lw=.5)
        axes[1, 1].plot(load_mat[-length:], lw=.5)
        # for ax in axes.ravel():
        #     ax.set_ylim([0, 255 if dtype == np.uint8 else 65535])
        plt.show()  # upper and lower plots should be identical

    # return dict(zip(dtypes, out_paths))
    return dict(dtype=path)

@_memory.cache
def concat_and_interpolate(mats, interp_npoints=5):
    # assumes each row of each mat is one time step and mats is a list

    print("mats: ", mats)

    dtype = mats[0].dtype

    first_vals = np.vstack([mat[0] for mat in mats])
    last_vals = np.vstack([mat[-1] for mat in mats])
    boundary_jumps = first_vals[1:] - last_vals[:-1]

    # print "first_vals: ", first_vals
    # print "last_vals: ", last_vals
    # print "boundary_jumps: ", boundary_jumps

    offsets = np.arange(1., interp_npoints + 1.) / (interp_npoints + 1)

    # print "offsets: ", boundary_jumps

    # multiply jumps by offsets to get interpolated values; note that
    # we reshape offsets oddly to get it to broadcast
    new_shape = list(np.ones(len(boundary_jumps.shape), dtype=np.int))
    new_shape.append(len(offsets))
    offsets = offsets.reshape(new_shape)
    boundary_jumps = boundary_jumps[..., np.newaxis]
    interp_samples = (offsets * boundary_jumps).astype(dtype)
    interp_samples += last_vals[:-1][..., np.newaxis]

    if len(mats[0].shape) < 2:
        mats = [mat[..., np.newaxis] for mat in mats]

    out_mats = [mats[0]]
    for i in range(1, len(mats)):
        if i == 1:
            print("interpolated samples shape: ", interp_samples[i - 1].T.shape)
            print("data matrix shape: ", mats[i].shape)
        out_mats.append(interp_samples[i - 1].T)
        out_mats.append(mats[i])

    return np.vstack(out_mats)

# ================================================================ main

def _test_concat_and_interpolate():  # TODO less janky unit tests
    X = np.arange(12).reshape(4, 3)
    mats = [X, X + 9 + 6]
    ret = concat_and_interpolate(mats)
    assert len(ret) == 2 * len(X) + 5
    assert np.array_equal(X, ret[:4])
    assert np.array_equal(X + 15, ret[-4:])

    X = np.arange(0, 31, 6).reshape(2, 3)
    ret = concat_and_interpolate(X)
    ans = np.array([0, 6, 12, 13, 14, 15, 16, 17, 18, 24, 30])[..., np.newaxis]
    assert np.array_equal(ret, ans)

def mat_from_recordings(recs):
    try:
        mats = [r.data for r in recs]  # recs is recording objects
    except AttributeError:
        mats = recs  # recs is just a data matrix (happens for UCR dataset)
        assert len(mats.shape) == 2  # fail fast if it isn't a data matrix

    return concat_and_interpolate(mats)

def main():
    write_normal_datasets = False
    write_ucr_datasets = False
    # write_normal_datasets = True
    write_ucr_datasets = True

    # delta_encode = False
    # zigzag_encode = False
    delta_encode = True
    zigzag_encode = True

    # STORAGE_ORDER = 'f'
    STORAGE_ORDER = 'c'

    dtypes = [np.uint8, np.uint16]
    # dtypes = [np.uint32]

    # store_as_dtype = None
    store_as_dtype = np.uint32

    if write_normal_datasets:
        funcs_and_names = [
            (ampds.all_gas_recordings, 'ampd_gas'),
            (ampds.all_water_recordings, 'ampd_water'),
            (ampds.all_power_recordings, 'ampd_power'),
            (ampds.all_weather_recordings, 'ampd_weather'),
            (uci_gas.all_recordings, 'uci_gas'),
            (pamap.all_recordings, 'pamap'),
            (msrc.all_recordings, 'msrc'),
        ]

        for func, name in funcs_and_names:
            recordings = func()
            print("data shapes: ", [r.data.shape for r in recordings])
            mat = mat_from_recordings(recordings)
            for dtype in dtypes:
                write_dataset(mat, name, order=STORAGE_ORDER, subdir=name,
                              dtype=dtype, delta_encode=delta_encode,
                              zigzag_encode=zigzag_encode,
                              store_as_dtype=store_as_dtype, verbose=1)

    if write_ucr_datasets:
        # i = 0 # TODO rm
        # for dset in ucr.origUCRDatasets():
        # for dset in list(ucr.allUCRDatasets())[:2]:
        for dset in ucr.allUCRDatasets():
            mat = concat_and_interpolate(dset.X)
            for dtype in dtypes:
                dtype2path = write_dataset(
                    mat, dset.name, order=STORAGE_ORDER, dtype=dtype,
                    subdir='ucr', delta_encode=delta_encode,
                    zigzag_encode=zigzag_encode, store_as_dtype=store_as_dtype,
                    verbose=1)

            # # break
            # if i == 2:
            #     print "mat shape: ", mat.shape
            #     mat_u8 = _quantize(mat, np.uint8).ravel()
            #     mat_u16 = _quantize(mat, np.uint16).ravel()

            #     # break

            #     out_mat_u8 = np.fromfile(dtype2path[np.uint8], dtype=np.uint8)
            #     out_mat_u16 = np.fromfile(dtype2path[np.uint16], dtype=np.uint16)

            #     print "mat size: ", mat_u8.size
            #     print "out_mat size: ", out_mat_u8.size
            #     print "mat[:20]: ", mat_u8[100:120]
            #     print "out_mat[:20]: ", out_mat_u8[100:120]

            #     import matplotlib.pyplot as plt; i = 0 # TODO rm
            #     _, axes = plt.subplots(2, 2, figsize=(8, 8))
            #     length = 1500
            #     axes[0, 0].plot(mat_u8[-length:])
            #     axes[0, 1].plot(mat_u16[-length:])
            #     axes[1, 0].plot(out_mat_u8[-length:])
            #     axes[1, 1].plot(out_mat_u16[-length:])
            #     plt.show()
            #     break
            # i += 1;

if __name__ == '__main__':
    _test_concat_and_interpolate()
    main()

max_stars_repo_name: dblalock/sprintz

max_issues_repo_name: memetb/sprintz

max_forks_repo_name: memetb/sprintz


text:
import numpy as np
from .. import Loop, LoopState

class SamplingLoop(Loop):
    def __init__(self, n_iterations, linear_constraints, n_skip):
        """
        Base class for a loop used for drawing samples
        """
        self.n_iterations = n_iterations
        super().__init__(linear_constraints, n_skip)

    def compute_next_point(self, x0):
        """
        Computes the next sample from the linearly constrained unit Gaussian
        :param x0: current state
        :return: new state
        """
        return NotImplementedError

    def is_converged(self):
        """ Stopping criterion for sampling core """
        return NotImplementedError

class SamplerState(LoopState):
    """
    Contains the state of the sampling loop, which includes a history of all evaluated locations
    """
    def __init__(self, x_init) -> None:
        self.samples = [x_init[:, i][:, None] for i in range(x_init.shape[-1])]
        self.iteration = 0
        super().__init__()

    def update(self, x_new) -> None:
        self.iteration += 1
        self.samples.append(x_new)

    @property
    def X(self):
        return np.hstack(self.samples)

max_stars_repo_name: alpiges/LinConGauss

max_issues_repo_name: alpiges/LinConGauss

max_forks_repo_name: alpiges/LinConGauss


text:
import numpy as np

class Empirical():
    def __init__(self, samples, perf_factor=1):
        self.samples = np.array(samples)
        assert perf_factor > 0, perf_factor
        if perf_factor != 1:
            self.samples /= perf_factor
        self.sidx = 0

    def draw(self):
        return np.random.choice(self.samples)

    def draw_sequential(self):
        sample = self.samples[self.sidx]
        self.sidx = (self.sidx + 1) % len(self.samples)
        return sample

    def mean(self):
        return self.samples.mean()

    def std(self):
        return self.samples.std(ddof=1)

    def min(self):
        return self.samples.min()

    def max(self):
        return self.samples.max()

def dist_from_dict(dist_dict, perf_factor=1):
    if dist_dict['type'] == 'empirical':
        return Empirical(dist_dict['samples'], perf_factor)
    else:
        raise ValueError(f'Unknown distribution type "{dist_dict["type"]}"')


text:
"""This file contains a wrapper class for augmentations from other projects.
Currently supported:
---
1. Albumentations: https://github.com/albumentations-team/albumentations
2. Imgaug: https://github.com/aleju/imgaug
"""
import numpy

from augraphy.base.augmentation import Augmentation

class ForeignAugmentation(Augmentation):
    """A wrapper for augmentations from other projects.

    :param foreignAugmentation: The fully-applied constructor for the foreign transform.
    :type foreignAugmentation: object
    :param p: The probability that augmentation will be applied.
    :type p: float, optional
    """

    def __init__(self, foreignAugmentation, p=1):
        self.augmentation = foreignAugmentation
        super().__init__(p=p)

    def __call__(self, image):
        image = image.copy()
        result = self.augmentation(image=image)
        output = self.handleForeignAugResult(result)
        return output

    def handleForeignAugResult(*res):
        """The argument to this depends on the foreign augmentation applied.
        If an Albumentation augmentation is used, the result is a dict with the
        output image at key "image".
        If an imgaug augmenter is used, the result is a numpy.ndarray.
        """
        if isinstance(res[1], dict):
            return res[1]["image"]
        else:
            return res[1]

max_stars_repo_name: RyonSayer/augraphy