text:
"""
    Classes for neural network models.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from .supervised_model import SupervisedModel
import numpy as np
from keras.models import Model
from keras.layers import Input, Dense
from sklearn.preprocessing import OneHotEncoder

class SimpleNN(SupervisedModel):
    def __init__(self, task_names,
                 n_features=1024,
                 batch_size=32,
                 epochs=200,
                 verbose=0):
        super(SimpleNN, self).__init__(task_names)
        # model params
        self.n_features = n_features
        self.batch_size = batch_size
        self.epochs = epochs
        self.verbose = verbose

        self.oh_enc = OneHotEncoder()
        self.oh_enc.fit(np.array([1,0]).reshape(-1,1))

        # setup model
        self.model_dict = {}
        for task_name in self.task_names:
            self.model_dict[task_name] = self._get_simple_model()

    def _get_simple_model(self):
        input = Input(shape=(self.n_features,))
        dense_1 = Dense(self.n_features, activation='relu')(input)
        dense_2 = Dense(self.n_features//2, activation='relu')(dense_1)
        softmax_output = Dense(2, activation='softmax')(dense_2)
        model = Model(inputs=input, outputs=softmax_output)

        model.compile(loss='categorical_crossentropy',
                      optimizer='adam')
        return model

    def fit(self, X_train, y_train):
        if X_train.shape[0] > 0:
            # perform random shuffling of training data (including X_train)
            shuffle_p = np.random.permutation(X_train.shape[0])
            X_train = X_train[shuffle_p]
            y_train = y_train[shuffle_p]

            # fit model
            for ti, task_name in enumerate(self.task_names):
                y_train_ti = y_train[:,ti].flatten()
                nan_indices = np.where(np.isnan(y_train_ti))[0] # remove NaN labes
                y_train_ti = np.delete(y_train_ti, nan_indices, axis=0)
                X_task_ti = np.delete(X_train, nan_indices, axis=0)

                y_train_ti = self.oh_enc.transform(y_train_ti.reshape(-1,1)).toarray()

                self.model_dict[task_name].fit(X_task_ti, y_train_ti,
                                               batch_size=self.batch_size,
                                               epochs=self.epochs,
                                               verbose=self.verbose)
            # set fit check
            self.check_fit = True

    def predict(self, X):
        y_pred = np.zeros(shape=(X.shape[0],len(self.task_names)))
        if self.check_fit:
            for ti, task_name in enumerate(self.task_names):
                y_pred[:,ti] = self.model_dict[task_name].predict(X,
                                                                  batch_size=self.batch_size,
                                                                  verbose=self.verbose)[:,1]
        else: # if model has not been fit, then assume 0.5 for all samples
            y_pred[:] = 0.5
        return y_pred


text:
import matplotlib.pyplot as plt
import numpy
import numpy as np

x = [-2, -1, 0, 1, 2]
y = [3, 4, 2, 1, 1]

# ///////////////////////////////////////////////////////////////////////////////////
# Пугливым не смотреть! Ниже програмный код!
# ///////////////////////////////////////////////////////////////////////////////////

n = len(x)
m = len(y)

print(f'n = {n}')
print('--------------------Вычислим коэффициенты системы--------------------\n')

print(f'Сумма чисел списка x: {sum(x)}')

def summa_xn(x_list, n_power):
    """Функция, которая считает сумму элементов в указанной степени"""
    return sum((elem ** n_power for elem in x_list))

def summa_x2(x):
    """Функция, которая считает сумму квадратов"""
    s = 0
    for i in x:
        s += i * i
    return s

print(f'Сумма квадратов чисел списка x: {summa_x2(x)}')

def summa_x3(x):
    """Функция, которая считает сумму кубов"""
    s = 0
    for i in x:
        s += i * i * i
    return s

print(f'Сумма кубов чисел списка x: {summa_x3(x)}')

def summa_x4(x):
    """Функция, которая считает сумму чисел в 4-й степени"""
    s = 0
    for i in x:
        s += i * i * i * i
    return s

print(f'Сумма чисел в 4-й степени списка x: {summa_x4(x)}\n')

print(f'Сумма чисел списка y: {sum(y)}')

def summa_xy(x, y):
    """Функция, которая находит сумму перемноженных элементов списков x и y"""
    s = list(map(lambda i, j: i * j, x, y))
    k = sum(s)
    return k

print(f'Сумма перемноженных чисел списков x и y: {summa_xy(x, y)}')

def summa_xxy(x, y):
    """Функция, которая считает сумму x**2 * y"""
    s = list(map(lambda i, j: (i ** 2) * j, x, y))
    k = sum(s)
    return k

print(f'Сумма x**2 * y: {summa_xxy(x, y)}\n')

print('---------------------------Решим систему---------------------------\n')

Matrix = numpy.array([[summa_x4(x), summa_x3(x), summa_x2(x)],
                      [summa_x3(x), summa_x2(x), sum(x)],
                      [summa_x2(x), sum(x), n]])  # Матрица (левая часть системы)
Vector = numpy.array([summa_xxy(x, y), summa_xy(x, y), sum(y)])  # Вектор (правая часть системы)

otvet = numpy.linalg.solve(Matrix, Vector)

print(f'a = {otvet[0]}')
print(f'b = {otvet[1]}')
print(f'c = {otvet[2]}\n')

def function_P(x):
    P = otvet[0] * x * x + otvet[1] * x + otvet[2]
    print(f'y({x}) = {P}\n')
    return P

def square_residual(x, y):
    s = list(map(lambda i, j: ((function_P(i) - j) ** 2), x, y))
    k = sum(s)
    return k

print(f'Квадратическая невязка = {square_residual(x, y)}')

x = np.linspace(-5, 2, 100)  # от -5 до 2 сделать 100 точек
y = otvet[0] * x * x + otvet[1] * x + otvet[2]  # y1 - тоже много точек

fig, ax = plt.subplots()  # будет 1 график, на нем:
ax.plot(x, y, color="blue", label="y(x)")  # функция y(x), синий, надпись y(x)
ax.set_xlabel("Ось абсцисс: x")  # подпись у горизонтальной оси х
ax.set_ylabel("Ось ординат: y")  # подпись у вертикальной оси y
ax.legend()  # показывать условные обозначения

plt.show()  # показать рисунок
fig.savefig('1.png')  # сохранить в файл 1.png

ext: py

max_stars_repo_name: Battle-Of-Two-K/NumericalMethods

max_issues_repo_name: Battle-Of-Two-K/NumericalMethods

max_forks_repo_name: Battle-Of-Two-K/NumericalMethods

reason: import numpy


text:
"""
Author: Rodrigo Loza
Email: lozuwaucb@gmail.com
Description: Implements common file, logic operations.
"""
import os
import datetime
import re
import json
import numpy as np
import cv2
import xml.etree.ElementTree as ET
# Local modules.
try:
  from .AssertDataTypes import *
except:
  from AssertDataTypes import *

class Util(object):
  def __init__(self):
    super(Util, self).__init__()

  @staticmethod
  def create_folder(folder_name = None):
    """
    Creates a folder.
    Args:
      folder_name: A string that contains the name of the folder to be created.
    Returns:
      None
    """
    # Assertions
    if (folder_name == None):
      raise ValueError("ERROR: Folder name parameter cannot be emtpy.")
    if (os.path.isdir(folder_name)):
      print("INFO: Folder already exists: {}".format(os.path.split(folder_name)[1]))
    else:
      try:
        os.mkdir(folder_name)
      except:
        raise Exception("ERROR: Folder {} could not be created.".format(folder_name))

  @staticmethod
  def create_random_name(name = None, length = None):
    """
    Generates a name based on the name parameter.
    Args:
      name: A string that contains the name of the image that is being labeled.
      length: An int that defines the length of the random string in the name
              to be created.
    Returns:
      A string that contains a new name with respect to the current time.
    """
    # Assertions
    if (name == None):
      raise ValueError("Name parameter cannot be empty.")
    if (length == None):
      raise ValueError("Length parameter cannot be empty.")
    # Local variables
    abc = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l",
      "m", "n", "o", "p", "q","r", "s", "t", "u", "v", "w", "x", "y", "z"]
    # Get current time.
    now = datetime.datetime.now()
    # Create random 4-length string.
    var_str = ""
    for i in range(4):
      random_index = int(np.random.rand()*(len(abc)-1))
      var_str += abc[random_index]
    # Append created variables.
    new_name = "{}_{}_{}_{}_{}_{}".format(name,
                                          now.year,
                                          now.month,
                                          now.hour,
                                          now.microsecond,
                                          var_str)
    # Return string
    return new_name

  @staticmethod
  def detect_file_extension(filename = None):
    """
    Detect the extension of a file.
    Args:
      filename: A string that contains the name of a file.
    Returns:
      A string that contains the extension of the file. It
      returns None if there is no extension.
    """
    # Assertions
    if (filename == None):
      raise ValueError("Filename cannot be empty.")
    # Logic
    if (filename.endswith(".jpg")):
      return ".jpg"
    elif (filename.endswith(".png")):
      return ".png"
    else:
      return None

  @staticmethod
  def save_img(frame = None, img_name = None, output_image_directory = None):
    """
    Saves an image and its annotation.
    Args:
      frame: A numpy/tensorflow tensor that contains an image.
      img_name: A string with a name that contains an image extension.
      output_image_directory: A string that contains the path to save the image.
    Returns:
      None
    Raises:
      -Exception: In case the file was not written to disk.
    """
    # Assertions.
    if (assertNumpyType(data = frame) == False):
      raise ValueError("Frame has to be a numpy array.")
    if (img_name == None):
      raise ValueError("img_name cannot be emtpy.")
    extension = Util.detect_file_extension(filename = img_name)
    if (extension == None):
      raise Exception("Your image extension is not valid. " +\
                      "Only jpgs and pngs are allowed. {}".format(extension))
    # Local variables.
    img_save_path = os.path.join(output_image_directory, img_name)
    # Logic.
    cv2.imwrite(img_save_path, frame)
    # Assert file has been written to disk.
    if (not os.path.isfile(img_save_path)):
      raise Exception("ERROR: Image was not saved. This happens " +\
                "sometimes when there are dozens of thousands of data " +\
                "points. Please run the script again and report this problem.")

  @staticmethod
  def save_annotation(filename = None, path = None, database_name = None, frame_size = None, data_augmentation_type = None, bounding_boxes = None, names = None, origin = None, output_directory = None):
    """
    Creates an XML file that contains the annotation's information of an image.
    This file's structure is based on the VOC format.
    Args:
      filename: A string that contains the name of a file. (Usually the name of the image).
      path: A string that contains the path to an image.
      database_name: A string that contains the name of a database.
      frame_size: A tuple that contains information about the size of an image.
      data_augmentation_type: A string that contains the type of augmentation that
                              is being used. Otherwise "Unspecified".
      bounding_boxes: A list of lists that contains the bounding boxes annotations.
      names: A list of strings that is parallel to bounding boxes. It depicts
            the name associated with each bounding box.
      origin: A string that contains information about the origin of the file.
      output_directory: A string that contains the path to a directory to save
                        the annotation.
    Returns:
      None
    """
    # Assertions
    if (filename == None):
      raise ValueError("Filename parameter cannot be empty.")
    if (path == None):
      raise ValueError("Path parameter cannot be empty.")
    if (database_name == None):
      raise ValueError("Database parameter cannot be empty.")
    if (frame_size == None):
      raise ValueError("Frame size parameter cannot be empty.")
    if (data_augmentation_type == None):
      raise ValueError("Data augmentation type parameter cannot be empty.")
    if (bounding_boxes == None):
      raise ValueError("Bounding boxes parameter cannot be empty.")
    if (names == None):
      raise ValueError("Names parameter cannot be empty.")
    if (origin == None):
      raise ValueError("Origin parameter cannot be empty.")
    if (output_directory == None):
      raise ValueError("Output directory parameter cannot be empty.")
    # XML VOC format
    annotation = ET.Element("annotation")
    # Image info
    ET.SubElement(annotation, "filename").text = str(filename)
    ET.SubElement(annotation, "origin").text = str(origin)
    ET.SubElement(annotation, "path").text = str(path)
    # Source
    source = ET.SubElement(annotation, "source")
    ET.SubElement(source, "database").text = str(database_name)
    # Size
    size = ET.SubElement(annotation, "size")
    ET.SubElement(size, "height").text = str(frame_size[0])
    ET.SubElement(size, "width").text = str(frame_size[1])
    if (len(frame_size) == 3):
      ET.SubElement(size, "depth").text = str(frame_size[2])
    # Data augmentation
    data_augmentation = ET.SubElement(annotation, "data_augmentation")
    ET.SubElement(data_augmentation, "type").text = str(data_augmentation_type)
    # Segmented
    ET.SubElement(annotation, "segmented").text = "0"
    # Objects
    for name, coordinate in zip(names, bounding_boxes):
      object_ = ET.SubElement(annotation, "object")
      ET.SubElement(object_, "name").text = str(name)
      ET.SubElement(object_, "pose").text = "Unspecified"
      ET.SubElement(object_, "truncated").text = "0"
      ET.SubElement(object_, "difficult").text = "0"
      bndbox = ET.SubElement(object_, "bndbox")
      xmin, ymin, xmax, ymax = coordinate
      ET.SubElement(bndbox, "xmin").text = str(xmin)
      ET.SubElement(bndbox, "ymin").text = str(ymin)
      ET.SubElement(bndbox, "xmax").text = str(xmax)
      ET.SubElement(bndbox, "ymax").text = str(ymax)
    # Write file
    tree = ET.ElementTree(annotation)
    extension = Util.detect_file_extension(filename)
    if (extension == None):
      raise Exception("Image's extension not supported {}".format(filename))
    tree.write(output_directory)
    # Assert file has been written to disk.
    if (not os.path.isfile(output_directory)):
      print(origin_information)
      print(img_name)
      print(xml_name)
      raise Exception("ERROR: Annotation was not saved. This happens " +\
                      "sometimes when there are dozens of thousands of data " +\
                      "points. Please run the script again and report this problem.")

  @staticmethod
  def save_lists_in_dataframe(columns = None, data = None, output_directory = None):
    """
    Save lists into a dataframe.
    Args:
      columns: A list of strings that contains the names of the columns
              for the dataframe.
      data: A list of lists that contains data.
      output_directory: A string that contains the path to where save
                        the dataframe.
    Returns:
      None
    """
    # Assertions
    if (columns == None):
      raise ValueError("ERROR: Paramater columns cannot be empty.")
    if (data == None):
      raise ValueError("ERROR: Paramater data cannot be empty.")
    if (output_directory == None):
      raise ValueError("ERROR: Paramater output_directory cannot be empty.")
    if (not os.path.isdir(output_directory)):
      raise Exception("ERROR: Path to {} does not exist.".format(output_directory))
    if (len(columns) != len(data)):
      raise Exception("ERROR: The len of the columns has to be the" +\
                      " same as data. Report this problem.")
    # Local import
    try:
      import pandas as pd
    except Exception as e:
      raise ImportError("ERROR: Pandas is not available, install it.")
    # Logic
    hashMap = {}
    for i in range(columns):
      hashMap[columns[i]] = data[i]
    df = pd.DataFrame(hashMap)
    df.to_excel(output_directory)

# import os
# import cv2
# import numpy as np

# images = os.listdir(".")
# hs = []
# ws = []
# for each in images:
#   frame = cv2.imread(each)
#   h = frame.shape[0]
#   w = frame.shape[1]
#   hs.append(h)
#   ws.append(w)

# hs = np.array(hs)
# ws = np.array(ws)

# print(hs.mean(), ws.mean())

# for each in images:
#   frame = cv2.imread(each)
#   cv2.imwrite(each, cv2.resize(frame, (int(ws.mean()), int(hs.mean()))))

max_stars_repo_name: Anoopkr/impy


text:
"""This is the Bokeh charts interface. It gives you a high level API
to build complex plot is a simple way.

This is the Scatter class which lets you build your Scatter charts
just passing the arguments to the Chart class and calling the proper
functions.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2012 - 2014, Continuum Analytics, Inc. All rights reserved.
#
# Powered by the Bokeh Development Team.
#
# The full license is in the file LICENCE.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import numpy as np

try:
    import pandas as pd
except:
    pd = None

from collections import OrderedDict
from ._chartobject import ChartObject, DataAdapter
from ..models import ColumnDataSource, Range1d

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------

class Scatter(ChartObject):
    """This is the Scatter class and it is in charge of plotting
    Scatter charts in an easy and intuitive way.

    Essentially, we provide a way to ingest the data, make the proper
    calculations and push the references into a source object.
    We additionally make calculations for the ranges. And finally add
    the needed glyphs (markers) taking the references from the source.

    Example:
        from collections import OrderedDict

        from bokeh.charts import Scatter
        xyvalues = OrderedDict()
        xyvalues['python'] = [(1, 2), (3, 3), (4, 7), (5, 5), (8, 26)]
        xyvalues['pypy'] = [(1, 12), (2, 23), (4, 47), (5, 15), (8, 46)]
        xyvalues['jython'] = [(1, 22), (2, 43), (4, 10), (6, 25), (8, 26)]

        scatter = Scatter(xyvalues, filename="scatter.html")
        scatter.title("Languages Scatter").legend("top_left")
        scatter.width(600).height(400).show()
    """
    def __init__(self, values, title=None, xlabel=None, ylabel=None,
                 legend=False, xscale="linear", yscale="linear", width=800,
                 height=600, tools=True, filename=False, server=False,
                 notebook=False, facet=False, xgrid=True, ygrid=True):
        """
        Args:
            values (iterable(tuples)): an iterable containing the data as
                (x, y) tuples.
            title (str, optional): the title of your chart. Defaults
                to None.
            xlabel (str, optional): the x-axis label of your chart.
                Defaults to None.
            ylabel (str, optional): the y-axis label of your chart.
                Defaults to None.
            legend (str, optional): the legend of your chart. The legend
                content is inferred from incoming input.It can be
                ``top_left``, ``top_right``, ``bottom_left``,
                ``bottom_right``. ``top_right`` is set if you set it
                 as True. Defaults to None.
            xscale (str, optional): the x-axis type scale of your chart.
                It can be ``linear``, ``datetime`` or ``categorical``.
                Defaults to ``datetime``.
            yscale (str, optional): the y-axis type scale of your chart.
                It can be ``linear``, ``datetime`` or ``categorical``.
                Defaults to ``linear``.
            width (int, optional): the width of your chart in pixels.
                Defaults to 800.
            height (int, optional): the height of you chart in pixels.
                Defaults to 600.
            tools (bool, optional): to enable or disable the tools in
                your chart. Defaults to True
            filename (str or bool, optional): the name of the file where
                your chart. will be written. If you pass True to this
                argument, it will use ``untitled`` as a filename.
                Defaults to False.
            server (str or bool, optional): the name of your chart in
                the server. If you pass True to this argument, it will
                use ``untitled`` as the name in the server.
                Defaults to False.
            notebook (bool, optional): whether to output to IPython notebook
                (default: False)
            facet (bool, optional): generate multiple areas on multiple
                separate charts for each series if True. Defaults to
                False
            xgrid (bool, optional): whether to display x grid lines
                (default: True)
            ygrid (bool, optional): whether to display y grid lines
                (default: True)

        Attributes:
            source (obj): datasource object for your plot,
                initialized as a dummy None.
            xdr (obj): x-associated datarange object for you plot,
                initialized as a dummy None.
            ydr (obj): y-associated datarange object for you plot,
                initialized as a dummy None.
            groups (list): to be filled with the incoming groups of data.
                Useful for legend construction.
            data (dict): to be filled with the incoming data and be passed
                to the ColumnDataSource in each chart inherited class.
                Needed for _set_And_get method.
            attr (list): to be filled with the new attributes created after
                loading the data dict.
                Needed for _set_And_get method.
        """
        self.values = values
        self.source = None
        self.xdr = None
        self.ydr = None
        self.groups = []
        self.data = dict()
        self.attr = []
        super(Scatter, self).__init__(
            title, xlabel, ylabel, legend, xscale, yscale, width, height,
            tools, filename, server, notebook, facet, xgrid, ygrid
        )

    def check_attr(self):
        """Check if any of the chained method were used.

        If they were not used, it assign the init parameters content
        by default.
        """
        super(Scatter, self).check_attr()

    def get_data(self):
        """Take the scatter.values data to calculate the chart properties
        accordingly. Then build a dict containing references to all the
        calculated points to be used by the marker glyph inside the
        ``draw`` method.
        """
        self.data = dict()
        # list to save all the attributes we are going to create
        self.attr = []
        # list to save all the groups available in the incomming input
        self.groups.extend(self.values.keys())
        # Grouping
        self.parse_data()

    @property
    def parse_data(self):
        """Parse data received from self.values and create correct x, y
        series values checking if input is a pandas DataFrameGroupBy
        object or one of the stardard supported types (that can be
        converted to a DataAdapter)
        """
        if pd is not None and \
                isinstance(self.values, pd.core.groupby.DataFrameGroupBy):
            return self._parse_groupped_data
        else:
            return self._parse_data

    def _parse_groupped_data(self):
        """Parse data in self.values in case it's a pandas
        DataFrameGroupBy and create the data 'x_...' and 'y_...' values
        for all data series
        """
        for i, val in enumerate(self.values.keys()):
            xy = self.values[val]
            self._set_and_get("x_", val, xy[:, 0])
            self._set_and_get("y_", val, xy[:, 1])

    def _parse_data(self):
        """Parse data in self.values in case it's an iterable (not a pandas
        DataFrameGroupBy) and create the data 'x_...' and 'y_...' values
        for all data series
        """
        for i, val in enumerate(self.values.keys()):
            x_, y_ = [], []
            xy = self.values[val]
            for value in self.values.index:
                x_.append(xy[value][0])
                y_.append(xy[value][1])

            self.set_and_get("x_", val, x_)
            self.set_and_get("y_", val, y_)

    def get_source(self):
        """Push the Scatter data into the ColumnDataSource and
        calculate the proper ranges."""
        self.source = ColumnDataSource(self.data)

        x_names, y_names = self.attr[::2], self.attr[1::2]

        endx = max(max(self.data[i]) for i in x_names)
        startx = min(min(self.data[i]) for i in x_names)
        self.xdr = Range1d(
            start=startx - 0.1 * (endx - startx),
            end=endx + 0.1 * (endx - startx)
        )
        endy = max(max(self.data[i]) for i in y_names)
        starty = min(min(self.data[i]) for i in y_names)
        self.ydr = Range1d(
            start=starty - 0.1 * (endy - starty),
            end=endy + 0.1 * (endy - starty)
        )

    def draw(self):
        """Use the marker glyphs to display the points.

        Takes reference points from data loaded at the ColumnDataSource.
        """
        duplets = list(self._chunker(self.attr, 2))
        colors = self._set_colors(duplets)

        for i, duplet in enumerate(duplets, start=1):
            self.chart.make_scatter(self.source, duplet[0], duplet[1], 'circle', colors[i - 1])

            if i < len(duplets):
                self.create_plot_if_facet()

        self.reset_legend()

    def _make_legend_glyph(self, source_legend, color):
        """Create a new glyph to represent one of the chart data series with the
        specified color

        The glyph is added to chart.glyphs.

        Args:
            source_legend (ColumnDataSource): source to be used when creating the glyph
            color (str): color of the glyph
        """
        self.chart.make_scatter(source_legend, "groups", None, 'circle', color)

    def _setup_show(self):
        """Prepare context before main show method is invoked.

        Customize show preliminary actions by handling DataFrameGroupBy
        values in order to create the series values and labels."""
        super(Scatter, self)._setup_show()

        # check if pandas is installed
        if pd:
            # if it is we try to take advantage of it's data structures
            # asumming we get an groupby object
            if isinstance(self.values, pd.core.groupby.DataFrameGroupBy):
                pdict = OrderedDict()

                for i in self.values.groups.keys():
                    self.labels = self.values.get_group(i).columns
                    xname = self.values.get_group(i).columns[0]
                    yname = self.values.get_group(i).columns[1]
                    x = getattr(self.values.get_group(i), xname)
                    y = getattr(self.values.get_group(i), yname)
                    pdict[i] = np.array([x.values, y.values]).T

                self.values = DataAdapter(pdict)
                self.labels = self.values.keys()

                # create axis labels from group by object only if the input
                # values is a DataFrameGroupBy
                if self._xlabel is None:
                    self._xlabel = self.labels[0]

                if self._ylabel is None:
                    self._ylabel = self.labels[1]

            else:
                self.values = DataAdapter(self.values)
                self.labels = self.values.keys()

        else:
            self.values = DataAdapter(self.values)
            self.labels = self.values.keys()


text:
import numpy as nmp
import theano.tensor as tns
import pymc3 as pmc

from GPyclone import aux

##
def get_model(x, r, R, vaf0, K=10):
    nsamples = r.shape[1]
    r, R, vaf0 = r[:, :, None], R[:, :, None], vaf0[:, :, None]
    idxs = aux.corr_vector_to_matrix_indices(K)
    with pmc.Model() as model:
        w = pmc.Dirichlet('w', nmp.ones(K))
        lw = tns.log(w)

        # alpha = pmc.Gamma('alpha', 1.0, 1.0)
        # u = pmc.Beta('u', 1.0, alpha, shape=K-1)
        # lw = aux.stick_breaking_log(u)

        rho = pmc.Gamma('rho', 1.0, 1.0)
        Cc = tns.fill_diagonal(pmc.LKJCorr('C', eta=2.0, n=K)[idxs], 1.0)
        Cr = aux.cov_quad_exp(x, 1.0, rho)
        mu_psi = pmc.MatrixNormal('mu_psi', mu=nmp.zeros((nsamples, K)), rowcov=Cr, colcov=Cc, shape=(nsamples, K))
        psi = pmc.Normal('psi', mu=mu_psi, sd=0.1, shape=(nsamples, K))
        phi = pmc.Deterministic('phi', pmc.invlogit(psi))

        # psi = pmc.MvNormal('psi', mu=nmp.zeros(K), tau=nmp.eye(K), shape=(nsamples, K))
        # phi = pmc.Deterministic('phi', pmc.invlogit(psi))

        theta = pmc.Deterministic('theta', vaf0 * phi[None, :, :])
        pmc.DensityDist('r', aux.binmixND_logp_fcn(R, theta, lw), observed=r)
    return model

max_stars_repo_name: dvav/clonosGP

max_issues_repo_name: dvav/clonosGP

max_forks_repo_name: dvav/clonosGP

reason: import numpy,import theano,import pymc3


text:
#!/usr/bin/env python
# coding: utf-8

"""

Automatical reprocessing .fg5 projects

Aleksei Polukhin
Department of Gravimetry and Geodynamics
Center of Geodesy, Cartography and SDI (TsNIIGAiK)
Mon Jan 23 15:10:50 2017

"""

from ldtp import *
import zeep
from astropy.time import Time
import sys
from os import remove, path, getcwd, system
from subprocess import Popen
from shutil import copyfile
from struct import unpack_from
import numpy as np
from re import finditer
from prettytable import PrettyTable

def get_datetime(filename):
    # Reading binary data
    with open(filename, "rb") as f:
        bin_data = f.read()

    # Searching datetime start byte
    date_key = r'\xe0.\x00\x00\x0b'
    matches = []
    for match in finditer(date_key, bin_data):
        matches.append(match.span())
    date_offset = matches[-1][0] - 8

    # Unpacking datetime
    time_values = {}
    hours, days = unpack_from("<II", bin_data, date_offset)
    secs = days * 45.0 * 60.0 + hours / (4294967295.0/45.0) * 60.0
    time_values["UNIX"] = secs - 2938117104000.0

    # Convert to ISO and MJD time formats
    t = Time(time_values["UNIX"], format="unix")
    t.format = "iso"
    t.out_subfmt = "date_hm"
    time_values["ISO"] = t.value
    t.format = "mjd"
    time_values["MJD"] = str(int(np.floor(t.value)))

    return time_values

def get_nsets(filename):
    # Reading binary data
    with open(filename, "rb") as f:
        bin_data = f.read()

    masks = {
        "project": r'\x24\x40\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00',
        "fact": r'\xF4\xBF\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
    }
    n_sets = {}
    # Searching datetime start byte
    for name, mask in masks.iteritems():
        matches = []
        for match in finditer(mask, bin_data):
            matches.append(match.span())
        date_offset = matches[-1][1]
        n = unpack_from("<h", bin_data, date_offset)
        n_sets[name] = int(n[0])

    return n_sets

def pole_coords(mjd):
    # Get values of pole coords from IERS
    try:
        client = zeep.Client("https://data.iers.org/eris/webservice/eop/eop.wsdl")
    except:
        print("No connection to Internet. Leaving with initial Pole coords")
        return -1
    get_eop = lambda bulletin: {pole: client.service.readEOP(pole, bulletin, mjd) for pole in ["x_pole", "y_pole"]}
    bltn = "EOP 14 C04 (IAU2000)"
    eop_values = get_eop(bltn)
    if eop_values["x_pole"] == None and eop_values["y_pole"] == None:
        print("Values for MJD %s not found in EOP 14 C04 (IAU2000). Taking values from Bulletin A..." % mjd)
        bltn = "Bulletin A"
        eop_values = get_eop(bltn)
    eop_values = {pole:str(round(float(eop_values[pole]) / 1000, 4)) for pole in ["x_pole", "y_pole"]}
    eop_values["BULLETIN"] = bltn

    return eop_values

def get_buttons(window):
    objects = getobjectlist(window)
    tbar_idx = objects.index("tbar0")
    buttons = objects[tbar_idx + 5 : tbar_idx + 8]
    buttons.append(objects[tbar_idx + 11])
    return buttons

def g9_proc(filename, vgrad):

    g9_bin = r"C:\Program Files\Micro-g LaCoste Inc\bin\g9.exe"
    if not path.isfile(g9_bin):
        g9_bin = r"C:\Program Files (x86)\Micro-g LaCoste Inc\bin\g9.exe"

    system("TASKKILL /F /IM g9.exe")
    Popen("%s %s" % (g9_bin, filename))

    wdw = "*Micro-g*"
    results = {}
    """
    frmSetup = 'frmSetup'
    frmETGTABSetup = 'frmETGTABSetup'
    frmOceanLoad = 'frmOceanLoad'
    btnSetupOk = u'btn\u041e\u041a'
    frmOverrideDialog = 'frmOverrideDialog'

    """
    frmSetup = "dlgSetup"
    frmETGTABSetup = 'dlgETGTABSetup'
    frmOceanLoad = 'dlgOceanLoad'
    btnSetupOk = 'btnOK'
    frmOverrideDialog = 'dlgOverrideDialog'

    wait(1)

    btnSetup, btnGo, btnStop, btnAbout = get_buttons(wdw)

    click(wdw, btnSetup)
    wait(1)

    # Information Tab
    #     Get Site Name and Code
    results["name"] = gettextvalue(frmSetup, 'txt0')
    results["code"] = gettextvalue(frmSetup, 'txt1')

    #     Get Setup Height
    results["h_setup"] = float(gettextvalue(frmSetup, 'txt7'))

    #     Set Nominal Pressure
    click(frmSetup, "btnSet")

    #     Set Gradient
    settextvalue(frmSetup, 'txt6', str(vgrad))

    #     Get Project Start DateTime
    project_datetime = get_datetime(filename)
    results["iso"] = project_datetime["ISO"]
    results["mjd"] = project_datetime["MJD"]

    #     Get Polar X/Y from IERS and set
    eop_data = pole_coords(project_datetime["MJD"])
    if eop_data == -1:
        results["bltn"] = "Default"

    else:
        settextvalue(frmSetup, 'txt8', eop_data["x_pole"])
        settextvalue(frmSetup, 'txt9', eop_data["y_pole"])
        results["bltn"] = eop_data["BULLETIN"]

    #     Set Transfer Height
    settextvalue(frmSetup, 'txt10', "130")

    # Acqusition Tab
    selecttab(frmSetup, 'ptl0', 'ptabAcquisition')
    #     Get Sets and Drops values
    results["sets"] = get_nsets(filename)["fact"]
    results["drops"] = gettextvalue(frmSetup, 'txt1')

    # Control Tab
    selecttab(frmSetup, 'ptl0', 'ptabControl')
    #     Calculate Tidal Corrections
    comboselect(frmSetup, 'cbo1', "ETGTAB")
    click(frmSetup, 'btnSetup1')
    click(frmETGTABSetup, 'btnRunOceanLoad')
    click(frmOceanLoad, 'btnOK')
    wait(1)
    click(frmETGTABSetup, 'btnOK')

    #    Uncheck box  Auto Peak Detection
    if verifycheck(frmSetup, 'chkAutoPeakDetection'):
        uncheck(frmSetup, 'chkAutoPeakDetection')

    # Push OK to close Setup window
    click(frmSetup, btnSetupOk)

    # Start processing
    click(wdw, btnGo)

    if guiexist(frmOverrideDialog):
        click(frmOverrideDialog, 'btnYes')

    # Waiting till processing ends
    wait(5)
    while int(gettextvalue('frmState', 'txt32')) != results["sets"]:
        wait(3)
    wait(5)

    results["gravity"] = float(gettextvalue('frmState', 'txt42'))
    results["SD"] = float(gettextvalue('frmState', 'txt43'))

    system("TASKKILL /F /IM g9.exe")

    #selectmenuitem(wdw, "mnuProject;mnuClose")

    results["SE"] = results["SD"] / np.sqrt(results["sets"])

    return results

def az_proc(project_azumuth, h_fact, h_mod, wzz):
    # Processing gravity at effective height
    g_eff = g9_proc(project_azumuth, wzz)
    g_eff["h_eff"] = h_fact + g_eff["h_setup"] - h_mod

    with open(path.splitext(project_azumuth)[0] + ".project.txt", 'a') as f:
        f.write(('\n\nReprocessing\nMJD:  {1}\nBulletin:  {2}		\nMeasurement Start:  {0}		\nGravity at Effective Height:  {3:.2f} uGal		\nStandard Deviation:  {4:.2f} uGal		\nStandard Error:  {5:.2f} uGal		\nEffective Height:  {6:.2f} cm		\nSetup Height:  {7:.2f} cm		\nHeight_mod:  {8:.2f} cm		\nFactory Height:  {9:.2f} cm').format(g_eff["iso"],
                g_eff["mjd"], g_eff["bltn"], g_eff["gravity"],
                g_eff["SD"], g_eff["SE"],  g_eff["h_eff"], g_eff["h_setup"],
                h_mod, h_fact))
    return g_eff

def main():
    h_mod = 7.08
    h_fact = 80.60
    wzz = 0.00

    # Process project at current directory
    project = getcwd()
    print project

    az_results = {}
    for az in ["north", "south"]:
        az_file = path.join(project, az, az + ".fg5")
        if path.isfile(az_file):
            az_results[az] = az_proc(az_file, h_fact, h_mod, wzz)
        else:
            print("Project file for %s azimuth not found" % az)
            continue

    az_results["mean"] = {}
    for key in ["gravity", "h_setup", "h_eff"]:
        az_results["mean"][key] = np.mean([az_results["north"][key], az_results["south"][key]])

    for key in ["SD", "SE"]:
        az_results[key] = np.sqrt(az_results["north"][key]**2 + az_results["south"][key]**2) / 2.0

    az_diff = az_results["north"]["gravity"] - az_results["south"]["gravity"]

    nt = Time.now()
    nt.format = "iso"
    nt.out_subfmt = "date"

    x = PrettyTable()
    x.field_names = ["Azimuth", "Start DateTime", "Sets", "Drops",
                         "g_eff", "SD", "SE", "h_setup"]
    for az in ["north", "south"]:
        x.add_row([az, az_results[az]["iso"], str(az_results[az]["sets"]),
                    az_results[az]["drops"],
                    "{:.2f}".format(az_results[az]["gravity"]),
                    "{:.2f}".format(az_results[az]["SD"]),
                    "{:.2f}".format(az_results[az]["SE"]),
                    "{:.2f}".format(az_results[az]["h_setup"])])

    x.add_row(["result", "", "", "",
                "{:.2f}".format(az_results["mean"]["gravity"]),
                "{:.2f}".format(az_results["SD"]),
                "{:.2f}".format(az_results["SE"]),
                "{:.2f}".format(az_results["mean"]["h_setup"])])

    with open(path.join(path.split(project)[0], "report.txt"), "w") as f:
        f.write("Recalculation Report\n\n")

        f.write(('Site Name:  {0}\nSite Code:  {1}\nCalculation Date:  {2}		\nGravity_eff:  {3:.2f} uGal		\nStandard Deviation:  {4:.2f} uGal		\nStandard Error:  {5:.2f} uGal		\nVertical Gradient:  {6:.2f} uGal/cm		\nEffective Height:  {7:.2f} cm		\nSetup Height:  {8:.2f} cm		\nHeight_mod:  {9:.2f} cm		\nFactory Height:  {10:.2f} cm\n\n').format(az_results["north"]["name"],
                    az_results["north"]["code"], nt.value, az_results["mean"]["gravity"],
                    az_results["SD"], az_results["SE"], wzz, az_results["mean"]["h_eff"],
                    az_results["mean"]["h_setup"], h_mod, h_fact))

        f.write(x.get_string() + "\n\n")

        f.write("Azimuth Difference (N-S):  %.2f uGal" % az_diff)

    for t_file in ["etgtab.out", "etgtab.prn", "ETCPOT.UFT"]:
        remove(t_file)

if __name__ == "__main__":
    main()

max_stars_repo_name: aipolukhin/gtools

max_issues_repo_name: aipolukhin/gtools

max_forks_repo_name: aipolukhin/gtools

reason: import numpy,from astropy


text:
"""Analysis visualization functions
"""

import numpy as np
from itertools import chain

from .._utils import string_types

def format_pval(pval, latex=True, scheme='default'):
    """Format a p-value using one of several schemes.

    Parameters
    ----------
    pval : float | array-like
        The raw p-value(s).
    latex : bool
        Whether to use LaTeX wrappers suitable for use with matplotlib.
    scheme : str
        A keyword indicating the formatting scheme. Currently supports "stars",
        "ross", and "default"; any other string will yield the same as
        "default".

    Returns
    -------
    pv : str | np.objectarray
        A string or array of strings of formatted p-values. If a list output is
        preferred, users may call ``.tolist()`` on the output of the function.
    """
    single_value = False
    if np.array(pval).shape == ():
        single_value = True
    pval = np.atleast_1d(np.asanyarray(pval))
    # add a tiny amount to handle cases where p is exactly a power of ten
    pval = pval + np.finfo(pval.dtype).eps
    expon = np.trunc(np.log10(pval)).astype(int)  # exponents
    pv = np.zeros_like(pval, dtype=object)
    if latex:
        wrap = '$'
        brk_l = '{{'
        brk_r = '}}'
    else:
        wrap = ''
        brk_l = ''
        brk_r = ''
    if scheme == 'ross':  # (exact value up to 4 decimal places)
        pv[pval >= 0.0001] = [wrap + 'p = {:.4f}'.format(x) + wrap
                              for x in pval[pval > 0.0001]]
        pv[pval < 0.0001] = [wrap + 'p < 10^' + brk_l + '{}'.format(x) +
                             brk_r + wrap for x in expon[pval < 0.0001]]
    elif scheme == 'stars':
        star = '{*}' if latex else '*'
        pv[pval >= 0.05] = wrap + '' + wrap
        pv[pval < 0.05] = wrap + star + wrap
        pv[pval < 0.01] = wrap + star * 2 + wrap
        pv[pval < 0.001] = wrap + star * 3 + wrap
    else:  # scheme == 'default'
        pv[pval >= 0.05] = wrap + 'n.s.' + wrap
        pv[pval < 0.05] = wrap + 'p < 0.05' + wrap
        pv[pval < 0.01] = wrap + 'p < 0.01' + wrap
        pv[pval < 0.001] = wrap + 'p < 0.001' + wrap
        pv[pval < 0.0001] = [wrap + 'p < 10^' + brk_l + '{}'.format(x) +
                             brk_r + wrap for x in expon[pval < 0.0001]]
    if single_value:
        pv = pv[0]
    return(pv)

def _instantiate(obj, typ):
    """Returns obj if obj is not None, else returns new instance of typ
    obj : an object
        An object (most likely one that a user passed into a function) that,
        if ``None``, should be initiated as an empty object of some other type.
    typ : an object type
        Expected values are list, dict, int, bool, etc.
    """
    return typ() if obj is None else obj

def barplot(h, axis=-1, ylim=None, err_bars=None, lines=False,
            groups=None, eq_group_widths=False, gap_size=0.2,
            brackets=None, bracket_text=None, bracket_inline=False,
            bracket_group_lines=False, bar_names=None, group_names=None,
            bar_kwargs=None, err_kwargs=None, line_kwargs=None,
            bracket_kwargs=None, pval_kwargs=None, figure_kwargs=None,
            smart_defaults=True, fname=None, ax=None):
    """Makes barplots w/ optional line overlays, grouping, & signif. brackets.

    Parameters
    ----------
    h : array-like
        If `h` is 2-dimensional, heights will be calculated as means along
        the axis given by `axis`. If `h` is of lower dimension, it is
        treated as raw height values. If `h` is a `pandas.DataFrame` and
        `bar_names` is ``None``, `bar_names` will be inferred from the
        DataFrame's `column` labels (if ``axis=0``) or `index` labels.
    axis : int
        The axis along which to calculate mean values to determine bar heights.
        Ignored if `h` is 0- or 1-dimensional.
    ylim : tuple | None
        y-axis limits passed to `matplotlib.pyplot.subplot.set_ylim`.
    err_bars : str | array-like | None
        Type of error bars to be added to the barplot. Possible values are
        ``'sd'`` for sample standard deviation, ``'se'`` for standard error of
        the mean, or ``'ci'`` for 95% confidence interval. If ``None``, no
        error bars will be plotted. Custom error bar heights are possible by
        passing an array-like object; in such cases `err_bars` must have the
        same dimensionality and shape as `h`.
    lines : bool
        Whether to plot within-subject data as lines overlaid on the barplot.
    groups : list | None
        List of lists containing the integers in ``range(num_bars)``, with
        sub-lists indicating the desired grouping. For example, if `h` has
        has shape (10, 4) and ``axis = -1`` then "num_bars" is 4; if you want
        the first bar isolated and the remaining three grouped, then specify
        ``groups=[[0], [1, 2, 3]]``.
    eq_group_widths : bool
        Should all groups have the same width? If ``False``, all bars will have
        the same width. Ignored if `groups` is ``None``, since the bar/group
        distinction is meaningless in that case.
    gap_size : float
        Width of the gap between groups (if `eq_group_width` is ``True``) or
        between bars, expressed as a proportion [0,1) of group or bar width.
        Half the width of `gap_size` will be added between the outermost bars
        and the plot edges.
    brackets : list of tuples | None
        Location of significance brackets. Scheme is similar to the
        specification of `groups`; a bracket between the first and second bar
        and another between the third and fourth bars would be specified as
        ``brackets=[(0, 1), (2, 3)]``. Brackets between groups of bars instead
        of individual bars are specified as lists within the tuple:
        ``brackets=[([0, 1], [2, 3])]`` draws a single bracket between group
        ``[0, 1]`` and group ``[2, 3]``. For best results, pairs of adjacent
        bars should come earlier in the list than non-adjacent pairs.
    bracket_text : str | list | None
        Text to display above brackets.
    bracket_inline : bool
        If ``True``, bracket text will be vertically centered along a broken
        bracket line. If ``False``, text will be above the line.
    bracket_group_lines : bool
        When drawing brackets between groups rather than single bars, should a
        horizontal line be drawn at each foot of the bracket to indicate this?
    bar_names : array-like | None
        Optional axis labels for each bar.
    group_names : array-like | None
        Optional axis labels for each group.
    bar_kwargs : dict
        Arguments passed to ``matplotlib.pyplot.bar()`` (ex: color, linewidth).
    err_kwargs : dict
        Arguments passed to ``matplotlib.pyplot.bar(error_kw)`` (ex: ecolor,
        capsize).
    line_kwargs : dict
        Arguments passed to ``matplotlib.pyplot.plot()`` (e.g., color, marker,
        linestyle).
    bracket_kwargs : dict
        arguments passed to ``matplotlib.pyplot.plot()`` (e.g., color, marker,
        linestyle).
    pval_kwargs : dict
        Arguments passed to ``matplotlib.pyplot.annotate()`` when drawing
        bracket labels.
    figure_kwargs : dict
        arguments passed to ``matplotlib.pyplot.figure()`` (e.g., figsize, dpi,
        frameon).
    smart_defaults : bool
        Whether to use pyplot default colors (``False``), or something more
        pleasing to the eye (``True``).
    fname : str | None
        Path and name of output file. File type is inferred from the file
        extension of `fname` and should work for any of the types supported by
        pyplot (pdf, eps, svg, png, raw).
    ax : matplotlib.pyplot.axes | None
        A ``matplotlib.pyplot.axes`` instance.  If ``None``, a new figure with
        a single subplot will be created.

    Returns
    -------
    p : handle for the ``matplotlib.pyplot.subplot`` instance.
    b : handle for the ``matplotlib.pyplot.bar`` instance.

    Notes
    -----
    Known limitations:
      1 Bracket heights don't get properly set when generating multiple
        subplots with ``sharey=True`` (matplotlib seems to temporarily force
        the ``ylim`` to +/- 0.6 in this case). Work around is to use
        ``sharey=False`` and manually set ``ylim`` for each subplot.
      2 Brackets that span groups cannot span partial groups. For example,
        if ``groups=[[0, 1, 2], [3, 4]]`` it is impossible to have a bracket
        at ``[(0, 1), (3, 4)]``...  it is only possible to do, e.g.,
        ``[0, (3, 4)]`` (single bar vs group) or  ``[(0, 1, 2), (3, 4)]``
        (full group vs full group).
      3 Bracket drawing is much better when adjacent pairs of bars are
        specified before non-adjacent pairs of bars.
    Smart defaults sets the following parameters:
        bar color: light gray (70%)
        error bar color: black
        line color: black
        bracket color: dark gray (30%)

    """
    from matplotlib import pyplot as plt, rcParams
    try:
        from pandas.core.frame import DataFrame
    except Exception:
        DataFrame = None

    # be nice to pandas
    if DataFrame is not None:
        if isinstance(h, DataFrame) and bar_names is None:
            bar_names = h.columns.tolist() if axis == 0 else h.index.tolist()
    # check arg errors
    if gap_size < 0 or gap_size >= 1:
        raise ValueError('Barplot argument "gap_size" must be in the range '
                         '[0, 1).')
    if err_bars is not None:
        if isinstance(err_bars, string_types) and \
                err_bars not in ['sd', 'se', 'ci']:
            raise ValueError('err_bars must be "sd", "se", or "ci" (or an '
                             'array of error bar magnitudes).')
    if brackets is not None:
        if any([len(x) != 2 for x in brackets]):
            raise ValueError('Each top-level element of brackets must have '
                             'length 2.')
        if not len(brackets) == len(bracket_text):
            raise ValueError('Mismatch between number of brackets and bracket '
                             'labels.')
    # handle single-element args
    if isinstance(bracket_text, string_types):
        bracket_text = [bracket_text]
    if isinstance(group_names, string_types):
        group_names = [group_names]
    # arg defaults: if arg is None, instantiate as given type
    brackets = _instantiate(brackets, list)
    bar_kwargs = _instantiate(bar_kwargs, dict)
    err_kwargs = _instantiate(err_kwargs, dict)
    line_kwargs = _instantiate(line_kwargs, dict)
    pval_kwargs = _instantiate(pval_kwargs, dict)
    figure_kwargs = _instantiate(figure_kwargs, dict)
    bracket_kwargs = _instantiate(bracket_kwargs, dict)
    # user-supplied Axes
    if ax is not None:
        bar_kwargs['axes'] = ax
    # smart defaults
    if smart_defaults:
        if 'color' not in bar_kwargs.keys():
            bar_kwargs['color'] = '0.7'
        if 'color' not in line_kwargs.keys():
            line_kwargs['color'] = 'k'
        if 'ecolor' not in err_kwargs.keys():
            err_kwargs['ecolor'] = 'k'
        if 'color' not in bracket_kwargs.keys():
            bracket_kwargs['color'] = '0.3'
    # fix bar alignment (defaults to 'center' in more recent versions of MPL)
    if 'align' not in bar_kwargs.keys():
        bar_kwargs['align'] = 'edge'
    # parse heights
    h = np.array(h)
    if len(h.shape) > 2:
        raise ValueError('Barplot "h" must have 2 or fewer dimensions.')
    heights = np.atleast_1d(h) if h.ndim < 2 else h.mean(axis=axis)
    # grouping
    num_bars = len(heights)
    if groups is None:
        groups = [[x] for x in range(num_bars)]
    groups = [list(x) for x in groups]  # forgive list/tuple mix-ups
    # calculate bar positions
    non_gap = 1 - gap_size
    offset = gap_size / 2.
    if eq_group_widths:
        group_sizes = np.array([float(len(_grp)) for _grp in groups], int)
        group_widths = [non_gap for _ in groups]
        group_edges = [offset + _ix for _ix in range(len(groups))]
        group_ixs = list(chain.from_iterable([range(x) for x in group_sizes]))
        bar_widths = np.repeat(np.array(group_widths) / group_sizes,
                               group_sizes).tolist()
        bar_edges = (np.repeat(group_edges, group_sizes) +
                     bar_widths * np.array(group_ixs)).tolist()
    else:
        bar_widths = [[non_gap for _ in _grp] for _grp in groups]
        # next line: offset + cumul. gap widths + cumul. bar widths
        bar_edges = [[offset + _ix * gap_size + _bar * non_gap
                      for _bar in _grp] for _ix, _grp in enumerate(groups)]
        group_widths = [np.sum(_width) for _width in bar_widths]
        group_edges = [_edge[0] for _edge in bar_edges]
        bar_edges = list(chain.from_iterable(bar_edges))
        bar_widths = list(chain.from_iterable(bar_widths))
    bar_centers = np.array(bar_edges) + np.array(bar_widths) / 2.
    group_centers = np.array(group_edges) + np.array(group_widths) / 2.
    # calculate error bars
    err = np.zeros(num_bars)  # default if no err_bars
    if err_bars is not None:
        if h.ndim == 2:
            if err_bars == 'sd':  # sample standard deviation
                err = h.std(axis)
            elif err_bars == 'se':  # standard error
                err = h.std(axis) / np.sqrt(h.shape[axis])
            else:  # 95% conf int
                err = 1.96 * h.std(axis) / np.sqrt(h.shape[axis])
        else:  # h.ndim == 1
            if isinstance(err_bars, string_types):
                raise ValueError('string arguments to "err_bars" ignored when '
                                 '"h" has fewer than 2 dimensions.')
            elif not h.shape == np.array(err_bars).shape:
                raise ValueError('When "err_bars" is array-like it must have '
                                 'the same shape as "h".')
            err = np.atleast_1d(err_bars)
        bar_kwargs['yerr'] = err
    # plot (bars and error bars)
    if ax is None:
        plt.figure(**figure_kwargs)
        p = plt.subplot(111)
    else:
        p = ax
    b = p.bar(bar_edges, heights, bar_widths, error_kw=err_kwargs,
              **bar_kwargs)
    # plot within-subject lines
    if lines:
        _h = h if axis == 0 else h.T
        xy = [(bar_centers, hts) for hts in _h]
        for subj in xy:
            p.plot(subj[0], subj[1], **line_kwargs)
    # draw significance brackets
    if len(brackets):
        brackets = [tuple(x) for x in brackets]  # forgive list/tuple mix-ups
        brk_offset = np.diff(p.get_ylim()) * 0.025
        brk_min_h = np.diff(p.get_ylim()) * 0.05
        # temporarily plot a textbox to get its height
        t = plt.annotate(bracket_text[0], (0, 0), **pval_kwargs)
        t.set_bbox(dict(boxstyle='round, pad=0.25'))
        plt.draw()
        bb = t.get_bbox_patch().get_window_extent()
        txth = np.diff(p.transData.inverted().transform(bb),
                       axis=0).ravel()[-1]
        if bracket_inline:
            txth = txth / 2.
        t.remove()
        # find highest points
        if lines and h.ndim == 2:  # brackets must be above lines & error bars
            apex = np.amax(np.r_[np.atleast_2d(heights + err),
                                 np.atleast_2d(np.amax(h, axis))], axis=0)
        else:
            apex = np.atleast_1d(heights + err)
        apex = np.maximum(apex, 0)  # for negative-going bars
        apex = apex + brk_offset
        gr_apex = np.array([np.amax(apex[_g]) for _g in groups])
        # boolean for whether each half of a bracket is a group
        is_group = [[hasattr(_b, 'append') for _b in _br] for _br in brackets]
        # bracket left & right coords
        brk_lr = [[group_centers[groups.index(_ix)] if _g
                   else bar_centers[_ix] for _ix, _g in zip(_brk, _isg)]
                  for _brk, _isg in zip(brackets, is_group)]
        # bracket L/R midpoints (label position)
        brk_c = [np.mean(_lr) for _lr in brk_lr]
        # bracket bottom coords (first pass)
        brk_b = [[gr_apex[groups.index(_ix)] if _g else apex[_ix]
                  for _ix, _g in zip(_brk, _isg)]
                 for _brk, _isg in zip(brackets, is_group)]
        # main bracket positioning loop
        brk_t = []
        for _ix, (_brk, _isg) in enumerate(zip(brackets, is_group)):
            # which bars does this bracket span?
            spanned_bars = list(chain.from_iterable(
                [_b if hasattr(_b, 'append') else [_b] for _b in _brk]))
            spanned_bars = range(min(spanned_bars), max(spanned_bars) + 1)
            # raise apex a bit extra if prev bracket label centered on bar
            prev_label_pos = brk_c[_ix - 1] if _ix else -1
            label_bar_ix = np.where(np.isclose(bar_centers, prev_label_pos))[0]
            if any(np.array_equal(label_bar_ix, x) for x in _brk):
                apex[label_bar_ix] += txth
            elif any(_isg):
                label_bar_less = np.where(bar_centers < prev_label_pos)[0]
                label_bar_more = np.where(bar_centers > prev_label_pos)[0]
                if len(label_bar_less) and len(label_bar_more):
                    apex[label_bar_less] += txth
                    apex[label_bar_more] += txth
            gr_apex = np.array([np.amax(apex[_g]) for _g in groups])
            # recalc lower tips of bracket: apex / gr_apex may have changed
            brk_b[_ix] = [gr_apex[groups.index(_b)] if _g else apex[_b]
                          for _b, _g in zip(_brk, _isg)]
            # calculate top span position
            _min_t = max(apex[spanned_bars]) + brk_min_h
            brk_t.append(_min_t)
            # raise apex on spanned bars to account for bracket
            apex[spanned_bars] = np.maximum(apex[spanned_bars],
                                            _min_t) + brk_offset
            gr_apex = np.array([np.amax(apex[_g]) for _g in groups])
        # draw horz line spanning groups if desired
        if bracket_group_lines:
            for _brk, _isg, _blr in zip(brackets, is_group, brk_b):
                for _bk, _g, _b in zip(_brk, _isg, _blr):
                    if _g:
                        _lr = [bar_centers[_ix]
                               for _ix in groups[groups.index(_bk)]]
                        _lr = (min(_lr), max(_lr))
                        p.plot(_lr, (_b, _b), **bracket_kwargs)
        # draw (left, right, bottom-left, bottom-right, top, center, string)
        for ((_l, _r), (_bl, _br), _t, _c, _s) in zip(brk_lr, brk_b, brk_t,
                                                      brk_c, bracket_text):
            # bracket text
            _t = float(_t)  # on newer Pandas it can be shape (1,)
            defaults = dict(ha='center', annotation_clip=False,
                            textcoords='offset points')
            for k, v in defaults.items():
                if k not in pval_kwargs.keys():
                    pval_kwargs[k] = v
            if 'va' not in pval_kwargs.keys():
                pval_kwargs['va'] = 'center' if bracket_inline else 'baseline'
            if 'xytext' not in pval_kwargs.keys():
                pval_kwargs['xytext'] = (0, 0) if bracket_inline else (0, 2)
            txt = p.annotate(_s, (_c, _t), **pval_kwargs)
            txt.set_bbox(dict(facecolor='w', alpha=0,
                              boxstyle='round, pad=0.2'))
            plt.draw()
            # bracket lines
            lline = ((_l, _l), (_bl, _t))
            rline = ((_r, _r), (_br, _t))
            tline = ((_l, _r), (_t, _t))
            if bracket_inline:
                bb = txt.get_bbox_patch().get_window_extent()
                txtw = np.diff(p.transData.inverted().transform(bb),
                               axis=0).ravel()[0]
                _m = _c - txtw / 2.
                _n = _c + txtw / 2.
                tline = [((_l, _m), (_t, _t)), ((_n, _r), (_t, _t))]
            else:
                tline = [((_l, _r), (_t, _t))]
            for x, y in [lline, rline] + tline:
                p.plot(x, y, **bracket_kwargs)
            # boost ymax if needed
            ybnd = p.get_ybound()
            if ybnd[-1] < _t + txth:
                p.set_ybound(ybnd[0], _t + txth)
    # annotation
    box_off(p)
    p.tick_params(axis='x', length=0, pad=12)
    p.xaxis.set_ticks(bar_centers)
    if bar_names is not None:
        p.xaxis.set_ticklabels(bar_names, va='baseline')
    if group_names is not None:
        ymin = ylim[0] if ylim is not None else p.get_ylim()[0]
        yoffset = -2.5 * rcParams['font.size']
        for gn, gp in zip(group_names, group_centers):
            p.annotate(gn, xy=(gp, ymin), xytext=(0, yoffset),
                       xycoords='data', textcoords='offset points',
                       ha='center', va='baseline')
    # axis limits
    p.set_xlim(0, bar_edges[-1] + bar_widths[-1] + gap_size / 2)
    if ylim is not None:
        p.set_ylim(ylim)
    # output file
    if fname is not None:
        from os.path import splitext
        fmt = splitext(fname)[-1][1:]
        plt.savefig(fname, format=fmt, transparent=True)
    # return handles for subplot and barplot instances
    plt.draw()
    return (p, b)

def box_off(ax):
    """Remove the top and right edges of a plot frame, and point ticks outward.

    Parameters
    ----------
    ax : matplotlib.axes.Axes
        A matplotlib plot or subplot object.
    """
    ax.get_xaxis().tick_bottom()
    ax.get_yaxis().tick_left()
    ax.tick_params(axis='x', direction='out')
    ax.tick_params(axis='y', direction='out')
    ax.spines['right'].set_color('none')
    ax.spines['top'].set_color('none')

def plot_screen(screen, ax=None):
    """Plot a captured screenshot

    Parameters
    ----------
    screen : array
        The N x M x 3 (or 4) array of screen pixel values.
    ax : matplotlib Axes | None
        If provided, the axes will be plotted to and cleared of ticks.
        If None, a figure will be created.

    Returns
    -------
    ax : matplotlib Axes
        The axes used to plot the image.
    """
    import matplotlib.pyplot as plt
    screen = np.array(screen)
    if screen.ndim != 3 or screen.shape[2] not in [3, 4]:
        raise ValueError('screen must be a 3D array with 3 or 4 channels')
    if ax is None:
        plt.figure()
        ax = plt.axes([0, 0, 1, 1])
    ax.imshow(screen)
    ax.axis('off')
    return ax

max_stars_repo_name: drammock/expyfun

max_issues_repo_name: drammock/expyfun

max_forks_repo_name: drammock/expyfun


text:
import numpy as np
import random

from model import Actor, Critic
from noise import OUNoise
from buffer import ReplayBuffer

import torch
import torch.nn.functional as F
import torch.optim as optim

device = 'cpu'

class MultiAgent:
    def __init__(self, config):

        self.random_seeds = config['random_seeds']
        self.params = config['params']
        self.memory = ReplayBuffer(self.params['action_size'],
                                   self.params['buffer_size'],
                                   self.params['batch_size'],
                                   device,
                                   self.random_seeds[0])
        self.params['memory'] = self.memory

        self.ddpg_agents = [Agent(self.params, self.random_seeds[i]) for i in range(2)]

        self.t_step = 0

    def act(self, states):
        actions = [agent.act(np.expand_dims(state, axis=0)) for agent, state in zip(self.ddpg_agents, states)]
        #actions = [agent.act(states) for agent in self.ddpg_agents]
        return actions

    def step(self, states, actions, rewards, next_states, dones):
        self.t_step += 1

        for state, action, reward, next_state, done in zip(states, actions, rewards, next_states, dones):
            self.memory.add(state, action, reward, next_state, done)

        if (len(self.memory) > self.params['batch_size']) and (self.t_step % self.params['num_steps_per_update'] == 0):
            for agent in self.ddpg_agents:
                experiences = self.memory.sample()
                agent.learn(experiences, self.params['gamma'])

    def reset(self):
        for agent in self.ddpg_agents:
            agent.reset()

class Agent():
    """Interacts with and learns from the environment."""

    def __init__(self, params, random_seed):
        """Initialize an Agent object.

        Params
        ======
            state_size (int): dimension of each state
            action_size (int): dimension of each action
            random_seed (int): random seed
        """
        self.n_agents = 1
        self.state_size = params['state_size']
        self.action_size = params['action_size']
        self.batch_size = params['batch_size']
        self.gamma = params['gamma']
        self.tau = params['tau']
        self.seed = random.seed(random_seed)

        # Actor Network (w/ Target Network)
        self.actor_local = Actor(self.state_size, self.action_size, random_seed).to(device)
        self.actor_target = Actor(self.state_size, self.action_size, random_seed).to(device)
        self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=params['lr_actor'])

        # Critic Network (w/ Target Network)
        self.critic_local = Critic(self.state_size, self.action_size, random_seed).to(device)
        self.critic_target = Critic(self.state_size, self.action_size, random_seed).to(device)
        self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=params['lr_critic'],
                                           weight_decay=params['weight_decay'])

        # Noise process
        self.noise = OUNoise(self.action_size, random_seed)

    def act(self, state, add_noise=True):
        """Returns actions for given state as per current policy."""
        state = torch.from_numpy(state).float().to(device)
        self.actor_local.eval()
        with torch.no_grad():
            action = self.actor_local(state).cpu().data.numpy()
        self.actor_local.train()
        if add_noise:
            action += self.noise.sample()
        return np.clip(action, -1, 1)

    def reset(self):
        self.noise.reset()

    def learn(self, experiences, gamma):
        """Update policy and value parameters using given batch of experience tuples.
        Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
        where:
            actor_target(state) -> action
            critic_target(state, action) -> Q-value

        Params
        ======
            experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
            gamma (float): discount factor
        """
        states, actions, rewards, next_states, dones = experiences

        # ---------------------------- update critic ---------------------------- #
        # Get predicted next-state actions and Q values from target models
        actions_next = self.actor_target(next_states)
        Q_targets_next = self.critic_target(next_states, actions_next)
        # Compute Q targets for current states (y_i)
        Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
        # Compute critic loss
        Q_expected = self.critic_local(states, actions)
        critic_loss = F.mse_loss(Q_expected, Q_targets)
        # Minimize the loss
        self.critic_optimizer.zero_grad()
        critic_loss.backward()
        self.critic_optimizer.step()

        # ---------------------------- update actor ---------------------------- #
        # Compute actor loss
        actions_pred = self.actor_local(states)
        actor_loss = -self.critic_local(states, actions_pred).mean()
        # Minimize the loss
        self.actor_optimizer.zero_grad()
        actor_loss.backward()
        self.actor_optimizer.step()

        # ----------------------- update target networks ----------------------- #
        self.soft_update(self.critic_local, self.critic_target, self.tau)
        self.soft_update(self.actor_local, self.actor_target, self.tau)

    def soft_update(self, local_model, target_model, tau):
        for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
            target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)

max_stars_repo_name: joyce-fang/deep-reinforcement-learning

max_issues_repo_name: joyce-fang/deep-reinforcement-learning

max_forks_repo_name: joyce-fang/deep-reinforcement-learning


text:
import StringIO
import os
import sys
import unittest

import numpy as np
import tigre
import tigre.algorithms as algs
from tigre.demos.Test_data import data_loader

dirname = os.path.dirname(__file__)

class TestStdout(unittest.TestCase):
    pass

def test_generator(algorithm, proj, geo, angles, niter):
    def test(self):
        capturedOutput = StringIO.StringIO()
        sys.stdout = capturedOutput
        getattr(algs, algorithm)(proj, geo, angles, niter=niter, verbose=False)
        self.assertIs(capturedOutput.getvalue(), "")
        sys.stdout = sys.__stdout__

    return test

if __name__ == "__main__":
    geo = tigre.geometry(mode="cone", default=True, high_quality=False)
    print(geo)

    true_img = data_loader.load_head_phantom(geo.nVoxel)
    angles = np.linspace(0, 2 * np.pi, 100)
    niter = 5
    proj = tigre.Ax(true_img, geo, angles)
    for alg in algs.__all__:
        if alg != "fbp":
            test_name = "test_print_%s" % (alg)
            test = test_generator(alg, proj, geo, angles, niter)
            setattr(TestStdout, test_name, test)

    unittest.main()

max_stars_repo_name: tsadakane/TIGRE

max_issues_repo_name: tsadakane/TIGRE

max_forks_repo_name: tsadakane/TIGRE


text:
# Importing Libraries

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os
import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator, load_img
from tensorflow.keras.layers import (Conv2D, BatchNormalization, Activation, MaxPooling2D, GlobalAveragePooling2D,
                          Dense, Flatten, Dropout)
from tensorflow.keras.optimizers import RMSprop, Adam, SGD
from tensorflow.keras import regularizers
from tensorflow.keras.callbacks import CSVLogger, ModelCheckpoint, ReduceLROnPlateau

# Importing Dataset and Plotting Images

path = '../input/140k-real-and-fake-faces/real_vs_fake/real-vs-fake'

def plot_img(path, set_):
    dir_ = os.path.join(path, 'train', set_)
    k = 0
    fig, ax = plt.subplots(3,3, figsize=(10,10))
    fig.suptitle(set_ + 'Faces')
    for j in range(3):
        for i in range(3):
            img = load_img(os.path.join(dir_, os.listdir(os.path.join(dir_))[k]))
            ax[j,i].imshow(img)
            ax[j,i].set_title("")
            ax[j,i].axis('off')
            k +=1
#  fig.tight_layout()
    plt.suptitle(set_ + ' Faces')
    return plt

plot_img(path, 'real').show()

### Fake Set

plot_img(path, 'fake').show()

# Train and Validation Set

bs = 64
row, col = 224, 224
train_datagen = ImageDataGenerator(rescale=1./255,
                                   horizontal_flip=True
                                  )
training_set = train_datagen.flow_from_directory(path + '/train',
                                                 class_mode='binary',
                                                 shuffle=True,
                                                 target_size=(row,col),
                                                 batch_size=bs
                                                )
val_test_datagen = ImageDataGenerator(rescale=1./255)

validation_set = val_test_datagen.flow_from_directory(path + '/valid',
                                                      class_mode='binary',
                                                      shuffle=True,
                                                      target_size=(row,col),
                                                      batch_size=bs
                                                     )
test_set = val_test_datagen.flow_from_directory(path + '/test',
                                                class_mode='binary',
                                                shuffle=True,
                                                target_size=(row,col),
                                                batch_size=bs
                                               )
training_set.class_indices

# Creating Model
from tensorflow.keras.applications import DenseNet121
def get_model():
    densenet = DenseNet121(weights='imagenet',
                           include_top=False,
                           input_shape=(224,224,3)
                          )
    model = tf.keras.models.Sequential([densenet,
                                        GlobalAveragePooling2D(),
                                        Dense(512, activation='relu'),
                                        BatchNormalization(),
                                        Dropout(0.3),
                                        Dense(1, activation='sigmoid')
                                      ])
    model.compile(optimizer=Adam(lr=0.001),
                loss='binary_crossentropy',
                metrics=['accuracy']
                )

    return model

model = get_model()
model.summary()

### Defining Model Checkpoints

checkpoint = ModelCheckpoint(filepath='spoffnet.h5',
                             save_best_only=True,
                             verbose=1,
                             mode='min',
                             moniter='val_loss'
                            )
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
                              factor=0.2,
                              patience=3,
                              verbose=1,
                              min_delta=0.0001
                             )
csv_logger = CSVLogger('training.log')

callbacks = [checkpoint, reduce_lr, csv_logger]

# Training Model

steps_per_epoch = training_set.n // training_set.batch_size
validation_steps = validation_set.n // validation_set.batch_size

hist = model.fit(training_set,
                    validation_data=validation_set,
                    callbacks=callbacks,
                    steps_per_epoch=steps_per_epoch,
                    validation_steps=validation_steps,
                    epochs=10
                   )

model.save("model10.h5")

# Accuracy On test set

test_loss, test_acc = model.evaluate(test_set)
print("validation accuracy :", str(test_acc*100)+"%")
print("validation loss :", test_loss)

# Confusion Matrix and Classification Report

y_pred = model.predict(test_set)
y_pred = (y_pred < 0.5).astype(np.int)

from sklearn.metrics import confusion_matrix, classification_report
cm = confusion_matrix(test_set.classes, y_pred)
print('Confusion Matrix')
print(cm)

names = ['fake', 'real']
print('Classification Report')
print(classification_report(test_set.classes, y_pred, target_names=names))


text:
import os
import sys
import numpy as np

###### print to log files #######
logfile_name = ''.join([os.environ['logfile_name'],'.log'])
flog=open(logfile_name, "a+")

fIn1=sys.argv[1]

fIn2=sys.argv[1]

ICstats = np.loadtxt(fIn1)
#print(ICstats)
#print(ICstats.shape)

motionICs = np.loadtxt(fIn2, delimiter=",",dtype=np.int32)
#print(motionICs)

peVar = np.zeros(len(motionICs))
ptVar = np.zeros(len(motionICs))

for i in range(0,len(motionICs)):
    ind = motionICs[i]
    peVar[i] = ICstats[ind-1,0]
    ptVar[i] = ICstats[ind-1,1]

peVar = np.sum(peVar)
ptVar = np.sum(ptVar)

print("%.2f percent of explained variance in removed motion components" % peVar)
flog.write("\n "+ str(peVar)+ " percent of explained variance in removed motion components")

print("%.2f percent of total variance in removed motion components" % ptVar)
flog.write("\n "+ str(ptVar)+ " percent of total variance in removed motion components")

flog.close()

max_stars_repo_name: IUSCA/IUSM-ConnPipe

max_issues_repo_name: IUSCA/IUSM-ConnPipe

max_forks_repo_name: IUSCA/IUSM-ConnPipe


text:
import numpy as np
import pandas as pd

import tensorflow as tf
from tensorflow import keras
import matplotlib.pyplot as plt

from StochasticProcesses import Stochastic_Processes
from Payoffs import Payoffs

from TD_net_v4 import TD_net, td_metrics

#%%

process_specs = {
        "processdicts" : [
                {"process" : "Bates", "s0" : 100, 'mu': 0.05, 'v0': 0.04, 'rho' : -0.2, 'theta': 0.04, 'xi': 0.04, 'kappa': 0.3, 'lamb': 0.75, 'k': 0.0625,
                 "randomize_spot" : [-20,20]
                 },
        ],
        "d_t" : 3 / 360,
        "N_steps" : 2,
        "batch_size" : 1000,
}

sample_size = 5000000

discretized_space = np.linspace(80,120,21)
discretized_time = range( 0, process_specs["N_steps"] + 1, 1 )

lower_bounds = discretized_space[:-1]
upper_bounds = discretized_space[1:]
bin_centers = ( lower_bounds + upper_bounds ) / 2

walks = Stochastic_Processes.generate_walks( **process_specs, N_walks = sample_size, verbose =False )
#walks_remove_dimension = walks[:,:,0]
#df_walks = pd.DataFrame(walks_remove_dimension)
#df_walks_sorted = df_walks.sort_values(by =10,ignore_index=True)
#np_sorted_walks = df_walks_sorted.to_numpy()
#walks = np_sorted_walks[..., np.newaxis]

#s = walks.shape
#timedim = np.tile( range( s[1] ), s[0] ).reshape(*s[0:-1],1)
#walks_with_time = np.concatenate( [walks,timedim], axis = -1 )

payoffs = np.maximum( walks[:,-1,0] - 100, 0 )

#%%

conditional_expectation = pd.DataFrame()

for t in discretized_time:

    walks_t = walks[:,t,0]

    for lb,ub,mid in zip(lower_bounds, upper_bounds, bin_centers):

        in_discrete_bin = ( lb <= walks_t ) * ( walks_t <= ub )

        subset_of_payoffs_with_condition = payoffs[in_discrete_bin]

        conditional_expectation.loc[t,mid] = np.mean( subset_of_payoffs_with_condition )

#%%

#output_shape = payoffs[0].shape
output_shape = ()

nodes_per_layer = 64
N_hidden_layers = 5
hidden_activation = keras.activations.relu
output_activation = keras.activations.linear

# TD MODEL
td_model = TD_net(

    #hidden layers
    nodes_per_layer = nodes_per_layer,
    N_hidden_layers = N_hidden_layers,
    hidden_activation = hidden_activation,

    #output
    output_shape = output_shape,
    output_activation = output_activation,
    row_wise_output = False
)

td_model.compile(
    optimizer = tf.keras.optimizers.SGD(),
    metrics = [td_metrics.Temporal_MAE, td_metrics.Prediction_MAE]
)

#%%

def input_creation(strike, n_steps, sample_size, td_input_count, central_weight ):

    walks = Stochastic_Processes.generate_walks( **process_specs, N_walks = sample_size, verbose =False )

    walks_remove_dimension = walks[:,:,0]
    df_walks = pd.DataFrame(walks_remove_dimension)

    df_walks_sorted = df_walks.sort_values(by =10,ignore_index=True)
    np_sorted_walks = df_walks_sorted.to_numpy()
    walks = np_sorted_walks[..., np.newaxis]

    s = walks.shape
    timedim = np.tile( range( s[1] ), s[0] ).reshape(*s[0:-1],1)
    walks_with_time = np.concatenate( [walks,timedim], axis = -1 )
    payoffs = np.maximum( walks[:,-1,0] - 100, 0 )

    central_td_inputs_count = int(central_weight * td_input_count)
    central_indexs = df_walks_sorted.index[abs(df_walks_sorted[n_steps] - strike) < 0.0001].tolist()
    central_index = central_indexs[int(len(central_indexs)/2)]
    lowest_central_index = central_index - int(central_td_inputs_count/2)
    highest_central_index = central_index + int(central_td_inputs_count/2)
    central_walks_td = walks_with_time[lowest_central_index : highest_central_index]
    central_payoffs_td = payoffs[lowest_central_index : highest_central_index]

    sample_size = len(walks)
    smallest = df_walks_sorted[n_steps][0]
    largest = df_walks_sorted[n_steps][sample_size-1]
    expected_range = largest - smallest

    walks_remaining = td_input_count - len(central_walks_td)

    numb_walks_left_sides = int(walks_remaining / 2)     # number of walks we can generate on the left / right
    numb_walks_right_sides = int(walks_remaining / 2)

    left_buckets = int(lowest_central_index / numb_walks_left_sides)
    right_buckets = int(((sample_size-1) - highest_central_index) / numb_walks_right_sides)

    indexs_left = [i for i in range(0,lowest_central_index,left_buckets)]
    indexs_right = [i for i in range(highest_central_index,sample_size-1,right_buckets)]

    side_indexs = indexs_left + indexs_right

    side_walks_td = (walks_with_time[side_indexs])
    side_payoffs_td = (payoffs[side_indexs])

    walks_td = np.vstack((central_walks_td, side_walks_td))
    payoffs_td = np.hstack((central_payoffs_td, side_payoffs_td))

    return walks_td,payoffs_td

"""
walks_td,payoffs_td = input_creation(100, 10, 20000000, 1000, 0.333)
#walks_td,payoffs_td = input_creation(100, 10, walks, 1000, 0.333)

td_hist = td_model.fit( # lr = 0.00001, lamb = 0.275, epochs = 30, layers = 5, nodes = 64

    walks_td,
    payoffs_td,

    lr = 0.00001 ,
    batch_size=2,
    lamb = 0.275,
    validation_split=0,
    epochs = 30,
    verbose = True,
    )

#%%

tds = []
mcs = []
xs = [i for i in range(81,121,2)]

time = 0
for space in range(81,121,2):

    td = float(td_model.predict([[[space,time]]]))
    tds.append(td)

    mc = conditional_expectation[space][time]
    mcs.append(mc)

fig, ax = plt.subplots(1,1)
ax.set_xlabel("Spot Price")
ax.set_ylabel("Expected Payoff")
ax.title.set_text(f"Expected payoff of td and mc - time {time}")

ax.plot(xs,tds,label = f"td",linestyle='--', marker='o')
ax.plot(xs,mcs,label = f"mcs",linestyle='--', marker='o')
ax.legend()

fig1, ax1 = plt.subplots(1,1)

ax1.set_xlabel("Spot Price")
ax1.set_ylabel("RMSE")
ax1.title.set_text("RMSE against spot price - varying times")

xs = [i for i in range(81,121,2)]

for time in range(0,12,2):

    rmses = []

    for space in range(81,121,2):

        t_pred = float(td_model.predict([[space,time]]))

        montecarlo = conditional_expectation[space][time]

        rmse = np.sqrt((montecarlo-t_pred)**2)
        rmses.append(rmse)

    ax1.plot(xs,rmses,label = f"t_{time}",linestyle='--', marker='o')

ax1.legend()

errors = pd.DataFrame()

for time in range(0,12,2):

    for space in range(81,121,2):

        t_pred = float(td_model.predict([[space,time]]))

        montecarlo = conditional_expectation[space][time]

        rmse = np.sqrt((montecarlo-t_pred)**2)

        errors.loc[time,space] = rmse
"""

#%%

fig2, ax2 = plt.subplots(1,1)

ax2.set_xlabel("Number of Steps")
ax2.set_ylabel("RMSE")
ax2.title.set_text("RMSE against number of steps - varying times")

num_steps = [i for i in range(1,21)]

for time in range(0,12,2):

    rmses = []
    steps = []

    for step in num_steps:

        process_specs = {
        "processdicts" : [
                {"process" : "Bates", "s0" : 100, 'mu': 0.05, 'v0': 0.04, 'rho' : -0.2, 'theta': 0.04, 'xi': 0.04, 'kappa': 0.3, 'lamb': 0.75, 'k': 0.0625,
                 "randomize_spot" : [-20,20]
                 },
        ],
        "d_t" : 3 / 360,
        "N_steps" : step,
        "batch_size" : 1000,
}

        sample_size = 5000000

        discretized_space = np.linspace(80,120,21)
        discretized_time = range( 0, process_specs["N_steps"] + 1, 1 )

        lower_bounds = discretized_space[:-1]
        upper_bounds = discretized_space[1:]
        bin_centers = ( lower_bounds + upper_bounds ) / 2

        walks = Stochastic_Processes.generate_walks( **process_specs, N_walks = sample_size, verbose =False )

        payoffs = np.maximum( walks[:,-1,0] - 100, 0 )

        conditional_expectation = pd.DataFrame()

        for t in discretized_time:

            walks_t = walks[:,t,0]

            for lb,ub,mid in zip(lower_bounds, upper_bounds, bin_centers):

                in_discrete_bin = ( lb <= walks_t ) * ( walks_t <= ub )

                subset_of_payoffs_with_condition = payoffs[in_discrete_bin]

                conditional_expectation.loc[t,mid] = np.mean( subset_of_payoffs_with_condition )

        steps.append(step)

        walks_td,payoffs_td = input_creation(100, 10, 20000000, 500, 0.333)

        td_hist = td_model.fit(

        walks_td,
        payoffs_td,

        lr = 0.00001 ,
        batch_size=2,
        lamb = 0.3,
        validation_split=0,
        epochs = 30,
        verbose = True,
        )

        errors = []

        for space in range(81,121,2):

            t_pred = float(td_model.predict([[space,time]]))
            montecarlo = conditional_expectation[space][time]
            error = np.sqrt((montecarlo-t_pred)**2)
            errors.append(error)

        rmse = sum(errors)/len(errors)
        rmses.append(rmse)

    ax2.plot(steps,rmses,label = f"time = {time}",linestyle='--', marker='o')

ax2.legend()

max_stars_repo_name: ArthurBook/TDLearning

max_issues_repo_name: ArthurBook/TDLearning

max_forks_repo_name: ArthurBook/TDLearning


text:
import os

import numpy
from sklearn.model_selection import train_test_split

from feature_extraction import NumericFeatureGetter
from rossmann_data import read_or_process_data
import numpy as np
from sklearn.neural_network import MLPRegressor

from sklearn.linear_model import ElasticNet, LinearRegression
from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from time import time

def rmspe(ys, yhats):
    return np.sqrt(np.mean([(yhat / y - 1) ** 2 for yhat, y in zip(yhats, ys)]))

def rmspe_xg(yhat, y):
    y = np.expm1(y.get_label())
    yhat = np.expm1(yhat)
    return rmspe(y, yhat)

def score_trained_regressor(
    regressor, train_data_dicts, y_train_list, test_data_dicts, y_test_list
):
    yhat = regressor.predict(test_data_dicts)
    yhat_train = regressor.predict(train_data_dicts)
    train_score = rmspe(np.expm1(y_train_list), np.expm1(yhat_train))
    test_score = rmspe(np.expm1(y_test_list), np.expm1(yhat))
    return train_score, test_score

if __name__ == "__main__":

    path = os.getcwd() + "/data"
    train_data_dicts, categorical_features, numerical_features = read_or_process_data(
        path
    )
    train_data_dicts, test_data_dicts = train_test_split(
        train_data_dicts, test_size=0.2, random_state=10
    )

    y_train_list = [d["target"] for d in train_data_dicts]
    y_test_list = [d["target"] for d in test_data_dicts]
    print("got %d train-samples" % len(train_data_dicts))
    print("got %d test-samples" % len(test_data_dicts))
    ###################################################################################################################
    #       just printing mix-max
    for feat_name in numerical_features:
        # print(pandas.Series(data=[d[feat_name] for d in train_data_dicts]).describe())
        features = [d[feat_name] for d in train_data_dicts]
        print(feat_name + "# min: %0.2f; max: %0.2f" % (min(features), max(features)))
    ###################################################################################################################
    #      defining Feature-Extractor objects

    num_getter = NumericFeatureGetter(numerical_features)
    preprocessed_numeric_features = Pipeline(
        [
            ("numeric_features", num_getter),
            ("preprocessed_numeric_feats", PCA(whiten=True))
            # ('preprocessed_numeric_feats', preprocessing.Normalizer())
        ]
    )
    # preprocessed_numeric_features.fit(train_data_dicts)
    for name, regressor in [
        ("lin_reg", LinearRegression()),
        ("elastic_net", ElasticNet(alpha=0.001, l1_ratio=0.7)),
        (
            "mlp",
            MLPRegressor(
                hidden_layer_sizes=(10,),
                batch_size=128,
                alpha=0.000001,
                early_stopping=True,
            ),
        ),
    ]:
        start = time()
        regression_pipeline = Pipeline(
            [
                ("feature_extractor", preprocessed_numeric_features),
                ("regressor", regressor),
            ]
        )
        target_array = numpy.array([d["target"] for d in train_data_dicts])
        regression_pipeline.fit(train_data_dicts, target_array)

        train_err, test_err = score_trained_regressor(
            regression_pipeline,
            train_data_dicts,
            y_train_list,
            test_data_dicts,
            y_test_list,
        )
        duration = time() - start
        assert isinstance(duration, float)
        print(
            name
            + "     train: %0.3f; test: %0.3f" % (train_err, test_err)
            + " took: %0.2f secs" % duration
        )

max_stars_repo_name: dertilo/regression

max_issues_repo_name: dertilo/regression

max_forks_repo_name: dertilo/regression


text:
import numpy as np
from sklearn.svm import OneClassSVM
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from azure.storage.blob import BlockBlobService
import pickle
import sys
import json
import datetime
import pandas as pd
import io

# query params
device = int(sys.argv[1])
tag = int(sys.argv[2])

# input/output params
config_file = sys.argv[3]
with open(config_file) as f:
    j = json.loads(f.read())

blob_account = j['blob_account']
blob_key = j['blob_key']
models_blob_container = j['models_blob_container']
data_blob_container = j['data_blob_container']
data_blob_name = j['data_blob']
predictions_blob_container = j['predictions_blob_container']

model_name = 'model_{0}_{1}'.format(device, tag)

blob_service = BlockBlobService(
    account_name=blob_account, account_key=blob_key)

# get data
data_blob = blob_service.get_blob_to_bytes(data_blob_container, data_blob_name)
data = pd.read_csv(io.BytesIO(data_blob.content))
data = data[(data['Device']==device) & (data['Tag']==tag)]
tss = data['TS']
vals = np.array(data['Value'])

# load model
model_blob = blob_service.get_blob_to_bytes(models_blob_container, model_name)
pipe = pickle.loads(model_blob.content)

# predict
preds = pipe.predict(vals.reshape(-1, 1))
preds = np.where(preds == 1, 0, 1) # 1 indicates an anomaly, 0 otherwise

# csv results
res = pd.DataFrame({'TS':tss,
                    'Device': np.repeat(device, len(preds)),
                    'Tag': np.repeat(tag, len(preds)),
                    'Val': vals,
                    'Prediction': preds})
res = res[['TS', 'Device', 'Tag', 'Val', 'Prediction']]

res_file_name = 'preds_{0}_{1}_{2}'.format(device,
                                               tag,
                                               datetime.datetime.now().strftime('%y%m%d%H%M%S')
                                               )

# save predictions
blob_service.create_blob_from_text(
    predictions_blob_container, res_file_name, res.to_csv(index=None))

max_stars_repo_name: Azure/BatchAIAnomalyDetection

max_issues_repo_name: Azure/BatchAIAnomalyDetection

max_forks_repo_name: Azure/BatchAIAnomalyDetection


text:
import matplotlib.pyplot as plt
import networkx as nx
from decision_graph import RDG, Node, Edge
from matplotlib.gridspec import GridSpec

def position_graphs_nodes(graph):
    '''
    determine the positioning of rdg nodes from graph structure.
    This is a hacky method that determines Y axis by node key rather than calculating an optimum based on overlaps
    '''

    node_x_positions = [(graph.nodes[node].key, graph.nodes[node].node_start) for node in graph.nodes]
    pos = {}
    for i in node_x_positions:
        pos[i[0]] = (i[1], i[0])

    highest_node = max(list(pos.keys()))
    pos[2] = (pos[2][0], pos[highest_node][1] + 1)

    return pos

def plot(graph, color_dict=None):
    G = nx.DiGraph()
    edges = graph.get_edges_from_to()

    pos = position_graphs_nodes(graph)

    endpoints = graph.get_endpoints()
    startpoints = graph.get_startpoints()
    translation_starts = graph.get_start_nodes()
    translation_stops = graph.get_stop_nodes()
    frameshifts = graph.get_frameshifts()

    G.add_edges_from(edges.keys())
    node_colors = []
    for node in G.nodes():
        if node in startpoints:
            node_colors.append((0, 0, 1))
        elif node in endpoints:
            node_colors.append((0.5,0,0.5))
        elif node in translation_starts:
            node_colors.append((0,1,0))
        elif node in translation_stops:
            node_colors.append((1,0,0))
        elif node in frameshifts:
            node_colors.append((1,0.5,0.3))
        else:
            node_colors.append((0,0,0))

    edge_colors = []
    for edge in G.edges:
        if graph.edges[edges[edge]].edge_type == "translated":
            frame = graph.edges[edges[edge]].frame
        else:
            frame = None

        if frame == 0:
            edge_colors.append((1,0,0))
        elif frame == 1:
            edge_colors.append((0,1,0))
        elif frame == 2:
            edge_colors.append((0,0,1))
        else:
            edge_colors.append((0,0,0))
    fig = plt.figure()
    fig.suptitle(f"Visualisation of an RDG for {graph.locus}")

    gs = GridSpec(2, 1, width_ratios=[1], height_ratios=[4, 1], hspace=0)
    ax1 = fig.add_subplot(gs[0])
    ax2 = fig.add_subplot(gs[1])

    nx.draw_networkx(G, pos=pos, ax=ax1, node_shape='o', node_size=10, node_color=node_colors, edge_color=edge_colors, with_labels=False)

    orfs_in_frame = {0:[], 1:[], 2:[]}

    orfs = graph.get_orfs()
    for orf in orfs:
        if len(orf) == 2:
            orfs_in_frame[orf[0]%3].append((orf[0], orf[1] - orf[0]))
        elif len(orf) == 3:
            orfs_in_frame[orf[0]%3].append((orf[0], orf[1] - orf[0]))
            orfs_in_frame[orf[1]%3].append((orf[1], orf[2] - orf[1]))

    height = 10
    yticks_heights = []
    yticks_labels = []
    for frame, color in zip(sorted(orfs_in_frame.keys()), [(1,0,0), (0,1,0),  (0,0,1)]):
        yticks_heights.append(height + 1)
        yticks_labels.append("Frame " + str(frame + 1))
        ax2.broken_barh(sorted(orfs_in_frame[frame]), (height, 2), facecolors=color, edgecolor='black',)
        height += 3

    ax2.tick_params(bottom=True, labelbottom=True)
    ax2.set_xlim(left=ax1.get_xlim()[0], right=ax1.get_xlim()[1])
    ax2.set_ylim(bottom=10, top=19)
    ax2.set_yticks(yticks_heights, labels=yticks_labels)

    plt.show()

if __name__ == "__main__":
    dg = RDG(name="Example Gene")
    dg.add_open_reading_frame(30, 90)
    dg.add_open_reading_frame(61, 400)
    dg.add_open_reading_frame(92, 150)

    # dg.add_open_reading_frame(550, 850)
    # dg.add_stop_codon_readthrough(850, 880)
    dg.add_frameshift(400, 450, 2)
    # print(dg.get_orfs())
    plot(dg)

max_stars_repo_name: JackCurragh/RDG

max_issues_repo_name: JackCurragh/RDG

max_forks_repo_name: JackCurragh/RDG

reason: import networkx


text:
#!/usr/bin/env python
"""venn.py: Python からはじめる数学入門

Usage:
  venn.py
"""
from matplotlib_venn import venn2
import matplotlib.pyplot as plt
from sympy import FiniteSet

def main():
    use_sympy = True
    if use_sympy:
        s1 = FiniteSet(*range(1, 20, 2))
        s2 = FiniteSet(2, 3, 5, 7, 11, 13, 17, 19)
    else:
        s1 = {i for i in range(1, 20, 2)}
        s2 = {2, 3, 5, 7, 11, 13, 17, 19}
    venn2(subsets=[s1, s2])
    plt.show()

if __name__ == '__main__':
    main()

max_stars_repo_name: showa-yojyo/notebook

max_issues_repo_name: showa-yojyo/note

max_forks_repo_name: showa-yojyo/note

reason: from sympy


text:
import json
import numpy as np

from . import portfolio

class Risk:

    def __init__(self, portfolio_obj):
        if not isinstance(portfolio_obj, portfolio.Portfolio):
            raise TypeError(f'{portfolio_obj} is not of type: Portfolio')

        self._port = portfolio_obj
        self._port.return_series()

        self.returns_df = self._port.returns_df
        self.bench = self._port.bench

        self.metrics = {}

    def all_metrics(self):
        self.daily_stats()
        self.beta()

        m = json.dumps(self.metrics, indent=4, sort_keys=False)
        print(m)

    def daily_stats(self):

        # 5 year
        avg_ret = self.returns_df['Percent_Ret'].mean()
        vol = self.returns_df['Percent_Ret'].std()

        # 3 year
        avg_ret_3 = self.returns_df['Percent_Ret'].tail(756).mean()
        vol_3 = self.returns_df['Percent_Ret'].tail(756).std()

        # 1 year
        avg_ret_1 = self.returns_df['Percent_Ret'].tail(252).mean()
        vol_1 = self.returns_df['Percent_Ret'].tail(252).std()

        daily_stats = {'fiveYear': {'meanReturn': avg_ret,
                                    'stdReturn': vol},
                       'threeYear': {'meanReturn': avg_ret_3,
                                     'stdReturn': vol_3},
                       'oneYear': {'meanReturn': avg_ret_1,
                                   'stdReturn': vol_1}}

        self.metrics['dailyStats'] = daily_stats

        return daily_stats

    def beta(self):
        # 5 year daily
        cov_5 = np.cov(self.returns_df['Percent_Ret'],
                       self.returns_df['Bench_P_Ret'])[0][1]
        var_5 = self.returns_df['Bench_P_Ret'].var()

        beta_daily_5 = cov_5 / var_5

        # 3 year daily
        cov_3 = np.cov(self.returns_df['Percent_Ret'].tail(756),
                       self.returns_df['Bench_P_Ret'].tail(756))[0][1]
        var_3 = self.returns_df['Bench_P_Ret'].tail(756).var()

        beta_daily_3 = cov_3 / var_3

        # 3 year daily
        cov_1 = np.cov(self.returns_df['Percent_Ret'].tail(252),
                       self.returns_df['Bench_P_Ret'].tail(252))[0][1]
        var_1 = self.returns_df['Bench_P_Ret'].tail(252).var()

        beta_daily_1 = cov_1 / var_1

        beta = {'fiveYearDaily': beta_daily_5,
                'threeYearDaily': beta_daily_3,
                'oneYearDaily': beta_daily_1}

        self.metrics['beta'] = beta

        return beta

max_stars_repo_name: matthewmercuri/pyfolio

max_issues_repo_name: matthewmercuri/pyfolio

max_forks_repo_name: matthewmercuri/pyfolio