text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function, absolute_import, division
from builtins import map, range, object, zip, sorted
from past.builtins import basestring

from . import TestBase
import unittest
import amplpy

def loadDietModel(ampl):
    ampl.eval('''
        set FOOD ;
        set NUTR ;
        # Parameters
        param cost { FOOD } > 0;
        param f_min { FOOD } >= 0;
        param f_max {j in FOOD } >= f_min [j];
        param n_min { NUTR } >= 0;
        param n_max {i in NUTR } >= n_min [i];
        param amt {NUTR , FOOD } >= 0;
        # Variables
        var Buy {j in FOOD } >= f_min [j], <= f_max [j];
        # Objective
        minimize total_cost : sum {j in FOOD } cost [j] * Buy[j];
        # Contraints
        subject to diet {i in NUTR }:
            n_min [i] <= sum {j in FOOD } amt[i,j] * Buy[j] <= n_max [i];
    ''')
    ampl.eval('''
        data;
        set NUTR := A C B1 B2 NA CAL;
        set FOOD := BEEF CHK FISH HAM MCH MTL SPG TUR ;
        param : cost f_min f_max :=
        BEEF 3.19 2 10
        CHK 2.59 2 10
        FISH 2.29 2 10
        HAM 2.89 2 10
        MCH 1.89 2 10
        MTL 1.99 2 10
        SPG 1.99 2 10
        TUR 2.49 2 10 ;
        param : n_min n_max :=
        A 700 20000
        C 700 20000
        B1 700 20000
        B2 700 20000
        NA 0 50000
        CAL 16000 24000 ;
        param amt (tr):
        A C B1 B2 NA CAL :=
        BEEF 60 20 10 15 938 295
        CHK 8 0 20 20 2180 770
        FISH 8 10 15 10 945 440
        HAM 40 40 35 10 278 430
        MCH 15 35 15 15 1182 315
        MTL 70 30 15 15 896 400
        SPG 25 50 25 15 1329 370
        TUR 60 20 15 10 1397 450 ;
    ''')

class TestEntities(TestBase.TestBase):
    """Test entities."""

    def testVariable(self):
        loadDietModel(self.ampl)
        ampl = self.ampl
        # ampl.solve()
        self.assertEqual(
            ampl.getVariable('Buy').numInstances(),
            ampl.getSet('FOOD').size()
        )
        f_min = ampl.getParameter('f_min')
        f_max = ampl.getParameter('f_max')
        Buy = ampl.getVariable('Buy')
        self.assertTrue(isinstance(Buy.getValues(), amplpy.DataFrame))
        self.assertTrue(isinstance(Buy.getValues(['val']), amplpy.DataFrame))
        for index, var in ampl.getVariable('Buy'):
            self.assertTrue(isinstance(var.value(), float))
            var.setValue(f_min[index])
            self.assertEqual(var.value(), f_min[index])
            var.fix()
            self.assertEqual(var.astatus(), 'fix')
            var.unfix()
            self.assertEqual(var.astatus(), 'in')
            var.fix(f_max[index])
            self.assertEqual(var.value(), f_max[index])
            self.assertTrue(isinstance(var.defeqn(), int))
            self.assertTrue(isinstance(var.dual(), float))
            self.assertTrue(isinstance(var.init(), float))
            self.assertTrue(isinstance(var.init0(), float))
            self.assertTrue(isinstance(var.lb(), float))
            self.assertGreaterEqual(var.lb(), f_min[index])
            self.assertTrue(isinstance(var.ub(), float))
            self.assertLessEqual(var.ub(), f_max[index])
            self.assertTrue(isinstance(var.lb0(), float))
            self.assertEqual(var.lb0(), f_min[index])
            self.assertTrue(isinstance(var.ub0(), float))
            self.assertEqual(var.ub0(), f_max[index])
            self.assertTrue(isinstance(var.lb1(), float))
            self.assertGreaterEqual(var.lb1(), f_min[index])
            self.assertTrue(isinstance(var.ub1(), float))
            self.assertLessEqual(var.ub1(), f_max[index])
            self.assertTrue(isinstance(var.lb2(), float))
            self.assertGreaterEqual(var.lb2(), f_min[index])
            self.assertTrue(isinstance(var.ub2(), float))
            self.assertLessEqual(var.ub2(), f_max[index])
            self.assertTrue(isinstance(var.lrc(), float))
            self.assertTrue(isinstance(var.urc(), float))
            self.assertTrue(isinstance(var.lslack(), float))
            self.assertTrue(isinstance(var.uslack(), float))
            self.assertTrue(isinstance(var.rc(), float))
            self.assertTrue(isinstance(var.slack(), float))
            self.assertTrue(isinstance(var.sstatus(), basestring))
            self.assertTrue(isinstance(var.status(), basestring))

    def testConstraint(self):
        loadDietModel(self.ampl)
        ampl = self.ampl
        # ampl.solve()
        self.assertEqual(
            ampl.getConstraint('diet').numInstances(),
            ampl.getSet('NUTR').size()
        )
        for index, con in ampl.getConstraint('diet'):
            self.assertTrue(isinstance(con.isLogical(), bool))
            con.drop()
            con.restore()
            self.assertTrue(isinstance(con.body(), float))
            self.assertTrue(isinstance(con.astatus(), basestring))
            self.assertEqual(con.astatus(), 'in')
            con.drop()
            self.assertEqual(con.astatus(), 'drop')
            con.restore()
            self.assertEqual(con.astatus(), 'in')
            self.assertTrue(isinstance(con.defvar(), int))
            self.assertTrue(isinstance(con.dinit(), float))
            self.assertTrue(isinstance(con.dinit0(), float))
            self.assertTrue(isinstance(con.dual(), float))
            self.assertTrue(isinstance(con.lb(), float))
            self.assertTrue(isinstance(con.ub(), float))
            self.assertTrue(isinstance(con.lbs(), float))
            self.assertTrue(isinstance(con.ubs(), float))
            self.assertTrue(isinstance(con.ldual(), float))
            self.assertTrue(isinstance(con.udual(), float))
            self.assertTrue(isinstance(con.lslack(), float))
            self.assertTrue(isinstance(con.uslack(), float))
            self.assertTrue(isinstance(con.slack(), float))
            self.assertTrue(isinstance(con.status(), basestring))
            self.assertTrue(isinstance(con.sstatus(), basestring))
            con.setDual(0)
            self.assertEqual(con.val(), None)
        ampl.eval('''
            var x;
            var y;
            s.t. xy: x <==> y;
        ''')
        self.assertTrue(ampl.getConstraint('xy').isLogical())
        self.assertTrue(isinstance(ampl.getConstraint('xy').val(), float))

    def testSet(self):
        loadDietModel(self.ampl)
        ampl = self.ampl
        self.assertEqual(
            ampl.getSet('NUTR').size(),
            len(ampl.getSet('NUTR').members()),
        )
        self.assertEqual(
            len(ampl.getSet('NUTR').members()),
            len(list(ampl.getSet('NUTR').members()))
        )
        self.assertTrue(ampl.getSet('NUTR').contains('A'))
        self.assertTrue(ampl.getSet('NUTR').contains(('B1',)))
        self.assertFalse(ampl.getSet('NUTR').contains('X'))
        for name, st in ampl.getSets():
            self.assertTrue(isinstance(st.arity(), int))
            self.assertEqual(st.arity(), 1)
            self.assertTrue(isinstance(st.size(), int))
            self.assertEqual(len(st.instances()), 1)
            self.assertEqual(
                len(dict(st.instances())),
                len(list(st.instances()))
            )
            self.assertTrue(isinstance(st.arity(), int))
        ampl.eval('model; set T{1..2}; set T2{1..2};')
        ampl.getSet('T')[1].setValues([1, 2])
        ampl.getSet('T')[2].setValues(['1', 2])
        ampl.getSet('T2')[1].setValues(ampl.getSet('T')[1].getValues())
        ampl.getSet('T2')[2].setValues(ampl.getSet('T')[2].getValues())
        ampl.eval('set T3 dimen 2; set T4 dimen 2;')
        ampl.getSet('T3').setValues([(1, 2)])

        try:
            import numpy as np
        except ImportError:
            pass
        else:
            ampl.getSet('T')[1].setValues(np.array([1, 2]))
            self.assertEqual(ampl.getSet('T')[1].size(), 2)
            ampl.getSet('T3').setValues(np.array([[1, 2], [3, 4]]))
            self.assertEqual(ampl.getSet('T3').size(), 2)

    def testParameter(self):
        loadDietModel(self.ampl)
        ampl = self.ampl
        self.assertEqual(
            ampl.getSet('FOOD').size(),
            ampl.getParameter('cost').numInstances()
        )
        ampl.eval('''
            param a;
            param b default a;
            param c symbolic;
            param d{1..2} symbolic;
            param d2{1..2} symbolic;
        ''')
        self.assertFalse(ampl.getParameter('a').hasDefault())
        self.assertTrue(ampl.getParameter('b').hasDefault())
        self.assertFalse(ampl.getParameter('a').isSymbolic())
        self.assertFalse(ampl.getParameter('b').isSymbolic())
        self.assertTrue(ampl.getParameter('c').isSymbolic())
        cost = ampl.getParameter('cost')
        self.assertEqual(cost.indexarity(), 1)
        self.assertEqual(cost.getIndexingSets(), ['FOOD'])
        for i, food in enumerate(ampl.getSet('FOOD').members()):
            cost[food] = i+1
            self.assertEqual(cost[food], i+1)
            cost.set(food, i+2)
            self.assertEqual(cost[food], i+2)
        a = ampl.getParameter('a')
        a.set(10)
        self.assertEqual(a.value(), 10)
        c = ampl.getParameter('c')
        b = ampl.getParameter('c')
        c.set('a')
        self.assertEqual(c.value(), 'a')
        self.assertEqual(b.value(), 'a')
        self.assertEqual(b.indexarity(), 0)
        self.assertEqual(a.isScalar(), True)
        self.assertEqual(b.isScalar(), True)
        self.assertEqual(c.isScalar(), True)
        d = ampl.getParameter('d')
        self.assertEqual(d.isScalar(), False)
        d.set(1, 'a')
        self.assertEqual(d[1], 'a')
        d.set(2, 'b')
        self.assertEqual(d[2], 'b')
        d.setValues({1: 'x', 2: 'y'})
        self.assertEqual(d[1], 'x')
        self.assertEqual(d[2], 'y')
        d.setValues(['xx', 'yy'])
        self.assertEqual(d[1], 'xx')
        self.assertEqual(d[2], 'yy')
        d2 = ampl.getParameter('d2')
        d2.setValues(d.getValues())
        self.assertEqual(d2[1], 'xx')
        self.assertEqual(d2[2], 'yy')
        self.assertEqual(cost.isScalar(), False)
        ampl.eval('param cost2{FOOD};')
        self.assertTrue(isinstance(cost.getValues(), amplpy.DataFrame))
        cost2 = ampl.getParameter('cost2')
        cost2.setValues(cost.getValues())
        for food in ampl.getSet('FOOD').members():
            self.assertEqual(cost2[food], cost[food])

        try:
            import numpy as np
        except ImportError:
            pass
        else:
            ampl.eval('param lst{1..3};')
            ampl.getParameter('lst').setValues(np.array([1, 2, 3]))

    def testPrecision(self):
        PI = 3.1415926535897932384626433832795028841971
        ampl = self.ampl
        ampl.eval('param x;')
        ampl.getParameter('x').set(PI)
        ampl.eval("display x;")  # invalidate cache
        self.assertEqual(ampl.param['x'].value(), PI)

    def testInfinity(self):
        inf = float('inf')
        ampl = self.ampl
        ampl.eval('param x;')
        ampl.getParameter('x').set(inf)
        self.assertEqual(ampl.getValue('x'), inf)
        ampl.eval('param y{1..3};')
        ampl.getParameter('y').setValues([inf]*3)
        for i in range(3):
            self.assertEqual(ampl.getValue('y[{}]'.format(i+1)), inf)

    def testObjective(self):
        loadDietModel(self.ampl)
        ampl = self.ampl
        obj = ampl.getObjective('total_cost')
        self.assertEqual(
            len(dict(obj)),
            obj.numInstances()
        )
        self.assertTrue(isinstance(obj.value(), float))
        self.assertTrue(isinstance(obj.astatus(), basestring))
        self.assertTrue(isinstance(obj.sstatus(), basestring))
        self.assertTrue(isinstance(obj.exitcode(), int))
        self.assertTrue(isinstance(obj.message(), basestring))
        self.assertTrue(isinstance(obj.result(), basestring))
        obj.drop()
        self.assertEqual(obj.astatus(), 'drop')
        obj.restore()
        self.assertEqual(obj.astatus(), 'in')
        self.assertFalse(obj.minimization())

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: sbebo/amplpy

max_issues_repo_name: sbebo/amplpy

max_forks_repo_name: sbebo/amplpy


text:
"""
Calculate evaluation metrics such as hits@k
"""
from typing import Any, Dict, List, Union

import numpy as np

def _hits_from_ndarray(nn_ind, gold, k, hits_counter):
    for hits_at_k in k:
        for i, _ in enumerate(nn_ind):
            if i in gold and gold[i] in nn_ind[i][:hits_at_k]:
                hits_counter[hits_at_k] += 1
    return hits_counter

def _hits_from_dict(nn_ind, gold, k, hits_counter):
    for hits_at_k in k:
        for i, v in nn_ind.items():
            if i in gold and gold[i] in v[:hits_at_k]:
                hits_counter[hits_at_k] += 1
    return hits_counter

def hits(
    nn_ind: Union[np.ndarray, Dict[Any, List]],
    gold: Dict[Any, Any],  # source -> target
    k=None,
) -> Dict[int, float]:
    """Show hits@k

    Parameters
    ----------
    nn_ind:  array or dict
        Contains the indices of the nearest neighbors for source entities.
    gold: dict
        Map of source indices to the respective target indices
    k: array
        Which ks should be evaluated

    Returns
    -------
    hits: dict
        k: relative hits@k values

    Examples
    --------
    >>> from kiez.evaluate import hits
    >>> import numpy as np
    >>> nn_ind = np.array([[1, 2, 3], [2, 3, 4], [3, 4, 5], [4, 5, 6]])
    >>> gold = {0: 2, 1: 4, 2: 3, 3: 4}
    >>> hits(nn_ind, gold)
    {1: 0.5, 5: 1.0, 10: 1.0}
    """
    if k is None:
        k = [1, 5, 10]
    k.sort()
    hits_counter = {hits_at_k: 0 for hits_at_k in k}
    if isinstance(nn_ind, (np.ndarray, list)):
        hits_counter = _hits_from_ndarray(nn_ind, gold, k, hits_counter)
    elif isinstance(nn_ind, Dict):
        hits_counter = _hits_from_dict(nn_ind, gold, k, hits_counter)
    return {hits_at_k: k_val / len(gold) for hits_at_k, k_val in hits_counter.items()}

ext: py

max_stars_repo_name: cthoyt/kiez

max_issues_repo_name: cthoyt/kiez

max_forks_repo_name: cthoyt/kiez

reason: import numpy


text:
"""

mweigert@mpi-cbg.de

"""
from __future__ import print_function, unicode_literals, absolute_import, division
import numpy as np
from itertools import product

def tile_iterator(im,
                 blocksize = (64, 64),
                 padsize = (64,64),
                 mode = "constant",
                 verbose = False):
    """

    iterates over padded tiles of an ND image
    while keeping track of the slice positions

    Example:
    --------
    im = np.zeros((200,200))
    res = np.empty_like(im)

    for padded_tile, s_src, s_dest in tile_iterator(im,
                              blocksize=(128, 128),
                              padsize = (64,64),
                              mode = "wrap"):

        #do something with the tile, e.g. a convolution
        res_padded = np.mean(padded_tile)*np.ones_like(padded_tile)

        # reassemble the result at the correct position
        res[s_src] = res_padded[s_dest]

    Parameters
    ----------
    im: ndarray
        the input data (arbitrary dimension)
    blocksize:
        the dimension of the blocks to split into
        e.g. (nz, ny, nx) for a 3d image
    padsize:
        the size of left and right pad for each dimension
    mode:
        padding mode, like numpy.pad
        e.g. "wrap", "constant"...

    Returns
    -------
        tile, slice_src, slice_dest

        tile[slice_dest] is the tile in im[slice_src]

    """

    if not(im.ndim == len(blocksize) ==len(padsize)):
        raise ValueError("im.ndim (%s) != len(blocksize) (%s) != len(padsize) (%s)"
                         %(im.ndim , len(blocksize) , len(padsize)))

    subgrids = tuple([int(np.ceil(1.*n/b)) for n,b in zip(im.shape, blocksize)])

    #if the image dimension are not divible by the blocksize, pad it accordingly
    pad_mismatch = tuple([(s*b-n) for n,s, b in zip(im.shape,subgrids,blocksize)])

    if verbose:
        print("tile padding... ")

    im_pad = np.pad(im,[(p,p+pm) for pm,p in zip(pad_mismatch,padsize)], mode = mode)

    # iterates over cartesian product of subgrids
    for i,index in enumerate(product(*[range(sg) for sg in subgrids])):
        # the slices
        # if verbose:
        #     print("tile %s/%s"%(i+1,np.prod(subgrids)))

        # dest[s_output] is where we will write to
        s_input = tuple([slice(i*b,(i+1)*b) for i,b in zip(index, blocksize)])

        s_output = tuple([slice(p,-p-pm*(i==s-1)) for pm,p,i,s in zip(pad_mismatch,padsize, index, subgrids)])

        s_output = tuple([slice(p,b+p-pm*(i==s-1)) for b,pm,p,i,s in zip(blocksize,pad_mismatch,padsize, index, subgrids)])

        s_padinput = tuple([slice(i*b,(i+1)*b+2*p) for i,b,p in zip(index, blocksize, padsize)])
        padded_block = im_pad[s_padinput]

        # print im.shape, padded_block.shape, s_output

        yield padded_block, s_input, s_output

if __name__ == '__main__':

    # simple test

    for n in [1,2,3]:
        print("n = %s"%n)
        im = np.random.uniform(-1,1,[103+13*_n for _n in range(n)])
        res = np.empty_like(im)

        for padded_tile, s_src, s_dest in tile_iterator(im,
                                  blocksize=(50,)*im.ndim,
                                  padsize = (64,)*im.ndim,
                                  mode = "wrap"):

            # reassemble the result at the correct position
            res[s_src] = padded_tile[s_dest]

        print("OK" if np.allclose(res, im) else "ERRRRRRRRRRROOOOOOOORRRRRR")

max_stars_repo_name: VolkerH/gputools

max_issues_repo_name: VolkerH/gputools

max_forks_repo_name: VolkerH/gputools


text:
from src.network import Network
import src.VisualizeNN as VisNN

import pandas as pd
import numpy as np
from matplotlib import pyplot as plt

def split_dataset(train_df: pd.DataFrame,
                  fraction: float = 0.2):
    """
    Split train data into train and validation.
    """

    #permute all samples
    train_df = train_df.sample(frac=1.0)

    #set validation fraction
    fraction = 0.2

    split_index = int(fraction*len(train_df))

    valid_df = train_df.iloc[:split_index]
    train_df = train_df.iloc[split_index:]

    return train_df, valid_df

def get_normalisation_scale(df: pd.DataFrame,
                            regression: bool = False):
    """
    Get normalization scales. Used later in normalize_dataset().
    """

    if regression:
        without_last_column = df
    else:
        # don't scale last column (class label)
        without_last_column = df.iloc[:, :-1]

    return (without_last_column.min(), without_last_column.max())

def normalized_dataset(df: pd.DataFrame,
                      df_min: pd.Series,
                      df_max: pd.Series,
                      regression: bool = False):
    """
    Normalize dataset using calculated min and max values from get_normalisation_scale().
    """

    # don't modify last column (class label)
    if regression is False:
        tmp_df = 2.0*((df.iloc[:,:-1]-df_min)/(df_max - df_min))-1.0
        tmp_df['cls'] = df['cls']
    else:
        tmp_df = 2.0*((df-df_min)/(df_max - df_min))-1.0
    return  tmp_df

def train_classification( net: Network,
               dataset: pd.DataFrame,
               max_epochs: int,
               learning_rate: float,
               batch_size: int = 1,
               multiclass: bool = False):
    """
    Train net for a classification task.
    The dataset consists of features and class label (in the last column).

    If the multiclass is True, uses one-hot encoding.
    """

    train_df, valid_df = split_dataset(dataset, 0.2)
    train_y_df = pd.get_dummies(train_df['cls'], dtype=float) if multiclass else train_df['cls'] - 1.0
    valid_y_df = pd.get_dummies(valid_df['cls'], dtype=float) if multiclass else valid_df['cls'] - 1.0
    y_dim = train_y_df.shape[1] if multiclass else 1

    train_losses = []
    validation_losses = []
    for epoch in range(max_epochs):
        train_loss = 0
        validation_loss = 0
        for i in range(0, len(train_df)-batch_size, batch_size):
            x = np.array(train_df.iloc[i:i+batch_size, :-1])
            y = np.reshape(np.array(train_y_df.iloc[i:i+batch_size]), (batch_size, y_dim))
            loss = net.fit(x, y, learning_rate, batch_size)
            train_loss +=loss

        for i in range(0, len(valid_df)-batch_size, batch_size):
            x = np.array(valid_df.iloc[i:i+batch_size, :-1])
            y = np.reshape(np.array(valid_y_df.iloc[i:i+batch_size]), (batch_size, y_dim))
            loss = net.validate(x, y, learning_rate, batch_size)
            validation_loss +=loss

        train_losses.append(train_loss/len(train_df))
        validation_losses.append(validation_loss/len(valid_df))
    return train_losses, validation_losses

def train_regression(net: Network,
                     dataset: pd.DataFrame,
                     max_epochs: int,
                     learning_rate: float,
                     batch_size: int = 1):
    """
    Train net for a regression task.
    The dataset consists of features and regression value (in the last column).
    """
    train_df, valid_df = split_dataset(dataset, 0.2)

    train_losses = []
    validation_losses = []
    for epoch in range(max_epochs):
        train_loss = 0
        validation_loss = 0
        for i in range(0, len(train_df)-batch_size, batch_size):
            x = np.array(train_df.iloc[i:i+batch_size, :-1])
            y = np.reshape(np.array(train_df.iloc[i:i+batch_size, -1]), (batch_size, 1))
            loss = net.fit(x, y, learning_rate, batch_size)
            train_loss +=loss

        for i in range(0, len(valid_df)-batch_size, batch_size):
            x = np.array(valid_df.iloc[i:i+batch_size, :-1])
            y = np.reshape(np.array(valid_df.iloc[i:i+batch_size, -1]), (batch_size, 1))
            loss = net.validate(x, y, learning_rate, batch_size)
            validation_loss +=loss

        train_losses.append(train_loss/len(train_df))
        validation_losses.append(validation_loss/len(valid_df))
    return train_losses, validation_losses

def plot_loss(train_losses, validation_losses):
    """
    Plots train and validation losses.
    """
    plt.plot(train_losses, label='train loss')
    plt.annotate('%0.4f' % train_losses[-1], xy=(1, train_losses[-1]), xytext=(20, 0),
                 xycoords=('axes fraction', 'data'), textcoords='offset points',
                 arrowprops=dict(arrowstyle= '-')
              )
    plt.plot(validation_losses, label='validation loss')
    plt.annotate('%0.4f' % validation_losses[-1], xy=(1, validation_losses[-1]), xytext=(20, 20),
                 xycoords=('axes fraction', 'data'), textcoords='offset points',
                 arrowprops=dict(arrowstyle= '-')
              )

    plt.legend()
    plt.figure()
    plt.show()

def plot_decision_boundary(network: Network,
                           test_df: pd.DataFrame,
                           h: float = .02):
    """
    Plots decision boundary of the classifier.
    h - spacing in a meshgrid.
    """

    x_min, x_max = test_df['x'].min(), test_df['x'].max()
    y_min, y_max = test_df['y'].min(), test_df['y'].max()

    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                        np.arange(y_min, y_max, h))

    Z = np.zeros(xx.ravel().shape)
    for i, (x, y) in enumerate(zip(xx.ravel(), yy.ravel())):
        z = network.predict(np.array([[x, y]]))
        if (z.shape[1] == 1):
            Z[i]  = 0 if z < 0.5 else 1
        else:
            Z[i] = np.argmax(z, axis=1)
    Z = Z.reshape(xx.shape)
    if z.shape[1] == 1:
        plt.scatter(test_df['x'], test_df['y'], c=test_df['cls'], cmap=plt.cm.viridis, alpha=0.9)
    else:
        plt.scatter(test_df['x'], test_df['y'], c=test_df['cls'], cmap=plt.cm.Dark2, alpha=0.9)
    plt.contourf(xx, yy, Z, cmap=plt.cm.Dark2, alpha=0.3)
    plt.show()

def plot_regression(network: Network,
                    test_df: pd.DataFrame,
                    num: float = 100):
    """
    Plot regression line of the network on top of the test data.
    num - number of points evaluated.
    """
    x_min, x_max = test_df['x'].min(), test_df['x'].max()

    xx = np.linspace(x_min, x_max, num)

    Z = np.zeros(xx.ravel().shape)
    for i, x  in enumerate(xx.ravel()):
        Z[i] = network.predict(np.array([[x]]))

    plt.scatter(test_df['x'], test_df['y'], c='c', alpha=0.1)
    plt.plot(xx, Z, c='orange', linewidth=2, alpha=0.9)
    plt.show()

def draw_weights(network: Network):
    w_max = 0
    w_min = 0
    dims = []

    for l in network.layers:
        if l.b is None:
            dims.append(l.w.shape[0])
            w_max = max(w_max, np.max(l.w))
            w_min = min(w_min, np.min(l.w))
        else:
            dims.append(l.w.shape[0] + 1)
            w_max = max(w_max, np.max(l.b), np.max(l.w))
            w_min = min(w_min, np.min(l.b), np.min(l.w))

    dims.append(l.w.shape[1])

    weights = []
    for l in network.layers:
        if l.b is None:
            weights.append(2*((l.w- w_min) /(w_max - w_min)) -1)
            #weights.append(2*((np.concatenate([l.w, np.zeros((1, l.w.shape[1]))+w_min]) - w_min) /(w_max - w_min)) -1)
        else:
            weights.append(2*((np.concatenate([np.concatenate([l.w, l.b]), np.zeros((l.w.shape[0] + 1, 1))], axis=1) - w_min) /(w_max - w_min))-1)
    for w in weights:
        print(w.shape)
    v = VisNN.DrawNN(dims, weights)
    v.draw()

def calculate_accuacy(network: Network,
                      df: pd.DataFrame,
                      multiclass: bool = False):

    good_counter = 0
    for i in range(len(df)):
        z = network.predict(np.array(df.iloc[i, :-1]))
        if multiclass:
            good_counter += 1 if np.argmax(z, axis=1) == df['cls'][i] - 1 else 0
        else:
            good_counter += 1 if (z.item() > 0.5 and df['cls'][i] == 2) or ( z.item() < 0.5 and df['cls'][i] == 1) else 0

    return good_counter/len(df)

def calculate_mse(network: Network,
                      df: pd.DataFrame):

    P = network.predict(np.array(df.iloc[:, :-1]))
    P = P.flatten()
    return np.average((P - df.iloc[:,-1])*(P - df.iloc[:,-1]))

max_stars_repo_name: YuseqYaseq/pw-deep-learning

max_issues_repo_name: YuseqYaseq/pw-deep-learning

max_forks_repo_name: YuseqYaseq/pw-deep-learning


text:
import numpy as np
from scipy import sparse

from numpy.testing import assert_array_almost_equal
from numpy.testing import assert_array_equal

from sklearn.utils import check_random_state
from sklearn.utils._testing import assert_warns
from sklearn.utils._testing import assert_raises_regexp
from sklearn.utils._testing import assert_raises
from sklearn.utils._testing import assert_allclose
from sklearn.datasets import make_regression
from sklearn.linear_model import LinearRegression, RANSACRegressor
from sklearn.linear_model import OrthogonalMatchingPursuit
from sklearn.linear_model._ransac import _dynamic_max_trials
from sklearn.exceptions import ConvergenceWarning

# Generate coordinates of line
X = np.arange(-200, 200)
y = 0.2 * X + 20
data = np.column_stack([X, y])

# Add some faulty data
rng = np.random.RandomState(1000)
outliers = np.unique(rng.randint(len(X), size=200))
data[outliers, :] += 50 + rng.rand(len(outliers), 2) * 10

X = data[:, 0][:, np.newaxis]
y = data[:, 1]

def test_ransac_inliers_outliers():

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, random_state=0)

    # Estimate parameters of corrupted data
    ransac_estimator.fit(X, y)

    # Ground truth / reference inlier mask
    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
                                   ).astype(np.bool_)
    ref_inlier_mask[outliers] = False

    assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)

def test_ransac_is_data_valid():
    def is_data_valid(X, y):
        assert X.shape[0] == 2
        assert y.shape[0] == 2
        return False

    rng = np.random.RandomState(0)
    X = rng.rand(10, 2)
    y = rng.rand(10, 1)

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5,
                                       is_data_valid=is_data_valid,
                                       random_state=0)

    assert_raises(ValueError, ransac_estimator.fit, X, y)

def test_ransac_is_model_valid():
    def is_model_valid(estimator, X, y):
        assert X.shape[0] == 2
        assert y.shape[0] == 2
        return False

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5,
                                       is_model_valid=is_model_valid,
                                       random_state=0)

    assert_raises(ValueError, ransac_estimator.fit, X, y)

def test_ransac_max_trials():
    base_estimator = LinearRegression()

    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, max_trials=0,
                                       random_state=0)
    assert_raises(ValueError, ransac_estimator.fit, X, y)

    # there is a 1e-9 chance it will take these many trials. No good reason
    # 1e-2 isn't enough, can still happen
    # 2 is the what ransac defines  as min_samples = X.shape[1] + 1
    max_trials = _dynamic_max_trials(
        len(X) - len(outliers), X.shape[0], 2, 1 - 1e-9)
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2)
    for i in range(50):
        ransac_estimator.set_params(min_samples=2, random_state=i)
        ransac_estimator.fit(X, y)
        assert ransac_estimator.n_trials_ < max_trials + 1

def test_ransac_stop_n_inliers():
    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, stop_n_inliers=2,
                                       random_state=0)
    ransac_estimator.fit(X, y)

    assert ransac_estimator.n_trials_ == 1

def test_ransac_stop_score():
    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, stop_score=0,
                                       random_state=0)
    ransac_estimator.fit(X, y)

    assert ransac_estimator.n_trials_ == 1

def test_ransac_score():
    X = np.arange(100)[:, None]
    y = np.zeros((100, ))
    y[0] = 1
    y[1] = 100

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=0.5, random_state=0)
    ransac_estimator.fit(X, y)

    assert ransac_estimator.score(X[2:], y[2:]) == 1
    assert ransac_estimator.score(X[:2], y[:2]) < 1

def test_ransac_predict():
    X = np.arange(100)[:, None]
    y = np.zeros((100, ))
    y[0] = 1
    y[1] = 100

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=0.5, random_state=0)
    ransac_estimator.fit(X, y)

    assert_array_equal(ransac_estimator.predict(X), np.zeros(100))

def test_ransac_resid_thresh_no_inliers():
    # When residual_threshold=0.0 there are no inliers and a
    # ValueError with a message should be raised
    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=0.0, random_state=0,
                                       max_trials=5)

    msg = ("RANSAC could not find a valid consensus set")
    assert_raises_regexp(ValueError, msg, ransac_estimator.fit, X, y)
    assert ransac_estimator.n_skips_no_inliers_ == 5
    assert ransac_estimator.n_skips_invalid_data_ == 0
    assert ransac_estimator.n_skips_invalid_model_ == 0

def test_ransac_no_valid_data():
    def is_data_valid(X, y):
        return False

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator,
                                       is_data_valid=is_data_valid,
                                       max_trials=5)

    msg = ("RANSAC could not find a valid consensus set")
    assert_raises_regexp(ValueError, msg, ransac_estimator.fit, X, y)
    assert ransac_estimator.n_skips_no_inliers_ == 0
    assert ransac_estimator.n_skips_invalid_data_ == 5
    assert ransac_estimator.n_skips_invalid_model_ == 0

def test_ransac_no_valid_model():
    def is_model_valid(estimator, X, y):
        return False

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator,
                                       is_model_valid=is_model_valid,
                                       max_trials=5)

    msg = ("RANSAC could not find a valid consensus set")
    assert_raises_regexp(ValueError, msg, ransac_estimator.fit, X, y)
    assert ransac_estimator.n_skips_no_inliers_ == 0
    assert ransac_estimator.n_skips_invalid_data_ == 0
    assert ransac_estimator.n_skips_invalid_model_ == 5

def test_ransac_exceed_max_skips():
    def is_data_valid(X, y):
        return False

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator,
                                       is_data_valid=is_data_valid,
                                       max_trials=5,
                                       max_skips=3)

    msg = ("RANSAC skipped more iterations than `max_skips`")
    assert_raises_regexp(ValueError, msg, ransac_estimator.fit, X, y)
    assert ransac_estimator.n_skips_no_inliers_ == 0
    assert ransac_estimator.n_skips_invalid_data_ == 4
    assert ransac_estimator.n_skips_invalid_model_ == 0

def test_ransac_warn_exceed_max_skips():
    global cause_skip
    cause_skip = False

    def is_data_valid(X, y):
        global cause_skip
        if not cause_skip:
            cause_skip = True
            return True
        else:
            return False

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator,
                                       is_data_valid=is_data_valid,
                                       max_skips=3,
                                       max_trials=5)

    assert_warns(ConvergenceWarning, ransac_estimator.fit, X, y)
    assert ransac_estimator.n_skips_no_inliers_ == 0
    assert ransac_estimator.n_skips_invalid_data_ == 4
    assert ransac_estimator.n_skips_invalid_model_ == 0

def test_ransac_sparse_coo():
    X_sparse = sparse.coo_matrix(X)

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, random_state=0)
    ransac_estimator.fit(X_sparse, y)

    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
                                   ).astype(np.bool_)
    ref_inlier_mask[outliers] = False

    assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)

def test_ransac_sparse_csr():
    X_sparse = sparse.csr_matrix(X)

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, random_state=0)
    ransac_estimator.fit(X_sparse, y)

    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
                                   ).astype(np.bool_)
    ref_inlier_mask[outliers] = False

    assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)

def test_ransac_sparse_csc():
    X_sparse = sparse.csc_matrix(X)

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, random_state=0)
    ransac_estimator.fit(X_sparse, y)

    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
                                   ).astype(np.bool_)
    ref_inlier_mask[outliers] = False

    assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)

def test_ransac_none_estimator():

    base_estimator = LinearRegression()

    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, random_state=0)
    ransac_none_estimator = RANSACRegressor(None, min_samples=2,
                                            residual_threshold=5,
                                            random_state=0)

    ransac_estimator.fit(X, y)
    ransac_none_estimator.fit(X, y)

    assert_array_almost_equal(ransac_estimator.predict(X),
                              ransac_none_estimator.predict(X))

def test_ransac_min_n_samples():
    base_estimator = LinearRegression()
    ransac_estimator1 = RANSACRegressor(base_estimator, min_samples=2,
                                        residual_threshold=5, random_state=0)
    ransac_estimator2 = RANSACRegressor(base_estimator,
                                        min_samples=2. / X.shape[0],
                                        residual_threshold=5, random_state=0)
    ransac_estimator3 = RANSACRegressor(base_estimator, min_samples=-1,
                                        residual_threshold=5, random_state=0)
    ransac_estimator4 = RANSACRegressor(base_estimator, min_samples=5.2,
                                        residual_threshold=5, random_state=0)
    ransac_estimator5 = RANSACRegressor(base_estimator, min_samples=2.0,
                                        residual_threshold=5, random_state=0)
    ransac_estimator6 = RANSACRegressor(base_estimator,
                                        residual_threshold=5, random_state=0)
    ransac_estimator7 = RANSACRegressor(base_estimator,
                                        min_samples=X.shape[0] + 1,
                                        residual_threshold=5, random_state=0)

    ransac_estimator1.fit(X, y)
    ransac_estimator2.fit(X, y)
    ransac_estimator5.fit(X, y)
    ransac_estimator6.fit(X, y)

    assert_array_almost_equal(ransac_estimator1.predict(X),
                              ransac_estimator2.predict(X))
    assert_array_almost_equal(ransac_estimator1.predict(X),
                              ransac_estimator5.predict(X))
    assert_array_almost_equal(ransac_estimator1.predict(X),
                              ransac_estimator6.predict(X))

    assert_raises(ValueError, ransac_estimator3.fit, X, y)
    assert_raises(ValueError, ransac_estimator4.fit, X, y)
    assert_raises(ValueError, ransac_estimator7.fit, X, y)

def test_ransac_multi_dimensional_targets():

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       residual_threshold=5, random_state=0)

    # 3-D target values
    yyy = np.column_stack([y, y, y])

    # Estimate parameters of corrupted data
    ransac_estimator.fit(X, yyy)

    # Ground truth / reference inlier mask
    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
                                   ).astype(np.bool_)
    ref_inlier_mask[outliers] = False

    assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)

def test_ransac_residual_loss():
    loss_multi1 = lambda y_true, y_pred: np.sum(np.abs(y_true - y_pred), axis=1)
    loss_multi2 = lambda y_true, y_pred: np.sum((y_true - y_pred) ** 2, axis=1)

    loss_mono = lambda y_true, y_pred : np.abs(y_true - y_pred)
    yyy = np.column_stack([y, y, y])

    base_estimator = LinearRegression()
    ransac_estimator0 = RANSACRegressor(base_estimator, min_samples=2,
                                        residual_threshold=5, random_state=0)
    ransac_estimator1 = RANSACRegressor(base_estimator, min_samples=2,
                                        residual_threshold=5, random_state=0,
                                        loss=loss_multi1)
    ransac_estimator2 = RANSACRegressor(base_estimator, min_samples=2,
                                        residual_threshold=5, random_state=0,
                                        loss=loss_multi2)

    # multi-dimensional
    ransac_estimator0.fit(X, yyy)
    ransac_estimator1.fit(X, yyy)
    ransac_estimator2.fit(X, yyy)
    assert_array_almost_equal(ransac_estimator0.predict(X),
                              ransac_estimator1.predict(X))
    assert_array_almost_equal(ransac_estimator0.predict(X),
                              ransac_estimator2.predict(X))

    # one-dimensional
    ransac_estimator0.fit(X, y)
    ransac_estimator2.loss = loss_mono
    ransac_estimator2.fit(X, y)
    assert_array_almost_equal(ransac_estimator0.predict(X),
                              ransac_estimator2.predict(X))
    ransac_estimator3 = RANSACRegressor(base_estimator, min_samples=2,
                                        residual_threshold=5, random_state=0,
                                        loss="squared_loss")
    ransac_estimator3.fit(X, y)
    assert_array_almost_equal(ransac_estimator0.predict(X),
                              ransac_estimator2.predict(X))

def test_ransac_default_residual_threshold():
    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       random_state=0)

    # Estimate parameters of corrupted data
    ransac_estimator.fit(X, y)

    # Ground truth / reference inlier mask
    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
                                   ).astype(np.bool_)
    ref_inlier_mask[outliers] = False

    assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)

def test_ransac_dynamic_max_trials():
    # Numbers hand-calculated and confirmed on page 119 (Table 4.3) in
    #   Hartley, R.~I. and Zisserman, A., 2004,
    #   Multiple View Geometry in Computer Vision, Second Edition,
    #   Cambridge University Press, ISBN: 0521540518

    # e = 0%, min_samples = X
    assert _dynamic_max_trials(100, 100, 2, 0.99) == 1

    # e = 5%, min_samples = 2
    assert _dynamic_max_trials(95, 100, 2, 0.99) == 2
    # e = 10%, min_samples = 2
    assert _dynamic_max_trials(90, 100, 2, 0.99) == 3
    # e = 30%, min_samples = 2
    assert _dynamic_max_trials(70, 100, 2, 0.99) == 7
    # e = 50%, min_samples = 2
    assert _dynamic_max_trials(50, 100, 2, 0.99) == 17

    # e = 5%, min_samples = 8
    assert _dynamic_max_trials(95, 100, 8, 0.99) == 5
    # e = 10%, min_samples = 8
    assert _dynamic_max_trials(90, 100, 8, 0.99) == 9
    # e = 30%, min_samples = 8
    assert _dynamic_max_trials(70, 100, 8, 0.99) == 78
    # e = 50%, min_samples = 8
    assert _dynamic_max_trials(50, 100, 8, 0.99) == 1177

    # e = 0%, min_samples = 10
    assert _dynamic_max_trials(1, 100, 10, 0) == 0
    assert _dynamic_max_trials(1, 100, 10, 1) == float('inf')

    base_estimator = LinearRegression()
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       stop_probability=-0.1)
    assert_raises(ValueError, ransac_estimator.fit, X, y)
    ransac_estimator = RANSACRegressor(base_estimator, min_samples=2,
                                       stop_probability=1.1)
    assert_raises(ValueError, ransac_estimator.fit, X, y)

def test_ransac_fit_sample_weight():
    ransac_estimator = RANSACRegressor(random_state=0)
    n_samples = y.shape[0]
    weights = np.ones(n_samples)
    ransac_estimator.fit(X, y, weights)
    # sanity check
    assert ransac_estimator.inlier_mask_.shape[0] == n_samples

    ref_inlier_mask = np.ones_like(ransac_estimator.inlier_mask_
                                   ).astype(np.bool_)
    ref_inlier_mask[outliers] = False
    # check that mask is correct
    assert_array_equal(ransac_estimator.inlier_mask_, ref_inlier_mask)

    # check that fit(X)  = fit([X1, X2, X3],sample_weight = [n1, n2, n3]) where
    #   X = X1 repeated n1 times, X2 repeated n2 times and so forth
    random_state = check_random_state(0)
    X_ = random_state.randint(0, 200, [10, 1])
    y_ = np.ndarray.flatten(0.2 * X_ + 2)
    sample_weight = random_state.randint(0, 10, 10)
    outlier_X = random_state.randint(0, 1000, [1, 1])
    outlier_weight = random_state.randint(0, 10, 1)
    outlier_y = random_state.randint(-1000, 0, 1)

    X_flat = np.append(np.repeat(X_, sample_weight, axis=0),
                       np.repeat(outlier_X, outlier_weight, axis=0), axis=0)
    y_flat = np.ndarray.flatten(np.append(np.repeat(y_, sample_weight, axis=0),
                                np.repeat(outlier_y, outlier_weight, axis=0),
                                          axis=0))
    ransac_estimator.fit(X_flat, y_flat)
    ref_coef_ = ransac_estimator.estimator_.coef_

    sample_weight = np.append(sample_weight, outlier_weight)
    X_ = np.append(X_, outlier_X, axis=0)
    y_ = np.append(y_, outlier_y)
    ransac_estimator.fit(X_, y_, sample_weight)

    assert_allclose(ransac_estimator.estimator_.coef_, ref_coef_)

    # check that if base_estimator.fit doesn't support
    # sample_weight, raises error
    base_estimator = OrthogonalMatchingPursuit()
    ransac_estimator = RANSACRegressor(base_estimator)
    assert_raises(ValueError, ransac_estimator.fit, X, y, weights)

def test_ransac_final_model_fit_sample_weight():
    X, y = make_regression(n_samples=1000, random_state=10)
    rng = check_random_state(42)
    sample_weight = rng.randint(1, 4, size=y.shape[0])
    sample_weight = sample_weight / sample_weight.sum()
    ransac = RANSACRegressor(base_estimator=LinearRegression(), random_state=0)
    ransac.fit(X, y, sample_weight=sample_weight)

    final_model = LinearRegression()
    mask_samples = ransac.inlier_mask_
    final_model.fit(
        X[mask_samples], y[mask_samples],
        sample_weight=sample_weight[mask_samples]
    )

    assert_allclose(ransac.estimator_.coef_, final_model.coef_, atol=1e-12)

max_stars_repo_name: ryuwd/scikit-learn

max_issues_repo_name: ryuwd/scikit-learn

max_forks_repo_name: ryuwd/scikit-learn

reason: import numpy,from numpy,from scipy


text:
#!/usr/bin/env python3

from smp_manifold_learning.scripts.train_model import train_model
from smp_manifold_learning.data.synthetic.synthetic_3D_unit_circle_loop_dataset_generator import generate_synth_3d_unit_circle_loop_dataset
import typing
import numpy as np
from smallab.experiment import Experiment
from smallab.runner import ExperimentRunner
from smallab.specification_generator import SpecificationGenerator
from time import time

class VAETrial(Experiment):
    def main(self, specification: typing.Dict) -> typing.Dict:
        name_hash = self.get_logging_folder() + "/" + self.get_logger_name(
        )[8:]
        save_output_filepath = name_hash + ".txt"

        if specification["dataset_filepath"][
                1]:  # not None, not an empty string
            t0 = time()

            (vae, train_nmse_per_dim, test_nmse_per_dim, re, kld, loss,
             off_manifold_nmse_per_dim) = train_model(
                 model_type=specification["model_type"],
                 dataset_filepath=specification["dataset_filepath"][0],
                 off_manifold_dataset_filepath=specification[
                     "dataset_filepath"][1],
                 input_size=specification["input_size"],
                 beta=specification["beta"],
                 use_batch_norm=specification["use_batch_norm"],
                 embedding_size=specification["embedding_size"],
                 hidden_sizes=specification["hidden_sizes"],
                 num_epochs=specification["num_epochs"],
                 nmse_threshold=specification["nmse_threshold"],
                 save_output_filepath=save_output_filepath,
                 n_data=specification["n_data"])

            t = time() - t0

            train_nmse = np.mean(train_nmse_per_dim)
            test_nmse = np.mean(test_nmse_per_dim)
            off_manifold_nmse = np.mean(off_manifold_nmse_per_dim)

            return {
                "vae": vae,
                "train_nmse_per_dim": train_nmse_per_dim,
                "test_nmse_per_dim": test_nmse_per_dim,
                "off_manifold_nmse_per_dim": off_manifold_nmse_per_dim,
                "train_nmse": train_nmse,
                "test_nmse": test_nmse,
                "off_manifold_nmse": off_manifold_nmse,
                "re": re,
                "kld": kld,
                "loss": loss,
                'time': t
            }
        else:
            t0 = time()

            (vae, train_nmse_per_dim, test_nmse_per_dim, re, kld,
             loss) = train_model(
                 model_type=specification["model_type"],
                 dataset_filepath=specification["dataset_filepath"][0],
                 off_manifold_dataset_filepath=None,
                 input_size=specification["input_size"],
                 beta=specification["beta"],
                 use_batch_norm=specification["use_batch_norm"],
                 embedding_size=specification["embedding_size"],
                 hidden_sizes=specification["hidden_sizes"],
                 num_epochs=specification["num_epochs"],
                 nmse_threshold=specification["nmse_threshold"],
                 save_output_filepath=save_output_filepath,
                 n_data=specification["n_data"])

            t = time() - t0

            train_nmse = np.mean(train_nmse_per_dim)
            test_nmse = np.mean(test_nmse_per_dim)

            return {
                "vae": vae,
                "train_nmse_per_dim": train_nmse_per_dim,
                "test_nmse_per_dim": test_nmse_per_dim,
                "train_nmse": train_nmse,
                "test_nmse": test_nmse,
                "re": re,
                "kld": kld,
                "loss": loss,
                'time': t
            }

if __name__ == '__main__':
    # comparisons to ECoMaNN ===================================
    model_type = "vae"
    n_trials = 3
    beta = 0.01
    thresh = 0.0001
    batch_norm = True
    n_epochs = 100
    hidden_sizes = [128, 64, 32, 16]
    # parameters for generating the circle loop data
    rand_seed = 38
    coord_cross_section = 0.0
    noise_level = 0.0

    do_sphere = True if input("Do sphere trials? y/n ") == 'y' else False
    do_circle = True if input("Do circle loop trials? y/n ") == 'y' else False
    do_3dof = True if input("Do plane trials? y/n ") == 'y' else False
    do_6dof = True if input("Do orient trials? y/n ") == 'y' else False

    names = {
        "sphere": "sphere_trials",
        "circle": "circle_trials",
        "3dof": "3DOF_trials",
        "6dof": "6DOF_trials"
    }

    # Sphere trials
    if do_sphere:
        generation_specification = {
            'dataset_filepath':
            [('../data/trajectories/synthetic_unit_sphere_wo_noise', '')],
            'input_size': [3],
            'model_type': [model_type],
            'beta': [beta],
            'use_batch_norm': [batch_norm],
            'embedding_size': [2],
            'num_epochs': [n_epochs],
            'nmse_threshold': [thresh],
            'hidden_sizes': [hidden_sizes],
            'n_trials': [_ for _ in range(n_trials)],
            'n_data': [None]  # use all data
        }

        specifications = SpecificationGenerator().generate(
            generation_specification)
        name = names["sphere"]
        runner = ExperimentRunner()
        runner.run(name, specifications, VAETrial(), force_pickle=True)

    # circleloop trials
    if do_circle:
        generate_synth_3d_unit_circle_loop_dataset(
            N_data=1000,
            rand_seed=rand_seed,
            z_coord=coord_cross_section,
            noise_level=noise_level,
            dataset_save_path='../data/trajectories/circle_loop.npy')
        generation_specification = {
            'dataset_filepath': [('../data/trajectories/circle_loop', None)],
            'input_size': [3],
            'model_type': [model_type],
            'beta': [beta],
            'use_batch_norm': [batch_norm],
            'embedding_size': [1],
            'num_epochs': [n_epochs],
            'nmse_threshold': [thresh],
            'hidden_sizes': [hidden_sizes],
            'n_trials': [_ for _ in range(n_trials)],
            'n_data': [None]  # use all data
        }

        specifications = SpecificationGenerator().generate(
            generation_specification)
        name = names["circle"]
        runner = ExperimentRunner()
        runner.run(name, specifications, VAETrial(), force_pickle=True)

    # 3DOF trials
    if do_3dof:
        generation_specification = {
            'dataset_filepath': [('../data/trajectories/3dof_v2_traj', None)],
            'input_size': [3],
            'model_type': [model_type],
            'beta': [beta],
            'use_batch_norm': [batch_norm],
            'embedding_size': [2],
            'num_epochs': [n_epochs],
            'nmse_threshold': [thresh],
            'hidden_sizes': [hidden_sizes],
            'n_trials': [_ for _ in range(n_trials)],
            'n_data': [None]  # use all data
        }

        specifications = SpecificationGenerator().generate(
            generation_specification)
        name = names["3dof"]
        runner = ExperimentRunner()
        runner.run(name, specifications, VAETrial(), force_pickle=True)

    # 6DOF trials
    if do_6dof:
        generation_specification = {
            'dataset_filepath': [('../data/trajectories/6dof_traj', None)],
            'input_size': [6],
            'model_type': [model_type],
            'beta': [beta],
            'use_batch_norm': [batch_norm],
            'embedding_size': [4],
            'num_epochs': [n_epochs],
            'nmse_threshold': [thresh],
            'hidden_sizes': [hidden_sizes],
            'n_trials': [_ for _ in range(n_trials)],
            'n_data': [None]
        }

        specifications = SpecificationGenerator().generate(
            generation_specification)
        name = names["6dof"]
        runner = ExperimentRunner()
        runner.run(name, specifications, VAETrial(), force_pickle=True)

    print("Done.")


text:
import numpy as np
import torch

def buffer_2_cluster_buffer(buffer, cluster_buffer_size, device):
    idxs = np.random.randint(0, buffer.size, cluster_buffer_size)
    obs_buffer = torch.as_tensor(buffer.obs_buf[idxs], dtype = torch.float32, device = device)
    acts_buffer = torch.as_tensor(buffer.acts_buf[idxs], dtype = torch.float32, device = device)
    rews_buffer = torch.as_tensor(buffer.rew_buf[idxs], dtype = torch.float32, device = device).unsqueeze(-1)
    state_action = torch.cat((obs_buffer, acts_buffer), dim=1).cpu()
    return state_action, rews_buffer

def expert_2_cluster_buffer(all_trajectories, cluster_buffer_size, device):
    buffer_size = all_trajectories['states'].shape[0]
    idxs = np.random.randint(0, buffer_size, cluster_buffer_size)
    # 存在device的问题，还没修改，现在的device恒为cpu，但是上面的off policy的device默认是gpu的
    obs_buffer = all_trajectories['states'][idxs]
    acts_buffer = all_trajectories['actions'][idxs]
    rews_buffer = torch.as_tensor(all_trajectories['rewards'][idxs], dtype = torch.float32, device = device).unsqueeze(-1)
    state_action = torch.cat((obs_buffer, acts_buffer), dim=1)
    return state_action, rews_buffer

max_stars_repo_name: PamiRL/SelectiveSampling

max_issues_repo_name: PamiRL/SelectiveSampling

max_forks_repo_name: PamiRL/SelectiveSampling


text:
import argparse
from math import ceil

import matplotlib
matplotlib.use('Agg')
from matplotlib.patches import FancyBboxPatch
from matplotlib import pylab as plt
from matplotlib import rc

from nipy import load_image
from nipy.core.api import xyz_affine
from nipy.labs.viz import plot_map

import numpy as np
from sys import stdout
import pickle

cdict = {'red': ((0.0, 0.0, 0.0),
                 (0.25, 0.2, 0.2),
                 (0.45, 0.0, 0.0),
                 (0.5, 0.5, 0.5),
                 (0.55, 0.0, 0.0),
                 (0.75, 0.8, 0.8),
                 (1.0,  1.0, 1.0)),
         'green': ((0.0, 0.0, 1.0),
                   (0.25, 0.0, 0.0),
                   (0.45, 0.0, 0.0),
                   (0.5, 0.5, 0.5),
                   (0.55, 0.0, 0.0),
                   (0.75, 0.0, 0.0),
                   (1.0,  1.0, 1.0)),
         'blue':  ((0.0, 0.0, 1.0),
                   (0.25, 0.8, 0.8),
                   (0.45, 0.0, 0.0),
                   (0.5, 0.5, 0.5),
                   (0.55, 0.0, 0.0),
                   (0.75, 0.0, 0.0),
                   (1.0,  0.0, 0.0)),}

cmap = matplotlib.colors.LinearSegmentedColormap('my_colormap',cdict,256)

def montage(nifti, anat, roi_dict, thr=2,
            fig=None, out_file=None, feature_dict=None,
            target_stat=None, target_value=None):
    if isinstance(anat, str):
        anat = load_image(anat)
    assert nifti is not None
    assert anat is not None
    assert roi_dict is not None

    texcol = 1
    bgcol = 0
    iscale = 2
    weights = nifti.get_data(); #weights = weights / weights.std(axis=3)
    features = weights.shape[-1]

    indices = [0]
    y = 8
    x = int(ceil(1.0 * features / y))

    font = {'size':8}
    rc('font',**font)

    if fig is None:
        fig = plt.figure(figsize=[iscale * y, iscale * x / 2.5])
    plt.subplots_adjust(left=0.01, right=0.99, bottom=0.01, top=0.99, wspace=0.1, hspace=0)

    for f in xrange(features):
        roi = roi_dict.get(f, None)
        if roi is None:
            continue
        coords = roi["top_clust"]["coords"]
        assert coords is not None

#        stdout.write("\rSaving montage: %d   " % f)
#        stdout.flush()

        feat = weights[:, :, :, f]
        feat = feat / feat.std()
        imax = np.max(np.absolute(feat)); imin = -imax
        imshow_args = {'vmax': imax, 'vmin': imin}

        # For some reason we need to do this.
        coords = ([-coords[0], -coords[1], coords[2]])

        ax = fig.add_subplot(x, y, f + 1)
        plt.axis('off')

        max_idx = np.unravel_index(np.argmax(feat), feat.shape)

        try: plot_map(feat,
                      xyz_affine(nifti),
                      anat=anat.get_data(),
                      anat_affine=xyz_affine(anat),
                      threshold=thr,
                      figure=fig,
                      axes=ax,
                      cut_coords=coords,
                      annotate=False,
                      cmap=cmap,
                      draw_cross=False,
                      **imshow_args)
        except Exception as e:
            print e

        plt.text(0.05, 0.8, str(f),
                 transform=ax.transAxes,
                 horizontalalignment='center',
                 color=(texcol,texcol,texcol))
        pos = [(0.05, 0.05), (0.4, 0.05), (0.8, 0.05)]
        colors = ["purple", "yellow", "green"]
        if feature_dict is not None and feature_dict.get(f, None) is not None:
            d = feature_dict[f]
            for i, key in enumerate([k for k in d if k != "real_id"]):
                plt.text(pos[i][0], pos[i][1], "%s=%.2f" % (key, d[key]) ,transform=ax.transAxes,
                         horizontalalignment="left", color=colors[i])
                if key == target_stat:
                    assert target_value is not None
                    if d[key] >= target_value:
                        p_fancy = FancyBboxPatch((0.1, 0.1), 2.5 - .1, 1 - .1,
                                                 boxstyle="round,pad=0.1",
                                                 ec=(1., 0.5, 1.),
                                                 fc="none")
                        ax.add_patch(p_fancy)
                    elif d[key] <= -target_value:
                        p_fancy = FancyBboxPatch((0.1, 0.1), iscale * 2.5 - .1, iscale - .1,
                                                 boxstyle="round,pad=0.1",
                                                 ec=(0., 0.5, 0.),
                                                 fc="none")
                        ax.add_patch(p_fancy)

#    stdout.write("\rSaving montage: DONE\n")
    if out_file is not None:
        plt.savefig(out_file, transparent=True, facecolor=(bgcol, bgcol, bgcol))
    else:
        plt.draw()

def make_argument_parser():
    """
    Creates an ArgumentParser to read the options for this script from
    sys.argv
    """
    parser = argparse.ArgumentParser()
    parser.add_argument("nifti", help="Nifti file to be processed.")
    parser.add_argument("--anat", default=None, help="Anat file for montage.")
    parser.add_argument("--rois", default=None, help="Pickled roi file.")
    parser.add_argument("--out", default=None, help="Output of montage.")
    parser.add_argument("--thr", default=2, help="Threshold for features.")
    return parser

def main(nifti_file, anat_file, roi_file, out_file, thr=2):
    iscale = 2
    nifti = load_image(nifti_file)
    anat = load_image(anat_file)
    roi_dict = pickle.load(open(roi_file, "rb"))
    montage(nifti, anat, roi_dict, out_file=out_file)

if __name__ == "__main__":
    parser = make_argument_parser()
    args = parser.parse_args()
    main(args.nifti, args.anat, args.rois, args.out, args.thr)


text:
import numpy as np
from numpy.random import default_rng

__all__ = ['limit', 'limit_inverse', 'wang', 'rand', 'reflect']

def limit(x, lower, upper, **_kwargs):
    r"""Repair solution and put the solution in the random position inside of the bounds of problem.

    Args:
        x (numpy.ndarray): Solution to check and repair if needed.
        lower (numpy.ndarray): Lower bounds of search space.
        upper (numpy.ndarray): Upper bounds of search space.

    Returns:
        numpy.ndarray: Solution in search space.

    """
    return np.clip(x, lower, upper, out=x)

def limit_inverse(x, lower, upper, **_kwargs):
    r"""Repair solution and put the solution in the random position inside of the bounds of problem.

    Args:
        x (numpy.ndarray): Solution to check and repair if needed.
        lower (numpy.ndarray): Lower bounds of search space.
        upper (numpy.ndarray): Upper bounds of search space.

    Returns:
        numpy.ndarray: Solution in search space.

    """
    ir = np.where(x < lower)
    x[ir] = upper[ir]
    ir = np.where(x > upper)
    x[ir] = lower[ir]
    return x

def wang(x, lower, upper, **_kwargs):
    r"""Repair solution and put the solution in the random position inside of the bounds of problem.

    Args:
        x (numpy.ndarray): Solution to check and repair if needed.
        lower (numpy.ndarray): Lower bounds of search space.
        upper (numpy.ndarray): Upper bounds of search space.

    Returns:
        numpy.ndarray: Solution in search space.

    """
    ir = np.where(x < lower)
    x[ir] = np.amin([upper[ir], 2 * lower[ir] - x[ir]], axis=0)
    ir = np.where(x > upper)
    x[ir] = np.amax([lower[ir], 2 * upper[ir] - x[ir]], axis=0)
    return x

def rand(x, lower, upper, rng=None, **_kwargs):
    r"""Repair solution and put the solution in the random position inside of the bounds of problem.

    Args:
        x (numpy.ndarray): Solution to check and repair if needed.
        lower (numpy.ndarray): Lower bounds of search space.
        upper (numpy.ndarray): Upper bounds of search space.
        rng (numpy.random.Generator): Random generator.

    Returns:
        numpy.ndarray: Fixed solution.

    """
    rng = default_rng(rng)
    ir = np.where(x < lower)
    x[ir] = rng.uniform(lower[ir], upper[ir])
    ir = np.where(x > upper)
    x[ir] = rng.uniform(lower[ir], upper[ir])
    return x

def reflect(x, lower, upper, **_kwargs):
    r"""Repair solution and put the solution in search space with reflection of how much the solution violates a bound.

    Args:
        x (numpy.ndarray): Solution to be fixed.
        lower (numpy.ndarray): Lower bounds of search space.
        upper (numpy.ndarray): Upper bounds of search space.

    Returns:
        numpy.ndarray: Fix solution.

    """
    ir = np.where(x > upper)
    x[ir] = lower[ir] + x[ir] % (upper[ir] - lower[ir])
    ir = np.where(x < lower)
    x[ir] = lower[ir] + x[ir] % (upper[ir] - lower[ir])
    return x

reason: import numpy,from numpy


text:
import numpy as np		#we use numpy for a lot of things

def main():
	i=0		#integer i and initialize i to 0
	n=10	#integer n, with value 10
	x=119.0	#float x with value 119

	#can use numpy to declare arrays quickly

	y=np.zeros(n,dtype=float)	#declares 10 zeros

	#we can iterate trough the elements of y
	#by passing an index

	for i in range(n):	#i in range [0,n-1]
		y[i] = 2.0 * float(i) + 1.0		#set y = 2i+1 as float

	#we can also simply iterate through an array
	for y_element in y:
		print(y_element)

#execute the main function
if __name__ == "__main__":
	main()


text:
import numpy as np
import math

def dotproduct(v1, v2):
    return sum((a * b) for a, b in zip(v1, v2))

def length(v):
    return math.sqrt(dotproduct(v, v))

def angle(v1, v2):
    try:
        if length(v1) * length(v2) == 0:
            radians = 0
        else:
            radians = math.acos(dotproduct(v1, v2) / (length(v1) * length(v2)))
    except Exception as e:
        # print str(e), ': math.acos(', dotproduct(v1, v2) / (length(v1) * length(v2)), '), v1 =', v1, ', v2 =', v2
        radians = 0
    return (float(radians) * 180.0) / math.pi

##### Feature base class #######

class Feature(object):
    name = "Feature"
    plugin = "Tracking Features"
    default_value = 0
    dimensionality = None

    def __init__(
        self,
        feats_name,
        default_value=None,
        delim="_",
        scales=[1.0, 1.0, 1.0],
        ndim=2,
        feat_dim=1,
    ):
        self.name += str(delim) + str(feats_name)
        self.feats_name = feats_name
        if default_value != None:
            self.default_value = default_value
        self.scales = scales
        self.ndim = ndim
        self.feat_dim = feat_dim

    def compute(self, feats_cur, feats_next, **kwargs):
        raise NotImplementedError("Feature not fully implemented yet.")

    def getName(self):
        return self.name

    def getPlugin(self):
        return self.plugin

    def dim(self):
        return self.dimensionality

class ParentChildrenRatio(Feature):
    name = "ParentChildrenRatio"
    dimensionality = 1

    def compute(self, feats_cur, feats_next, **kwargs):
        if len(feats_next) < 2:
            return np.array(len(feats_cur) * [self.default_value])
        result = np.array(feats_cur) / np.array(feats_next[0] + feats_next[1])
        for i in range(len(result)):
            if math.isnan(result[i]):
                result[i] = self.default_value
        return result

    def dim(self):
        return self.dimensionality * self.feat_dim

class ChildrenRatio(Feature):
    name = "ChildrenRatio"
    dimensionality = 1

    def compute(self, feats_cur, feats_next, **kwargs):
        if len(feats_next) < 2:
            return np.array(len(feats_cur) * [self.default_value])
        ratio = np.array(feats_next[0]) / np.array(feats_next[1])
        for i in range(len(ratio)):
            if math.isnan(ratio[i]):
                ratio[i] = self.default_value
            if ratio[i] > 1 and ratio[i] != 0:
                ratio[i] = 1.0 / ratio[i]
        return ratio

    def dim(self):
        return self.dimensionality * self.feat_dim

class SquaredDistances(Feature):
    name = "SquaredDistances"

    def compute(self, feats_cur, feats_next, **kwargs):
        return feats_cur

    def dim(self):
        return self.ndim

class ParentChildrenAngle(Feature):
    name = "ParentChildrenAngle"
    dimensionality = 1

    def compute(self, feats_cur, feats_next, **kwargs):
        angles = []
        for idx, com1 in enumerate(feats_next):
            v1 = (com1 - feats_cur) * self.scales[0 : com1.shape[0]]
            for com2 in feats_next[idx + 1 :]:
                v2 = (com2 - feats_cur) * self.scales[0 : com2.shape[0]]
                ang = angle(v1, v2)
                if ang > 180:
                    assert ang <= 360.01, "the angle must be smaller than 360 degrees"
                    ang = 360 - ang
                angles.append(ang)

        if len(angles) == 0:
            angles = [self.default_value]

        return max(angles)

class ParentIdentity(Feature):
    name = ""

    def compute(self, feats_cur, feats_next, **kwargs):
        return feats_cur

class FeatureManager(object):

    feature_mappings = {
        "ParentIdentity": ParentIdentity,
        "SquaredDistances": SquaredDistances,
        "ChildrenRatio": ChildrenRatio,
        "ParentChildrenRatio": ParentChildrenRatio,
        "ParentChildrenAngle": ParentChildrenAngle,
    }

    def __init__(
        self,
        scales=[1.0, 1.0, 1.0],
        n_best=3,
        com_name_cur="RegionCenter",
        com_name_next="RegionCenter",
        size_name="Count",
        delim="_",
        template_size=50,
        ndim=2,
        size_filter=4,
        squared_distance_default=9999,
    ):
        self.scales = scales[0:ndim]
        self.n_best = n_best
        self.com_name_cur = com_name_cur
        self.com_name_next = com_name_next
        self.size_name = size_name
        self.delim = delim
        self.template_size = template_size
        self.ndim = ndim
        self.size_filter = size_filter
        self.squared_distance_default = squared_distance_default

    def _getBestSquaredDistances(
        self, com_cur, coms_next, size_filter=None, sizes_next=[], default_value=9999
    ):
        """ returns the squared distances to the objects in the neighborhood of com_curr, optionally with size filter """
        squaredDistances = []

        for label_next in coms_next.keys():
            assert label_next in sizes_next.keys()
            if size_filter != None and sizes_next[label_next] >= size_filter:
                dist = np.linalg.norm(coms_next[label_next] - com_cur * self.scales)
                squaredDistances.append([label_next, dist])

        squaredDistances = np.array(squaredDistances)
        # sort the array in the second column in ascending order
        squaredDistances = np.array(
            sorted(squaredDistances, key=lambda a_entry: a_entry[1])
        )

        # initialize with label -1 and default value
        result = np.array(
            [[-1, default_value] for x in range(self.n_best)], dtype=np.float32
        )
        if squaredDistances.shape[0] != 0:
            result[
                0 : min(squaredDistances.shape[0], result.shape[0]), :
            ] = squaredDistances[0 : min(squaredDistances.shape[0], result.shape[0]), :]

        return result

    def computeFeatures_at(
        self, feats_cur, feats_next, img_next, feat_names, label_image_filename=None
    ):
        """
        **Parameters:**

        * if `label_image_filename` is given, it is used to filter the objects from the feature dictionaries
          that belong to that label image only (in the JST setting)
        """

        #        n_labels = list(feats_cur.values())[0].shape[0]
        result = {}

        # find available features
        vigra_feat_names = set([self.com_name_cur, self.com_name_next, self.size_name])
        feat_classes = {}

        for name in feat_names:
            name_split = name.split(self.delim)
            if "SquaredDistances" in name_split:
                continue

            if len(name_split) != 2:
                raise ValueError(
                    "tracking features consist of an operator and a feature name only, given name={}".format(
                        name_split
                    )
                )
            if len(feats_cur[name_split[1]].shape) > 1:
                feat_dim = feats_cur[name_split[1]].shape[1]
            else:
                feat_dim = 1
            feat_classes[name] = self.feature_mappings[name_split[0]](
                name_split[1], delim=self.delim, ndim=self.ndim, feat_dim=feat_dim
            )

            shape = (list(feats_cur.values())[0].shape[0], feat_classes[name].dim())
            result[name] = np.ones(shape) * feat_classes[name].default_value

            vigra_feat_names.add(name_split[1])

        # initialize squared distances
        for idx in range(self.n_best):
            name = "SquaredDistances_" + str(idx)
            result[name] = (
                np.ones((list(feats_cur.values())[0].shape[0], 1))
                * self.squared_distance_default
            )

        # construct mapping which we only need if label_image_filename was given and the features 'filename' and 'id' exist
        if (
            label_image_filename is not None
            and "filename" in feats_next
            and "id" in feats_next
        ):
            global_indices_current_label_image_only = [
                l
                for l, f in enumerate(feats_next["filename"])
                if f == label_image_filename
            ]
            local_to_global_index_map = dict(
                [
                    (feats_next["id"][l], l)
                    for l in global_indices_current_label_image_only
                ]
            )

        # for every object in this frame, check which objects are in the vicinity in the next frame
        valid_indices = [0]
        for label_cur, com_cur in enumerate(feats_cur[self.com_name_cur]):
            if (
                label_image_filename is not None
                and "filename" in feats_cur
                and feats_cur["filename"][label_cur] != label_image_filename
            ):
                # in the JST context, only look at objects from a given segmentation hypotheses set
                continue
            if label_cur == 0:
                continue

            valid_indices.append(label_cur)
            feats_next_subset = {}
            for k in vigra_feat_names:
                feats_next_subset[k] = {}

            if feats_next is not None and img_next is not None:
                # find roi around the center of the current object
                idx_cur = [round(x) for x in com_cur]

                roi = []
                for idx, coord in enumerate(idx_cur):
                    start = max(coord - self.template_size / 2, 0)
                    stop = min(coord + self.template_size / 2, img_next.shape[idx])
                    roi.append(slice(int(start), int(stop)))

                # find all coms in the neighborhood of com_cur by checking the next frame's labelimage in the roi
                subimg_next = img_next[roi]
                labels_next = np.unique(subimg_next).tolist()

                # if 'id' in features, map the labels first -- because labels_next refers image object ids,
                # whereas the features are the union of objects from several segmentations
                if "id" in feats_next:
                    labels_next = [
                        local_to_global_index_map[l] for l in labels_next if l != 0
                    ]

                for l in labels_next:
                    if l != 0:
                        for n in vigra_feat_names:
                            feats_next_subset[n][l] = np.array(
                                [feats_next[n][l]]
                            ).flatten()

            sq_dist_label = self._getBestSquaredDistances(
                com_cur,
                feats_next_subset[self.com_name_next],
                self.size_filter,
                feats_next_subset[self.size_name],
                default_value=self.squared_distance_default,
            )

            feats_next_subset_best = {}
            for n in vigra_feat_names:
                feats_next_subset_best[n] = []
                for idx, row in enumerate(sq_dist_label):
                    l = row[0]
                    if l != -1:
                        feats_next_subset_best[n].append(feats_next_subset[n][l])

            # first add squared distances
            for idx in range(self.n_best):
                name = "SquaredDistances_" + str(idx)
                result[name][label_cur] = sq_dist_label[idx][1]

            # add all other features
            for name, feat_class in feat_classes.items():
                if feat_class.feats_name == "SquaredDistances":
                    f_next = sq_dist_label[0:2, 1]
                    f_cur = None
                else:
                    f_cur = np.array(
                        [feats_cur[feat_class.feats_name][label_cur]]
                    ).flatten()
                    f_next = np.array(
                        [feats_next_subset_best[feat_class.feats_name]]
                    ).reshape((-1, f_cur.shape[0]))
                result[name][label_cur] = feat_class.compute(f_cur, f_next)

        # return only valid labels
        for feature_name in result:
            result[feature_name] = result[feature_name][valid_indices]

        return result

if __name__ == "__main__":
    import vigra
    import numpy as np

    img_cur = vigra.readImage("/home/mschiegg/tmp/segmentImage.tif")
    img_next = img_cur

    labels_cur = vigra.analysis.labelImage(img_cur)
    feats_cur = vigra.analysis.extractRegionFeatures(
        labels_cur.astype(np.float32),
        labels_cur.astype(np.uint32),
        features="all",
        ignoreLabel=0,
    )

    feat_names = [
        "ParentChildrenRatio_Count",
        "ParentChildrenRatio_Mean",
        "ChildrenRatio_Count",
        "ChildrenRatio_Mean",
        "ParentChildrenAngle_RegionCenter",
        "ChildrenRatio_SquaredDistances",
    ]
    fm = FeatureManager()
    res = fm.computeFeatures_at(feats_cur, feats_cur, img_cur, feat_names)

max_stars_repo_name: m-novikov/hytra

max_issues_repo_name: m-novikov/hytra

max_forks_repo_name: m-novikov/hytra


text:
import numpy as np
import scipy.stats

from sklearn.preprocessing import quantile_transform, robust_scale

def load_dataset(oid_file, feature_file, feature_names_file=None):
    """
    Load a dataset.

    Parameters
    ----------
    oid_file: file with stored numpy array of oids
    feature_file: file with stored numpy array of corresponding featues
    feature_names_file: file with features' names (Optional, default: None).

    Return
    ------
    Loaded dataset.
    """
    oid = np.memmap(oid_file, mode='r', dtype=np.uint64)

    if feature_names_file is not None:
        with open(feature_names_file) as fo:
            names = fo.read().split()

        dt = [(name, np.float32) for name in names]
        features = np.memmap(feature_file, mode='r', dtype=dt, shape=oid.shape)
    else:
        features = np.memmap(feature_file, mode='r', dtype=np.float32)
        features = features.reshape(oid.size, -1)

    return oid, features

def concat_datasets(*args):
    """
    Concatenate datasets.

    Parameters
    ----------
    args: pairs of datasets (oids, features).

    Return
    ------
    Resulting dataset, also a pair (oids, features).
    """
    oids, features = zip(*args)
    return np.concatenate(oids, axis=0), np.concatenate(features, axis=0)

def scale_values(features, algorithm=None):
    """
    Scale the table of features.

    Parameters
    ----------
    features: Matrix of N-by-M values with N experiments and M features.
    algorithm: Scaling algorithm: 'minmax', 'std'(default), 'pca' or 'norm'.
    Minmax algorithm scales to the [0; 1] interval, while std algorithm
    scales to the zero mean and unitary standard deviation. PCA algorithm
    performs linear transform to principal axes of covariance ellipsoid.
    'norm' transforms the empirical distribution of features to a more normal
    one.

    Return
    ------
    N-by-M matrix of scaled features.
    """
    algorithm = algorithm or 'std'

    if algorithm == 'std':
        mean = features.mean(axis=0)
        std = features.std(axis=0)
        return (features - mean) / np.maximum(std, np.finfo(np.float).eps)
    elif algorithm == 'pca':
        mean = features.mean(axis=0)
        u, _, _ = np.linalg.svd(features - mean, full_matrices=False)
        return u
    elif algorithm == 'minmax':
        minis = features.min(axis=0)
        maxis = features.max(axis=0)
        delta = maxis - minis
        delta[delta == 0] = 1.0
        return (features - minis) / delta
    elif algorithm == 'norm':
        return quantile_transform(features, output_distribution='normal', copy=True)
    else:
        raise ValueError('Unkown scale algorithm: {}'.format(algorithm))

max_stars_repo_name: snad-space/zwad

max_issues_repo_name: snad-space/zwad

max_forks_repo_name: snad-space/zwad

reason: import numpy,import scipy


text:
import matplotlib
matplotlib.use('Agg')  # backend
from matplotlib import pyplot as plt
import numpy as np

try:
    data = np.loadtxt('cond.txt')
except:
    print "*** Error, could not read file cond.txt with loadtxt"
    data = None

if data is not None:
    n = data[:,0]     # first column
    cond = data[:,1]  # second column
else:
    n = np.linspace(1,20,20)

cond_formula = 0.01133 * np.exp(3.49*n)

plt.figure()

# semilog plot:
plt.semilogy(n,cond_formula,'ko-',label='from formula')
if data is not None:
    plt.semilogy(n,cond,'rx-',label='estimated')
plt.legend(loc='upper left')
plt.title("Condition number of n by n Hilbert matrix")
plt.xlabel('n')

fname = 'cond.png'
plt.savefig(fname)
print "Created ",fname

max_stars_repo_name: philipwangdk/HPC

max_issues_repo_name: philipwangdk/HPC

max_forks_repo_name: philipwangdk/HPC


text:
# import nltk
# from nltk.corpus import stopwords
# from nltk.stem import WordNetLemmatizer
import csv
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn import tree
from sklearn.linear_model import SGDClassifier
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier

# 预处理
def preprocessing(text):
    # # text=text.decode("utf-8")
    # tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
    # stops = stopwords.words('english')
    # tokens = [token for token in tokens if token not in stops]
    #
    # tokens = [token.lower() for token in tokens if len(token) >= 3]
    # lmtzr = WordNetLemmatizer()
    # tokens = [lmtzr.lemmatize(token) for token in tokens]
    # preprocessed_text = ' '.join(tokens)
    preprocessed_text = text
    return preprocessed_text

# 读取数据集
file_path = r'C:\Users\Administrator\Desktop\SMSSpamCollectionjsn.txt'
sms = open(file_path, 'r', encoding='utf-8')
sms_data = []
sms_label = []
csv_reader = csv.reader(sms, delimiter='\t')
for line in csv_reader:
    sms_label.append(line[0])
    sms_data.append(preprocessing(line[1]))
sms.close()
# print(sms_data)

# 按0.7：0.3比例分为训练集和测试集，再将其向量化
dataset_size = len(sms_data)
trainset_size = int(round(dataset_size * 0.7))
print('dataset_size:', dataset_size, '   trainset_size:', trainset_size)
x_train = np.array([''.join(el) for el in sms_data[0:trainset_size]])
y_train = np.array(sms_label[0:trainset_size])

x_test = np.array(sms_data[trainset_size + 1:dataset_size])
y_test = np.array(sms_label[trainset_size + 1:dataset_size])

vectorizer = TfidfVectorizer(min_df=2, ngram_range=(1, 2), stop_words='english', strip_accents='unicode', norm='l2')

X_train = vectorizer.fit_transform(x_train)
X_test = vectorizer.transform(x_test)

# 朴素贝叶斯分类器
clf = MultinomialNB().fit(X_train, y_train)
y_nb_pred = clf.predict(X_test)

print(y_nb_pred)
print('nb_confusion_matrix:')
cm = confusion_matrix(y_test, y_nb_pred)
print(cm)
print('nb_classification_report:')
cr = classification_report(y_test, y_nb_pred)
print(cr)

feature_names = vectorizer.get_feature_names()
coefs = clf.coef_
intercept = clf.intercept_
coefs_with_fns = sorted(zip(coefs[0], feature_names))
n = 10
top = zip(coefs_with_fns[:n], coefs_with_fns[:-(n + 1):-1])
for (coef_1, fn_1), (coef_2, fn_2) in top:
    print('\t%.4f\t%-15s\t\t%.4f\t%-15s' % (coef_1, fn_1, coef_2, fn_2))

# 决策树
clf = tree.DecisionTreeClassifier().fit(X_train.toarray(), y_train)
y_tree_pred = clf.predict(X_test.toarray())

print('tree_confusion_matrix:')
cm = confusion_matrix(y_test, y_tree_pred)
print(cm)
print('tree_classification_report:')
print(classification_report(y_test, y_tree_pred))

# SGD
clf = SGDClassifier(alpha=0.0001, n_iter=50).fit(X_train, y_train)
y_SGD_pred = clf.predict(X_test)
print('SGD_confusion_matrix:')
cm = confusion_matrix(y_test, y_SGD_pred)
print(cm)
print('SGD_classification_report:')
print(classification_report(y_test, y_SGD_pred))

# svm
clf = LinearSVC().fit(X_train, y_train)
y_svm_pred = clf.predict(X_test)
print('svm_confusion_matrix:')
cm = confusion_matrix(y_test, y_svm_pred)
print(cm)
print('svm_classification_report:')
print(classification_report(y_test, y_svm_pred))

# RandomForestClassifier
clf = RandomForestClassifier(n_estimators=10)
clf.fit(X_train, y_train)
y_RF_pred = clf.predict(X_test)
print('RF_confusion_matrix:')
print(confusion_matrix(y_test, y_RF_pred))
print('RF_classification_report:')
print(classification_report(y_test, y_RF_pred))

max_stars_repo_name: ivitan/LearnPython

max_issues_repo_name: ivitan/LearnPython

max_forks_repo_name: ivitan/LearnPython


text:
# Copyright 2019 The Cacophony Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from io import BytesIO
from datetime import datetime, timedelta, timezone

import numpy as np
from pytest import approx

from cptv import CPTVWriter, CPTVReader
from cptv.frame import Frame

def test_round_trip_header_defaults():
    buf = BytesIO()

    w = CPTVWriter(buf)
    w.write_header()
    w.close()

    buf.seek(0, 0)

    r = CPTVReader(buf)
    assert r.version == 2
    assert r.x_resolution == 160
    assert r.y_resolution == 120
    assert not r.device_name
    assert not r.device_id
    assert (datetime.now(tz=timezone.utc) - r.timestamp) < timedelta(minutes=1)
    assert r.latitude == 0
    assert r.longitude == 0

    assert r.accuracy == 0
    assert r.altitude == 0
    assert r.fps == 0
    assert r.model is None
    assert r.brand is None
    assert r.firmware is None
    assert r.camera_serial == 0
    assert r.background_frames == 0

def test_round_trip_header():
    buf = BytesIO()

    w = CPTVWriter(buf)
    w.timestamp = datetime(2018, 7, 6, 5, 4, 3, tzinfo=timezone.utc)
    w.device_name = b"hello"
    w.device_id = 42
    w.latitude = 142.3
    w.longitude = -39.2
    w.loc_timestamp = datetime(2018, 9, 6, 5, 4, 3, tzinfo=timezone.utc)

    w.preview_secs = 3
    w.motion_config = b"blob"
    w.accuracy = 20
    w.altitude = 200
    w.fps = 30
    w.model = b"ultra"
    w.brand = b"laser"
    w.firmware = b"killer"
    w.camera_serial = 221
    back_frame = random_frame(60, 30)
    back_frame.background_frame = True
    w.background_frame = back_frame
    w.write_header()
    for i in range(10):
        frame = random_frame(60, 30)
        w.write_frame(frame)
    w.close()

    buf.seek(0, 0)

    r = CPTVReader(buf)
    assert r.version == 2
    assert r.x_resolution == 160
    assert r.y_resolution == 120
    assert r.timestamp == w.timestamp
    assert r.device_name == w.device_name
    assert r.device_id == w.device_id
    assert r.latitude == approx(w.latitude, 0.000001)
    assert r.longitude == approx(w.longitude, 0.000001)
    assert r.loc_timestamp == w.loc_timestamp
    assert r.preview_secs == w.preview_secs
    assert r.motion_config == w.motion_config

    assert r.accuracy == w.accuracy
    assert r.altitude == w.altitude
    assert r.fps == w.fps
    assert r.model == w.model
    assert r.brand == w.brand
    assert r.firmware == w.firmware
    assert r.camera_serial == w.camera_serial
    assert r.background_frames == 1
    count = 0
    for frame in r:
        if count == 0:
            assert frame.background_frame
        else:
            assert frame.background_frame == False
        count += 1
    assert count == 11

def test_one_frame():
    check_frames([random_frame(60, 30)])

def test_random_frames():
    check_frames([random_frame(60, 30), random_frame(61, 31), random_frame(62, 32)])

def test_minimal_change():
    frame0 = random_frame(0, 0)

    # Change one pixel
    pix1 = frame0.pix.copy()
    pix1[0, 0] += 1
    frame1 = new_frame(pix1)

    check_frames([frame0, frame1])

def test_step_change():
    frame0 = random_frame(0, 0)

    pix1 = frame0.pix.copy()
    pix1 += 1
    frame1 = new_frame(pix1)

    check_frames([frame0, frame1])

def test_large_value():
    frame0 = random_frame(0, 0)

    pix1 = frame0.pix.copy()
    pix1[0] += 32767
    frame1 = new_frame(pix1)

    check_frames([frame0, frame1])

def check_frames(frames):
    buf = BytesIO()
    w = CPTVWriter(buf)
    w.write_header()
    for frame in frames:
        w.write_frame(frame)
    w.close()

    buf.seek(0, 0)

    r = CPTVReader(buf)
    count = 0
    for in_frame, out_frame in zip(frames, r):
        assert in_frame == out_frame
        count += 1

    assert count == len(frames)

def new_frame(pix):
    return Frame(pix, timedelta(seconds=0), timedelta(seconds=0), 0.0, 0.0)

def random_frame(time_on, last_ffc_time, min_value=None, max_value=None):
    if min_value is None:
        min_value = 3000
    if max_value is None:
        max_value = 6000
    return Frame(
        np.random.randint(min_value, max_value, (120, 160), "uint16"),
        timedelta(seconds=time_on),
        timedelta(seconds=last_ffc_time),
        np.random.randint(2800, 3000),
        np.random.randint(2800, 3000),
    )

max_stars_repo_name: gferraro/python-cptv

max_issues_repo_name: gferraro/python-cptv

max_forks_repo_name: gferraro/python-cptv