text:
# Agglomerative Clustering Demo
# Author: Animesh Gupta

import superimport

import numpy as np
from scipy.cluster.hierarchy import dendrogram, linkage
import matplotlib.pyplot as plt
import pyprobml_utils as pml

X = np.array([[1,2],
    [2.5,4.5],
    [2,2],
    [4,1.5],
    [4,2.5],])

labels = range(1, 6)
plt.figure(figsize=(10, 6))
plt.yticks(np.linspace(0,5,11))
plt.ylim(0,5)
plt.grid(color='gray', linestyle='dashed')
plt.scatter(X[:,0],X[:,1], label='True Position')

for label, x, y in zip(labels, X[:, 0], X[:, 1]):
    plt.annotate(
        label,
        xy=(x, y), xytext=(-3, 3),
        textcoords='offset points', ha='right', va='bottom', fontsize=25, color="red")
pml.savefig("agglom_demo_data.pdf", dpi=300)

linked = linkage(X, 'single')

labelList = range(1, 6)

plt.figure(figsize=(10, 7))
dendrogram(linked,
            orientation='top',
            labels=labelList,
            distance_sort='descending',
            show_leaf_counts=True)
pml.savefig("agglom_demo_dendrogram.pdf", dpi=300)

plt.show()

max_stars_repo_name: vipavlovic/pyprobml

max_issues_repo_name: vipavlovic/pyprobml

max_forks_repo_name: vipavlovic/pyprobml

reason: import numpy,from scipy


text:
import numpy as np
import torch
from torch import nn

def MetropolisWithAccept(energy,x,length,tranCore = None):
    torch.set_grad_enabled(False)
    shape = [i if no==0 else 1 for no,i in enumerate(x.shape)]
    if tranCore is None:
        def t(x):
            return x + torch.randn_like(x)
        tranCore = t
    E = energy(x)
    for l in range(length):
        xnew = tranCore(x)
        Enew = energy(xnew)

        diff = E-Enew
        accept = (diff.exp()>=diff.uniform_()).to(x)

        E = accept*Enew + (1.-accept)*E
        acceptMask = accept.reshape(shape)
        x = acceptMask*xnew+(1.-acceptMask)*x
    torch.set_grad_enabled(True)

    return x, accept

def Metropolis(*args,**kwargs):
    x, _ = MetropolisWithAccept(*args,**kwargs)
    return x

class MetroplolisSampler(nn.Module):
    def __init__(self,energy,nvars,thermalSteps = 25,tranCore = None):
        super(MetroplolisSampler,self).__init__()
        if tranCore is None:
            def t(x):
                return x + torch.randn_like(x)
            tranCore = t
        self.nvars = nvars
        self.energy = energy
        self.inital = Metropolis(self.energy,torch.randn(nvars),thermalSteps,self.tranCore)

    def step(self):
        self.inital = Metropolis(self.energy,torch.randn(nvars),thermalSteps,self.tranCore)
        return self.inital

ext: py

max_stars_repo_name: jameswind/neuralCT

max_issues_repo_name: jameswind/neuralCT

max_forks_repo_name: jameswind/neuralCT

reason: import numpy


text:
import numpy as np
from .base import Filter

class XYZFilter(Filter):
    def extract_info(self):
        self.points = self.pyntcloud.xyz

class BoundingBoxFilter(XYZFilter):
    """
    Compute a bounding box filter for the given points.

    Parameters
    ----------

    min_x, max_x, min_y, max_y, min_z, max_z: float
        The bounding box limits for each coordinate.
        If some limits are missing, the default values are -infinite
        for the min_i and infinite for the max_i.

    """

    def __init__(self, *, pyntcloud, min_x=-np.inf, max_x=np.inf, min_y=-np.inf,
                 max_y=np.inf, min_z=-np.inf, max_z=np.inf):
        super().__init__(pyntcloud=pyntcloud)
        self.min_x, self.max_x = min_x, max_x
        self.min_y, self.max_y = min_y, max_y
        self.min_z, self.max_z = min_z, max_z

    def compute(self):

        bound_x = np.logical_and(self.points[:, 0] > self.min_x,
                                 self.points[:, 0] < self.max_x)
        bound_y = np.logical_and(self.points[:, 1] > self.min_y,
                                 self.points[:, 1] < self.max_y)
        bound_z = np.logical_and(self.points[:, 2] > self.min_z,
                                 self.points[:, 2] < self.max_z)

        bb_filter = np.logical_and(np.logical_and(bound_x, bound_y), bound_z)

        return bb_filter

max_stars_repo_name: bernssolg/pyntcloud-master

max_issues_repo_name: bernssolg/pyntcloud-master

max_forks_repo_name: bernssolg/pyntcloud-master


text:
from PyQt5 import QtGui, QtCore, QtWidgets
import numpy as np
import pandas as pd
import os
import sys
import matplotlib
matplotlib.use('Qt5Agg')
import matplotlib.pyplot as plt
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg
from matplotlib.backends.backend_qt5agg import NavigationToolbar2QT
from matplotlib.figure import Figure
from matplotlib.patches import Rectangle

def set_or_none(iterable):
    if iterable is not None and not isinstance(iterable,set):
        iterable = set(iterable)
    return iterable

class SpectrumArtist():
    show_flagged = True
    show_unselected = True

    def __init__(self,artist):
        self.artist = artist
        self._flagged = False
        self._selected = False
        self._visible = True
        self.style = '-'
        self.color = 'k'

    @property
    def flagged(self):
        return self._flagged

    @flagged.setter
    def flagged(self,value):
        self._flagged = value
        self.color = 'r' if self._flagged else 'k'
        self._update_look()

    @property
    def selected(self):
        return self._selected

    @selected.setter
    def selected(self,value):
        self._selected = value
        self.style = '--' if self._selected else '-'
        self._update_look()

    @property
    def visible(self):
        return self._visible

    @visible.setter
    def visible(self,value):
        self._visible = value
        if self._visible:
            self.artist.set_linestyle(self.style)
        else:
            self.artist.set_linestyle('None')

    def _calculate_visibility(self):
        visible = True
        if not self.selected and not self.show_unselected:
            visible = False
        if self.flagged and not self.show_flagged:
            visible = False
        self.visible = visible

    def _update_look(self):
        self._calculate_visibility()
        if self.visible:
            self.artist.set_color(self.color)
            self.artist.set_linestyle(self.style)

class CollectionCanvas(FigureCanvasQTAgg):
    """Ultimately, this is a QWidget (as well as a FigureCanvasAgg, etc.)."""

    selected = QtCore.pyqtSignal(tuple)

    def __init__(self, parent=None, width=5, height=4, dpi=100):
        fig = Figure(figsize=(width, height), dpi=dpi)
        self.ax = fig.add_subplot(111)
        self.ax.grid(True)
        self._flag_style = 'r'
        self._unselected_style = '-'

        fig.tight_layout()
        FigureCanvasQTAgg.__init__(self, fig)
        self.setParent(parent)

        FigureCanvasQTAgg.setSizePolicy(self,
                                   QtWidgets.QSizePolicy.Expanding,
                                   QtWidgets.QSizePolicy.Expanding)
        FigureCanvasQTAgg.updateGeometry(self)

    @property
    def show_unselected(self):
        pass

    @show_unselected.setter
    def show_unselected(self,value):
        SpectrumArtist.show_unselected = value

    @property
    def show_flagged(self):
        pass

    @show_flagged.setter
    def show_flagged(self,value):
        SpectrumArtist.show_flagged = value

    def rectangleStartEvent(self,event):
        self._rect = None
        self._rect_start = event

    def rectangleMoveEvent(self,event):
        try:
            dx = event.xdata - self._rect_start.xdata
            dy = event.ydata - self._rect_start.ydata
        except TypeError:
            #we're out of canvas bounds
            return

        if self._rect is not None:
            self._rect.remove()

        self._rect = Rectangle((self._rect_start.xdata,self._rect_start.ydata),
                dx,dy, color='k',ls='--',lw=1,fill=False)
        self.ax.add_patch(self._rect)
        self.ax.draw_artist(self._rect)

    def rectangleEndEvent(self,event):
        class FakeEvent(object):
            def __init__(self,x,y):
                self.xdata, self.ydata = x, y

        if self._rect is not None:
            self._rect.remove()
        else:
            #make a small, fake rectangle
            dy = (self.ax.get_ylim()[1]-self.ax.get_ylim()[0])/100.
            self._rect_start = FakeEvent(event.xdata-10,event.ydata+dy)
            event = FakeEvent(event.xdata+10,event.ydata-dy)

        x0 = min(self._rect_start.xdata,event.xdata)
        x1 = max(self._rect_start.xdata,event.xdata)
        y0 = min(self._rect_start.ydata,event.ydata)
        y1 = max(self._rect_start.ydata,event.ydata)
        self.selected.emit((x0,x1,y0,y1))

    def _onMouseDown(self,event):
        if self.ax.get_navigate_mode() is None:
            self._bg_cache = self.copy_from_bbox(self.ax.bbox)
            self.clicked = True
            self.rectangleStartEvent(event)

    def _onMouseUp(self,event):
        if self.ax.get_navigate_mode() is None:
            self.restore_region(self._bg_cache)
            self.blit(self.ax.bbox)
            self.clicked = False
            self.rectangleEndEvent(event)

    def _onMouseMove(self,event):
        if self.ax.get_navigate_mode() is None:
            if(self.clicked):
                self.restore_region(self._bg_cache)
                self.rectangleMoveEvent(event)
                self.blit(self.ax.bbox)

    def setupMouseNavigation(self):
        self.clicked = False
        self.select_mode = 'rectangle'
        self._bg_cache = None

        self._cids = [
            self.mpl_connect('button_press_event',self._onMouseDown),
            self.mpl_connect('button_release_event',self._onMouseUp),
            self.mpl_connect('motion_notify_event',self._onMouseMove),
        ]

    def suspendMouseNavigation(self):
        for cid in self._cids:
            self.mpl_disconnect(cid)

    def update_selected(self,selected_keys,only_add=False):
        # better lookup time
        selected_keys = set_or_none(selected_keys)
        if only_add:
            # if we're only adding, just select
            for key in selected_keys:
                self.artist_dict[key].selected = True
        else:
            # otherwise, unselect everything that isn't selected
            keys = self.artist_dict.keys()
            for key in keys:
                self.artist_dict[key].selected = key in selected_keys
        self.draw()

    def set_flagged(self,flagged_keys,selected_keys=None,flag=True):
        # better lookup time
        flagged_keys = set_or_none(flagged_keys)
        selected_keys = set_or_none(selected_keys)
        for key in flagged_keys:
            self.artist_dict[key].flagged = flag

        self.draw()

    def add_flagged(self,unflagged_keys,selected_keys=None):
        self.set_flagged(unflagged_keys,selected_keys,True)

    def remove_flagged(self,unflagged_keys,selected_keys=None):
        self.set_flagged(unflagged_keys,selected_keys,False)

    def update_artists(self,collection,new_lim=False):
        if collection is None:
            return
        # save limits
        if new_lim == False:
            xlim = self.ax.get_xlim()
            ylim = self.ax.get_ylim()
        # plot
        self.ax.clear()
        collection.plot(ax=self.ax, style='k', picker=1)
        #self.ax.set_title(collection.name)
        keys = [s.name for s in collection.spectra]
        artists = self.ax.lines
        self.artist_dict = {key:SpectrumArtist(artist)
                for key,artist in zip(keys,artists)}
        for key in collection.flags:
            self.artist_dict[key].flagged = True
        self.ax.legend().remove()
        self.ax.grid(True)
        self.draw()

class ToolBar(NavigationToolbar2QT):
    def __init__(self,canvas_,parent,ax):
        NavigationToolbar2QT.__init__(self,canvas_,parent,coordinates=False)
        self._xlim = (0,1)
        self._ylim = (0,1)
        self._ax = ax
        self._canvas_ = canvas_
        self._addActions()

    def home(self):
        """Override home method to return to home of most recent plot"""
        self._ax.set_xlim(*self._xlim)
        self._ax.set_ylim(*self._ylim)
        self._canvas_.draw()

    def setHome(self,xlim,ylim):
        self._xlim = xlim
        self._ylim = ylim

    def _rebind_save(self):
        # find the save button from the matplotlib toolbar and rebind its action
        # This will probably break at some point
        action = [a for a in self.__actions if "Save" in a.toolTip()][0]
        print(action.toolTip())
        action.triggered.disconnect()
        action.setToolTip("Export Dataset")
        self.icons["save"] = action

    def _update_pan_zoom(self):
        def pan2():
            self.icons["select"].setChecked(False)
            self.pan()
        def zoom2():
            self.icons["select"].setChecked(False)
            self.zoom()

        self._actions["pan"].triggered.disconnect()
        self._actions["pan"].triggered.connect(pan2)
        self._actions["zoom"].triggered.disconnect()
        self._actions["zoom"].triggered.connect(zoom2)

    @property
    def __actions(self):
        return [child for child in self.children()
                   if isinstance(child, QtWidgets.QAction)]

    def _addActions(self):
        path = os.path.split(os.path.abspath(__file__))[0]
        dir_ = os.path.join(path,"Assets")
        self.icons = {}
        def _icon_of(name,fname,description, idx=None):
            icon = QtGui.QIcon(os.path.join(dir_,fname))
            if idx is None:
                action = self.addAction(icon,description)
            else:
                action = QtWidgets.QAction(icon,description)
                self.insertAction(self.__actions[idx],action)
            self.icons[name] = action
            return action
        self._rebind_save()
        _icon_of("select","icons8-cursor-32.png","Select spectra with left mouse",5)
        _icon_of("load","icons8-opened-folder-32.png","Load Collection")
        _icon_of("flag","icons8-flag-filled-32.png","Flag Selection")
        _icon_of("unflag","icons8-empty-flag-32.png","Unflag Selection")
        _icon_of("vis","icons8-show-flag-32.png","Show/Hide Flags")
        _icon_of("export","icons8-flag-save-32.png","Export Flags")
        _icon_of("operators","icons8-math-32.png","Operator Configuration")
        _icon_of("stats","icons8-normal-distribution-histogram-32.png","Plot Statistics")
        _icon_of("stitch","icons8-stitch-32.png","Stitch")
        _icon_of("jump","icons8-jump-correct-32.png","Jump Correct")
        _icon_of("interpolate","icons8-interpolate-32.png","Interpolate")
        _icon_of("proximal","icons8-proximal-join.png","Proximal Join")
        _icon_of("reset","icons8-restart-32.png","Revert Operators")
        self.insertSeparator(self.icons['flag'])
        self.insertSeparator(self.icons['operators'])

        self.icons["select"].setCheckable(True)
        self._update_pan_zoom()

    def triggered(self,key):
        return self.icons[key].triggered

    def returnToSelectMode(self):
        if self._ax.get_navigate_mode() == 'PAN':
            #Turn panning off
            self.pan()
        elif self._ax.get_navigate_mode() == 'ZOOM':
            #Turn zooming off
            self.zoom()


text:
import pytest
import json
import os
import numpy as np
import pandas as pd
import taxcalc as tc
from taxcrunch import cruncher as cr

CURR_PATH = os.path.abspath(os.path.dirname(__file__))
input_path = os.path.join(CURR_PATH, "test_adjustment.json")
reform_path = os.path.join(CURR_PATH, "test_reform.json")
no_reform_path = os.path.join(CURR_PATH, "test_adjustment_noreform.json")

def test_baseline_choice():
    c = cr.Cruncher(baseline=reform_path)
    assert isinstance(c, cr.Cruncher)

def test_reform_choice():
    c = cr.Cruncher(
        inputs=no_reform_path, custom_reform=reform_path
    )
    assert isinstance(c, cr.Cruncher)
    with pytest.raises(AttributeError):
        cr.Cruncher(
            inputs=input_path, custom_reform=reform_path
        )

def test_basic_table(cr_data):
    table = cr_data.basic_table()
    assert isinstance(table, pd.DataFrame)

def test_calc_table(cr_data):
    table = cr_data.calc_table()
    # remove commas and convert to numeric
    table = table.replace(',','',regex=True)
    for col in table.columns:
        table[col] = table[col].apply(pd.to_numeric)
    assert isinstance(table, pd.DataFrame)
    table_path = os.path.join(CURR_PATH, "expected_calc_table.csv")
    # table.to_csv(table_path)
    expected_table = pd.read_csv(table_path, index_col=0)
    diffs = False
    for col in table.columns:
        assert np.allclose(table[col], expected_table[col])

def test_mtr_table(cr_data):
    table = cr_data.mtr_table()
    assert isinstance(table, pd.DataFrame)
    assert abs(table.all(axis=None)) < 1

max_stars_repo_name: hdoupe/Tax-Cruncher-ARP

max_issues_repo_name: hdoupe/Tax-Cruncher-ARP

max_forks_repo_name: hdoupe/Tax-Cruncher-ARP


text:
import numpy as np

class Lidar_measurement_model(object):

    """
    position should be in asteroid centered reference frame
    Asteroid represented as a point cloud of vertices, but cut in half to avoid spurious readings

    """
    def __init__(self, shape_model, max_range=10000):
        self.shape_model = shape_model
        self.max_range = max_range
        print('Half Shape Model: ', np.min(shape_model,axis=0), np.max(shape_model,axis=0), shape_model.shape[0])
        m = shape_model.shape[0]
        D = []
        for i in range(m):
            v = shape_model[i]
            sm_f = shape_model[np.arange(m)!=i]
            d = np.min(np.linalg.norm(sm_f-v,axis=1))
            D.append(d)
        print('Shape Model Vertex Spacing (mean / min / max): ', np.mean(D),np.min(D), np.max(D))
        self.miss_threshold = 2 *  np.max(D)

    # asteroid body-centered frame

    def get_range(self, pos, vel, dvec):
        test_dot = np.dot(-pos/np.linalg.norm(pos) , dvec)
        if test_dot < 0:
            return self.max_range, 0.0, None

        closest, dist = self.closest_vertex(self.shape_model, pos, dvec)
        if dist > self.miss_threshold:
            return self.max_range, 0.0, None

        range = np.linalg.norm(closest - pos)
        dopplar = -pos.dot(vel)/np.linalg.norm(pos-closest)
        #print('*closest: ', closest)
        #print('pos/vel: ', pos,vel)
        #print('range: ', range)
        #print('dopplar: ', dopplar)
        return range, dopplar, closest

    def ortho_proj(self, V,p,d):
        # vertices, position, dvec
        Q=V-p
        qdot = Q.dot(d)
        proj = (qdot * np.expand_dims(d, axis=1)).T
        oproj = Q-proj
        return oproj

    def closest_vertex(self, V,p,d):
        dist = np.linalg.norm(self.ortho_proj(V,p,d),axis=1)
        idx = np.argmin(dist)
        return V[idx], dist[idx]

max_stars_repo_name: CHEN-yongquan/RL-Meta-Learning-ACTA

max_issues_repo_name: CHEN-yongquan/RL-Meta-Learning-ACTA

max_forks_repo_name: CHEN-yongquan/RL-Meta-Learning-ACTA


text:
'''

Synopsis
--------
Simulate how often we would expect a demographic-only method to be correct,
given counts of number of possible levels of each named categorical variable
such as 'age' or 'edu' or 'income', by drawing a population with uniform distribution
over each categorical variable.

Usage
-----
## What accuracy if we only used age and had 2 levels and 100 subjects?
$ simulate_acc_of_categorical_reid.py --n_subjects 100 --use_fields age --n_possible_age 2

## What accuracy if we only used age and had 100 levels and 100 subjects?
$ simulate_acc_of_categorical_reid.py --n_subjects 100 --use_fields age --n_possible_age 50

$ simulate_acc_of_categorical_reid.py --n_subjects 10000 --use_fields ALL

'''

import argparse
import numpy as np
import pandas as pd

## Num possible levels of each categorical variable.
# These are roughly consistent with NHANES dataset

FIELD_NAMES_AND_COUNTS={
    'age':60,
    'gender':2,
    'income':16,
    'edu':7,
    'race':5,
    'country_of_origin':4,
    }

def build_random_subject(
        seed=0,
        subj_uid=0,
        field_names_and_counts=FIELD_NAMES_AND_COUNTS,
        ):
    prng = np.random.RandomState(seed)
    subj_dict = dict(subj_uid=subj_uid)
    for key, n_possib in field_names_and_counts.items():
        subj_dict[key] = prng.choice(n_possib)
    return subj_dict

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--n_subjects', type=int, default=10000)
    parser.add_argument('--seed_offset', type=int, default=0)
    parser.add_argument('--n_possible_age', type=int, default=FIELD_NAMES_AND_COUNTS['age'])
    parser.add_argument('--n_possible_income', type=int, default=FIELD_NAMES_AND_COUNTS['income'])
    parser.add_argument('--n_possible_race', type=int, default=FIELD_NAMES_AND_COUNTS['race'])
    parser.add_argument('--n_possible_edu', type=int, default=FIELD_NAMES_AND_COUNTS['edu'])
    parser.add_argument('--use_fields', type=str, default='ALL')
    args = parser.parse_args()
    use_fields = args.use_fields.split(',')
    if args.use_fields == 'ALL':
        use_fields = FIELD_NAMES_AND_COUNTS.keys()
    field_names_and_counts = dict()
    for field in use_fields:
        field_names_and_counts[field] = FIELD_NAMES_AND_COUNTS[field]
    for key, val in args.__dict__.items():
        if key.startswith('n_possible'):
            field = key[len('n_possible_'):]
            if field in use_fields:
                field_names_and_counts[field] = val
    for field, count in field_names_and_counts.items():
        print("%5s %3d possible" % (field, count))

    n_subjects = args.n_subjects
    seed_offset = args.seed_offset * n_subjects

    subj_dict_list = list()
    for subj_uid in range(n_subjects):
        subj_dict = build_random_subject(
            seed=subj_uid + seed_offset,
            field_names_and_counts=field_names_and_counts,
            subj_uid=subj_uid)
        subj_dict_list.append(subj_dict)

    my_df = pd.DataFrame(subj_dict_list)
    my_df = my_df.set_index('subj_uid')

    print(my_df.head())

    row_strs = [str(s) for s in my_df.values]
    uvals_U, counts_U = np.unique(row_strs, return_counts=True)

    n_subj_total = 0
    n_correct_total = 0
    print("How often would we expect to identify a subject by guessing?")
    for count in np.unique(counts_U):
        n_subj_with_ct = np.sum(counts_U[counts_U == count])
        n_subj_total += n_subj_with_ct
        pr_correct_per_guess = 1.0/float(count)
        est_correct_guess = n_subj_with_ct * pr_correct_per_guess
        n_correct_total += est_correct_guess

        print("count %3d cur_subj %5d cur_correct %5d | total_subj %5d total_correct %5d total_acc %.3f" % (
            count, n_subj_with_ct, est_correct_guess, n_subj_total, n_correct_total, n_correct_total/float(n_subj_total)))


text:
#!/usr/bin/env python
import rospy
from geometry_msgs.msg import Twist
from geometry_msgs.msg import PoseWithCovarianceStamped
from geometry_msgs.msg import PoseStamped
import tf
import numpy as np
import math

current_pose = None
target_pose = None

def update_current_pose(data):
    global current_pose
    current_pose = data.pose.pose

def update_target_pose(data):
    global target_pose
    target_pose = data.pose

rospy.init_node('dewey_nav', anonymous=True)
publisher = rospy.Publisher('cmd_vel', Twist, queue_size = 1)
rospy.Subscriber("amcl_pose", PoseWithCovarianceStamped, update_current_pose)
rospy.Subscriber("move_base_simple/goal", PoseStamped, update_target_pose)
r = rospy.Rate(25)
while not rospy.is_shutdown():
    if(current_pose and target_pose):
        # Assuming counter clockwise rotation
        # if zeta2 (calculated with arcsin) is negative
        # Than the rotation is counter clockwise
        # if zeta2 is positive
        # Than the rotation is clockwise
        vector_a = np.array([target_pose.position.x - current_pose.position.x, target_pose.position.y - current_pose.position.y])
        vector_b = np.array([current_pose.position.x, current_pose.position.y])

        vector_a_hat = vector_a / np.linalg.norm(vector_a)
        vector_b_hat = vector_b / np.linalg.norm(vector_b)
        zeta1 = math.acos(np.dot(vector_a_hat, vector_b_hat))
        zeta2 = math.asin((vector_a_hat[0] - math.cos(zeta1) * vector_b_hat[0]) / -vector_b_hat[1])
        print zeta1, zeta2
        propotion = 0.3 * np.sign(zeta2) * zeta1

        #print "propotion set to ", propotion
        # twist = Twist()
        # twist.linear.x = 0
        # twist.linear.y = 0
        # twist.linear.z = 0
        # twist.angular.x = 0
        # twist.angular.y = 0
        # twist.angular.z = propotion
        # publisher.publish(twist)
    r.sleep()


text:
import unittest
import numpy as np
from collections import defaultdict

from utils.matching import stitch_component, align_component
from utils.roi import ROI

class MyTestCase(unittest.TestCase):
    def test_stitch_component(self):
        component = defaultdict(list)
        roi1 = ROI(scan=(48, 54),
                   rt=(48, 54),
                   i=[1.]*7,
                   mz=[100] * 7,
                   mzmean=100)
        roi2 = ROI(scan=(25, 30),
                   rt=(25, 30),
                   i=list(np.random.randn(6)),
                   mz=[100] * 6,
                   mzmean=100)
        component['sample'] = [roi1, roi2]
        new_component = stitch_component(component)
        self.assertEqual(roi1.i, list(new_component['sample'][0].i[-7:]))
        self.assertEqual(roi2.i, list(new_component['sample'][0].i[:6]))

    def test_align_component_simple(self):
        component = defaultdict(list)
        roi1_sample1 = ROI(scan=(-5, 1),
                           rt=(5, 10),
                           i=[0, 2, 2, 0, 1, 1, 0],
                           mz=[105] * 7,
                           mzmean=105)
        component['sample1'] = [roi1_sample1]

        roi1_sample2 = ROI(scan=(-6, 0),
                           rt=(5, 10),
                           i=[1, 1, 0, 0.9, 0.8, 0, 0],
                           mz=[105] * 7,
                           mzmean=105)
        component['sample2'] = [roi1_sample2]
        group = align_component(component)
        shifts = dict()
        for sample, shift in zip(group.samples, group.shifts):
            shifts[sample] = shift
        self.assertEqual({'sample1': 0, 'sample2': 2}, shifts)

    def test_align_component_complex(self):
        component = defaultdict(list)
        roi1_sample1 = ROI(scan=(11, 20),
                           rt=(11, 20),
                           i=[0] + [1, 2, 3, 4, 3, 2, 1] + [0] * 2,
                           mz=[100] * 10,
                           mzmean=100)
        component['sample1'] = [roi1_sample1]

        roi1_sample2 = ROI(scan=(12, 19),
                           rt=(12, 19),
                           i=[0] + [1, 2, 3, 2, 3, 2, 1],
                           mz=[100] * 8,
                           mzmean=100)
        roi2_sample2 = ROI(scan=(25, 31),
                           rt=(25, 31),
                           i=[0] + [1, 1, 1, 1, 1] + [0],
                           mz=[100] * 7,
                           mzmean=100)
        component['sample2'] = [roi1_sample2, roi2_sample2]

        roi1_sample3 = ROI(scan=(10, 28),
                           rt=(10, 28),
                           i=[0] + [1, 2, 3, 3, 3, 2, 1] + [0] * 3 + [1, 1, 1, 1, 1] + [0]*3,
                           mz=[100] * 19,
                           mzmean=100)
        component['sample3'] = [roi1_sample3]

        group = align_component(component)
        shifts = dict()
        for sample, shift in zip(group.samples, group.shifts):
            shifts[sample] = shift
        self.assertEqual({'sample1': 0, 'sample2': -1, 'sample3': 1}, shifts)

    def test_align_component_strange(self):
        component = defaultdict(list)
        roi1_sample1 = ROI(scan=(10, 20),
                           rt=(10, 20),
                           i=np.random.randn(11),
                           mz=[100] * 11,
                           mzmean=100)
        component['sample1'] = [roi1_sample1]

        roi1_sample2 = ROI(scan=(100, 108),
                           rt=(100, 108),
                           i=np.random.randn(9)+10,
                           mz=[100] * 9,
                           mzmean=100)
        component['sample2'] = [roi1_sample2]

        group = align_component(component)
        shifts = dict()
        for sample, shift in zip(group.samples, group.shifts):
            shifts[sample] = shift
        self.assertEqual([0, 0], group.shifts)

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: vivekmathema/peakonly

max_issues_repo_name: vivekmathema/peakonly

max_forks_repo_name: vivekmathema/peakonly


text:
import jax
from jax import numpy as jnp

@jax.pmap
def many_args(*args):
    x = 0
    for i in range(len(args)):
        x += args[i]
    return x

N = 110

args = [
  jnp.ones((4, 10)) for _ in range(N)
]

out = many_args(*args)
print(out)

reason: import numpy,import jax,from jax


text:
# Pandas is an API

# Pandas is actually set of software components , much of which is not
# even written in Python.
import pandas as pd
import numpy as np
import plotly.graph_objects as go
from plotly.offline import plot
import matplotlib.pyplot as plt
import datetime
from pycoingecko import CoinGeckoAPI
from mplfinance.original_flavor import candlestick2_ohlc

# You create a dictionary, this is just data.
dict_={'a':[11,21,31],'b':[12,22,32]}

# When you create a Pandas object with the Dataframe constructor in
# API lingo, this is an "instance". The data in the dictionary is passed
# along to the pandas API. You then use the dataframe to communicate with
# the API.
df=pd.DataFrame(dict_)
type(df)

# When you call the method head the dataframe communicates with the API
# displaying the first few rows of the dataframe.
df.head()

# When you call the method mean,the API will calculate the mean and return
# the value.
df.mean()

# REST APIs
# Rest API’s function by sending a request, the request is communicated via
# HTTP message. The HTTP message usually contains a JSON file. This
# contains instructions for what operation we would like the service or
# resource to perform. In a similar manner, API returns a response, via an
# HTTP message, this response is usually contained within a JSON.
# In cryptocurrency a popular method to display the movements of the price
# of a currency.

# In this lab, we will be using the CoinGecko API to create one of these
# candlestick graphs for Bitcoin. We will use the API to get the price data
# for 30 days with 24 observation per day, 1 per hour. We will find the
# max, min, open, and close price per day meaning we will have 30
# candlesticks and use that to generate the candlestick graph. Although
# we are using the CoinGecko API we will use a Python client/wrapper for
# the API called PyCoinGecko. PyCoinGecko will make performing the requests
# easy and it will deal with the enpoint targeting. Lets start off by
# getting the data we need. Using the
# get_coin_market_chart_by_id(id, vs_currency, days). id is the name of the
# coin you want, vs_currency is the currency you want the price in, and
# days is how many days back from today you want.
cg = CoinGeckoAPI()

bitcoin_data = cg.get_coin_market_chart_by_id(id='bitcoin', vs_currency='usd', days=30)

print(type(bitcoin_data ))

# The response we get is in the form of a JSON which includes the price,
# market caps, and total volumes along with timestamps for each
# observation. We are focused on the prices so we will select that data.
bitcoin_price_data = bitcoin_data['prices']

bitcoin_price_data[0:5]

# Finally lets turn this data into a Pandas DataFrame.
data = pd.DataFrame(bitcoin_price_data, columns=['TimeStamp', 'Price'])

# Now that we have the DataFrame we will convert the timestamp to datetime
# and save it as a column called Date. We will map our unix_to_datetime to
# each timestamp and convert it to a readable datetime.
data['Date'] = pd.to_datetime(data['TimeStamp'], unit='ms')

# Using this modified dataset we can now group by the Date and find the min,
# max, open, and close for the candlesticks.
candlestick_data = data.groupby(data.Date.dt.date, as_index=False).agg({"Price": ['min', 'max', 'first', 'last']})

# Finally we are now ready to use plotly to create our Candlestick Chart.
fig = go.Figure(data=[go.Candlestick(x=data['Date'],
                open=candlestick_data['Price']['first'],
                high=candlestick_data['Price']['max'],
                low=candlestick_data['Price']['min'],
                close=candlestick_data['Price']['last'])
                ])

fig.update_layout(xaxis_rangeslider_visible=False)

fig.show()


text:
"""
    Copyright (c) 2018 dibghosh AT stanford edu
    Permission is hereby granted, free of charge, to any person obtaining
    a copy of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom the Software
    is furnished to do so, subject to the following conditions:
    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.
    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
    OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
    IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
    CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
    TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
    OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
import sys
print(sys.path)
print(sys.executable)
import os
import numpy as np
import json
import random
import matplotlib.pyplot as plt
from IPython.display import clear_output

import keras
from keras.utils.data_utils import get_file
from keras import backend as K
from keras.models import model_from_json
from keras import regularizers
import functools

import tensorflow as tf

import numpy as np
import os
import time
from keras.preprocessing import image
from keras.layers import GlobalAveragePooling2D, Dense, Dropout,Activation,Flatten
from keras.applications import ResNet50
# from keras.applications.inception_v3 import InceptionV3
from keras.callbacks import TensorBoard, EarlyStopping, LearningRateScheduler
from keras import optimizers

from keras.layers import Input
from keras.models import Model
from keras.utils import np_utils
from sklearn.utils import shuffle

# lscpu Core(s) per socket: 2
NUM_PARALLEL_EXEC_UNITS = 2
# sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))
sess = tf.Session(
    config=tf.ConfigProto(
        log_device_placement=True,
        intra_op_parallelism_threads=NUM_PARALLEL_EXEC_UNITS,
        inter_op_parallelism_threads=2,
        allow_soft_placement=True,
#         gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5),
        device_count = {'GPU': 1, 'CPU': NUM_PARALLEL_EXEC_UNITS }
    )
)
keras.backend.set_session(sess)

NUM_CLASSES = 46
TRAIN_DATA_SIZE = 5000
TEST_DATA_SIZE = 1000
VAL_DATA_SIZE = 1000
img_h = 224
img_w = 224
# inception_img_h = 299
# inception_img_w = 299

np.random.seed(seed=1234)
random.seed(1234)

class ImageClass():
    "Stores the paths to images for a given class"
    def __init__(self, name, image_paths):
        self.name = name
        self.image_paths = image_paths

    def __str__(self):
        return self.name + ', ' + str(len(self.image_paths)) + ' images'

    def __len__(self):
        return len(self.image_paths)

def get_dataset(path, has_class_directories=True):
    dataset = []
    path_exp = os.path.expanduser(path)
    classes = [path for path in os.listdir(path_exp) \
                    if os.path.isdir(os.path.join(path_exp, path))]
    classes.sort()
    nrof_classes = len(classes)
    for i in range(nrof_classes):
        class_name = classes[i]
        facedir = os.path.join(path_exp, class_name)
        image_paths = get_image_paths(facedir)
        dataset.append(ImageClass(class_name, image_paths))

    return dataset

def get_image_paths(facedir):
    image_paths = []
    if os.path.isdir(facedir):
        images = os.listdir(facedir)
        image_paths = [os.path.join(facedir,img) for img in images]
    return image_paths

def get_image_paths_and_labels(dataset):
    image_paths_flat = []
    labels_flat = []
    for i in range(len(dataset)):
        image_paths_flat += dataset[i].image_paths
        labels_flat += [i] * len(dataset[i].image_paths)
    return image_paths_flat, labels_flat

def preprocess_input(x, dim_ordering='default'):
    if dim_ordering == 'default':
        dim_ordering = K.image_dim_ordering()
    assert dim_ordering in {'tf', 'th'}

    if dim_ordering == 'th':
        x[:, 0, :, :] -= 103.939
        x[:, 1, :, :] -= 116.779
        x[:, 2, :, :] -= 123.68
        # 'RGB'->'BGR'
        x = x[:, ::-1, :, :]
    else:
        x[:, :, :, 0] -= 103.939
        x[:, :, :, 1] -= 116.779
        x[:, :, :, 2] -= 123.68
        # 'RGB'->'BGR'
        x = x[:, :, :, ::-1]
    return x

from glob import glob
from keras.preprocessing import image
from tqdm import tqdm_notebook, tqdm # Iteration visualization

def load_dataset(data_dir_list, mode, max_per_class=100):
    """
        loads images in memory. Expensive method. Doesn't scale well
    """

    img_data_list, labels =[],[]
    images_per_class = max_per_class

    for category in tqdm(data_dir_list):
        img_dir = "../deepfashion/dataset/%s/%s/*.jpg" % (mode, category)
#         print("Loading category =%s from path=%s" % (category, img_dir))
        img_list=glob(img_dir)
        if not max_per_class:
            # take all images
            images_per_class = len(img_list)
        print ('Found {} images out of {} for category {}'.format(images_per_class, len(img_list), category))
        for img_path in img_list[:images_per_class]:
            labels.append(category)
            img = image.load_img(img_path, target_size=(img_h, img_w))
            x = image.img_to_array(img)
            x = np.expand_dims(x, axis=0)
            x = preprocess_input(x)
            img_data_list.append(x)

    img_data = np.array(img_data_list)
    img_data=np.rollaxis(img_data,1,0)
    img_data=img_data[0]
    return img_data, labels

### write both training + validation graphs in same plot
# https://stackoverflow.com/questions/47877475/keras-tensorboard-plot-train-and-validation-scalars-in-a-same-figure

class TrainValTensorBoard(TensorBoard):
    def __init__(self, log_dir='./logs', **kwargs):
        # Make the original `TensorBoard` log to a subdirectory 'training'
        training_log_dir = os.path.join(log_dir, 'training')
        super(TrainValTensorBoard, self).__init__(training_log_dir, **kwargs)

        # Log the validation metrics to a separate subdirectory
        self.val_log_dir = os.path.join(log_dir, 'validation')

    def set_model(self, model):
        # Setup writer for validation metrics
        self.val_writer = tf.summary.FileWriter(self.val_log_dir)
        super(TrainValTensorBoard, self).set_model(model)

    def on_epoch_end(self, epoch, logs=None):
        # Pop the validation logs and handle them separately with
        # `self.val_writer`. Also rename the keys so that they can
        # be plotted on the same figure with the training metrics
        logs = logs or {}
        val_logs = {k.replace('val_', ''): v for k, v in logs.items() if k.startswith('val_')}
        for name, value in val_logs.items():
            summary = tf.Summary()
            summary_value = summary.value.add()
            summary_value.simple_value = value.item()
            summary_value.tag = name
            self.val_writer.add_summary(summary, epoch)
        self.val_writer.flush()

        # Pass the remaining logs to `TensorBoard.on_epoch_end`
        logs = {k: v for k, v in logs.items() if not k.startswith('val_')}
        super(TrainValTensorBoard, self).on_epoch_end(epoch, logs)

    def on_train_end(self, logs=None):
        super(TrainValTensorBoard, self).on_train_end(logs)
        self.val_writer.close()

def save_model_to_disk(model, model_name="model"):
    # serialize model to JSON
    model_json = model.to_json()
    with open("%s.json" % model_name, "w") as json_file:
        json_file.write(model_json)
    # serialize weights to HDF5
    model.save_weights("%s.h5" % model_name)
    print("Saved %s to disk" % model_name)

def load_model_from_disk(model_name):
    # load json and create model
    json_file = open('%s.json' % model_name, 'r')
    loaded_model_json = json_file.read()
    json_file.close()
    loaded_model = model_from_json(loaded_model_json)
    # load weights into new model
    loaded_model.load_weights("%s.h5" % model_name)
    print("Loaded %s from disk" % model_name)
    return loaded_model

# apply learning rate decay
# using step decay function and LearningRateScheduler callback to take
# step decay function as argument and return updated learning rates for use in SGD optimizer.
def step_decay(epoch):
    initial_lrate = 0.0001
    drop = 0.5
    epochs_drop = 10.0
    lrate = initial_lrate * math.pow(drop,
                                     math.floor((1+epoch)/epochs_drop))
    return lrate

# also create a learning rate decay plotter
class LossHistory(keras.callbacks.Callback):
    def on_train_begin(self, logs={}):
        self.losses = []
        self.lr = []

    def on_epoch_end(self, batch, logs={}):
        self.losses.append(logs.get("loss"))
        self.lr.append(step_decay(len(self.losses)))

class TrainingPlot(keras.callbacks.Callback):

    def __init__(self, num_epochs, batch_size, **kwargs):
        self.num_epochs = num_epochs
        self.batch_size = batch_size
        super(TrainingPlot, self).__init__(**kwargs)

    # This function is called when the training begins
    def on_train_begin(self, logs={}):
        # Initialize the lists for holding the logs, losses and accuracies
        self.losses = []
        self.acc = []
        self.val_losses = []
        self.val_acc = []
        self.logs = []

    # This function is called at the end of each epoch
    def on_epoch_end(self, epoch, logs={}):

        # Append the logs, losses and accuracies to the lists
        self.logs.append(logs)
        self.losses.append(logs.get('loss'))
        self.acc.append(logs.get('acc'))
        self.val_losses.append(logs.get('val_loss'))
        self.val_acc.append(logs.get('val_acc'))

        # Before plotting ensure at least 2 epochs have passed
        if len(self.losses) > 1:

            # Clear the previous plot
            clear_output(wait=True)
            N = np.arange(0, len(self.losses))

            # You can chose the style of your preference
            # print(plt.style.available) to see the available options
            plt.style.use("seaborn")

            # Plot train loss, train acc, val loss and val acc against epochs passed
            plt.figure()
            plt.plot(N, self.losses, label = "train_loss")
            plt.plot(N, self.acc, label = "train_acc")
            plt.plot(N, self.val_losses, label = "val_loss")
            plt.plot(N, self.val_acc, label = "val_acc")
            plt.title("Training Loss and Accuracy [Epoch {}]".format(epoch))
            plt.xlabel("Epoch #")
            plt.ylabel("Loss/Accuracy")
            plt.legend()
            plt.savefig('resnet50_{}_{}_{}.png'.format(self.batch_size, self.num_epochs, time.time()))
            plt.close()

train_data_dir = os.listdir("../deepfashion/dataset/train/")
val_data_dir = os.listdir("../deepfashion/dataset/val/")
test_data_dir = os.listdir("../deepfashion/dataset/test/")

images_per_class = 400
train_data, train_labels = load_dataset(train_data_dir, "train", images_per_class)
print("[*] loaded %s training images" % (len(train_labels)))
val_data, val_labels = load_dataset(val_data_dir, "val", images_per_class)
print("[*] loaded %s validation images" % (len(val_labels)))
test_data, test_labels = load_dataset(test_data_dir, "test", max_per_class=None)
print("[*] loaded %s test images" % (len(test_labels)))

# train_set = get_dataset(train_data_dir)
# val_set = get_dataset(val_data_dir)

# nrof_classes = len(train_set)

# print('Number of classes : %s' % nrof_classes)

# # prepare data
# # convert class labels to on-hot encoding
# # Get a list of image paths and their labels
# train_image_list, train_label_list = get_image_paths_and_labels(train_set)
# assert len(image_list) > 0, 'The training set should not be empty'
# val_image_list, val_label_list = get_image_paths_and_labels(val_set)

# we will use the encoders from the scikit-learn library.
# Specifically, the LabelEncoder of creating an integer encoding of labels
# and the OneHotEncoder for creating a one hot encoding of integer encoded values.
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder

label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(train_labels)
onehot_encoder = OneHotEncoder(sparse=False)
integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
train_onehot_encoded = onehot_encoder.fit_transform(integer_encoded)

label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(val_labels)
onehot_encoder = OneHotEncoder(sparse=False)
integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
val_onehot_encoded = onehot_encoder.fit_transform(integer_encoded)

label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(test_labels)
onehot_encoder = OneHotEncoder(sparse=False)
integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
test_onehot_encoded = onehot_encoder.fit_transform(integer_encoded)

### network
# keras BN bug - https://github.com/keras-team/keras/pull/9965
# K.clear_session()
# K.set_learning_phase(0)

base_model = ResNet50(
        weights='imagenet',
        include_top=False,
        input_shape=(img_h, img_w, 3))

# base_model = InceptionV3(
#         weights='imagenet',
#         include_top=False,
#         input_shape=(inception_img_h, inception_img_w, 3))

base_model.summary()

L2_RATE_KERNEL = 0.01
L2_RATE_ACTIVITY = 0.01

last_layer = base_model.output
# add a global spatial average pooling layer
x = GlobalAveragePooling2D()(last_layer)
# add fully-connected & dropout layers
x = Dense(1024,
          activation='relu',
          name='fc-1')(x)
# a softmax layer for 46 classes
predictions = Dense(NUM_CLASSES, activation='softmax',name='output_layer')(x)

# this is the model we will train
custom_resnet_model = Model(inputs=base_model.input, outputs=predictions)

custom_resnet_model.summary()

# first: train only the top layers (which were randomly initialized)
# i.e. freeze all layers
for layer in custom_resnet_model.layers:
    layer.trainable = False

# custom_resnet_model.layers[-1].trainable

custom_resnet_model.compile(loss='categorical_crossentropy', optimizer=optimizers.Adam(), metrics=['accuracy'])

# train settings
TRAIN_BATCH_SIZE = 128
WARM_UP_EPOCHS = 1
FINAL_EPOCHS = 100
GRAD_CLIP_THRESHOLD = 0.5
ALPHA_LEARNING_RATE = 0.001

tensorboard = TensorBoard(log_dir="./deepfashion/tboard-resnet50-logs/{}_{}_{}".format(TRAIN_BATCH_SIZE, FINAL_EPOCHS, time.time()), write_graph=True)

t=time.time()
with tf.device('/gpu:0'):
    hist = custom_resnet_model.fit(
        train_data,
        train_onehot_encoded,
        batch_size=TRAIN_BATCH_SIZE,
        epochs=WARM_UP_EPOCHS,
        verbose=1,
        validation_data=(val_data, val_onehot_encoded))
print('Training time (secs): %s' % (time.time() - t))

with tf.device('/gpu:0'):
    (loss, accuracy) = custom_resnet_model.evaluate(test_data,
                                                     test_onehot_encoded,
                                                     batch_size=TRAIN_BATCH_SIZE,
                                                     verbose=1)

print("[INFO] pre fine-tune loss={:.4f}, pre fine-tune accuracy: {:.4f}%".format(loss,accuracy * 100))

# at this point, the top layers are well trained and we can start fine-tuning
# convolutional layers from inception V3. We will freeze the bottom N layers
# and train the remaining top layers.

# let's visualize layer names and layer indices to see how many layers
# we should freeze:
for i, layer in enumerate(base_model.layers):
    print(i, layer.name)

# Test # 1: we chose to train the top 1 resnet blocks, i.e. we will freeze
# the first 163 layers and unfreeze the rest: (add_15)
# for layer in base_model.layers[:163]:
#     layer.trainable = False
# for layer in base_model.layers[163:]:
#     layer.trainable = True

# Test # 2: we chose to train the top 2 resnet blocks, i.e. we will freeze
# the first 153 layers and unfreeze the rest: (add_14)
# for layer in base_model.layers[:153]:
#     layer.trainable = False
# for layer in base_model.layers[153:]:
#     layer.trainable = True

# Test # 3: we chose to train the top 3 resnet blocks, i.e. we will freeze
# the first 143 layers and unfreeze the rest: (add_13)
# for layer in base_model.layers[:141]:
#     layer.trainable = False
# for layer in base_model.layers[141:]:
#     layer.trainable = True

# Test # 4: we chose to train the top 4 resnet blocks, i.e. we will freeze
# the first 143 layers and unfreeze the rest: (add_12)
for layer in base_model.layers[:131]:
    layer.trainable = False
for layer in base_model.layers[131:]:
    layer.trainable = True

# we chose to train the top 2 inception blocks, i.e. we will freeze
# the first 172 layers and unfreeze the rest:
# for layer in model.layers[:172]:
#     layer.trainable = False
# for layer in model.layers[172:]:
#     layer.trainable = True

# UNUSED: Store the model on disk
# model_name = 'resnet50_{}_{}_{}.h5'.format(TRAIN_BATCH_SIZE, EPOCHS, time.time())
# save_model_to_disk(custom_resnet_model, model_name)

# print('STATIC LEARNING_PHASE = 1')
# K.clear_session()
# K.set_learning_phase(1)

# UNUSED: custom_resnet_model = load_model_from_disk(model_name)

# we need to recompile the model for these modifications to take effect
# we use SGD with a low learning rate
# from keras.optimizers import SGD
# custom_resnet_model.compile(optimizer=SGD(lr=0.0001, momentum=0.9), loss='categorical_crossentropy', metrics=['accuracy'])

# stop if val_loss stops improving for 10 epochs
# early_stopping = EarlyStopping(verbose=1, patience=10, monitor='val_loss')
# add top-K accuracy reporting
top3_acc = functools.partial(keras.metrics.top_k_categorical_accuracy, k=3)
top3_acc.__name__ = 'top3_acc'

opti_grad_clip=optimizers.Adam(lr=ALPHA_LEARNING_RATE)
# opti_grad_clip=optimizers.RMSprop(lr=2e-3)
custom_resnet_model.compile(loss='categorical_crossentropy', optimizer=opti_grad_clip, metrics=['accuracy', 'top_k_categorical_accuracy', top3_acc])

# init plotter
plot_losses = TrainingPlot(FINAL_EPOCHS, TRAIN_BATCH_SIZE)

# learning rate
# loss_history = LossHistory()
# lrate = LearningRateScheduler(step_decay)
lr_decay = LearningRateScheduler(schedule=lambda epoch: ALPHA_LEARNING_RATE * (0.9 ** epoch))

# we train our model again (this time fine-tuning the top 2 inception blocks
# alongside the top Dense layers
with tf.device('/gpu:0'):
    hist = custom_resnet_model.fit(
        train_data,
        train_onehot_encoded,
        batch_size=TRAIN_BATCH_SIZE,
        epochs=FINAL_EPOCHS,
        verbose=1,
        validation_data=(val_data, val_onehot_encoded),
        callbacks=[tensorboard, plot_losses, lr_decay])

with tf.device('/gpu:0'):
    (loss, accuracy, top_5, top_3) = custom_resnet_model.evaluate(test_data,
                                                    test_onehot_encoded,
                                                    batch_size=TRAIN_BATCH_SIZE,
                                                    verbose=1)

print("[INFO] final loss={:.4f}, final accuracy: {:.4f}, final top_5: {:.4f}, final top_3: {:.4f}%".format(loss, accuracy * 100, top_5, top_3))

# let's visualize layer names and layer indices to see how many layers
# we should freeze:
# for i, layer in enumerate(base_model.layers):
#     print(i, layer.name)


text:
import sys
import datetime
import os
import glob
import tables
import json
import numpy as np

# project imports
sys.path.insert(0, os.path.dirname(os.path.abspath('')))
from core.data import open_data_file
from core.model import isensee2017_model
from core.generator import data_generator, get_number_of_steps, get_number_of_patches
from core.training import train_model
from core.metrics import weighted_soft_dice_loss  # , weighted_dice_coefficient_loss
from unet.unet_config import get_kfold_configuration, get_unet_configuration, load_kconfig, unet_model_root_path
from unet.train import kfold_training_and_validation_generators
import nibabel as nib
from core.augment import get_image, scale_image, resample_to_img

def scaling_example(data_file_train):
    # Investigate what scaling of image does
    image = get_image(data_file_train.root.data[0][0], data_file_train.root.affine[0])
    image_scaled_down = resample_to_img(scale_image(image, 0.50), image, interpolation="continuous")
    image_scaled_up = resample_to_img(scale_image(image, 1.50), image, interpolation="continuous")

    image.to_filename(os.path.join('C:\school\RnD', "generator_orig.nii.gz"))
    image_scaled_down.to_filename(os.path.join('C:\school\RnD', "generator_down.nii.gz"))
    image_scaled_up.to_filename(os.path.join('C:\school\RnD', "generator_up.nii.gz"))

def generator_experiment(mconfig, unet_subfolder=None):
    ## DEBUG cheats
    mconfig["patch_shape"] = None

    # open files
    data_path_train = os.path.join(mconfig["kfold_path"],
                                   'config_{}_train.h5'.format(mconfig["kfold_index"]))
    data_path_validation = os.path.join(mconfig["kfold_path"],
                                        'config_{}_validation.h5'.format(mconfig["kfold_index"]))
    print(data_path_train)
    print(mconfig["kfold_path"])
    assert os.path.exists(data_path_train)
    assert os.path.exists(data_path_validation)
    data_file_train = open_data_file(data_path_train)
    data_file_validation = open_data_file(data_path_validation)

    # get training and testing generators
    train_generator, validation_generator, n_train_steps, n_validation_steps = kfold_training_and_validation_generators(
        data_file_train,
        data_file_validation,
        batch_size=1,
        n_labels=2,
        labels=(0, 1),
        non_healthy=True,
        patch_shape=mconfig["patch_shape"],
        validation_batch_size=mconfig["validation_batch_size"],
        validation_patch_overlap=mconfig["validation_patch_overlap"],
        training_patch_start_offset=mconfig["training_patch_start_offset"],
        permute=True,
        augment=mconfig["augment"],
        skip_blank=mconfig["skip_blank"],
        augment_flip=mconfig["flip"],
        augment_distortion_factor=mconfig["distort"]
    )

    print(train_generator)
    output1 = (next(train_generator))
    print(np.unique(output1[1][0][0], return_counts=True))
    # print(np.unique(output1[1][0][1]))
    # print(np.unique(output1[1][0][2]))
    # print(np.unique(output1[1][0][3]))
    #
    # output1 = (next(train_generator))
    # print(np.unique(output1[1][0][0]))
    # print(np.unique(output1[1][0][1]))
    # print(np.unique(output1[1][0][2]))
    # print(np.unique(output1[1][0][3]))

    # # Investigate what scaling of image does
    # scaling_example(data_file_train)

    # Close datafiles
    data_file_train.close()
    data_file_validation.close()

def main(kfold_subfolder, unet_subfolder):
    # Get configuration
    kconfig = load_kconfig(kfold_subfolder)
    mconfig = get_unet_configuration(kconfig)

    # Train model
    generator_experiment(mconfig, unet_subfolder)

if __name__ == "__main__":
    kfold_subfolder = "experiment05"
    unet_subfolder = "train_exp_05"
    main(kfold_subfolder, unet_subfolder)

max_stars_repo_name: ggammelgaard/hrpqct-methods

max_issues_repo_name: ggammelgaard/hrpqct-methods

max_forks_repo_name: ggammelgaard/hrpqct-methods


text:
# Copyright (c) OpenMMLab. All rights reserved.
import os.path as osp
import tempfile
import warnings
from collections import OrderedDict

import json_tricks as json
import numpy as np
from mmcv import Config, deprecated_api_warning

from mmpose.datasets.builder import DATASETS
from ..base import Kpt2dSviewRgbImgTopDownDataset

@DATASETS.register_module()
class InterHand2DDataset(Kpt2dSviewRgbImgTopDownDataset):
    """InterHand2.6M 2D dataset for top-down hand pose estimation.

    "InterHand2.6M: A Dataset and Baseline for 3D Interacting Hand Pose
    Estimation from a Single RGB Image", ECCV'2020.
    More details can be found in the `paper
    <https://arxiv.org/pdf/2008.09309.pdf>`__ .

    The dataset loads raw features and apply specified transforms
    to return a dict containing the image tensors and other information.

    InterHand2.6M keypoint indexes::

        0: 'thumb4',
        1: 'thumb3',
        2: 'thumb2',
        3: 'thumb1',
        4: 'forefinger4',
        5: 'forefinger3',
        6: 'forefinger2',
        7: 'forefinger1',
        8: 'middle_finger4',
        9: 'middle_finger3',
        10: 'middle_finger2',
        11: 'middle_finger1',
        12: 'ring_finger4',
        13: 'ring_finger3',
        14: 'ring_finger2',
        15: 'ring_finger1',
        16: 'pinky_finger4',
        17: 'pinky_finger3',
        18: 'pinky_finger2',
        19: 'pinky_finger1',
        20: 'wrist'

    Args:
        ann_file (str): Path to the annotation file.
        camera_file (str): Path to the camera file.
        joint_file (str): Path to the joint file.
        img_prefix (str): Path to a directory where images are held.
            Default: None.
        data_cfg (dict): config
        pipeline (list[dict | callable]): A sequence of data transforms.
        dataset_info (DatasetInfo): A class containing all dataset info.
        test_mode (str): Store True when building test or
            validation dataset. Default: False.
    """

    def __init__(self,
                 ann_file,
                 camera_file,
                 joint_file,
                 img_prefix,
                 data_cfg,
                 pipeline,
                 dataset_info=None,
                 test_mode=False):

        if dataset_info is None:
            warnings.warn(
                'dataset_info is missing. '
                'Check https://github.com/open-mmlab/mmpose/pull/663 '
                'for details.', DeprecationWarning)
            cfg = Config.fromfile('configs/_base_/datasets/interhand2d.py')
            dataset_info = cfg._cfg_dict['dataset_info']

        super().__init__(
            ann_file,
            img_prefix,
            data_cfg,
            pipeline,
            dataset_info=dataset_info,
            test_mode=test_mode)

        self.ann_info['use_different_joint_weights'] = False
        self.camera_file = camera_file
        self.joint_file = joint_file
        self.db = self._get_db()

        print(f'=> num_images: {self.num_images}')
        print(f'=> load {len(self.db)} samples')

    @staticmethod
    def _cam2pixel(cam_coord, f, c):
        """Transform the joints from their camera coordinates to their pixel
        coordinates.

        Note:
            - N: number of joints

        Args:
            cam_coord (ndarray[N, 3]): 3D joints coordinates
                in the camera coordinate system
            f (ndarray[2]): focal length of x and y axis
            c (ndarray[2]): principal point of x and y axis

        Returns:
            img_coord (ndarray[N, 3]): the coordinates (x, y, 0)
                in the image plane.
        """
        x = cam_coord[:, 0] / (cam_coord[:, 2] + 1e-8) * f[0] + c[0]
        y = cam_coord[:, 1] / (cam_coord[:, 2] + 1e-8) * f[1] + c[1]
        z = np.zeros_like(x)
        img_coord = np.concatenate((x[:, None], y[:, None], z[:, None]), 1)
        return img_coord

    @staticmethod
    def _world2cam(world_coord, R, T):
        """Transform the joints from their world coordinates to their camera
        coordinates.

        Note:
            - N: number of joints

        Args:
            world_coord (ndarray[3, N]): 3D joints coordinates
                in the world coordinate system
            R (ndarray[3, 3]): camera rotation matrix
            T (ndarray[3]): camera position (x, y, z)

        Returns:
            cam_coord (ndarray[3, N]): 3D joints coordinates
                in the camera coordinate system
        """
        cam_coord = np.dot(R, world_coord - T)
        return cam_coord

    def _get_db(self):
        """Load dataset.

        Adapted from 'https://github.com/facebookresearch/InterHand2.6M/'
            'blob/master/data/InterHand2.6M/dataset.py'
        Copyright (c) FaceBook Research, under CC-BY-NC 4.0 license.
        """
        with open(self.camera_file, 'r') as f:
            cameras = json.load(f)
        with open(self.joint_file, 'r') as f:
            joints = json.load(f)
        gt_db = []
        bbox_id = 0
        for img_id in self.img_ids:
            num_joints = self.ann_info['num_joints']

            ann_id = self.coco.getAnnIds(imgIds=img_id, iscrowd=False)
            ann = self.coco.loadAnns(ann_id)[0]
            img = self.coco.loadImgs(img_id)[0]

            capture_id = str(img['capture'])
            camera_name = img['camera']
            frame_idx = str(img['frame_idx'])
            image_file = osp.join(self.img_prefix, self.id2name[img_id])

            camera_pos, camera_rot = np.array(
                cameras[capture_id]['campos'][camera_name],
                dtype=np.float32), np.array(
                    cameras[capture_id]['camrot'][camera_name],
                    dtype=np.float32)
            focal, principal_pt = np.array(
                cameras[capture_id]['focal'][camera_name],
                dtype=np.float32), np.array(
                    cameras[capture_id]['princpt'][camera_name],
                    dtype=np.float32)
            joint_world = np.array(
                joints[capture_id][frame_idx]['world_coord'], dtype=np.float32)
            joint_cam = self._world2cam(
                joint_world.transpose(1, 0), camera_rot,
                camera_pos.reshape(3, 1)).transpose(1, 0)
            joint_img = self._cam2pixel(joint_cam, focal, principal_pt)[:, :2]
            joint_img = joint_img.reshape(2, -1, 2)

            joint_valid = np.array(
                ann['joint_valid'], dtype=np.float32).reshape(2, -1)
            # if root is not valid -> root-relative 3D pose is also not valid.
            # Therefore, mark all joints as invalid
            for hand in range(2):
                joint_valid[hand, :] *= joint_valid[hand][-1]

                if np.sum(joint_valid[hand, :]) > 11:
                    joints_3d = np.zeros((num_joints, 3), dtype=np.float32)
                    joints_3d_visible = np.zeros((num_joints, 3),
                                                 dtype=np.float32)
                    joints_3d[:, :2] = joint_img[hand, :, :]
                    joints_3d_visible[:, :2] = np.minimum(
                        1, joint_valid[hand, :].reshape(-1, 1))

                    # use the tightest bbox enclosing all keypoints as bbox
                    bbox = [img['width'], img['height'], 0, 0]
                    for i in range(num_joints):
                        if joints_3d_visible[i][0]:
                            bbox[0] = min(bbox[0], joints_3d[i][0])
                            bbox[1] = min(bbox[1], joints_3d[i][1])
                            bbox[2] = max(bbox[2], joints_3d[i][0])
                            bbox[3] = max(bbox[3], joints_3d[i][1])

                    bbox[2] -= bbox[0]
                    bbox[3] -= bbox[1]

                    # use 1.5bbox as input
                    center, scale = self._xywh2cs(*bbox, 1.5)

                    gt_db.append({
                        'image_file': image_file,
                        'center': center,
                        'scale': scale,
                        'rotation': 0,
                        'joints_3d': joints_3d,
                        'joints_3d_visible': joints_3d_visible,
                        'dataset': self.dataset_name,
                        'bbox': bbox,
                        'bbox_score': 1,
                        'bbox_id': bbox_id
                    })
                    bbox_id = bbox_id + 1
        gt_db = sorted(gt_db, key=lambda x: x['bbox_id'])

        return gt_db

    @deprecated_api_warning(name_dict=dict(outputs='results'))
    def evaluate(self, results, res_folder=None, metric='PCK', **kwargs):
        """Evaluate interhand2d keypoint results. The pose prediction results
        will be saved in ``${res_folder}/result_keypoints.json``.

        Note:
            - batch_size: N
            - num_keypoints: K
            - heatmap height: H
            - heatmap width: W

        Args:
            results (list[dict]): Testing results containing the following
                items:

                - preds (np.ndarray[N,K,3]): The first two dimensions are \
                    coordinates, score is the third dimension of the array.
                - boxes (np.ndarray[N,6]): [center[0], center[1], scale[0], \
                    scale[1],area, score]
                - image_paths (list[str]): For example, ['Capture12/\
                    0390_dh_touchROM/cam410209/image62434.jpg']
                - output_heatmap (np.ndarray[N, K, H, W]): model outputs.
            res_folder (str, optional): The folder to save the testing
                results. If not specified, a temp folder will be created.
                Default: None.
            metric (str | list[str]): Metric to be performed.
                Options: 'PCK', 'AUC', 'EPE'.

        Returns:
            dict: Evaluation results for evaluation metric.
        """
        metrics = metric if isinstance(metric, list) else [metric]
        allowed_metrics = ['PCK', 'AUC', 'EPE']
        for metric in metrics:
            if metric not in allowed_metrics:
                raise KeyError(f'metric {metric} is not supported')

        if res_folder is not None:
            tmp_folder = None
            res_file = osp.join(res_folder, 'result_keypoints.json')
        else:
            tmp_folder = tempfile.TemporaryDirectory()
            res_file = osp.join(tmp_folder.name, 'result_keypoints.json')

        kpts = []
        for result in results:
            preds = result['preds']
            boxes = result['boxes']
            image_paths = result['image_paths']
            bbox_ids = result['bbox_ids']

            batch_size = len(image_paths)
            for i in range(batch_size):
                image_id = self.name2id[image_paths[i][len(self.img_prefix):]]

                kpts.append({
                    'keypoints': preds[i].tolist(),
                    'center': boxes[i][0:2].tolist(),
                    'scale': boxes[i][2:4].tolist(),
                    'area': float(boxes[i][4]),
                    'score': float(boxes[i][5]),
                    'image_id': image_id,
                    'bbox_id': bbox_ids[i]
                })
        kpts = self._sort_and_unique_bboxes(kpts)

        self._write_keypoint_results(kpts, res_file)
        info_str = self._report_metric(res_file, metrics)
        name_value = OrderedDict(info_str)

        if tmp_folder is not None:
            tmp_folder.cleanup()

        return name_value

max_stars_repo_name: ALISCIFP/mmpose

max_issues_repo_name: ALISCIFP/mmpose

max_forks_repo_name: ALISCIFP/mmpose


text:
from numba import types
from numba.core import cgutils
from numba.core.typing.arraydecl import get_array_index_type
from numba.extending import lower_builtin, type_callable
from numba.np.arrayobj import make_array, normalize_indices, basic_indexing

__all__ = ["atomic_add", "atomic_sub", "atomic_max", "atomic_min"]

def atomic_rmw(context, builder, op, arrayty, val, ptr):
    assert arrayty.aligned  # We probably have to have aligned arrays.
    dataval = context.get_value_as_data(builder, arrayty.dtype, val)
    return builder.atomic_rmw(op, ptr, dataval, "monotonic")

def declare_atomic_array_op(iop, uop, fop):
    def decorator(func):
        @type_callable(func)
        def func_type(context):
            def typer(ary, idx, val):
                out = get_array_index_type(ary, idx)
                if out is not None:
                    res = out.result
                    if context.can_convert(val, res):
                        return res

            return typer

        @lower_builtin(func, types.Buffer, types.Any, types.Any)
        def func_impl(context, builder, sig, args):
            """
            array[a] = scalar_or_array
            array[a,..,b] = scalar_or_array
            """
            aryty, idxty, valty = sig.args
            ary, idx, val = args

            if isinstance(idxty, types.BaseTuple):
                index_types = idxty.types
                indices = cgutils.unpack_tuple(builder, idx, count=len(idxty))
            else:
                index_types = (idxty,)
                indices = (idx,)

            ary = make_array(aryty)(context, builder, ary)

            # First try basic indexing to see if a single array location is denoted.
            index_types, indices = normalize_indices(context, builder, index_types, indices)
            dataptr, shapes, _strides = basic_indexing(
                context, builder, aryty, ary, index_types, indices, boundscheck=context.enable_boundscheck,
            )
            if shapes:
                raise NotImplementedError("Complex shapes are not supported")

            # Store source value the given location
            val = context.cast(builder, val, valty, aryty.dtype)
            op = None
            if isinstance(aryty.dtype, types.Integer) and aryty.dtype.signed:
                op = iop
            elif isinstance(aryty.dtype, types.Integer) and not aryty.dtype.signed:
                op = uop
            elif isinstance(aryty.dtype, types.Float):
                op = fop
            if op is None:
                raise TypeError("Atomic operation not supported on " + str(aryty))
            return atomic_rmw(context, builder, op, aryty, val, dataptr)

        return func

    return decorator

@declare_atomic_array_op("add", "add", "fadd")
def atomic_add(ary, i, v):
    """
    Atomically, perform `ary[i] += v` and return the previous value of `ary[i]`.

    i must be a simple index for a single element of ary. Broadcasting and vector operations are not supported.

    This should be used from numba compiled code.
    """
    orig = ary[i]
    ary[i] += v
    return orig

@declare_atomic_array_op("sub", "sub", "fsub")
def atomic_sub(ary, i, v):
    """
    Atomically, perform `ary[i] -= v` and return the previous value of `ary[i]`.

    i must be a simple index for a single element of ary. Broadcasting and vector operations are not supported.

    This should be used from numba compiled code.
    """
    orig = ary[i]
    ary[i] -= v
    return orig

@declare_atomic_array_op("max", "umax", None)
def atomic_max(ary, i, v):
    """
    Atomically, perform `ary[i] = max(ary[i], v)` and return the previous value of `ary[i]`.
    This operation does not support floating-point values.

    i must be a simple index for a single element of ary. Broadcasting and vector operations are not supported.

    This should be used from numba compiled code.
    """
    orig = ary[i]
    ary[i] = max(ary[i], v)
    return orig

@declare_atomic_array_op("min", "umin", None)
def atomic_min(ary, i, v):
    """
    Atomically, perform `ary[i] = min(ary[i], v)` and return the previous value of `ary[i]`.
    This operation does not support floating-point values.

    i must be a simple index for a single element of ary. Broadcasting and vector operations are not supported.

    This should be used from numba compiled code.
    """
    orig = ary[i]
    ary[i] = min(ary[i], v)
    return orig

max_stars_repo_name: swreinehr/katana

max_issues_repo_name: swreinehr/katana

max_forks_repo_name: swreinehr/katana

reason: from numba


text:
from typing import List
import numpy as np
import random
from bpemb import BPEmb
import tensorflow as tf
import pandas as pd
import os
from typing import Union
import ast

def convert_sparse_matrix_to_sparse_tensor(X):
    coo = X.tocoo()
    indices = np.mat([coo.row, coo.col]).transpose()
    return tf.SparseTensor(indices, coo.data, coo.shape)

def binary_search_p(func, ground_truth, test_set, iterations=20):
    """
    Find the best value of p, given the scoring function func
    """
    best_p = -10
    best_func_score = func(ground_truth, test_set >= best_p)
    p_values = np.arange(-1, 2, 0.1)
    for p_value in p_values:
        value = 20 ** p_value

        current_score = func(ground_truth, test_set >= - value)
        if current_score >= best_func_score:
            best_func_score = current_score
            best_p = -value
    return best_p, best_func_score

def subsample_df_by_column(df: pd.DataFrame, column: str, values: List, num_samples_per_value: int) -> pd.DataFrame:
    """
    Takes a DataFrame. For each value in `values` it takes the sub frame which has this value in the column `column`
    and samples `num_samples_per_value` rows.

    Parameters
    ----------
    df: pd.DataFrame
        The pd.DataFrame to look at.
    column: str
        The column, where we search for the values
    values: List
        Typically each value is a user ID and we want a limited sample, so we don't get skewed results.
    num_samples_per_value: int
        The amount of rows to sample for each value
    Returns
    -------
    subsample_df: pd.DataFrame
        The returned DataFrame.
    """
    assert column in df.columns.values
    subsample_df = pd.DataFrame([], columns=df.columns.values)

    for sample_id in values:
        sample_df = df[df[column] == sample_id]
        if sample_df.shape[0] > num_samples_per_value:
            sample_df = sample_df.sample(n=num_samples_per_value, replace=False)
        subsample_df = pd.concat([subsample_df, sample_df])

    return subsample_df.reset_index(drop=True)

def pd_set_minus(df: pd.DataFrame, df_minus: pd.DataFrame) -> pd.DataFrame:
    """
    Definition set minus: Let A, B be sets. Then A - B holds every element which is in A, but not in B
    In that manner we return every row of df, which is not in df_minus.

    Parameters
    ----------
    df: pd.DataFrame
        The original DataFrame.
    df_minus: pd.DataFrame
        The DataFrame who's rows we exclude from df.

    Returns
    -------
    df - df_minus: pd.DataFrame

    """
    df = pd.merge(df, df_minus, how='outer', indicator=True).query('_merge == "left_only"').drop(columns=['_merge'])
    return df.reset_index(drop=True)

def get_tensorboard_dir(path: str = 'data/models/krpn', prefix: str = 'model') -> str:
    """
    Analyzes the already existing folders in `path` and returns a new directory name to use for tensorboard
    logging.

    Parameters
    ----------
    path: str
        Top level path for all tensorboard outputs
    prefix: str
        String to prefix the models.

    Returns
    -------
    model_dir: str
        A new directory name to log to Tensorboard.
    """

    if not os.path.exists(path):
        os.mkdir(path)

    files = os.listdir(path)
    nums = []
    for file in files:
        nums.append(int(file.replace('{}_'.format(prefix), '')))
    if len(nums) > 0:
        num = np.max(np.array(nums)) + 1
    else:
        num = 0
    model_dir = '{}/{}_{}'.format(path, prefix, num)
    return model_dir

def get_prediction_df(estimator, evaluate_input_fn, data, data_df,
                      columns=['class_ids', 'probabilities', 'logits']):
    """Run all experiments in data and report the prediction results in a
    pd.DataFrame

    Parameters
    ----------
    estimator : tf.estimator
        Previously trained or downloaded Estimator
    evaluate_input_fn :
        Function providing the input to the estimato. See train.py for options.
    data : type
        Description of parameter `data`.
    data_df : pd.DataFrame
        Description of parameter `data_df`.
    columns : type
        Description of parameter `columns`.

    Returns
    -------
    pred_df: pd.DataFrame
        DataFrame holding predcition values and settings for each data split,
        e.g. the train split.

    """

    predictions = estimator.predict(input_fn=lambda: evaluate_input_fn(data[0],data[1], 64))

    preds = []
    i = 0
    for pred_dict in predictions:
        row = [data_df.iloc[i]['kg_uid'], data_df.iloc[i]['ext_uid']]
        for column in columns:
            col = pred_dict[column]
            row.append(col)
        i = i + 1
        preds.append(row)

    columns = ['kg_uid', 'ext_uid'] + columns
    preds_df = pd.DataFrame(preds, columns=columns)
    assert preds_df.shape[0] == data_df.shape[0]
    pred_df = pd.merge(data_df, preds_df, how='left', on=['kg_uid', 'ext_uid'])
    assert pred_df.shape[0] == preds_df.shape[0]

    return pred_df

def add_paths_to_df(df, path_df: Union[str, pd.DataFrame], left_on_col, right_on_col):
    if isinstance(path_df, str):
        path_df = pd.read_csv(path_df, sep=';')
        path_df['paths'] = path_df['paths'].apply(lambda x: eval(x))

    assert isinstance(path_df, pd.DataFrame)
    df = pd.merge(df, path_df, left_on=left_on_col, right_on=right_on_col, how='left').drop(columns=[right_on_col])
    df = df.dropna()
    # df['num_paths'] = df['paths'].apply(lambda x: len(x))
    return df

def add_config_elts_as_cols(row: pd.Series, elts: List[str] = []) -> pd.Series:
    """Takes a row of a pandas DataFrame and creates a new columns which
    hold information from the config dictrionary in row['config'].

    Parameters
    ----------
    row : pd.Series
        Row of a result DataFrame
    elts : List[str]
        Elements of the RunConfig which we want to put into its own column.

    Returns
    -------
    pd.Series
        New row after added columns.

    """

    if type(row['config']) == str:
        config = ast.literal_eval(row['config'])
    else:
        config = row['config']
    for elt in elts:
        if elt in config:
            row[elt] = config[elt]
        else:
            row[elt] = ''
    return row

def cols_to_f1(row):
    add = float(row['precision']) + float(row['recall'])
    mult = 2 * float(row['precision']) * float(row['recall'])
    if add == 0:
        return 0
    return mult / add

def get_standard_binary_estimator_spec(logits, labels, mode, learning_rate):
    """

    Parameters
    ----------
    logits
    labels
    mode
    learning_rate

    Returns
    -------

    """
    # labels is None, if we are in predicition mode
    if labels is not None:
        labels = tf.reshape(labels, [-1, 1])

    probabilities = tf.nn.sigmoid(logits)
    predicted_classes = tf.round(probabilities)
    probabilities = tf.concat([probabilities, 1 - probabilities], axis=1)
    # print(predicted_classes.get_shape())
    if mode == tf.estimator.ModeKeys.PREDICT:
        predictions = {
            'class_ids': tf.squeeze(predicted_classes),
            'probabilities': probabilities,
            'logits': logits
        }

        return tf.estimator.EstimatorSpec(mode, predictions=predictions)

    # Compute loss.
    labels = tf.cast(labels, dtype=tf.float32)
    loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits))

    l2_loss = tf.losses.get_regularization_loss()
    loss += l2_loss

    # Compute evaluation metrics.
    accuracy = tf.metrics.accuracy(labels=labels, predictions=predicted_classes, name='acc_op')
    precision = tf.metrics.precision(labels=labels, predictions=predicted_classes, name='prec_op')
    recall = tf.metrics.recall(labels=labels, predictions=predicted_classes, name='rec_op')
    f1_score = tf.contrib.metrics.f1_score(labels=labels, predictions=predicted_classes, name='f1_op')
    auc_score = tf.metrics.auc(labels=labels, predictions=predicted_classes, name='auc_op')
    #ap_score = tf.metrics.average_precision_at_k(labels=labels, predictions=predicted_classes, k=10, name='ap_score')

    metrics = {'accuracy': accuracy, 'f1_score': f1_score, 'precision': precision, 'recall': recall,
               'auc': auc_score}#, 'ap_score': ap_score}

    for metric in metrics:
        if metric != 'loss':
            tf.summary.scalar(metric, metrics[metric][1])

    if mode == tf.estimator.ModeKeys.EVAL:
        return tf.estimator.EstimatorSpec(
            mode, loss=loss, eval_metric_ops=metrics)

    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
    train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())

    return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)


text:
import numpy as np
import scipy as sp
from scipy import misc
import time
import logging

from liblinearutil import predict

from features import eDN_features, dist_to_cntr_features, whiten_features

EDN_INSIZE = (512, 384)

class EDNSaliencyModel(object):

    def __init__(self, descriptions, svmModel, biasToCntr=False):
        self.descriptions = descriptions
        self.svm = svmModel['svm']
        self.whitenParams = svmModel['whitenParams']
        self.biasToCntr = biasToCntr
        nFeatures = np.sum([d['desc'][-1][0][1]['initialize']['n_filters']
                        for d in self.descriptions if d != None])
        diff = self.svm.get_nr_feature() - nFeatures
        if diff != 0:
            if diff == 1 and not biasToCntr:
                raise ValueError("The number of features in eDN and svm "
                                 "models does not match! Is the center bias "
                                 "correctly set?")

    def saliency(self, img, normalize=True):
        """Computes eDN saliency map for single image or image sequence"""

        descs = self.descriptions

        # rescale image to typical input size
        imgSize = img.shape[:2]
        rescale_factor = 0.5*EDN_INSIZE[0]/max(imgSize) + \
                         0.5*EDN_INSIZE[1]/min(imgSize)

        # single image:
        # img = misc.imresize(image, rescale_factor, 'bicubic')

        # image sequence (or single image)
        scaledImg = np.zeros([a*rescale_factor for a in imgSize] + [img.shape[2]])
        for j in xrange(img.shape[2]/3):
            scaledImg[:,:,j*3:j*3+3] = misc.imresize(img[:,:,j*3:j*3+3],
                                                     rescale_factor, 'bicubic')
        img = scaledImg

        # compute eDN features for description(s)
        t1 = time.time()
        fMapEDN, fMapSize = eDN_features(img, descs)
        t2 = time.time()
        logging.info("Feature computation took %0.3fs" % (t2-t1))

        if self.biasToCntr:
            fMapCntr = dist_to_cntr_features(img, fMapSize)
            fMap = np.hstack((fMapCntr, fMapEDN))
        else:
            fMap = fMapEDN

        fMapW, fwParams = whiten_features(fMap, self.whitenParams)

        # SVM prediction
        t1 = time.time()
        bs, pAcc, pred = predict([], fMapW.tolist(), self.svm, options="-q")
        t2 = time.time()
        logging.info("Prediction took %0.3fs" % (t2-t1))
        pred = np.array(pred)

        # reshaping and upscaling
        pred = pred.reshape(fMapSize, order='F')
        predLarge = sp.ndimage.interpolation.zoom(pred,
            (imgSize[0]/float(pred.shape[0]),
            imgSize[1]/float(pred.shape[1])))

        # normalization
        if normalize:
            rescaled = (255.0 / (predLarge.max()-predLarge.min()) *
                       (predLarge-predLarge.min())).astype(np.uint8)
            return rescaled
        else:
            return predLarge

reason: import numpy,import scipy,from scipy


text:
"""Pipeline onnx client NLI cpu"""

import collections
import csv
import json
import os
import numpy as np

from mfactcheck.multi_nli.data import NLIProcessor, convert_examples_to_features
from mfactcheck.multi_nli.config_util import _get_nli_configs
from .base import Pipeline
from mfactcheck.utils.log_helper import LogHelper
from mfactcheck.utils.predict_utils import predictions_aggregator

LogHelper.setup()
logger = LogHelper.get_logger(os.path.splitext(os.path.basename(__file__))[0])

class MultiNLIPipeline(Pipeline):
    def __init__(self, module="nli", args=None, args_parser=_get_nli_configs):
        super().__init__(module, args, args_parser)
        self.processor = NLIProcessor()
        self.label_list = self.processor.get_labels()
        self.num_labels = len(self.label_list)

    def __call__(self):
        """Classify verification label given input claim-sentence pairs"""

        eval_examples = self.processor.get_dev_examples(
            self.args.data_dir, self.args.predict_rte_file
        )
        # eval_examples = eval_examples[0:20]  # debugging
        self.num_eg = len(eval_examples)

        eval_data = convert_examples_to_features(
            eval_examples,
            self.label_list,
            self.args.max_seq_length,
            self.tokenizer,
            "classification",
        )

        preds, labels, new_guids, guids_map = super().__call__(eval_data, self.num_eg)
        preds = np.argmax(preds, axis=1)  # 0 = Support; 1 = Refute; 2 = NEI

        # Implements the logic rules to get one verification prediction per claim from 5 separate predictions
        predictions_aggregator(
            logger, self.args, preds, labels, new_guids, guids_map, compute_acc=False
        )