text:
# -*- coding: utf-8 -*-

from __future__ import division, print_function

import numpy as np
import pandas as pd

import matplotlib
import matplotlib.pyplot as pl

from peerless import plot_setup  # NOQA

names = ["id", "binary", "mass", "radius", "period", "a", "e", "per", "long",
         "asc", "incl", "teq", "age", "method", "year", "updated", "ra", "dec",
         "dist", "s_mass", "s_radius", "s_feh", "s_teff", "s_age"]

oec = pd.read_csv("open_exoplanet_catalogue.txt", skiprows=30, names=names)

for k, _ in oec.groupby("method"):
    print(k, np.sum(oec.method == k))

G = 2945.4625385377644
oec["approx_period"] = 2*np.pi*np.sqrt((215.1*oec.a)**3/G/oec.s_mass)

names = ["transit", "RV", "microlensing"]
hists = {}
m0 = np.isfinite(oec.period)
oec.period[~m0] = oec.approx_period[~m0]
m0 = np.isfinite(oec.period)
bins = np.linspace(np.log10(1.0), np.log10(1e4), 15)
for k in names:
    m = (oec.method == k) & m0
    h, bins = np.histogram(np.log10(oec[m].period), bins)
    hists[k] = h
print(hists["microlensing"])

w = np.diff(10**bins)
bins = 10**bins
bins = bins[:-1]

fig, ax = pl.subplots(1, 1, figsize=(6, 4))
ax.bar(bins, hists["transit"], width=w, color="#6baed6", edgecolor="none",
       label="transit")
ax.bar(bins, hists["RV"], width=w, color="#fd8d3c", edgecolor="none",
       bottom=hists["transit"],
       label="RV")
ax.bar(bins, hists["microlensing"], width=w, color="#74c476", edgecolor="none",
       bottom=hists["transit"] + hists["RV"],
       label="microlensing")
ax.set_xscale("log")
ax.set_xlim(bins[0], bins[-1] + w[-1])

ax.set_xlabel("orbital period [days]")
ax.set_ylabel("number of discoveries")

pl.legend(fontsize=14, loc="upper right")

ax.xaxis.set_major_formatter(matplotlib.ticker.FormatStrFormatter("%.0f"))

fig.savefig("method_comp.pdf", bbox_inches="tight")

max_stars_repo_name: dfm/single-transits

max_issues_repo_name: dfm/single-transits

max_forks_repo_name: dfm/single-transits

reason: import numpy


text:
from testtools import TestCase
import pandas as pd
import numpy as np

from hybrid_search_engine import index

class TestIndex(TestCase):
    def test_build_index(self):
        df = pd.DataFrame({
            "Id": [1, 2, 3],
            "Title": ["Cat", "Dog", "Cat"],
            "Data": ["Cat is a mammal", "Mammal is an animal", "Feline"],
            "Tag": ["Animal", "Animal", None]
        })
        filtering_columns = ["Tag"]
        columns = ["Title", "Data"]
        id_column = "Id"

        index_df, documents_df = index.build_index_from_df(df, columns=columns, id_column=id_column,
                                                           filtering_columns=filtering_columns)
        expected_index = pd.DataFrame({
            "token": ["cat", "mammal", "dog", "animal", "feline"],
            "Title": [[0, 2], [], [1], [], []],
            "Data": [[0], [0, 1], [], [1], [2]],
            "Title TF": [[1, 1], [], [1], [], []],
            "Data TF": [[1], [1, 1], [], [1], [1]]
        })
        cols = ["Title", "Data", "Title TF", "Data TF"]

        for c in cols:
            expected_index[c] = expected_index[c].apply(np.array)

        self.assertEqual(expected_index["token"].values.tolist(), index_df["token"].values.tolist())

        for c in ["Title", "Data", "Title TF", "Data TF"]:
            expected_values = expected_index[c].values.tolist()
            actual_values = index_df[c].values.tolist()

            for v1, v2 in zip(expected_values, actual_values):
                self.assertTrue((v1 == v2).all())

        expected_norm_values = np.array([
            [1., 1 / np.sqrt(2)],
            [1., 1 / np.sqrt(2)],
            [1., 1.]
        ])

        expected_ids = np.array([1, 2, 3])

        self.assertTrue((expected_norm_values == documents_df[[f"{c} Norm" for c in columns]].values).all())
        self.assertTrue((expected_ids == documents_df[id_column].values).all())

ext: py

max_stars_repo_name: DomnenkoB/Hybrid-Search-Engine

max_issues_repo_name: DomnenkoB/Hybrid-Search-Engine

max_forks_repo_name: DomnenkoB/Hybrid-Search-Engine


text:
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 21 15:11:15 2021

@author: Paul Valcke
"""

import numpy as np

# ---------------------------
_DESCRIPTION = """
    DESCRIPTION : This is a Goodwin model based on extensive variables.
    Inflation not integrated to the process
    TYPICAL BEHAVIOR : Convergence toward solow point ( good equilibrium) or debt crisis
    LINKTOARTICLE :
    """   # Description that will be shown when the user load the model or browse through models

_DPARAM = {}  # SHOULD BE REMOVED WITH NEWFORMALISM

# ---------------------------
# user-defined model
# contains the logical core
_LOGICS = {
    # differential equations (ode)
    'ode': {
        'a': {
            'logic': lambda alpha=0, itself=0: alpha * itself,
            'com': 'Exogenous technical progress as an exponential',
        },
        'N': {
            'logic': lambda beta=0, itself=0: beta * itself,
            'com': 'Exogenous population as an exponential',
        },
        'K': {
            'logic': lambda I=0, itself=0, delta=0: I - itself * delta,
            'com': 'Capital evolution from investment and depreciation',
        },
        'D': {
            'logic': lambda I=0, Pi=0: I - Pi,
            'com': 'Debt as Investment-Profit difference',
        },
        'W': {
            'logic': lambda phillips=0, itself=0: itself*phillips,
            'com': 'salary through negociation'
        },
        'p': {
            'logic': lambda itself=0, inflation=0: itself*inflation,
            'com': 'NO INFLATION FOR THE MOMENT',
        },
    },

    # Intermediary relevant functions
    'func': {
        'Y': {
            'logic': lambda K=0, nu=1: K / nu,
            'com': 'Leontiev optimized production function ',
        },
        'GDP': {
            'logic': lambda Y=0, p=0: Y*p,
            'com': 'Leontiev ',
        },
        'inflation': {
            'logic': lambda p=0: 0,
            'com': 'INFLATION NOT CODED',
        },
        'L': {
            'logic': lambda K=0, a=1, nu=1: K / (a * nu),
            'com': 'Full instant employement based on capital',
        },
        'Pi': {
            'logic': lambda Y=0, W=0, L=0, r=0, D=0: Y - W * L - r * D,
            'com': 'Profit for production-Salary-debt logic',
        },
        'lambda': {
            'logic': lambda L=0, N=1: L / N,
            'com': 'employement rate',
        },
        'omega': {
            'logic': lambda W=0, L=0, Y=1: W * L / Y,
            'com': 'wage share',
        },
        'phillips': {
            'logic': lambda phi0=0, phi1=0, lamb=0: -phi0 + phi1 / (1 - lamb)**2,
            'com': 'Wage increase rate through employement',
        },
        'kappa': {
            'logic': lambda k0=0, k1=0, k2=0, Pi=0, Y=1: k0 + k1 * np.exp(k2*Pi/Y),
            'com': 'Relative GDP investment through relative profit',
        },
        'I': {
            'logic': lambda GDP=0, kappa=0: GDP * kappa,
            'com': '',
        },
        'd': {
            'logic': lambda D=0, GDP=1: D / GDP,
            'com': 'private debt ratio',
        },
        'pi': {
            'logic': lambda omega=0, r=0, d=0: 1 - omega - r * d,
            'com': '',
        },
    },
}

_PRESETS = {
    'default': {
        'fields': {
            'a': 1,
            'N': 1,
            'K': 2.7,
            'D': 0.2,
            'W': .85,
            'p': 1,
            'alpha': 0.02,
            'beta': 0.025,
            'nu': 3,
            'delta': .005,
            'phinull': .04,
            'k0': -0.0065,
            'k1': np.exp(-5),
            'k2': 20,
            'r': 0.03,
        },
        'com': (
            'This is a run that should give simple '
            'convergent oscillations'),
        'plots': [],
    },
    'crisis': {
        'fields': {
            'a': 1,
            'N': 1,
            'K': 2.7,
            'D': 10,
            'W': .85,
            'p': 1,

            'r': 0.0
        },
        'com': (
            'This is a run that should create a debt crisis'
        ),
        'plots': [],
    },
}

_FUNC_ORDER = None  # user-defined function order (optional)

_FUNC_PARAMS = None  # user-defined list of parameters (optional)

max_stars_repo_name: MattArran/GEMMES

max_issues_repo_name: MattArran/GEMMES

max_forks_repo_name: MattArran/GEMMES


text:
# --------------
#Importing header files
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

#Path of the file
path

#Code starts here
data = pd.read_csv(path)
data.rename(columns={'Total':'Total_Medals'},inplace=True)
data.head(10)

# --------------
#Code starts here

data['Better_Event'] = np.where(data['Total_Summer']>=data['Total_Winter'],'Summer','Winter')
data['Better_Event'] =np.where(data['Total_Summer'] ==data['Total_Winter'],'Both',data['Better_Event'])
print(data.head())
better_event = data['Better_Event'].value_counts()
for key,value in better_event.items():
    if value==better_event.max():
        better_event=key
        break
print(better_event)

# --------------
#Code starts here

top_countries = data[['Country_Name','Total_Summer','Total_Winter','Total_Medals']]
top_countries.drop(top_countries.index[146],inplace=True)
def top_ten(top_countries,Column):
    return list(top_countries.nlargest(10,Column)['Country_Name'])

top_10_summer = top_ten(top_countries,'Total_Summer')
top_10_winter = top_ten(top_countries,'Total_Winter')
top_10 = top_ten(top_countries,'Total_Medals')
common = list(set(top_10).intersection(set(top_10_summer)).intersection(set(top_10_winter)))
# common = pd.merge(common,top_10_winter,how='inner')

# --------------
#Code starts here
top_df = data[data['Country_Name'].isin(top_10)]
summer_df = data[data['Country_Name'].isin(top_10_summer)]
winter_df = data[data['Country_Name'].isin(top_10_winter)]

fig,(ax_1,ax_2,ax_3) = plt.subplots(3,1,figsize=(15,10))
top_df['Total_Medals'].value_counts().plot(kind='bar',ax = ax_1)
summer_df['Total_Summer'].value_counts().plot(kind='bar',ax = ax_2)
winter_df['Total_Winter'].value_counts().plot(kind='bar',ax = ax_3)
ax_1.set_title('Overall Top Performer')

ax_2.set_title('Summer Top Performer')
ax_3.set_title('Winter Top  Performer')

# --------------
#Code starts here

summer_df['Golden_Ratio'] = summer_df['Gold_Summer']/summer_df['Total_Summer']

summer_max_ratio = summer_df['Golden_Ratio'].max()
summer_country_gold = list(summer_df['Country_Name'][summer_df['Golden_Ratio']==summer_max_ratio])[0]

winter_df['Golden_Ratio'] = winter_df['Gold_Winter']/winter_df['Total_Winter']

winter_max_ratio = winter_df['Golden_Ratio'].max()
winter_country_gold = list(winter_df['Country_Name'][winter_df['Golden_Ratio']==winter_max_ratio])[0]

top_df['Golden_Ratio'] = top_df['Gold_Total']/top_df['Total_Medals']

top_max_ratio = top_df['Golden_Ratio'].max()

top_country_gold = list(top_df['Country_Name'][top_df['Golden_Ratio']==top_max_ratio])[0]

# --------------
#Code starts here
data_1 = data.drop(data.index[146])
data_1['Total_Points'] = data_1['Gold_Total']*3+data_1['Silver_Total']*2+data_1['Bronze_Total']
most_points = data_1['Total_Points'].max()
best_country = list(data_1['Country_Name'][data_1['Total_Points']==most_points])[0]

# --------------
#Code starts here
best = data[data['Country_Name']==best_country]
best = best[['Gold_Total','Silver_Total','Bronze_Total']]
best.plot(kind='bar',stacked=True)
plt.xlabel('United States')
plt.ylabel('Medals Tally')
plt.xticks(rotation=45)


text:
"""
    Unit test for the pf_xarray class.
    This verifies methods with the pf_xarray tools module of parflow.
    It uses existing test data .pfb files in the parflow repository for testing.
"""

import sys
import os
import unittest
import numpy as np
import xarray as xr
rootdir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
print(rootdir)
sys.path.append(rootdir)
from pf_xarray.pf_backend import ParflowBackendEntrypoint
from pf_xarray.io import ParflowBinaryReader, read_stack_of_pfbs, read_pfb, write_pfb
from pfb_peek import PFBPeek

EXAMPLE_PFB_FILE_PATH_0 = f"{rootdir}/pf_xarray/tests/data/forsyth5.out.press.00000.pfb"
EXAMPLE_PFB_FILE_PATH_1 = f"{rootdir}/pf_xarray/tests/data/forsyth5.out.press.00001.pfb"
TEMP_DIRECTORY = "/tmp/parflow_xarray"

class TestPFXArray(unittest.TestCase):
    def test_open_dataset(self):
        """Test reading a pfb file using xarray open_dataset."""

        ds = xr.open_dataset(
            EXAMPLE_PFB_FILE_PATH_0, name='forsyth5.out.press', engine=ParflowBackendEntrypoint)

        # Verify the sizes of the dimensions from the loaded xarray
        self.assertEqual(3, len(ds.dims))
        self.assertEqual(21, ds.dims["z"])
        self.assertEqual(46, ds.dims["x"])
        self.assertEqual(46, ds.dims["y"])
        # Verify the checkum of values in the data array is what is expected
        da = ds.to_array()
        self.assertEqual(-444360000, int(np.sum(da)))
        # Verify that the shape of the data set is 1 variable with the expected 3D dimensions
        self.assertEqual((1, 21, 46, 46), da.shape)

    def test_read_pfb_file_list(self):
        """Test reading a list of pfb files."""

        pfb_file_list = [
            EXAMPLE_PFB_FILE_PATH_0,
            EXAMPLE_PFB_FILE_PATH_1
        ]
        da = read_stack_of_pfbs(pfb_file_list)

        # Verify that the shape of the data set is 2 time rows with the expected 3D dimensions
        self.assertEqual((2, 21, 46, 46), da.shape)
        # Verify the sizes of the dimensions from the loaded xarray
        # Verify the checkum of values in the data array is what is expected
        self.assertEqual(-887197545, int(np.sum(da)))

    def test_write_pfb_file(self):
        """Test writing a single pfb file."""

        with ParflowBinaryReader(EXAMPLE_PFB_FILE_PATH_0) as pfb:
            da = pfb.read_all_subgrids()
            header = pfb.header
            header['p'] = '8'
            header['q'] = '5'
            header['z'] = '1'
            if not os.path.exists(TEMP_DIRECTORY):
                os.makedirs(TEMP_DIRECTORY)
            file_name = f'{TEMP_DIRECTORY}/example0.pfb'
            if os.path.exists(file_name):
                os.remove(file_name)
            write_pfb(file_name, da, header=header)
            if not os.path.exists(file_name):
                self.fail(f"PFB File '{file_name}' was not created.")
            total = 0
            with PFBPeek() as s:
                header = s.open_pfb(file_name)
                self.assertEqual(46, header.get('nx'))
                self.assertEqual(46, header.get('ny'))
                self.assertEqual(21, header.get('nz'))
                number_of_subgrids = int(header.get('n_subgrids', 0))
                self.assertEqual(40, number_of_subgrids)
                for i in range(0, number_of_subgrids):
                    sg_header = s.read_subgrid_header()
                    data = s.read_subgrid_data(sg_header)
                    total = total + data.sum()
                    if i == 1:
                        self.assertEqual(6, sg_header.get('ix'))
                        self.assertEqual(0, sg_header.get('iy'))
                        self.assertEqual(1, sg_header.get('iz'))
                        self.assertEqual(6, sg_header.get('nx'))
                        self.assertEqual(10, sg_header.get('ny'))
                        self.assertEqual(21, sg_header.get('nz'))
            self.assertEqual(-444360000.0, int(total))
            os.remove(file_name)

    def test_local(self):
        da = np.arange(1, 25, dtype=np.float64)
        da.resize([2, 4, 3])
        header = {}
        header['nx'] = '3'
        header['ny'] = '4'
        header['nz'] = '2'
        header['p'] = '2'
        header['q'] = '2'
        header['r'] = '1'
        if not os.path.exists(TEMP_DIRECTORY):
            os.makedirs(TEMP_DIRECTORY)
        file_name = f'{TEMP_DIRECTORY}/local.pfb'
        if os.path.exists(file_name):
            os.remove(file_name)
        if os.path.exists(file_name + '.dist'):
            os.remove(file_name + '.dist')
        write_pfb(file_name, da, header=header, dist=True)
        if not os.path.exists(file_name):
            self.fail(f"PFB File '{file_name}' was not created.")
        if not os.path.exists(file_name + '.dist'):
            self.fail(f"PFB File '{file_name}.dist' was not created.")
        with PFBPeek() as s:
            header = s.open_pfb(file_name)
            self.assertEqual(3, header.get('nx'))
            self.assertEqual(4, header.get('ny'))
            self.assertEqual(2, header.get('nz'))
            sg_header = s.read_subgrid_header()
            data = s.read_subgrid_data(sg_header)
            checksum = data.sum()
            self.assertEqual(72, checksum)
            self.assertEqual(2, sg_header.get('nx'))
            self.assertEqual(2, sg_header.get('ny'))
            self.assertEqual(2, sg_header.get('nz'))
            sg_header = s.read_subgrid_header()
            data = s.read_subgrid_data(sg_header)
            self.assertEqual(1, sg_header.get('nx'))
            self.assertEqual(2, sg_header.get('ny'))
            self.assertEqual(2, sg_header.get('nz'))
            self.assertEqual(2, sg_header.get('ix'))
            self.assertEqual(0, sg_header.get('iy'))
            self.assertEqual(0, sg_header.get('iz'))
            checksum = data.sum()
            self.assertEqual(42, checksum)
            self.assertEqual(3, data[0][0][0])
        os.remove(file_name + '.dist')
        os.remove(file_name)

    def test_empty_array(self):
        da = np.arange(0, 0, dtype=np.float64)
        da.resize([0, 0, 0])
        header = {}
        header['p'] = '1'
        header['q'] = '1'
        header['r'] = '1'
        if not os.path.exists(TEMP_DIRECTORY):
            os.makedirs(TEMP_DIRECTORY)
        file_name = f'{TEMP_DIRECTORY}/empty.pfb'
        if os.path.exists(file_name):
            os.remove(file_name)
        write_pfb(file_name, da, header=header)
        if not os.path.exists(file_name):
            self.fail(f"PFB File '{file_name}' was not created.")
        with PFBPeek() as s:
            header = s.open_pfb(file_name)
            self.assertEqual(0, header.get('nx'))
            self.assertEqual(0, header.get('ny'))
            self.assertEqual(0, header.get('nz'))
            self.assertEqual(0, header.get('nz'))
            number_of_subgrids = int(header.get('n_subgrids', 0))
            self.assertEqual(1, number_of_subgrids)
        os.remove(file_name)

if __name__ == '__main__':
    unittest.main()


text:
"""
 =====================
 Demo: demo_agent_manager_save
 =====================
"""
import numpy as np
from rlberry.envs.benchmarks.ball_exploration import PBall2D
from rlberry.agents.torch.ppo import PPOAgent
from rlberry.manager import AgentManager, plot_writer_data, evaluate_agents

if __name__ == "__main__":
    # --------------------------------
    # Define train and evaluation envs
    # --------------------------------
    train_env = (PBall2D, dict())
    eval_env = (PBall2D, dict())

    # -----------------------------
    # Parameters
    # -----------------------------
    N_EPISODES = 100
    GAMMA = 0.99
    HORIZON = 50
    BONUS_SCALE_FACTOR = 0.1
    MIN_DIST = 0.1

    params_ppo = {"gamma": GAMMA, "horizon": HORIZON, "learning_rate": 0.0003}

    eval_kwargs = dict(eval_horizon=HORIZON, n_simulations=20)

    # -------------------------------
    # Run AgentManager and save results
    # --------------------------------
    ppo_stats = AgentManager(
        PPOAgent,
        train_env,
        fit_budget=N_EPISODES,
        init_kwargs=params_ppo,
        eval_kwargs=eval_kwargs,
        n_fit=4,
        output_dir="dev/",
        parallelization="process",
    )
    ppo_stats.fit()  # fit the 4 agents
    ppo_stats_fname = ppo_stats.save()
    del ppo_stats

    # -------------------------------
    # Load and plot results
    # --------------------------------
    ppo_stats = AgentManager.load(ppo_stats_fname)

    # learning curves
    plot_writer_data(
        ppo_stats,
        tag="episode_rewards",
        preprocess_func=np.cumsum,
        title="Cumulative Rewards",
        show=False,
    )

    # compare final policies
    output = evaluate_agents([ppo_stats], n_simulations=15)
    print(output)

max_stars_repo_name: riccardodv/rlberry

max_issues_repo_name: riccardodv/rlberry

max_forks_repo_name: riccardodv/rlberry


text:
# -*- coding: utf-8 -*-
"""
maps viewer
to use with common pyQt5 gui
"""

import sys
from PyQt5 import QtCore
from PyQt5.QtWidgets import QMainWindow, QApplication, QWidget, QCheckBox,\
                            QFileDialog, QMessageBox, QLabel, QComboBox,\
                            QRadioButton, QButtonGroup,\
                            QVBoxLayout, QHBoxLayout
from PyQt5.QtCore import Qt

from matplotlib import cm
from matplotlib.colors import ListedColormap, LinearSegmentedColormap

import matplotlib.pyplot as plt
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg
from matplotlib.figure import Figure

import numpy as np

from utils_ViscoIndent import load_AFM_data_pickle_short

class maps_view(QMainWindow):

    def __init__(self, parent=None):
        super(maps_view, self).__init__(parent)
        self.MAPviewdialog = parent  # share variables
        if not hasattr(self.MAPviewdialog, 'commongui'):
            self.initUI()

    def initUI(self):
        if not hasattr(self.MAPviewdialog, 'commongui'):
            if 'Pars' in globals():
                print('Data found in workspace')
                strData = 'Data found in workspace. Reuse or open new?'
                choice = QMessageBox.question(QMessageBox(), "Choose Data", strData, QMessageBox.Yes | QMessageBox.Open)
                if choice == QMessageBox.Yes:
                    global Pars, Results, Data
                    self.Pars = Pars
                    self.Results = Results
                    self.Data = Data
                else:
                    options = QFileDialog.Options()
                    self.fileName, _ = QFileDialog.getOpenFileName(self, "QFileDialog.getOpenFileName()", "", "All Files (*);;Python Files (*.py)", options=options)
                    self.Pars, self.Data, self.Results = load_AFM_data_pickle_short(self.fileName)
            else:
                options = QFileDialog.Options()
                self.fileName, _ = QFileDialog.getOpenFileName(self, "QFileDialog.getOpenFileName()", "", "All Files (*);;Python Files (*.py)", options=options)
                # self.Pars, self.Data, self.Results = open_vars(self.fileName[:-4])
                self.Pars, self.Data, self.Results = load_AFM_data_pickle_short(self.fileName)

            # self.Results = self.Results[:np.size(self.Data, 0)]  # size correction
            # self.Results = self.Results.drop(self.Results.index[np.size(self.Data, 0):])  # size correction
            # self.Results.Pixel = self.Data[:, 0]  # remove nans
        else:
            self.Pars = self.MAPviewdialog.Pars
            self.Data = self.MAPviewdialog.Data
            self.Results = self.MAPviewdialog.Results
            # self.Results = self.Results[:np.size(self.Data,0)]  # size correction
            # self.Results.Pixel = self.Data[:, 0]  # remove nans

        self.cmaplist = plt.colormaps()
        top = cm.get_cmap('Oranges', 64)
        bottom = cm.get_cmap('viridis', 512)
        newcolors = np.vstack((top(np.linspace(0, 1, 64)),
                               bottom(np.linspace(0, 1, 512))))
        self.newcmp = ListedColormap(newcolors, name='viridis_topo')
        self.cmaplist.append('viridis_topo')

        # self.pixN = np.asarray(self.Data[:, 0], int) # take from results somehow...
        self.pixN = np.asarray(self.Results.Pixel, int)

        # self.Res_fields = list(self.Results.columns)
        self.Res_fields = list(self.Results.select_dtypes(include=['float32']).columns)

        self.fig = fig = Figure()
        self.canvas1 = FigureCanvasQTAgg(fig)
        self.ax1 = fig.add_subplot(111, title="Map")
        img1 = self.ax1.imshow(self.Pars.topo, interpolation='nearest', cmap='viridis', origin='lower')
        self.cb = fig.colorbar(img1)

        self.boxROI = QCheckBox("Show ROIs", self)
        self.boxROI.stateChanged.connect(self.clickBoxROI)

        self.labelLog = QLabel('Log or linear scale:')
        self.btngroupLogopts = [QRadioButton("linear"),
                                QRadioButton("log")]
        self.btngroupLog = QButtonGroup()

        self.cbRes = QComboBox()
        self.cbRes.addItems(self.Res_fields)
        self.cbRes.addItems(['topography', 'corrected topography'])
        self.cbRes.currentIndexChanged.connect(self.cbRes_changed)
        self.cbRes.setCurrentIndex(1)

        self.layoutLog = QVBoxLayout()
        self.layoutLog.addWidget(self.labelLog)
        for i in range(len(self.btngroupLogopts)):
            self.layoutLog.addWidget(self.btngroupLogopts[i])
            self.btngroupLog.addButton(self.btngroupLogopts[i], i)
            self.btngroupLogopts[i].clicked.connect(self.cbRes_changed)
        self.btngroupLogopts[0].setChecked(True)

        self.setGeometry(300, 300, 900, 600)
        self.setWindowTitle('Maps')

        layoutA = QVBoxLayout()
        layoutA.addWidget(self.canvas1)

        layoutB = QVBoxLayout()
        layoutB.addWidget(self.cbRes)
        layoutB.addWidget(self.boxROI)
        layoutB.addLayout(self.layoutLog)
        layoutB.setAlignment(Qt.AlignTop)  # alignment of widgets

        layoutM = QHBoxLayout()
        layoutM.addLayout(layoutA)
        layoutM.addLayout(layoutB)

        widget = QWidget()
        widget.setLayout(layoutM)
        self.setCentralWidget(widget)
        self.show()
        self.activateWindow()
        self.select_pts()

    def cbRes_changed(self):

        self.ax1.clear()
        if self.cbRes.currentText() == 'topography':
            self.currentmap = self.Pars.topo
            cmapS = self.newcmp
        elif self.cbRes.currentText() == 'corrected topography':
            self.currentmap = self.Pars.topo2
            cmapS = self.newcmp
        else:
            self.res_to_map(self.cbRes.currentText())
            cmapS = 'viridis'
        # self.currentmap = self.Pars.topo2
        if self.btngroupLog.checkedId() == 1:
            img1 = self.ax1.imshow(np.log(self.currentmap), interpolation='nearest', cmap=cmapS, origin='lower')  # 'jet'
        else:
            img1 = self.ax1.imshow(self.currentmap, interpolation='nearest', cmap=cmapS, origin='lower')  # 'jet'
        self.cb.remove()
        self.cb = self.fig.colorbar(img1)
        self.canvas1.draw()
        if self.boxROI.isChecked():
            self.showROI()

    def res_to_map(self, keyV):
        fullResults = np.full([self.Pars.topo.size], np.nan)
        # pixN = Results.Pixel[:np.size(Data,0)].values
        # pixN = np.asarray(Results.Pixel[1:np.size(Data,0)], int)

        # fullResults[pixN] = Results.EHertz.values[:np.size(Data,0)]
        fullResults[self.pixN] = self.Results[keyV].values
        self.currentmap = np.reshape(fullResults, self.Pars.topo.shape)

    def clickBoxROI(self, state):

        if state == QtCore.Qt.Checked:
            # self.showROI()
            self.cbRes_changed()
        else:
            self.cbRes_changed()
            pass

    def showROI(self):
        regions = self.Pars.ROIPars.reg_nums
        region_numbers = self.Pars.ROIPars.reg_nums_all
        map_regions = np.reshape(region_numbers, self.Pars.topo.shape)
        for ii in range(0, self.Pars.ROIPars.num_of_ROIs):
            xc = np.arange(self.Pars.topo.shape[0])
            yc = np.arange(self.Pars.topo.shape[1])
            self.ax1.contour(xc, yc, np.reshape(regions[ii], self.Pars.topo.shape))
            map_regions = np.reshape(np.copy(region_numbers), self.Pars.topo.shape)
            map_regions[map_regions != ii+1] = 0
            labelXcoorF = np.maximum.reduce(map_regions, 0)
            labelXcoor = np.mean(xc[(labelXcoorF == ii+1)])
            labelYcoorF = np.maximum.reduce(map_regions, 1)
            labelYcoor = np.mean(yc[(labelYcoorF == ii+1)])
            self.ax1.text(labelXcoor, labelYcoor, str(ii+1),
                          horizontalalignment='center',
                          verticalalignment='center', fontsize=20)
            self.canvas1.draw()

    def select_pts(self):
        self._cid = self.fig.canvas.mpl_connect("axes_enter_event", self._start_ginput)

    def _start_ginput(self, event):
        self.fig.canvas.mpl_disconnect(self._cid)
        # Workaround for now; patch makes this unnecessary.
        self.fig.show = lambda: None
        for ii in range(1, 100):
            self.pts = self.fig.ginput(1, timeout=0)
            pts = np.asarray(self.pts).squeeze()
            pts = pts.round()  # round pts coordinates
            pts = pts.astype(int)  # coordinates to int
            print(pts)
            if len(pts) > 0:
                PixN = pts[0] + (pts[1])*self.Pars.topo.shape[0]
                print(PixN)
            else:
                break
            try:
                numinarray = np.where(self.pixN == PixN)[0][0]
                print(numinarray)
                if hasattr(self.MAPviewdialog, 'commongui'):
                    self.MAPviewdialog.slider.setValue(numinarray)
                else:
                    print(PixN)
            except:
                print('data point is not in the array')
            ii = ii+1
        # self.select_pts()

    def closeEvent(self, event):

        if not hasattr(self.MAPviewdialog, 'commongui'):
            # global Pars = self.Pars
            print('module exit')
            QApplication.quit()
        else:
            self.MAPviewdialog.Pars = self.Pars
            print('module_PLpart_finished')
            self.close()

if __name__ == '__main__':
    import matplotlib.pyplot as plt
    try:
        del app
    except:
        print('noapp')

    app = QApplication(sys.argv)
    main = maps_view()
    main.show()
    sys.exit(app.exec_())

max_stars_repo_name: yu-efremov/ViscoIndent

max_issues_repo_name: yu-efremov/ViscoIndent

max_forks_repo_name: yu-efremov/ViscoIndent


text:
import logging
import os

# import pickle
from shutil import copyfile
import time

# import json
import numpy as np
import matplotlib.pyplot as plt
from tensorboardX import SummaryWriter
import torch

def _create_logger(root_dir, file_name="log.txt"):
    log_file = os.path.join(root_dir, file_name)
    log_format = "%(asctime)s  %(levelname)5s  %(message)s"
    logging.basicConfig(level=logging.DEBUG, format=log_format, filename=log_file)
    console = logging.StreamHandler()
    console.setLevel(logging.DEBUG)
    console.setFormatter(logging.Formatter(log_format))
    logging.getLogger(__name__).addHandler(console)
    return logging.getLogger(__name__)

class Logger:
    def __init__(self, cfg):
        cfg["log_dir"] = os.path.abspath(os.path.expanduser(cfg["log_dir"]))

        # main log
        if "use_timestamp" in cfg.keys() and cfg["use_timestamp"] is False:
            dir_name = cfg["tag"]
        else:
            timestamp = time.strftime("%Y%m%d_%H%M%S", time.localtime())
            dir_name = f"{cfg['tag']}_{timestamp}"
        self.__log_dir = os.path.join(cfg["log_dir"], dir_name)
        os.makedirs(self.__log_dir, exist_ok=True)

        self.__log = _create_logger(self.__log_dir, cfg["log_fname"])
        self.log_debug(f"Log directory: {self.__log_dir}")

        # backup important files (e.g. config.yaml)
        self.__backup_dir = os.path.join(self.__log_dir, "backup")
        os.makedirs(self.__backup_dir, exist_ok=True)
        for file_ in cfg["backup_list"]:
            self.log_debug(f"Backup {file_}")
            copyfile(
                os.path.abspath(file_),
                os.path.join(self.__backup_dir, os.path.basename(file_)),
            )

        # for storing results (network output etc.)
        self.__output_dir = os.path.join(self.__log_dir, "output")
        os.makedirs(self.__output_dir, exist_ok=True)

        # for storing ckpt
        self.__ckpt_dir = os.path.join(self.__log_dir, "ckpt")
        os.makedirs(self.__ckpt_dir, exist_ok=True)

        # for tensorboard
        tb_dir = os.path.join(self.__log_dir, "tb")
        os.makedirs(tb_dir, exist_ok=True)
        self.__tb = SummaryWriter(log_dir=tb_dir)

        # the sigterm checkpoint
        self.__sigterm_ckpt = os.path.join(
            cfg["log_dir"], f"sigterm_ckpt_{cfg['tag']}.pth"
        )

        gpu = (
            os.environ["CUDA_VISIBLE_DEVICES"]
            if "CUDA_VISIBLE_DEVICES" in os.environ.keys()
            else "ALL"
        )
        self.log_info(f"CUDA_VISIBLE_DEVICES={gpu}")

    def flush(self):
        self.__tb.flush()

    def close(self):
        self.__tb.close()
        handlers = self.__log.handlers[:]
        for handler in handlers:
            handler.close()
            self.__log.removeHandler(handler)

    """
    Python log
    """
    def log_error(self, s):
        self.__log.error(s)

    def log_warning(self, s):
        self.__log.warning(s)

    def log_info(self, s):
        self.__log.info(s)

    def log_debug(self, s):
        self.__log.debug(s)

    """
    Add to tensorboard
    """

    def add_scalar(self, key, val, step):
        self.__tb.add_scalar(key, val, step)

    def add_fig(self, key, fig, step, close_fig=False):
        """Convert a python fig to np.ndarry and add it to tensorboard"""
        fig.canvas.draw()
        im = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep="")
        im = im.reshape(fig.canvas.get_width_height()[::-1] + (3,))
        im = im.transpose(2, 0, 1)  # (3, H, W)
        im = im.astype(np.float32) / 255.0
        self.add_im(key, im, step)

        if close_fig:
            plt.close(fig)

    def add_im(self, key, im, step):
        """Add an image to tensorboard. The image should be as np.ndarray
        https://tensorboardx.readthedocs.io/en/latest/tutorial.html#add-image
        """
        self.__tb.add_image(key, im, step)

    """
    Save to system
    """

    def get_save_dir(self, epoch, split):
        return os.path.join(self.__output_dir, split, f"e{epoch:06d}")

    # def save_dict(self, dict_, file_name, epoch, split):
    #     """Save the dictionary to a pickle file. Single value items in the dictionary
    #     are stored in addition as a json file for easy inspection.
    #     """
    #     json_dict = {}
    #     for key, val in dict_.items():
    #         if not isinstance(val, (np.ndarray, tuple, list, dict)):
    #             json_dict[key] = str(val)

    #     save_dir = self.get_save_dir(epoch, split)
    #     json_fname = os.path.join(save_dir, f"{file_name}.json")
    #     os.makedirs(os.path.dirname(json_fname), exist_ok=True)
    #     with open(json_fname, "w") as fp:
    #         json.dump(json_dict, fp, sort_keys=True, indent=4)
    #     self.log_info(f"Dictonary saved to {json_fname}.")

    #     pickle_fname = os.path.join(save_dir, f"{file_name}.pkl")
    #     with open(pickle_fname, "wb") as fp:
    #         pickle.dump(dict_, fp, protocol=pickle.HIGHEST_PROTOCOL)
    #     self.log_info(f"Dictonary saved to {pickle_fname}.")

    # def save_fig(self, fig, file_name, epoch, split, close_fig=True):
    #     fname = os.path.join(self.get_save_dir(epoch, split), f"{file_name}.png")
    #     os.makedirs(os.path.dirname(fname), exist_ok=True)
    #     fig.savefig(fname)
    #     if close_fig:
    #         plt.close(fig)

    # def save_file(self, file_str, file_name, epoch, split):
    #     fname = os.path.join(self.get_save_dir(epoch, split), f"{file_name}.txt")
    #     os.makedirs(os.path.dirname(fname), exist_ok=True)
    #     with open(fname, "w") as f:
    #         f.write(file_str)

    """
    Save and load checkpoints
    """

    def save_ckpt(self, fname, model, optimizer, epoch, step):
        if not os.path.dirname(fname):
            fname = os.path.join(self.__ckpt_dir, fname)

        if model is not None:
            if isinstance(model, torch.nn.DataParallel):
                model_state = model.module.state_dict()
            else:
                model_state = model.state_dict()
        else:
            model_state = None
        optim_state = optimizer.state_dict() if optimizer is not None else None

        ckpt_dict = {
            "epoch": epoch,
            "step": step,
            "model_state": model_state,
            "optimizer_state": optim_state,
        }
        torch.save(ckpt_dict, fname)
        self.log_info(f"Checkpoint saved to {fname}.")

    def load_ckpt(self, fname, model, optimizer=None):
        ckpt = torch.load(fname)
        epoch = ckpt["epoch"] if "epoch" in ckpt.keys() else 0
        step = ckpt["step"] if "step" in ckpt.keys() else 0

        model.load_state_dict(ckpt["model_state"])

        if optimizer is not None:
            optimizer.load_state_dict(ckpt["optimizer_state"])

        self.log_info(f"Load checkpoint {fname}: epoch {epoch}, step {step}.")

        return epoch, step

    def save_sigterm_ckpt(self, model, optimizer, epoch, step):
        """Save a checkpoint, which another process can use to continue the training,
        if the current process is terminated or preempted. This checkpoint should
        be saved in a process-agnoistic directory such that it can be located by
        both processes.
        """
        self.save_ckpt(self.__sigterm_ckpt, model, optimizer, epoch, step)

    def load_sigterm_ckpt(self, model, optimizer):
        return self.load_ckpt(self.__sigterm_ckpt, model, optimizer)

    def sigterm_ckpt_exists(self):
        return os.path.isfile(self.__sigterm_ckpt)


text:
import numpy as np
import scipy.signal as signal

# Default config
config = {
    "Low Pass Filter Cut-Off Frequency (normalized)": 0.025,
    "Low Pass Filter Order": 2,
    "Low Pass Filter Type": 'butter',
}

########################################## Utils ########################################################
lp_filter = signal.iirfilter(config["Low Pass Filter Order"],
                             config["Low Pass Filter Cut-Off Frequency (normalized)"],
                             btype='lowpass',
                             output='sos',
                             ftype=config["Low Pass Filter Type"],
                             fs=2.0)

def LowPassFilter(x):
    pad = 1000
    filtered_x = signal.sosfiltfilt(lp_filter, np.pad(x,(pad,pad),mode='reflect'))[pad:-pad]
    return np.clip(filtered_x, 1e-30, None) # all features should be positive, negative values are due to gibbs phenomena
#########################################################################################################

#######################  Functions to calculate instantaneous features ###################################
#
#   INPUT:
#   Two 1d numpy arrays with same length. The x array represents each signal sample and t array represents
#   the time of each sample.
#
#   OUTPUT:
#   1d numpy array representing the calculated feature value at each time sample.
#
#   NOTE:
#   The function name will be used for identification of the feature in pandas dataframes (column name)
#
def Variance(t, x):
    # assumes mean = 0
    return LowPassFilter(x**2)

def AbsoluteValue(t, x):
    return LowPassFilter(np.abs(x))

def MeanOfAmplitude(t, x):
    return LowPassFilter(np.abs(np.gradient(x,t)))

def PeakEnvelope(t, x):
    n = x.shape[0]
    dx = np.gradient(x, t)
    pk_intensity = np.zeros(n)
    for i in range(1,n):
        if (dx[i-1]*dx[i] < 0.0):
            pk_intensity[i] = np.abs((x[i]+x[i-1])/2.0)
    return LowPassFilter(pk_intensity)

def PeakIntensity(t, x):
    n = x.shape[0]
    dx = np.gradient(x, t)
    pk_intensity_t = [t[0]]
    pk_intensity = [np.abs(x[0])]
    for i in range(1,n):
        if (dx[i-1]*dx[i] < 0.0):
            pk_intensity.append(np.abs(x[i]))
            pk_intensity_t.append(t[i])
    pk_intensity_t.append(t[-1]+(t[1]-t[0]))
    pk_intensity.append(np.abs(x[-1]))
    return LowPassFilter(np.interp(t, pk_intensity_t, pk_intensity))

def InstantaneousAmplitude(t, x):
    return LowPassFilter(np.abs(signal.hilbert(x)))

def InstantaneousFrequency(t, x):
    fs = 1.0/(t[1]-t[0])
    analytical_signal = signal.hilbert(x)
    instantaneous_phase = LowPassFilter(np.unwrap(np.angle(analytical_signal)))
    instantaneous_frequency = np.gradient(instantaneous_phase) / (2.0 * np.pi) * fs
    return LowPassFilter(instantaneous_frequency)

def EnvelopePeakIntensity(t,x):
    return PeakIntensity(t, np.abs(signal.hilbert(x)))

def EnvelopePeakEnvelope(t,x):
    return PeakEnvelope(t, np.abs(signal.hilbert(x)))

reason: import numpy,import scipy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jul  2 15:28:36 2021

@author: Briana
"""

import numpy as np
import numpy.ma as ma
import math
from astropy.io import fits
from astropy.wcs import WCS
import matplotlib.pyplot as plt

# need to give it dataframe containing fiber information for one field with
# columns:
# flux, flux_err, ra, dec
# need to also supply fiber diameter, new pixel size, max distance, kernal
# sigma (degrees)
# optional to supply the new grid dimensions (xmin, xmax, ymin, ymax)
# otherwise builds grid from min and max coordinates of the field

class fibers_to_grid():

    def __init__(self, fib_ra, fib_dec, fiberd,
                 regrid_size, max_radius, kern_sig):
        self.fiberd = fiberd
        self.regrid_size = regrid_size
        self.max_dist = max_radius**2
        self.kern_sig = kern_sig

        self.fib_ra = fib_ra
        self.fib_dec = fib_dec

        self.fib_flux = None
        self.fib_flux_err = None

        self.grid_bounds = None
        self.new_pix_area = None
        self.regrid_x = None
        self.regrid_Y = None
        self.x_grid = None
        self.y_grid = None
        self.avg_dec = None

        self.nx = None
        self.ny = None

        self.weight_grid = None
        self.flux_grid = None
        self.error_grid = None

    def build_new_grid(self, grid=(0, 0, 0, 0)):

        try:
            xmin, xmax, ymin, ymax = grid
            if len(set(list(grid))) == 1:
                # min and max x,y values in the dither
                print('BUILDING grid from fiber coodinates')
                xmin, xmax = (self.fib_ra.min(), self.fib_ra.max())
                ymin, ymax = (self.fib_dec.min(), self.fib_dec.max())
        except:
            print('grid must be in format (xmin, xmax, ymin, ymax)')

        # adjust min and mas to include outside of edges fiber radius
        xmin = xmin - self.fiberd / 2.0
        xmax = xmax + self.fiberd / 2.0
        ymin = ymin - self.fiberd / 2.0
        ymax = ymax + self.fiberd / 2.0

        self.grid_bounds = (xmin, xmax, ymin, ymax)

        # size of pixels in new grid
        # need to multiply cos(dec) term to ra so get about same
        # number of pixels in x and y
        avg_y = np.average([ymax, ymin])
        self.avg_dec = avg_y
        self.regrid_x = self.regrid_size/np.cos(np.deg2rad(avg_y))
        self.regrid_y = self.regrid_size

        # defind number of dimenstions
        # nx, ny are the new grid dimensions
        self.nx = int((xmax-xmin)/(self.regrid_x))+1
        self.ny = int((ymax-ymin)/(self.regrid_y))+1

        # new pixel area
        self.new_pix_area = self.regrid_x*self.regrid_y

        # build new grid
        x_range = xmin+(np.arange(self.nx)+0.5)*self.regrid_x
        y_range = ymin+(np.arange(self.ny)+0.5)*self.regrid_y

        self.x_grid, self.y_grid = np.meshgrid(x_range, y_range)

        return self.x_grid, self.y_grid

    def shepards_kernal(self, fib_flux, fib_flux_err):
        if self.x_grid is None:
            self.build_new_grid()

        self.fib_flux = fib_flux
        self.fib_flux_err = fib_flux_err

        fib_sa = self.fib_flux/(((self.fiberd/2.0)**2)*math.pi)
        fib_sa_err = self.fib_flux_err/(((self.fiberd/2.0)**2)*math.pi)

        nx, ny = self.nx, self.ny
        xmin, ymin = self.grid_bounds[0], self.grid_bounds[2]

        I = np.zeros((ny, nx))
        W = np.zeros((ny, nx))
        E = np.zeros((ny, nx))

        # iterate through each pixel in the mastergrid
        for iy, ix in np.ndindex(I.shape):

            # find the x and y value of that pixel based on the grid size
            x = xmin + (ix + 0.5) * self.regrid_x
            y = ymin + (iy + 0.5) * self.regrid_y

            # distance modulous
            dist2_allfibs = ((x - self.fib_ra)**2) + ((y - self.fib_dec)**2)
            # find indices of fibers within user defined
            # max distance from pixel
            fib_inds = np.where(dist2_allfibs < self.max_dist)

            # if there are fibers within max distance from pixel
            if len(fib_inds[0]) > 0:

                dist2 = dist2_allfibs[fib_inds]
                sa = fib_sa[fib_inds]
                sa_err = fib_sa_err[fib_inds]

                nan_mask = np.where(~np.isnan(sa))
                dist2 = dist2[nan_mask]
                sa = sa[nan_mask]
                sa_err = sa_err[nan_mask]

                weights = np.exp(-0.5*dist2/(self.kern_sig**2))

                I[iy][ix] = np.sum(self.new_pix_area * weights * sa)
                E[iy][ix] = np.sum((self.new_pix_area**2) * (weights**2) * (sa_err**2))
                W[iy][ix] = np.sum(weights)

            # if no fibers within max distance of pixel
            else:

                I[iy][ix] = np.NaN
                E[iy][ix] = np.NaN
                W[iy][ix] = np.NaN

        self.weight_grid = W
        self.flux_grid = I/W
        self.error_grid = np.sqrt(E)/W

        return self.flux_grid, self.error_grid

    def build_wcs(self, wave_dict=None):
        npix_dec, npix_ra = np.shape(self.flux_grid)

        ra_ref_pix = int(npix_ra/2)
        dec_ref_pix = int(npix_dec/2)

        wcs_dict = {'CUNIT1': 'deg', 'CUNIT2': 'deg',
                    'CTYPE1': 'RA---TAN', 'CTYPE2': 'DEC--TAN',
                    'CRPIX1': ra_ref_pix, 'CRPIX2': dec_ref_pix,
                    'CRVAL1': self.x_grid[dec_ref_pix, ra_ref_pix],
                    'CRVAL2': self.y_grid[dec_ref_pix, ra_ref_pix],
                    'CDELT1': (self.regrid_x*-1), 'CDELT2': self.regrid_y}

        if isinstance(wave_dict, dict):
            wcs_dict = {**wcs_dict, **wave_dict}
            print('Creating 3D WCS')
        else:
            print('Creating 2D WCS')

        new_wcs = WCS(wcs_dict)
        return new_wcs

    def plot_results(self, plot_error=False, plot_weights=False,
                     savepath=None):

        fig, ax = plt.subplots(1, 1)
        ax.pcolor(self.x_grid, self.y_grid, self.flux_grid)
        ax.scatter(self.fib_ra, self.fib_dec, color='black', s=1)
        for i in range(len(self.fib_flux)):
            circle1 = plt.Circle((self.fib_ra[i], self.fib_dec[i]),
                                 self.fiberd/2.0, ec='grey', fill=False)
            ax.add_artist(circle1)

        plt.show()
        return None

    def save_interp_image(self, hdr, outfile):
        hdu_f = fits.PrimaryHDU(self.flux_grid, header=hdr)
        hdu_f.writeto(outfile, overwrite=True)
        hdu_f.close()

        hdu_e = fits.PrimaryHDU(self.error_grid, header=hdr)
        hdu_e.writeto(outfile, overwrite=True)
        hdu_e.close()

    def test_flux_conservation(self):

        num_fib = len(self.fib_flux)
        flux_grid_mask = ma.masked_invalid(self.flux_grid)
        num_pix = flux_grid_mask.count()
        print('# fibers:', num_fib, '# pixels:', num_pix)

        fib_area = num_fib*(((self.fiberd/2.0)**2)*np.pi)
        pix_area = num_pix*(self.regrid_x*self.regrid_y)
        print('total pix_area/fib_area:', pix_area/fib_area)

        pre_interp = np.sum(self.fib_flux)/fib_area
        post_interp = np.nansum(self.flux_grid)/pix_area
        convs_ratio = post_interp/pre_interp

        print('Flux pre-interp:', pre_interp)
        print('Flux post-interp:', post_interp)
        print('Flux_ratio:', convs_ratio)
        return convs_ratio

max_stars_repo_name: BrianaLane/Interferon

max_issues_repo_name: BrianaLane/Interferon

max_forks_repo_name: BrianaLane/Interferon

reason: import numpy,from astropy


text:
# -*- coding: utf-8 -*-

#
# Authors: Dmitriy Serdyuk

import itertools
import numpy

from six.moves import range
from itertools import chain
from scipy import fft
from scipy.signal import stft

FS = 44100            # samples/second
DEFAULT_WINDOW_SIZE = 2048    # fourier window size
OUTPUT_SIZE = 128               # number of distinct notes
STRIDE = 512          # samples between windows
WPS = FS / float(512)   # windows/second

class MusicNet(object):
    def __init__(self, filename, in_memory=True, window_size=4096,
                 output_size=84, feature_size=1024, sample_freq=11000,
                 complex_=False, fourier=False, stft=False, fast_load=False,
                 rng=None, seed=123):
        if not in_memory:
            raise NotImplementedError
        self.filename = filename

        self.window_size = window_size
        self.output_size = output_size
        self.feature_size = feature_size
        self.sample_freq = sample_freq
        self.complex_ = complex_
        self.fourier = fourier
        self.stft = stft
        self.fast_load = fast_load

        if rng is not None:
            self.rng = rng
        else:
            self.rng = numpy.random.RandomState(seed)

        self._train_data = {}
        self._valid_data = {}
        self._test_data = {}
        self._loaded = False

        self._eval_sets = {}

    def splits(self):
        with open(self.filename, 'rb') as f:
            # This should be fast
            all_inds = numpy.load(f).keys()
        test_inds = ['2303', '2382', '1819']
        valid_inds = ['2131', '2384', '1792',
                      '2514', '2567', '1876']
        train_inds = [ind for ind in all_inds
                      if ind not in test_inds and ind not in test_inds]
        return train_inds, valid_inds, test_inds

    @classmethod
    def note_to_class(cls, note):
        return note - 21

    @property
    def train_data(self):
        if self._train_data == {}:
            self.load()
        return self._train_data

    @property
    def valid_data(self):
        if self._valid_data == {}:
            self.load()
        return self._valid_data

    @property
    def test_data(self):
        if self._test_data == {}:
            self.load()
        return self._test_data

    def load(self, filename=None, reload=False):
        if filename is None:
            filename = self.filename
        if self._loaded and not reload:
            return

        with open(filename, 'rb') as f:
            train_inds, valid_inds, test_inds = self.splits()
            data_file = numpy.load(f)
            if self.fast_load:
                train_inds = train_inds[:6]
                train_data = {}
                for ind in chain(train_inds, valid_inds, test_inds):
                    train_data[ind] = data_file[ind]
            else:
                train_data = dict(data_file)

            # test set
            test_data = {}
            for ind in test_inds:
                if ind in train_data:
                    test_data[ind] = train_data.pop(ind)

            # valid set
            valid_data = {}
            for ind in valid_inds:
                valid_data[ind] = train_data.pop(ind)

            self._train_data = train_data
            self._valid_data = valid_data
            self._test_data = test_data

    def construct_eval_set(self, data, step=128):
        n_files = len(data)
        pos_per_file = 7500
        features = numpy.empty([n_files * pos_per_file, self.window_size])
        outputs = numpy.zeros([n_files * pos_per_file, self.output_size])

        features_ind = 0
        labels_ind = 1

        for i, ind in enumerate(data):
            print(ind)
            audio = data[ind][features_ind]

            for j in range(pos_per_file):
                if j % 1000 == 0:
                    print(j)
                # start from one second to give us some wiggle room for larger
                # segments
                index = self.sample_freq + j * step
                features[pos_per_file * i + j] = audio[index:
                                                       index + self.window_size]

                # label stuff that's on in the center of the window
                s = int((index + self.window_size / 2))
                for label in data[ind][labels_ind][s]:
                    note = label.data[1]
                    outputs[pos_per_file * i + j, self.note_to_class(note)] = 1
        return features, outputs

    @property
    def feature_dim(self):
        dummy_features = numpy.zeros((1, self.window_size, 1))
        dummy_output = numpy.zeros((1, self.output_size))
        dummy_features, _ = self.aggregate_raw_batch(
            dummy_features, dummy_output)
        return dummy_features.shape[1:]

    def aggregate_raw_batch(self, features, output):
        """Aggregate batch.

        All post processing goes here.

        Parameters:
        -----------
        features : 3D float tensor
            Input tensor
        output : 2D integer tensor
            Output classes

        """
        channels = 2 if self.complex_ else 1
        features_out = numpy.zeros(
            [features.shape[0], self.window_size, channels])
        if self.fourier:
            if self.complex_:
                data = fft(features, axis=1)
                features_out[:, :, 0] = numpy.real(data[:, :, 0])
                features_out[:, :, 1] = numpy.imag(data[:, :, 0])
            else:
                data = numpy.abs(fft(features, axis=1))
                features_out = data
        elif self.stft:
            _, _, data = stft(features, nperseg=120, noverlap=60, axis=1)
            length = data.shape[1]
            n_feats = data.shape[3]
            if self.complex_:
                features_out = numpy.zeros(
                    [len(self.train_data), length, n_feats * 2])
                features_out[:, :, :n_feats] = numpy.real(data)
                features_out[:, :, n_feats:] = numpy.imag(data)
            else:
                features_out = numpy.abs(data[:, :, 0, :])
        else:
            features_out = features
        return features_out, output

    def train_iterator(self):
        features = numpy.zeros([len(self.train_data), self.window_size])

        while True:
            output = numpy.zeros([len(self.train_data), self.output_size])
            for j, ind in enumerate(self.train_data):
                s = self.rng.randint(
                    self.window_size / 2,
                    len(self.train_data[ind][0]) - self.window_size / 2)
                data = self.train_data[ind][0][s - self.window_size / 2:
                                               s + self.window_size / 2]
                features[j, :] = data
                for label in self.train_data[ind][1][s]:
                    note = label.data[1]
                    output[j, self.note_to_class(note)] = 1
            yield self.aggregate_raw_batch(features[:, :, None], output)

    def eval_set(self, set_name):
        if not self._eval_sets:
            for name in ['valid', 'test']:
                data = self.valid_data if name == 'valid' else self.test_data
                x, y = self.construct_eval_set(data)
                x, y = self.aggregate_raw_batch(x[:, :, None], y)
                self._eval_sets[name] = (x, y)
        return self._eval_sets[set_name]

reason: import numpy,from scipy


text:
# Copyright 2020 - 2021 MONAI Consortium
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest

import numpy as np
import torch
from parameterized import parameterized

from monai.transforms import RandAffine

TEST_CASES = [
    [
        dict(as_tensor_output=False, device=None),
        {"img": torch.arange(27).reshape((3, 3, 3))},
        np.arange(27).reshape((3, 3, 3)),
    ],
    [
        dict(as_tensor_output=False, device=None, spatial_size=-1),
        {"img": torch.arange(27).reshape((3, 3, 3))},
        np.arange(27).reshape((3, 3, 3)),
    ],
    [
        dict(as_tensor_output=False, device=None),
        {"img": torch.arange(27).reshape((3, 3, 3)), "spatial_size": (2, 2)},
        np.array([[[2.0, 3.0], [5.0, 6.0]], [[11.0, 12.0], [14.0, 15.0]], [[20.0, 21.0], [23.0, 24.0]]]),
    ],
    [
        dict(as_tensor_output=True, device=None),
        {"img": torch.ones((1, 3, 3, 3)), "spatial_size": (2, 2, 2)},
        torch.ones((1, 2, 2, 2)),
    ],
    [
        dict(
            prob=0.9,
            rotate_range=(np.pi / 2,),
            shear_range=[1, 2],
            translate_range=[2, 1],
            as_tensor_output=True,
            padding_mode="zeros",
            spatial_size=(2, 2, 2),
            device=None,
        ),
        {"img": torch.ones((1, 3, 3, 3)), "mode": "bilinear"},
        torch.tensor([[[[0.3658, 1.0000], [1.0000, 1.0000]], [[1.0000, 1.0000], [1.0000, 0.9333]]]]),
    ],
    [
        dict(
            prob=0.9,
            rotate_range=(np.pi / 2,),
            shear_range=[1, 2],
            translate_range=[2, 1],
            scale_range=[0.1, 0.2],
            as_tensor_output=True,
            device=None,
        ),
        {"img": torch.arange(64).reshape((1, 8, 8)), "spatial_size": (3, 3)},
        torch.tensor([[[18.7362, 15.5820, 12.4278], [27.3988, 24.2446, 21.0904], [36.0614, 32.9072, 29.7530]]]),
    ],
]

ARR_NUMPY = np.arange(9 * 10).reshape(1, 9, 10)
ARR_TORCH = torch.Tensor(ARR_NUMPY)
TEST_CASES_SKIPPED_CONSISTENCY = []
for im in (ARR_NUMPY, ARR_TORCH):
    for as_tensor_output in (True, False):
        for in_dtype_is_int in (True, False):
            TEST_CASES_SKIPPED_CONSISTENCY.append((im, as_tensor_output, in_dtype_is_int))

class TestRandAffine(unittest.TestCase):
    @parameterized.expand(TEST_CASES)
    def test_rand_affine(self, input_param, input_data, expected_val):
        g = RandAffine(**input_param)
        g.set_random_state(123)
        result = g(**input_data)
        self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected_val, torch.Tensor))
        if isinstance(result, torch.Tensor):
            np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
        else:
            np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)

    @parameterized.expand(TEST_CASES_SKIPPED_CONSISTENCY)
    def test_skipped_transform_consistency(self, im, as_tensor_output, in_dtype_is_int):
        t1 = RandAffine(prob=0, as_tensor_output=as_tensor_output)
        t2 = RandAffine(prob=1, spatial_size=(10, 11), as_tensor_output=as_tensor_output)

        # change dtype to int32 or float32
        if in_dtype_is_int:
            im = im.astype("int32") if isinstance(im, np.ndarray) else im.int()
        else:
            im = im.astype("float32") if isinstance(im, np.ndarray) else im.float()

        out1 = t1(im)
        out2 = t2(im)

        # check same type
        self.assertEqual(type(out1), type(out2))
        # check matching dtype
        self.assertEqual(out1.dtype, out2.dtype)

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: KohYoungResearchAmerica/MONAI

max_issues_repo_name: KohYoungResearchAmerica/MONAI

max_forks_repo_name: KohYoungResearchAmerica/MONAI


text:
from __future__ import absolute_import
import unittest
from scipy.stats import lognorm
from utils import runner

from runner.tools.dist import dist_todict, dist_fromkw
from runner.tools.dist import dist_todict2, dist_fromkw2, DiscreteDist
from runner.param import Param

class TestDistScipy(unittest.TestCase):

    kw = {'loc': 10, 'name': 'lognorm', 'scale': 11, 'shapes': (2,)}

    def test_todict(self):
        self.assertEqual(dist_todict(lognorm(2, loc=10, scale=11)), self.kw)
        self.assertEqual(dist_todict(lognorm(2, 10, 11)), self.kw)
        self.assertEqual(dist_todict(lognorm(2, 10, scale=11)), self.kw)
        self.assertEqual(dist_todict(lognorm(2, loc=10, scale=11)), self.kw)

        default = {'loc': 0, 'name': 'lognorm', 'scale': 1, 'shapes': (2,)}
        self.assertEqual(dist_todict(lognorm(2, loc=0)), default)
        self.assertEqual(dist_todict(lognorm(2)), default)
        self.assertEqual(dist_todict(lognorm(2, 0)), default)

    def test_roundtrip(self):
        self.assertEqual(dist_todict(dist_fromkw(**self.kw)), self.kw)

class TestDistDiscrete(unittest.TestCase):

    kw = {'name':'discrete', 'values': [1,2,3]}

    def test_fromkw(self):
        d = dist_fromkw2(**self.kw)
        self.assertIsInstance(d, DiscreteDist)
        self.assertEqual(d.values.tolist(), self.kw['values'])

    def test_roundtrip(self):
        self.assertEqual(dist_todict2(dist_fromkw2(**self.kw)), self.kw)

class TestParamIO(unittest.TestCase):
    def setUp(self):
        self.a = Param.parse('a=N?3,2')
        self.b = Param.parse('b=U?-1,1')

    def test_asdict(self):
        self.assertEqual(self.a.as_dict(), {
            'name':'a',
            'dist_name':'norm',
            'dist_loc':3,
            'dist_scale':2,
        })

        self.assertEqual(self.b.as_dict(), {
            'name':'b',
            'dist_name':'uniform',
            'dist_loc':-1,
            'dist_scale':2,
        })

    def test_roundtrip(self):
        self.assertEqual(Param.fromkw(**self.a.as_dict()), self.a)
        self.assertEqual(Param.fromkw(**self.b.as_dict()), self.b)

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: alex-robinson/runner

max_issues_repo_name: alex-robinson/runner

max_forks_repo_name: alex-robinson/runner

reason: from scipy


text:
import numpy as np
import math
import matplotlib.pyplot as mplt
from vispy import scene, visuals
from vispy.plot import Fig
from vispy.color import Color
from mpl_toolkits.mplot3d import Axes3D
from cardiachelpers import meshhelper
from cardiachelpers import mathhelper
from cardiachelpers import stackhelper
import subprocess

def segmentRender(all_data_endo, all_data_epi, apex_pt, basal_pt, septal_pts, origin, transform, landmarks=True, ax=None, scar=None):
	"""Display the segmentation contours and user-indicated points

	args:
		all_data_endo (array): All endocardial contour data including slice values.
		all_data_epi (array): All epicardial contour data including slice values.
		apex_pt (array): Apical point
		basal_pt (array): Basal point
		septal_pts (array): Septal points (including midpoint)
		landmarks (bool): Whether to plot the pinpoints.
		ax (mplotlib3d axes): Axes on which to plot (default None)
	returns:
		ax (mplotlib3d axes): The axes of the contour
	"""
	if not ax:
	#	fig = mplt.figure()
	#	ax = fig.add_subplot(111, projection='3d')
		canvas = scene.SceneCanvas(keys='interactive', show=True)
		ax = canvas.central_widget.add_view()
		ax.bgcolor = '#ffffff'
		ax.camera = 'arcball'
		ax.padding = 0
	all_data_endo = np.array(all_data_endo).squeeze()
	all_data_epi = np.array(all_data_epi).squeeze()
	data_endo = stackhelper.rotateDataCoordinates(all_data_endo[:, :3], apex_pt, basal_pt, septal_pts)[0]
	data_epi = stackhelper.rotateDataCoordinates(all_data_epi[:, :3], apex_pt, basal_pt, septal_pts)[0]
	# Subtract origin and transform data
	apex_transform = np.dot((apex_pt - origin), np.transpose(transform))
	basal_transform = np.dot((basal_pt - origin), np.transpose(transform))
	septal_transform1 = np.dot((septal_pts[0, :] - origin), np.transpose(transform))
	septal_transform2 = np.dot((septal_pts[1, :] - origin), np.transpose(transform))
	septal_transform3 = np.dot((septal_pts[2, :] - origin), np.transpose(transform))
	# Set up bins as the unique data in all_data_endo third column (the slices)
	bins = np.unique(all_data_endo[:, 3])
	for jz in range(bins.size):
		# Get the indices that match the current bin and append then append the first value
		endo_tracing = np.where(all_data_endo[:, 3] == bins[jz])[0]
		endo_tracing = np.append(endo_tracing, endo_tracing[0])
		# Pull x, y, z from endo and epi and plot
		x = data_endo[endo_tracing, 2]
		y = data_endo[endo_tracing, 1]
		z = data_endo[endo_tracing, 0]
		endo_plot = np.column_stack((x, y, -z))
		endo_markers = scene.visuals.Line(pos=endo_plot, color='yellow', connect='strip', parent=ax.scene)
		#ax.plot(x, y, -z, 'y-')
		# Epi plotting
		epi_tracing = np.where(all_data_epi[:, 3] == bins[jz])[0]
		epi_tracing = np.append(epi_tracing, epi_tracing[0])
		x = data_epi[epi_tracing, 2]
		y = data_epi[epi_tracing, 1]
		z = data_epi[epi_tracing, 0]
		epi_plot = np.column_stack((x, y, -z))
		epi_markers = scene.visuals.Line(pos=epi_plot, color='cyan', connect='strip', parent=ax.scene)
		#ax.plot(x, y, -z, 'c-')
	if landmarks:
		# Plot the apex, basal, and septal points
		ab = np.array([apex_transform, basal_transform])
		si = np.array([septal_transform2, septal_transform3])
		ab_pos = np.column_stack((ab[:, 2], ab[:, 1], -ab[:, 0]))
		si_pos = np.column_stack((si[:, 2], si[:, 1], -si[:, 0]))
		septal_pos = np.column_stack((septal_transform1[2], septal_transform1[1], -septal_transform1[0]))
		scene.visuals.Line(pos=ab_pos, color='black', connect='strip', parent=ax.scene)
		scene.visuals.Markers(pos=si_pos, face_color='black', symbol='disc', parent=ax.scene)
		scene.visuals.Markers(pos=septal_pos, face_color='red', symbol='disc', parent=ax.scene)
		#ax.plot(ab[:, 2], ab[:, 1], -ab[:, 0], 'k-.')
		#ax.scatter(si[:, 2], si[:, 1], -si[:, 0], 'bo', s=50)
		#ax.scatter(septal_transform1[2], septal_transform1[1], -septal_transform1[0], 'ro', s=50)

	return(ax)

def surfaceRender(nodal_mesh, focus, ax=None):
	"""Plot surface mesh on optionally-passed axes

	args:
		nodal_mesh (array): Mesh to be plotted
		ax (mplot3d axes object): Axes on which to plot the mesh
	returns:
		ax (mplot3d axes object): Axes containing the surface plot contained in a figure
	"""
	# If no axes were passed, generate new set of axes
	if not ax:
		#fig = mplt.figure()
		#ax = fig.add_subplot(111, projection='3d')
		canvas = scene.SceneCanvas(keys='interactive', show=False, size=(900, 900), position=(100, 100))
		ax = canvas.central_widget.add_view()
		ax.bgcolor = '#ffffff'
		ax.camera = 'arcball'
		ax.camera.set_range(x=(-30, 30), y=(-30, 30), z=(-30, 30))
		ax.padding = 0
	else:
		canvas = ax.canvas

	# Sort the mesh by first 3 columns
	nodal_mesh = nodal_mesh[nodal_mesh[:, 0].argsort()]
	nodal_mesh = nodal_mesh[nodal_mesh[:, 1].argsort(kind='mergesort')]
	nodal_mesh = nodal_mesh[nodal_mesh[:, 2].argsort(kind='mergesort')]

	# Set up number of divisions and calculate e for each division (as a ratio)
	num_div = 20
	e = [i/num_div for i in range(num_div + 1)]
	# Convert angular values from degrees to radians
	rads = math.pi/180
	nodal_mesh[:, 1:3] *= rads
	# Store the shapes and sizes of the mesh values
	m = nodal_mesh.shape[0]
	size_nodal_nu = np.where(nodal_mesh[:, 2] == 0)[0].size
	size_nodal_phi = m/size_nodal_nu
	# Get the mu and theta values from the mesh
	nodal_nu = nodal_mesh[:size_nodal_nu, 1]
	nodal_phi = nodal_mesh[::size_nodal_nu, 2]
	# Convert apex node from prolate to cartesian, then plot with scatter
	if min(nodal_nu) == 0:
		x, y, z = mathhelper.prolate2cart(nodal_mesh[0, 0], nodal_mesh[0, 1], nodal_mesh[0, 2], focus)
		scene.visuals.Markers(pos=np.column_stack((z, y, -x)), face_color='black', symbol='disc', parent=ax.scene)
		#ax.scatter(z, y, -x)
		start_nu = 1
	else:
		start_nu = 0
	# Plot circumferential element boundaries
	line_pos_arr = np.empty([0, 3])
	line_con_arr = np.empty([0, 2])
	for i in range(start_nu, size_nodal_nu):
		for j in range(int(size_nodal_phi)):
			# Define nodal values for interpolation
			if j == size_nodal_phi-1:
				ind0 = i
				p0 = 2*math.pi
			else:
				ind0 = (j+1)*size_nodal_nu + i
				p0 = nodal_phi[j+1]
			ind1 = (j)*size_nodal_nu + i
			p1 = nodal_phi[j]
			# Get mu and dM/dm1
			m0 = nodal_mesh[ind0, 0]
			dm0 = nodal_mesh[ind0, 3]
			m1 = nodal_mesh[ind1, 0]
			dm1 = nodal_mesh[ind1, 3]
			# Convert to cartesian
			n0x, n0y, n0z = mathhelper.prolate2cart(nodal_mesh[ind0, 0], nodal_mesh[ind0, 1], nodal_mesh[ind0, 2], focus)
			# Plot the node
			scene.visuals.Markers(pos=np.column_stack((n0z, n0y, -n0x)), face_color='black', symbol='disc', parent=ax.scene)
			#ax.scatter(n0z, n0y, -n0x)
			# Plot the arc segments
			for k in range(2, len(e)):
				# Determine starting point to use
				if k == 2:
					pt_x, pt_y, pt_z = n0x, n0y, n0z
				else:
					pt_x, pt_y, pt_z = x_here, y_here, z_here
				# Get lambda
				hm0 = 1 - 3*(e[k]**2) + 2*(e[k]**3)
				hdm0 = e[k]*(e[k] - 1)**2
				hm1 = (e[k]**2)*(3 - 2*e[k])
				hdm1 = (e[k]**2)*(e[k] - 1)
				m = hm0 * m0 + hdm0 * dm0 + hm1 * m1 + hdm1 * dm1
				# Get theta
				p_here = p0 - e[k]*(p0 - p1)
				# Convert to cartesian
				x_here, y_here, z_here = mathhelper.prolate2cart(m, nodal_nu[i], p_here, focus)
				# Create vectors
				x = np.append(pt_x, x_here)
				y = np.append(pt_y, y_here)
				z = np.append(pt_z, z_here)
				# Plot segments
				line_con_arr = np.vstack((line_con_arr, np.array([[line_pos_arr.shape[0]+i, line_pos_arr.shape[0]+i+1] for i in range(z.shape[0]-1)])))
				line_pos_arr = np.vstack((line_pos_arr, np.column_stack((z, y, -x))))
				#ax.plot(z, y, -x, 'k-.')
	# Plot longitudinal element boundaries
	for i in range(int(size_nodal_phi)):
		for j in range(size_nodal_nu-1):
			# Define nodal values needeed for interpolation
			ind0 = i*size_nodal_nu + j
			ind1 = ind0 + 1
			n0 = nodal_nu[j]
			n1 = nodal_nu[j+1]
			# Get lambda and dL/de2
			m0 = nodal_mesh[ind0, 0]
			dm0 = nodal_mesh[ind0, 4]
			m1 = nodal_mesh[ind1, 0]
			dm1 = nodal_mesh[ind1, 4]
			# Convert nodal points to cartesian
			n0x, n0y, n0z = mathhelper.prolate2cart(nodal_mesh[ind0, 0], nodal_mesh[ind0, 1], nodal_mesh[ind0, 2], focus)
			# Plot arc
			for k in range(2, len(e)):
				# Determine point to use
				if k == 2:
					pt_x, pt_y, pt_z = n0x, n0y, n0z
				else:
					pt_x, pt_y, pt_z = x_here, y_here, z_here
				# Get lambda
				hm0 = 1 - 3*(e[k]**2) + 2*(e[k]**3)
				hdm0 = e[k]*(e[k] - 1)**2
				hm1 = (e[k]**2)*(3 - 2*e[k])
				hdm1 = (e[k]**2)*(e[k] - 1)
				m = hm0 * m0 + hdm0 * dm0 + hm1 * m1 + hdm1 * dm1
				# Get nu
				n_here = n0 + e[k]*(n1-n0)
				# Convert to cartesian
				x_here, y_here, z_here = mathhelper.prolate2cart(m, n_here, nodal_phi[i], focus)
				# Append the vectors for plotting
				x = np.append(pt_x, x_here)
				y = np.append(pt_y, y_here)
				z = np.append(pt_z, z_here)
				# Plot the segment
				line_con_arr = np.vstack((line_con_arr, np.array([[line_pos_arr.shape[0]+i, line_pos_arr.shape[0]+i+1] for i in range(z.shape[0]-1)])))
				line_pos_arr = np.vstack((line_pos_arr, np.column_stack((z, y, -x))))
				#ax.plot(z, y, -x, 'k-.')
	scene.visuals.Line(pos=line_pos_arr, color='black', connect=line_con_arr, parent=ax.scene)
	canvas.show()
	return(ax)

def displayScarTrace(scar, origin, transform, ax=None):
	"""Plots scar trace overlay onto a passed axis.
	"""
	if not ax:
		#fig = mplt.figure()
		#ax = fig.add_subplot(111, projection='3d')
		canvas = scene.SceneCanvas(keys='interactive', show=False, size=(900, 900), position=(100, 100))
		ax = canvas.central_widget.add_view()
		ax.bgcolor = '#ffffff'
		ax.camera = 'arcball'
		ax.camera.set_range(x=(-30, 30), y=(-30, 30), z=(-30, 30))
		ax.padding = 0
	else:
		canvas = ax.canvas
	line_con_arr = np.empty([0, 2])
	line_pos_arr = np.empty([0, 3])
	for scar_slice in scar:
		# Append the first point to the end to make a circular contour
		cur_scar = np.append(scar_slice, np.expand_dims(scar_slice[0, :], 0), axis=0)
		# Transform the data using the same transformation as endo/epi contours
		data_scar = np.dot((cur_scar - np.array([origin for i in range(cur_scar.shape[0])])), np.transpose(transform))
		# Plot
		x = data_scar[:, 2]
		y = data_scar[:, 1]
		z = data_scar[:, 0]
		line_con_arr = np.vstack((line_con_arr, np.array([[line_pos_arr.shape[0]+i, line_pos_arr.shape[0]+i+1] for i in range(z.shape[0]-1)])))
		line_pos_arr = np.vstack((line_pos_arr, np.column_stack((x, y, -z))))
		#ax.plot(x, y, -z, 'r-')
	scene.visuals.Line(pos=line_pos_arr, color='red', connect=line_con_arr, parent=ax.scene)
	canvas.show()
	return(ax)

def displayDensePts(dense_pts, dense_slices, origin, transform, dense_displacement_all=False, dense_plot_quiver=0, timepoint=-1, ax=None):
	"""Shows DENSE pointts and (optionally) displacements in a 3D graph.
	"""
	# If no axes were passed, generate axes
	if not ax:
		#fig = mplt.figure()
		#ax = fig.add_subplot(111, projection='3d')
		canvas = scene.SceneCanvas(keys='interactive', show=False, size=(900, 900), position=(100, 100))
		ax = canvas.central_widget.add_view()
		ax.bgcolor = '#ffffff'
		ax.camera = 'arcball'
		ax.camera.set_range(x=(-30, 30), y=(-30, 30), z=(-30, 30))
		ax.padding = 0
	else:
		canvas = ax.canvas

	# Pull appropriate timepoint (or set to False if displacement is undesired)
	if timepoint >= 0:
		try:
			dense_displacement = dense_displacement_all[timepoint]
		except(IndexError):
			warnings.warn('Timepoint not in range of values! Displacement will be ignored.')
			timepoint = -1
			dense_displacement = False
		except(TypeError):
			warnings.warn('Displacement not passed, but timepoint requested! Displacement will be ignored.')
			dense_displacement = False
	else:
		dense_displacement = False

	for i in range(len(dense_slices)):
		cur_slice = dense_slices[i]
		cur_dense_pts = np.column_stack((dense_pts[i], [cur_slice - origin[2]]*dense_pts[i].shape[0]))
		#data_dense = np.dot(cur_dense_pts, np.transpose(transform))
		x = cur_dense_pts[:, 0]
		y = cur_dense_pts[:, 1]
		z = cur_dense_pts[:, 2]
		ax.scatter(x, y, -z, ',')

		if dense_displacement:
			dense_displacement_slice = dense_displacement[i]
			if dense_plot_quiver == 1:
				ax.quiver(x, y, -z, dense_displacement_slice[:, 0], dense_displacement_slice[:, 1], [0]*dense_displacement_slice.shape[0])
	return(ax)

def nodeRender(nodes, ax=None):
	"""Display the nodes passed in as a 3D scatter plot."""
	if not ax:
		#fig = mplt.figure()
		#ax = fig.add_subplot(111, projection='3d')
		canvas = scene.SceneCanvas(keys='interactive', show=False, size=(900, 900), position=(100, 100))
		ax = canvas.central_widget.add_view()
		ax.bgcolor = '#ffffff'
		ax.camera = 'arcball'
		ax.camera.set_range(x=(-30, 30), y=(-30, 30), z=(-30, 30))
		ax.padding = 0
	else:
		canvas = ax.canvas
	x = nodes[:, 2]
	y = nodes[:, 1]
	z = nodes[:, 0]
	#ax.scatter(x, y, -z)
	scene.visuals.Markers(pos=np.column_stack((x, y, -z)), face_color='yellow', parent=ax.scene)
	return(ax)

def displayMeshPostview(file_name, executable_name):
	"""Launch PostView with specific file selected.

	The string for the subprocess must point to your installation of PostView.
	"""
	p = subprocess.Popen([executable_name, file_name])
	return(p)

def plotListData(input_list, title_list=False, ax=None, color='k'):
	if not ax:
		#fig = mplt.figure()
		#ax = fig.add_subplot(111)
		fig = Fig()
		ax = fig[0, 0]
	x_vals = list(range(len(input_list)))
	ax.plot(np.column_stack((x_vals, input_list)), color=color)
	#if title_list:
	#	ax.legend(title_list)
	return(ax)

def plot3d(input_arr, ax=None, type='plot'):
	if not ax:
		fig = mplt.figure()
		ax=fig.add_subplot(111, projection='3d')
	if type == 'scatter':
		ax.scatter(input_arr[:, 0], input_arr[:, 1], input_arr[:, 2])
	else:
		ax.plot(input_arr[:, 0], input_arr[:, 1], input_arr[:, 2])
	return(ax)

def visPlot3d(input_arr, view=None, type='line'):
	if not view:
		canvas = scene.SceneCanvas(keys='interactive', show=True)
		view = canvas.central_widget.add_view()
		view.bgcolor = '#ffffff'
		view.camera = 'arcball'
		view.padding = 0
	if type == 'markers':
		markers = scene.visuals.Markers(pos=input_arr, face_color='black', symbol='disc', parent=view.scene)
	elif type == 'line':
		markers = scene.visuals.Line(pos=input_arr, color='black', connect='strip', parent=view.scene)
	return(view)

max_stars_repo_name: cardiacbiomechanicsgroup/lvdatamap

max_issues_repo_name: cardiacbiomechanicsgroup/lvdatamap

max_forks_repo_name: cardiacbiomechanicsgroup/lvdatamap


text:
import numpy as np

import foodwebviz as fw

from trophic_levels import TROPHIC_LEVELS_REAL # TODO

def test_calc_trophic_levels(food_web):
    np.testing.assert_almost_equal(np.array([1.0, 2.0, 1.0]), fw.calculate_trophic_levels(food_web), 4)

# def test_calc_trophic_levels2(self):
#     import glob
#     from foodwebs.foodweb_io import readFW_SCOR

#     food_webs = [readFW_SCOR(net_path) for net_path in glob.glob('./data/*')]
#     for web, trophic in zip(food_webs, TROPHIC_LEVELS_REAL):
#         web_trophic = web._calcTrophicLevels()
#         np.testing.assert_almost_equal(trophic, web_trophic, 7)

max_stars_repo_name: lpawluczuk/foodwebviz

max_issues_repo_name: lpawluczuk/foodwebviz

max_forks_repo_name: lpawluczuk/foodwebviz


text:
# This file is part of PyOP2
#
# PyOP2 is Copyright (c) 2012, Imperial College London and
# others. Please see the AUTHORS file in the main source directory for
# a full list of copyright holders.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * The name of Imperial College London or that of other
#       contributors may not be used to endorse or promote products
#       derived from this software without specific prior written
#       permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTERS
# ''AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.

import pytest
import numpy

from pyop2 import op2

def _seed():
    return 0.02041724

nnodes = 4096
nele = nnodes // 2

@pytest.fixture(scope='module')
def node():
    return op2.Set(nnodes, 'node')

@pytest.fixture(scope='module')
def ele():
    return op2.Set(nele, 'ele')

@pytest.fixture(scope='module')
def dnode(node):
    return op2.DataSet(node, 1, 'dnode')

@pytest.fixture(scope='module')
def dnode2(node):
    return op2.DataSet(node, 2, 'dnode2')

@pytest.fixture(scope='module')
def dele(ele):
    return op2.DataSet(ele, 1, 'dele')

@pytest.fixture(scope='module')
def dele2(ele):
    return op2.DataSet(ele, 2, 'dele2')

@pytest.fixture
def d1(dnode):
    return op2.Dat(dnode, numpy.zeros(nnodes), dtype=numpy.int32)

@pytest.fixture
def d2(dnode2):
    return op2.Dat(dnode2, numpy.zeros(2 * nnodes), dtype=numpy.int32)

@pytest.fixture
def vd1(dele):
    return op2.Dat(dele, numpy.zeros(nele), dtype=numpy.int32)

@pytest.fixture
def vd2(dele2):
    return op2.Dat(dele2, numpy.zeros(2 * nele), dtype=numpy.int32)

@pytest.fixture(scope='module')
def node2ele(node, ele):
    vals = numpy.arange(nnodes) / 2
    return op2.Map(node, ele, 1, vals, 'node2ele')

class TestVectorMap:

    """
    Vector Map Tests
    """

    def test_sum_nodes_to_edges(self):
        """Creates a 1D grid with edge values numbered consecutively.
        Iterates over edges, summing the node values."""

        nedges = nnodes - 1
        nodes = op2.Set(nnodes, "nodes")
        edges = op2.Set(nedges, "edges")

        node_vals = op2.Dat(
            nodes, numpy.array(range(nnodes), dtype=numpy.uint32), numpy.uint32, "node_vals")
        edge_vals = op2.Dat(
            edges, numpy.array([0] * nedges, dtype=numpy.uint32), numpy.uint32, "edge_vals")

        e_map = numpy.array([(i, i + 1)
                            for i in range(nedges)], dtype=numpy.uint32)
        edge2node = op2.Map(edges, nodes, 2, e_map, "edge2node")

        kernel_sum = """
        static void sum(unsigned int* edge, unsigned int *nodes) {
        *edge = nodes[0] + nodes[1];
        }
        """
        op2.par_loop(op2.Kernel(kernel_sum, "sum"), edges,
                     edge_vals(op2.WRITE),
                     node_vals(op2.READ, edge2node))

        expected = numpy.asarray(
            range(1, nedges * 2 + 1, 2))
        assert all(expected == edge_vals.data)

    def test_read_1d_vector_map(self, node, d1, vd1, node2ele):
        vd1.data[:] = numpy.arange(nele)
        k = """
        static void k(int *d, int *vd) {
        *d = vd[0];
        }"""
        op2.par_loop(op2.Kernel(k, 'k'), node,
                     d1(op2.WRITE),
                     vd1(op2.READ, node2ele))
        assert all(d1.data[::2] == vd1.data)
        assert all(d1.data[1::2] == vd1.data)

    def test_write_1d_vector_map(self, node, vd1, node2ele):
        k = """
        static void k(int *vd) {
        vd[0] = 2;
        }
        """

        op2.par_loop(op2.Kernel(k, 'k'), node,
                     vd1(op2.WRITE, node2ele))
        assert all(vd1.data == 2)

    def test_inc_1d_vector_map(self, node, d1, vd1, node2ele):
        vd1.data[:] = 3
        d1.data[:] = numpy.arange(nnodes).reshape(d1.data.shape)

        k = """
        static void k(int *vd, int *d) {
        vd[0] += *d;
        }"""
        op2.par_loop(op2.Kernel(k, 'k'), node,
                     vd1(op2.INC, node2ele),
                     d1(op2.READ))
        expected = numpy.zeros_like(vd1.data)
        expected[:] = 3
        expected += numpy.arange(
            start=0, stop=nnodes, step=2).reshape(expected.shape)
        expected += numpy.arange(
            start=1, stop=nnodes, step=2).reshape(expected.shape)
        assert all(vd1.data == expected)

    def test_read_2d_vector_map(self, node, d2, vd2, node2ele):
        vd2.data[:] = numpy.arange(nele * 2).reshape(nele, 2)
        k = """
        static void k(int d[2], int vd[1][2]) {
        d[0] = vd[0][0];
        d[1] = vd[0][1];
        }"""
        op2.par_loop(op2.Kernel(k, 'k'), node,
                     d2(op2.WRITE),
                     vd2(op2.READ, node2ele))
        assert all(d2.data[::2, 0] == vd2.data[:, 0])
        assert all(d2.data[::2, 1] == vd2.data[:, 1])
        assert all(d2.data[1::2, 0] == vd2.data[:, 0])
        assert all(d2.data[1::2, 1] == vd2.data[:, 1])

    def test_write_2d_vector_map(self, node, vd2, node2ele):
        k = """
        static void k(int vd[1][2]) {
        vd[0][0] = 2;
        vd[0][1] = 3;
        }
        """

        op2.par_loop(op2.Kernel(k, 'k'), node,
                     vd2(op2.WRITE, node2ele))
        assert all(vd2.data[:, 0] == 2)
        assert all(vd2.data[:, 1] == 3)

    def test_inc_2d_vector_map(self, node, d2, vd2, node2ele):
        vd2.data[:, 0] = 3
        vd2.data[:, 1] = 4
        d2.data[:] = numpy.arange(2 * nnodes).reshape(d2.data.shape)

        k = """
        static void k(int vd[1][2], int d[2]) {
        vd[0][0] += d[0];
        vd[0][1] += d[1];
        }"""
        op2.par_loop(op2.Kernel(k, 'k'), node,
                     vd2(op2.INC, node2ele),
                     d2(op2.READ))

        expected = numpy.zeros_like(vd2.data)
        expected[:, 0] = 3
        expected[:, 1] = 4
        expected[:, 0] += numpy.arange(start=0, stop=2 * nnodes, step=4)
        expected[:, 0] += numpy.arange(start=2, stop=2 * nnodes, step=4)
        expected[:, 1] += numpy.arange(start=1, stop=2 * nnodes, step=4)
        expected[:, 1] += numpy.arange(start=3, stop=2 * nnodes, step=4)
        assert all(vd2.data[:, 0] == expected[:, 0])
        assert all(vd2.data[:, 1] == expected[:, 1])

if __name__ == '__main__':
    import os
    pytest.main(os.path.abspath(__file__))


text:
# packages
import os, sys, datetime
sys.path.append("C:/BERTVision/code/torch")
from data.bert_processors.processors import Tokenize_Transform
from common.evaluators.bert_glue_evaluator import BertGLUEEvaluator
from utils.collate import collate_BERT
from torch.cuda.amp import autocast
import torch
from torch.utils.data import DataLoader
from tqdm.auto import tqdm
from tqdm.notebook import trange
import numpy as np

class BertFreezeTrainer(object):
    '''
    This class handles the training of classification models with BERT
    architecture.
    Parameters
    ----------
    model : object
        A HuggingFace Classification BERT transformer
    tokenizer: object
        A HuggingFace tokenizer that fits the HuggingFace transformer
    optimizer: object
        A compatible Torch optimizer
    processor: object
        A Torch Dataset processor that emits data
    scheduler: object
        The learning rate decreases linearly from the initial lr set
    args: object
        A argument parser object; see args.py
    scaler: object
        A gradient scaler object to use FP16
    Operations
    -------
    This trainer:
        (1) Trains the weights
        (2) Generates dev set loss
        (3) Creates start and end logits and collects their original index for scoring
        (4) Writes their results and saves the file as a checkpoint
    '''
    def __init__(self, model, optimizer, processor, scheduler, args, kwargs, scaler, logger):
        # pull in objects
        self.args = args
        self.kwargs = kwargs
        self.model = model
        self.optimizer = optimizer
        self.processor = processor
        self.scheduler = scheduler
        self.scaler = scaler
        self.logger = logger

        # shard the large datasets:
        if any([self.args.model == 'QQP',
                self.args.model == 'QNLI',
                self.args.model == 'MNLI',
                self.args.model == 'SST'
                ]):
            # turn on sharding
            self.train_examples = self.processor(type='train', transform=Tokenize_Transform(self.args, self.logger), shard=True, kwargs=self.kwargs)

        else:
            # create the usual processor
            self.train_examples = self.processor(type='train', transform=Tokenize_Transform(self.args, self.logger))

        # declare progress
        self.logger.info(f"Initializing {self.args.model}-train with {self.args.max_seq_length} token length")

        # create a timestamp for the checkpoints
        timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")

        # create a location to save the files
        make_path = os.path.join(self.args.save_path, self.args.checkpoint, self.args.model)
        os.makedirs(make_path, exist_ok=True)
        self.snapshot_path = os.path.join(self.args.save_path, self.args.checkpoint, self.args.model, '%s.pt' % timestamp)

        # determine the number of optimization steps
        self.num_train_optimization_steps = int(
            len(self.train_examples) / args.batch_size) * args.epochs

        # create placeholders for model metrics and early stopping if desired
        self.iterations, self.nb_tr_steps, self.tr_loss = 0, 0, 0
        self.best_dev_f1, self.unimproved_iters, self.dev_loss = 0, 0, np.inf
        self.early_stop = False

        # use the same frozen weights each epoch and batch
        self.freeze = []#self.args.freeze

        # retrieve a freeze value between 0 and 1
        #self.freeze_p = self.args.freeze_p
        self.freeze_p = np.random.uniform(0.05, 0.95)

        # declare progress
        self.logger.info(f"Freezing this % of params now: {self.freeze_p}")

        # if locked_masks is here, then hold the same weights frozen constant
        self.locked_masks = {
                        name: (
                            torch.tensor(np.random.choice([False, True],
                                                          size=torch.numel(weight),
                                                          # [no freeze, freeze]
                                                          p=[0.0, 1.0])
                                         .reshape(weight.shape))
                            # freeze everything, but if you share a name with self.freeze; variably turn on some weights
                            if any(weight in name for weight in self.freeze) or name.startswith('p') else
                            torch.tensor(np.random.choice([False, True],
                                                          size=torch.numel(weight),
                                                          p=[(1-self.freeze_p), self.freeze_p])
                                         .reshape(weight.shape))
                          )
                        for name, weight in self.model.named_parameters()
                        }

    def train_epoch(self, train_dataloader):

        # if locked_masks is here, then vary the amount / actual weights frozen
        #self.freeze_p = np.random.uniform(0.05, 0.95)

        # set the model to train
        self.model.train()
        # pull data from data loader
        for step, batch in enumerate(tqdm(train_dataloader, desc="Training")):
            # and sent it to the GPU
            input_ids, attn_mask, token_type_ids, labels, idxs = (
                batch['input_ids'].to(self.args.device),
                batch['attention_mask'].to(self.args.device),
                batch['token_type_ids'].to(self.args.device),
                batch['labels'].to(self.args.device),
                batch['idx'].to(self.args.device)
            )

            # FP16
            with autocast():
                # forward
                out = self.model(
                                 input_ids=input_ids,
                                 attention_mask=attn_mask,
                                 token_type_ids=token_type_ids,
                                 labels=labels
                                 )

            # loss
            if self.args.n_gpu > 1:
                loss = out.loss.mean()
            else:
                loss = out.loss

            # backward
            self.scaler.scale(out.loss).backward()
            # zero gradients of interest
            for name, weight in self.model.named_parameters():
                if weight.grad is not None and name in self.locked_masks:
                    weight.grad[self.locked_masks[name]] = 0
            self.scaler.step(self.optimizer)
            self.scaler.update()
            self.scheduler.step()
            self.optimizer.zero_grad()

            # update metrics
            self.tr_loss += loss.item()
            self.nb_tr_steps += 1

        # gen. loss
        avg_loss = self.tr_loss / self.nb_tr_steps

        # print end of trainig results
        self.logger.info(f"Training complete! Loss: {avg_loss}")

    def train(self):
        '''
        This function handles the entirety of the training, dev, and scoring.
        '''
        # tell the user general metrics
        self.logger.info(f"Number of examples: {len(self.train_examples)}")
        self.logger.info(f"Batch size: {len(self.train_examples)}")
        self.logger.info(f"Number of optimization steps: {self.num_train_optimization_steps}")

        # storage containers for training metrics
        self.epoch_loss = list()
        self.epoch_metric = list()
        self.epoch = list()
        self.epoch_freeze_p = list()

        # instantiate dataloader
        train_dataloader = DataLoader(self.train_examples,
                                      batch_size=self.args.batch_size,
                                      shuffle=True,
                                      num_workers=self.args.num_workers,
                                      drop_last=False,
                                      collate_fn=collate_BERT)
        # for each epoch
        if any([self.args.model == 'SST',
                self.args.model == 'MSR',
                self.args.model == 'RTE',
                self.args.model == 'QNLI',
                self.args.model == 'QQP',
                self.args.model == 'WNLI'
                ]):

            for epoch in trange(int(self.args.epochs), desc="Epoch"):
                # train
                self.train_epoch(train_dataloader)
                # get dev loss
                dev_acc, dev_precision, dev_recall, dev_f1, dev_loss = BertGLUEEvaluator(self.model, self.processor, self.args, self.logger).get_loss(type='dev')
                # print validation results
                self.logger.info("Epoch {0: d}, Dev/Acc {1: 0.3f}, Dev/Pr. {2: 0.3f}, Dev/Re. {3: 0.3f}, Dev/F1 {4: 0.3f}, Dev/Loss {5: 0.3f}",
                                 epoch+1, dev_acc, dev_precision, dev_recall, dev_f1, dev_loss)

                self.epoch_loss.append(dev_loss)
                self.epoch_metric.append(dev_acc)
                self.epoch.append(epoch+1)
                self.epoch_freeze_p.append(self.freeze_p)

            return self.epoch_loss, self.epoch_metric, self.epoch, self.epoch_freeze_p

        elif any([self.args.model == 'CoLA']):

            for epoch in trange(int(self.args.epochs), desc="Epoch"):
                # train
                self.train_epoch(train_dataloader)
                # get dev loss
                matthews, dev_loss = BertGLUEEvaluator(self.model, self.processor, self.args, self.logger).get_loss(type='dev')
                # print validation results
                self.logger.info("Epoch {0: d}, Dev/Matthews {1: 0.3f}, Dev/Loss {2: 0.3f}",
                                 epoch+1, matthews, dev_loss)

                self.epoch_loss.append(dev_loss)
                self.epoch_metric.append(matthews)
                self.epoch.append(epoch+1)
                self.epoch_freeze_p.append(self.freeze_p)

            return self.epoch_loss, self.epoch_metric, self.epoch, self.epoch_freeze_p

        elif any([self.args.model == 'STSB']):

            for epoch in trange(int(self.args.epochs), desc="Epoch"):
                # train
                self.train_epoch(train_dataloader)
                # get dev loss
                pearson, spearman, dev_loss = BertGLUEEvaluator(self.model, self.processor, self.args, self.logger).get_loss(type='dev')
                # print validation results
                self.logger.info("Epoch {0: d}, Dev/Pearson {1: 0.3f}, Dev/Spearman {2: 0.3f}, Dev/Loss {3: 0.3f}",
                                 epoch+1, pearson, spearman, dev_loss)

                # take the average of the two tests
                avg_corr = (pearson + spearman) / 2

                self.epoch_loss.append(dev_loss)
                self.epoch_metric.append(avg_corr)
                self.epoch.append(epoch+1)
                self.epoch_freeze_p.append(self.freeze_p)

            return self.epoch_loss, self.epoch_metric, self.epoch, self.epoch_freeze_p

        elif any([self.args.model == 'MNLI']):

            for epoch in trange(int(self.args.epochs), desc="Epoch"):
                # train
                self.train_epoch(train_dataloader)
                # matched
                dev_acc1, dev_precision, dev_recall, dev_f1, dev_loss1 = BertGLUEEvaluator(self.model, self.processor, self.args, self.logger).get_loss(type='dev_matched')
                # print validation results
                self.logger.info("Epoch {0: d}, Dev/Acc {1: 0.3f}, Dev/Pr. {1: 0.3f}, Dev/Re. {1: 0.3f}, Dev/F1 {1: 0.3f}, Dev/Loss {1: 0.3f}",
                                 epoch+1, dev_acc1, dev_precision, dev_recall, dev_f1, dev_loss1)

                # matched
                dev_acc2, dev_precision, dev_recall, dev_f1, dev_loss2 = BertGLUEEvaluator(self.model, self.processor, self.args, self.logger).get_loss(type='dev_mismatched')
                # print validation results
                self.logger.info("Epoch {0: d}, Dev/Acc {1: 0.3f}, Dev/Pr. {1: 0.3f}, Dev/Re. {1: 0.3f}, Dev/F1 {1: 0.3f}, Dev/Loss {1: 0.3f}",
                                 epoch+1, dev_acc2, dev_precision, dev_recall, dev_f1, dev_loss2)

                # compute average loss
                dev_loss = (dev_loss1 + dev_loss2) / 2
                # compute average acc
                dev_acc = (dev_acc1 + dev_acc2) / 2

                self.epoch_loss.append(dev_loss)
                self.epoch_metric.append(dev_acc)
                self.epoch.append(epoch+1)
                self.epoch_freeze_p.append(self.freeze_p)

            return self.epoch_loss, self.epoch_metric, self.epoch, self.epoch_freeze_p

#

max_stars_repo_name: siduojiang/BERTVision

max_issues_repo_name: siduojiang/BERTVision