text:
import pt_utils
import numpy as np

from scipy.io.wavfile import write

just_freqs = np.array([1.0,
             1.5,
             1.125,
             1.6875,
             1.265625,
             1.8984375,
             1.423828125,
             1.06787109375,
             1.601806640625,
             1.20135498046875,
             1.802032470703125,
             1.3515243530273438])

# =============================================================================
# NOTEGROUPS TO WAV FILES, IN
# =============================================================================

# A SINGLE PILE OF FREQUENCIES FROM A NOTEGROUP
def notegroup_wavepile(notegroup, Fs=44100, duration=2, chromatic=False, from_middle_c=0, temperament="equal", shepard=False):
    ng = notegroup

    if (chromatic == False):
        ng = pt_utils.f_circle_to_c_chrom(ng)

    notenums = pt_utils.bit_locs(ng)

    return ordered_notegroup_wavepile(notenums, Fs=Fs, duration=duration, chromatic=chromatic, from_middle_c=from_middle_c, temperament=temperament, shepard=shepard)

# A SINGLE PILE OF FREQUENCIES FROM A SET OF INTEGERS 0-11

def ordered_notegroup_wavepile(notenums, Fs=44100, duration=2, chromatic=False, from_middle_c=0, temperament="equal", shepard=False):
    freqs = []

    for a_note in notenums:
        p_class = a_note
        freqs.append(freq_for_chrom_pitchnum(p_class, from_middle_c=from_middle_c, temperament=temperament))
        if (shepard == True):
            freqs.append(freq_for_chrom_pitchnum(p_class, from_middle_c=from_middle_c-2, temperament=temperament))
            freqs.append(freq_for_chrom_pitchnum(p_class, from_middle_c=from_middle_c-1, temperament=temperament))

    return pile_freq_sequence(freqs, Fs=Fs, duration=duration)

# A SINGLE SEQUENCE OF FREQUENCIES FROM A NOTEGROUP

def notegroup_wavestep(notegroup, Fs=44100, duration=2, chromatic=False, from_middle_c=0, temperament="equal"):
    ng = notegroup

    if (chromatic == False):
        ng = pt_utils.f_circle_to_c_chrom(ng)

    notenums = pt_utils.bit_locs(ng)

    return ordered_notegroup_wavestep(notenums, Fs=Fs, duration=duration, chromatic=chromatic, from_middle_c=from_middle_c, temperament=temperament)

# A SINGLE SEQUENCE OF FREQUENCIES FROM A SET OF INTEGERS 0-11

def ordered_notegroup_wavestep(notenums, Fs=44100, duration=2, chromatic=False, from_middle_c=0, temperament="equal", shepard=False):
    freqs = []
    for a_note in notenums:
        freqs.append(freq_for_chrom_pitchnum(a_note, from_middle_c=from_middle_c, temperament=temperament))
        if (shepard == True):
            freqs.append(freq_for_chrom_pitchnum(a_note, from_middle_c=from_middle_c-2, temperament=temperament))
            freqs.append(freq_for_chrom_pitchnum(a_note, from_middle_c=from_middle_c-1, temperament=temperament))

    return link_freq_sequence(freqs, duration=duration)

#========================================
# LINKED SEQUENCES -- 'ORDERED' IS PITCH CLASS NUMBERS, NOT BINARY 'NOTEGROUPS'

def link_wavepile_sequences(notegroup_list, Fs=44100, duration=2, chromatic=False, from_middle_c=0, temperament="equal", shepard=False):
    '''
    return a wave file with the signals in sequence

    Returns
    -------
    a sequence of chords spread evenly over duration

    '''

    # set the length of each segment
    stepdur = duration/len(notegroup_list)

    # get a signal of that length for each notegroup
    signal = np.empty(0)

    for i, a_notegroup in enumerate(notegroup_list):
        signal = np.concatenate((signal, notegroup_wavepile(a_notegroup, Fs=Fs, duration=stepdur, chromatic=chromatic, from_middle_c=from_middle_c, temperament=temperament, shepard=shepard)), axis=0)

    return signal

def link_ordered_wavepile_sequences(notegroup_list, Fs=44100, duration=2, chromatic=False, from_middle_c=0, temperament="equal"):
    '''
    return a wave file with the signals in sequence

    Returns
    -------
    a sequence of chords spread evenly over duration

    '''

    # set the length of each segment
    stepdur = duration/len(notegroup_list)

    # get a signal of that length for each notegroup
    signal = np.empty(0)

    for i, a_notegroup in enumerate(notegroup_list):
        signal = np.concatenate((signal, ordered_notegroup_wavepile(a_notegroup, Fs=Fs, duration=stepdur, chromatic=chromatic, from_middle_c=from_middle_c, temperament=temperament)), axis=0)

    return signal

def link_wavestep_sequences(notegroup_list, Fs=44100, duration=2, chromatic=False, from_middle_c=0, temperament="equal"):
    '''
    return a wave file with the notegroup signals in sequence

    Returns
    -------
    a sequence of chords spread evenly over duration

    '''

    # set the length of each segment
    stepdur = duration/len(notegroup_list)

    # get a signal of that length for each notegroup
    signal = np.empty(0)

    for i, a_notegroup in enumerate(notegroup_list):
        signal = np.concatenate((signal, notegroup_wavestep(a_notegroup, Fs=Fs, duration=stepdur, chromatic=chromatic, from_middle_c=from_middle_c, temperament=temperament)), axis=0)

    return signal

def link_ordered_wavestep_sequences(notegroup_list, Fs=44100, duration=2, chromatic=False, from_middle_c=0, temperament="equal"):
    '''
    return a wave file with the signals in sequence

    Returns
    -------
    a sequence of chords spread evenly over duration

    '''

    # set the length of each segment
    stepdur = duration/len(notegroup_list)

    # get a signal of that length for each notegroup
    signal = np.empty(0)

    for i, a_notegroup in enumerate(notegroup_list):
        signal = np.concatenate((signal, ordered_notegroup_wavestep(a_notegroup, Fs=Fs, duration=stepdur, chromatic=chromatic, from_middle_c=from_middle_c, temperament=temperament)), axis=0)

    return signal

#========================================
# RAW FREQUENCY FUNCTIONS
def freq_for_chrom_pitchnum(pitchnum, from_middle_c=0, temperament="equal"):
    '''
    Returns a frequency value in the octave above middle C for a chromatic number 0-11
    '''

    p_num = pitchnum

    middle_c_freq = 262
    if (temperament == "just"):
        base_freq = just_freqs[(p_num * 7)  % 12]
    else:
        base_freq =  pow(2, p_num/12)
    return middle_c_freq * base_freq * pow(2, from_middle_c)

def freq_4_note(a_note, from_middle_c=0, temperament="equal"):
    return 2 * np.pi * freq_for_chrom_pitchnum(a_note, from_middle_c=from_middle_c, temperament=temperament)

# FREQUENCY TO SIGNAL-OF-LENGTH FUNCTIONS
def signal_4_freq(freq, Fs=44100, duration=2, in_out_env=True, amp=1.0, phi=0.0):
    t = np.linspace(0, duration, int(Fs * duration))
    signal = amp * np.sin((2 * np.pi * freq) * t + phi)

    if in_out_env == True:
        ramp_len = 40
        ramp = np.linspace(0, 1, num=ramp_len)
        signal[0:ramp_len] *= ramp
        signal[len(signal)-ramp_len:] *= ramp[::-1]

    return signal

def link_freq_sequence(freqs, duration=2):
    stepdur = duration/len(freqs)
    signal = np.empty(0)

    for a_freq in freqs:
        signal = np.concatenate((signal, signal_4_freq(a_freq, duration=stepdur)), axis=0)

    return signal

def pile_freq_sequence(freqs, Fs=44100, duration=2):
    t = np.linspace(0, duration, int(Fs * duration))
    signal = np.zeros_like(t)

    for a_freq in freqs:
        signal += signal_4_freq(a_freq, duration=duration)

    return signal

# numpy sequence becomes wave file.
def norm_wave_write(seq, filename, sr=44100):
    # normalize the sum of the waves...
    seq /= seq.max()*1.5
    write(filename, sr, seq)

max_stars_repo_name: timsum/harmonypartition

max_issues_repo_name: timsum/harmonypartition

max_forks_repo_name: timsum/harmonypartition

reason: import numpy,from scipy


text:
#coding=utf8

########################################################################

import numpy
import vtk

########################################################################

def getABPointsFromTTTSectors(ugrid_sectors,
                              verbose=True):

    if (verbose): print ('*** getABPointsFromTTTSectors ***')

    nb_points = ugrid_sectors.GetNumberOfPoints()
    nb_cells = ugrid_sectors.GetNumberOfCells()

    nb_csects = 12
    nb_rsects = 3
    nb_slices = nb_points / (nb_rsects+1) / (nb_csects+1)
    if (verbose): print ("nb_slices =", nb_slices)

    zmin = ugrid_sectors.GetPoint(0)[2]
    zmax = ugrid_sectors.GetPoint(ugrid_sectors.GetNumberOfPoints()-1)[2]

    dist_btw_slices = abs(zmin-zmax)/(nb_slices-1)
    if (verbose): print ("dist_btw_slices =", dist_btw_slices)

    A = ugrid_sectors.GetPoints().GetPoint(0)
    B = ugrid_sectors.GetPoints().GetPoint(6)
    C = ugrid_sectors.GetPoints().GetPoint(3)
    D = ugrid_sectors.GetPoints().GetPoint(9)

    #print A
    #print B
    #print C
    #print D

    Px = ((A[0]*B[1]-A[1]*B[0])*(C[0]-D[0])-(A[0]-B[0])*(C[0]*D[1]-C[1]*D[0]))/((A[0]-B[0])*(C[1]-D[1])-(A[1]-B[1])*(C[0]-D[0]))
    Py = ((A[0]*B[1]-A[1]*B[0])*(C[1]-D[1])-(A[1]-B[1])*(C[0]*D[1]-C[1]*D[0]))/((A[0]-B[0])*(C[1]-D[1])-(A[1]-B[1])*(C[0]-D[0]))

    #print Px
    #print Py

    A = [Px, Py, zmin]
    B = [Px, Py, zmax]

    if (verbose): print ("A =", A)
    if (verbose): print ("B =", B)

    points_AB = vtk.vtkPoints()
    points_AB.InsertNextPoint(A)
    points_AB.InsertNextPoint(B)

    cells_AB = vtk.vtkCellArray()
    cell_AB  = vtk.vtkVertex()
    cell_AB.GetPointIds().SetId(0, 0)
    cells_AB.InsertNextCell(cell_AB)
    cell_AB.GetPointIds().SetId(0, 1)
    cells_AB.InsertNextCell(cell_AB)

    AB_ugrid = vtk.vtkUnstructuredGrid()
    AB_ugrid.SetPoints(points_AB)
    AB_ugrid.SetCells(vtk.VTK_VERTEX, cells_AB)

    return points_AB

ext: py

max_stars_repo_name: WeiXuanChan/heartFEM

max_issues_repo_name: WeiXuanChan/heartFEM

max_forks_repo_name: WeiXuanChan/heartFEM

reason: import numpy


text:
import numpy as np, pandas as pd

from .Viz import *
from .ModelsML import ts_LSTM, DecisionTree, scale
from .functions import *
from .base import *

from functools import reduce, partial
from sklearn.model_selection import RepeatedKFold
from sklearn.ensemble import RandomForestRegressor
from sklearn.datasets import make_regression
from sklearn.decomposition import TruncatedSVD
from sklearn import tree

class RegressorCollection():
    """Regressor Types:
        RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=2,
           max_features='auto', max_leaf_nodes=None,
           min_impurity_decrease=0.0, min_impurity_split=None,
           min_samples_leaf=1, min_samples_split=2,
           min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,
           oob_score=False, random_state=0, verbose=0, warm_start=False)
        RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=2,
           max_features='auto', max_leaf_nodes=None,
           min_impurity_decrease=0.0, min_impurity_split=None,
           min_samples_leaf=1, min_samples_split=2,
           min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,
           oob_score=False, random_state=0, verbose=0, warm_start=False)
        RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=2,
           max_features='auto', max_leaf_nodes=None,
           min_impurity_decrease=0.0, min_impurity_split=None,
           min_samples_leaf=1, min_samples_split=2,
           min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,
           oob_score=False, random_state=0, verbose=0, warm_start=False)
        RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=2,
           max_features='auto', max_leaf_nodes=None,
           min_impurity_decrease=0.0, min_impurity_split=None,
           min_samples_leaf=1, min_samples_split=2,
           min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,
           oob_score=False, random_state=0, verbose=0, warm_start=False)"""

    def __init__(self, X=None, y=None,
                 ireg=DecisionTree(ttype='generic',max_depth=8), *args, **kwargs):

        self.corr = X.corr()
        self.ireg = ireg

        if 'oh_features' in list(kwargs.keys()):
            self.oh_features = kwargs['oh_features']

        else:
            self.oh_features = list(filter(
                    lambda x: x not in self.corr.columns.tolist(), X.columns))

        self.X = MLDataFrame(X)
        self.y = MLDataFrame(y)

        self.X.add_oh(self.oh_features)

        if 'scale_cols' in list(kwargs.keys()):
            self.scale_cols = kwargs['scale_cols']
            if 'scale_types' in list(kwargs.keys()):
                self.scale_types = kwargs['scale_types']
            else:
                self.scale_types = ['minmax']*len(self.scale_cols)

            self.X.add_scaled(self.X.oh, scale_cols=self.scale_cols,
                              scale_types=self.scale_types)
            self.y.add_scaled(self.y)

        else:
             self.X.add_scaled(self.X.oh)
             self.y.add_scaled(self.y)

        self.models = dict(zip([arg.__name__ for arg in args], args))
        self.fitted = {}

    def rank_importance(self, out_file='tree.dot',show=True):
        self.initial_tree = self.ireg.fit(self.X.oh, self.y)
        tree.export_graphviz(self.initial_tree, out_file=out_file)
        self.feature_importances_ = pd.DataFrame(self.initial_tree.feature_importances_,
                                                 index=self.X.oh.columns,
                                                 columns=['Features']).sort_values(
                                                         'Features',ascending=False).to_dict()

    def fit(self, X, y):
        for key in self.models.keys():
            self.fitted[key] = self.models[key].fit()

    def gen_pc(self, dims=2, topx=6):
        svd = TruncatedSVD(algorithm='randomized', n_components=2, n_iter=5,
                           random_state=0)

        fi = pd.DataFrame(self.feature_importances_).sort_values(
                'Features', ascending=False).index[:20].tolist()

        rd = svd.fit_transform(self.X.scaled)
        ncols = rd.shape[1]

        self.principle_comp = MLDataFrame(
                pd.DataFrame(rd, columns=['component' + str(i) for i in range(dims)]))

        self.principle_comp.add_scaled(self.principle_comp)

max_stars_repo_name: benzomo/PyPackages

max_issues_repo_name: benzomo/PyPackages

max_forks_repo_name: benzomo/PyPackages


text:
import numpy as np
import pandas as pd
import datetime as dt
import time
import matplotlib.pyplot as plt
import seaborn as sns
import vnpy.analyze.data.data_prepare as dp
from jqdatasdk import *
from vnpy.trader.database import database_manager
from mpl_toolkits.axisartist.parasite_axes import HostAxes, ParasiteAxes
import matplotlib.dates as dates
from matplotlib import ticker
import math

def interval_show(df, forecast_start, forecast_end):
    """展示🐂顶周期"""
    plt.style.use('fivethirtyeight')
    # 1、拉取数据
    df['date'] = pd.to_datetime(df['date'])  # 转换时间类型
    df.set_index(['date'], inplace=True)
    df.index.name = None  # 去掉索引列名

    df.close.plot(figsize=(14, 7))
    for top_date in ('2007-10-01', '2015-06-01'):
        plt.axvline(top_date, color='r', alpha=0.7)
    if forecast_start is not None and forecast_end is not None:
        # TODO: 向df中添加date数据
        # 添加预测时间到图形中
        date_span = pd.date_range(forecast_start, forecast_end)
        value_span = [df.close.max() for x in date_span]
        plt.fill_between(date_span, value_span, color='orange', alpha=0.8)

    plt.show()

def regplot_trend(df):
    """线性回归，看趋势"""
    df['date'] = pd.to_datetime(df['date'])  # 转换时间类型
    df.set_index(['date'], inplace=True)
    df.index.name = None  # 去掉索引列名
    plt.figure(figsize=(14, 7))
    sns.regplot(x=np.arange(0, df.shape[0]), y=df.close.values)
    plt.show()

def forecast_bull_time():
    """简单预测下一次牛顶"""
    best_dates = [dt.datetime.strptime(best_date, '%Y-%m-%d').date() for best_date in
                  ('1993-02-01', '2001-06-01', '2007-10-01', '2015-06-01')]
    # 计算牛顶的平均时间
    gap_dates = [days.days for days in np.diff(best_dates)]
    print('牛顶平均间隔时间：', ['{} days'.format(day) for day in gap_dates])
    # 计算平均时间间隔
    mean_gap = np.mean(gap_dates)
    print('平均时间间隔{}天，既{}年'.format(round(mean_gap, 2), round(mean_gap / 365, 2)))
    # 再计算平均值偏差
    std_gap = np.std(gap_dates)
    print('平均时间偏差{}天，既{}年'.format(round(std_gap, 2), round(std_gap / 365, 2)))
    # 计算下一个牛市出现的合理时间区间
    early_date = best_dates[-1] + dt.timedelta(round(mean_gap - std_gap, 2))
    late_date = best_dates[-1] + dt.timedelta(round(mean_gap + std_gap, 2))
    print('下一次牛市时间预计区间 {} ~ {}'.format(early_date, late_date))
    return early_date, late_date

def model_1(df):
    """
    定投模型1
    1、每月第一个交易日定投1000
    2、不考虑买入的份额是否够一手，按照小数结算
    3、不考虑交易中的滑点和手续费相关
    """
    # 获取每个月第一个交易日
    pre_year = 0
    pre_month = 0
    first_day = []
    for index, row in df.iterrows():
        if index.year != pre_year or index.month != pre_month:
            first_day.append(index)
        pre_year = index.year
        pre_month = index.month
    index_df = df.loc[first_day]
    # 按照预定模型计算
    day_df = index_df.copy()
    day_df['pct_change'] = day_df.close.pct_change()
    day_df = day_df[['close', 'pct_change']]
    save_money = []  # 存入金额
    hold_money = []  # 拥有金额
    save_base = 1000  # 每次存入金额
    for i in range(len(day_df)):
        if i == 0:
            save_money.append(save_base)
            hold_money.append(save_base)
        else:
            save_money.append(save_money[-1] + save_base)
            hold_money.append(hold_money[-1] * (1 + day_df['pct_change'][i]) + save_base)
    day_df['save_money'] = save_money
    day_df['hold_money'] = hold_money
    day_df['return_money'] = day_df['hold_money'] - day_df['save_money']  # 累计收入
    day_df[['save_money', 'hold_money', 'return_money']].plot(figsize=(14, 7))
    plt.legend(['save_money', 'hold_money', 'return_money'])
    plt.show()
    # TODO: 增加复合年化利率计算

def model2(symbol, alias, start_date, end_date):
    """
    定投模型2：
    1、PE处于适中估值，不做任何操作，每月定投资金放入现金池
    2、PE处于低估，持续定投
    3、PE高于适中估值，卖出并且放入现金池中
    """
    # 获取交易日价格和金融数据
    df = dp.load_bar_data(symbol, alias, start_date=start_date, end_data=end_date)
    datas = database_manager.load_finance_data(symbol + '.' + alias, start_date, end_date)
    df_finance = pd.DataFrame(columns=('code', 'datetime', 'pe', 'pb'))
    for data in datas:
        df_finance = df_finance.append({'code': data.code, 'datetime': data.datetime, 'pe': data.pe, 'pb': data.pb},
                                       ignore_index=True)
    if len(df) == len(df_finance):
        df['pe'] = df_finance['pe'].copy()
        df['pb'] = df_finance['pb'].copy()
    df['date'] = pd.to_datetime(df['date'])  # 转换时间类型
    df.set_index(['date'], inplace=True)
    df.index.name = None  # 去掉索引列名
    # 获取每个月第一个交易日
    pre_year = 0
    pre_month = 0
    first_day = []
    for index, row in df.iterrows():
        if index.year != pre_year or index.month != pre_month:
            first_day.append(index)
        pre_year = index.year
        pre_month = index.month
    index_df = df.loc[first_day]
    day_df = index_df.copy()
    day_df['pct_change'] = day_df.close.pct_change()

    # day_df = day_df[['close', 'pb', 'pe' 'pct_change']]
    # print(day_df.head(10))
    miden_estimation = (12.17798, 12.93906)
    save_money = []  # 每月定存
    back_money = []  # 回收资金
    hold_money = []  # 持仓资金
    base_money = 1000  # 定投基准
    for i in range(len(day_df)):
        pe = day_df['pe'][i]  # 估值位
        print(pe)
        if i == 0:  # 初始买入
            # 1.计算买入金额
            save_money.append(base_money)
            # 2. 计算回收金额
            back_money.append(0)
            # 3. 计算持仓变化
            hold_money.append(base_money)
            continue

        if pe <= miden_estimation[0]:  # 执行买入计算
            # 1.计算买入金额
            save_money.append(base_money)
            # 2. 计算回收金额
            back_money.append(0)
            # 3. 计算持仓变化
            hold_money.append(hold_money[-1] * (1 + day_df['pct_change'][i]) + base_money)
        elif pe >= miden_estimation[-1]:  # 执行卖出计算
            # 1. 计算买入金额
            save_money.append(0)
            # 2. 计算回收金额
            back_money.append(base_money)
            # 3. 计算持仓变化
            hold_money.append(hold_money[-1] * (1 + day_df['pct_change'][i]) - base_money)
        else:
            # 1.计算买入金额
            save_money.append(0)
            # 2. 计算回收金额
            back_money.append(0)
            # 3. 计算持仓变化
            hold_money.append(hold_money[-1] * (1 + day_df['pct_change'][i]))

    day_df['save_money'] = save_money  # 定投金额
    day_df['save_money_cumsum'] = day_df['save_money'].cumsum()  # 定投累计金额
    day_df['hold_money'] = hold_money  # 持仓金额
    day_df['back_money'] = back_money  # 回收金额
    day_df['back_money_cumsum'] = day_df['back_money'].cumsum()  # 累计回收金额
    day_df['total_money'] = day_df['hold_money'] + day_df['back_money_cumsum']  # 总资金
    day_df['return_money'] = day_df['total_money'] - day_df['save_money_cumsum']  # 持续收益
    day_df['return_rate'] = (day_df['total_money'] / day_df['save_money_cumsum']) - 1  # 持续收益率
    day_df[['save_money_cumsum', 'total_money', 'back_money_cumsum', 'return_money']].plot(figsize=(14, 7))
    plt.legend(['save_money_cumsum', 'total_money', 'back_money_cumsum', 'return_money'])
    plt.show()

    print('累计投入: {}元'.format(day_df['save_money_cumsum'][-1]))
    print('累计收益: {}元'.format(day_df['return_money'][-1]))
    print('最终本息累积: {}元'.format(day_df['total_money'][-1]))
    print('绝对收益率为: {}%'.format((day_df['return_money'][-1] / day_df['save_money_cumsum'][-1]) * 100))

# 指定日期的指数PE(市值加权)
def get_index_pe_date(index_code, date):
    auth('13277099856', '1221gzcC')
    stocks = get_index_stocks(index_code, date)
    q = query(valuation).filter(valuation.code.in_(stocks))
    df = get_fundamentals(q, date)
    df = df[df['pe_ratio'] > 0]
    if len(df) > 0:
        # pe = len(df)/sum([1/p if p>0 else 0 for p in df.pe_ratio])
        # pe = df['pe_ratio'].size/(1/df['pe_ratio']).sum()
        pe = df['circulating_market_cap'].sum() / (df['circulating_market_cap'] / df['pe_ratio']).sum()
        return pe
    else:
        return float('NaN')

# 指定日期的指数PB(市值加权)
def get_index_pb_date(index_code, date):
    auth('13277099856', '1221gzcC')
    stocks = get_index_stocks(index_code, date)
    q = query(valuation).filter(valuation.code.in_(stocks))
    df = get_fundamentals(q, date)
    df = df[df['pb_ratio'] > 0]
    if len(df) > 0:
        # pb = len(df)/sum([1/p if p>0 else 0 for p in df.pb_ratio])
        # pb = df['pb_ratio'].size/(1/df['pb_ratio']).sum()
        pb = df['circulating_market_cap'].sum() / (df['circulating_market_cap'] / df['pb_ratio']).sum()
        return pb
    else:
        return float('NaN')

def draw_pe_pb(code, start_date, end_date):
    """画出pe、pb的趋势图"""
    datas = database_manager.load_finance_data(code, start_date, end_date)
    df = pd.DataFrame(columns=('code', 'datetime', 'pe', 'pb'))
    for data in datas:
        df = df.append({'code': data.code, 'datetime': data.datetime, 'pe': data.pe, 'pb': data.pb}, ignore_index=True)
    # 设置索引
    df['datetime'] = pd.to_datetime(df['datetime'])
    df.set_index(['datetime'], inplace=True)
    df[['pe', 'pb']].plot(figsize=(14, 7), subplots=True)
    # TODO: PE和close对比
    # df[['pe', 'pb']].plot(secondary_y=['pb'], alpha=.8)
    plt.show()

def draw_price_pe(symbol, alias, start_date, end_date):
    """画出价格与pe关系"""
    df = dp.load_bar_data(symbol, alias, start_date=start_date, end_data=end_date)
    datas = database_manager.load_finance_data(symbol + '.' + alias, start_date, end_date)
    df_finance = pd.DataFrame(columns=('code', 'datetime', 'pe', 'pb'))
    for data in datas:
        df_finance = df_finance.append({'code': data.code, 'datetime': data.datetime, 'pe': data.pe, 'pb': data.pb},
                                       ignore_index=True)
    if len(df) == len(df_finance):
        df['pe'] = df_finance['pe'].copy()
        df['pb'] = df_finance['pb'].copy()
    df['date'] = pd.to_datetime(df['date'])
    df.set_index(['date'], inplace=True)
    _, axs = plt.subplots(ncols=2, figsize=(14, 5))
    df[['close', 'pe']].plot(secondary_y=['pe'], ax=axs[0], alpha=.8)
    df[['close', 'pb']].plot(secondary_y=['pb'], ax=axs[1], alpha=.8)
    plt.show()

def analyze_pe_pb_dis(symbol, alias, start_date, end_date):
    """分析PE\PB分布"""
    datas = database_manager.load_finance_data(symbol + '.' + alias, start_date, end_date)
    df_finance = pd.DataFrame(columns=('code', 'datetime', 'pe', 'pb'))
    for data in datas:
        df_finance = df_finance.append({'code': data.code, 'datetime': data.datetime, 'pe': data.pe, 'pb': data.pb},
                                       ignore_index=True)

    _, axs = plt.subplots(nrows=2, ncols=2, figsize=(14, 7))
    sns.distplot(df_finance.pe, ax=axs[0][0])
    sns.boxplot(df_finance.pe, ax=axs[0][1])
    sns.distplot(df_finance.pb, ax=axs[1][0])
    sns.boxplot(df_finance.pb, ax=axs[1][1])
    # 展示PE/PB关系
    sns.jointplot(x='pb', y='pe', data=df_finance, height=7)
    # 将PE分成十个分位，查看各个分位PE数量
    pe_array = df_finance.pe.values
    value_counts = pd.cut(pe_array, 10).value_counts()
    print(value_counts)
    low = value_counts[0:4].sum()
    medin = value_counts[4:6].sum()
    high = value_counts[6:10].sum()
    print('比值({}：{}：{})'.format(low, medin, high))
    plt.figure(figsize=(14, 4))
    sns.barplot(x=np.arange(0, len(value_counts)), y=value_counts.values)
    plt.show()

def show_quantile(symbol, alias, start_date, end_date):
    """展示分位区间"""
    datas = database_manager.load_finance_data(symbol + '.' + alias, start_date, end_date)
    df_finance = pd.DataFrame(columns=('code', 'datetime', 'pe', 'pb'))
    for data in datas:
        df_finance = df_finance.append({'code': data.code, 'datetime': data.datetime, 'pe': data.pe, 'pb': data.pb},
                                       ignore_index=True)

    _df = pd.DataFrame()
    df = df_finance.copy()
    df.index.name = None
    _df['pe'] = df.pe
    _df = _df
    p_high = [_df.pe.quantile(i / 10) for i in [4, 5, 6]]
    for p_h, i in zip(p_high, [4, 5, 6]):
        _df[str(i / 10 * 100) + '%'] = p_h

    print(_df.head(10))
    _df['datetime'] = df_finance.datetime.copy()
    _df['datetime'] = pd.to_datetime(_df['datetime'])  # 转换时间类型
    _df.set_index(['datetime'], inplace=True)

    _df.plot(figsize=(14, 7))
    plt.show()

def dym_quantile(n):
    """
        动态分位图
        当前投资已5年内历史数据计算百分位，价格合适购入
    """
    # 这里的计算按一年244个交易日计算
    windows = int(n * 244)  # 将时间取整数
    start_date = dt.datetime(2006, 1, 1)
    end_date = dt.datetime(2020, 4, 1)
    df = dp.load_bar_data('000300', 'XSHG', start_date=start_date, end_data=end_date)
    df_finance = dp.load_finance_data('000300.XSHG', start_date=start_date, end_date=end_date)
    if len(df) == len(df_finance):
        print('yes!!!, len:%s' % len(df))
        df['pe'] = df_finance['pe']
    df['quantile'] = df_finance['pe'].rolling(windows).apply(lambda x: pd.Series(x).rank().iloc[-1] /
                                                                       pd.Series(x).shape[0], raw=True)
    # df['date'] = pd.to_datetime(df['date'])  # 转换时间类型
    # df.set_index(['date'], inplace=True)
    # df.index.name = None  # 去掉索引列名
    df.dropna(inplace=True)

    # 画出适中估值区间

    # plt.figure()
    # 创建第一个画板
    fig = plt.figure(figsize=(16, 9))

    host = HostAxes(fig, [0.15, 0.1, 0.65, 0.8])
    par1 = ParasiteAxes(host, sharex=host)
    par2 = ParasiteAxes(host, sharex=host)
    host.parasites.append(par1)
    host.parasites.append(par2)

    host.set_xlabel('Date')
    host.set_ylabel('Close')
    host.axis['right'].set_visible(False)

    par1.axis['right'].set_visible(True)
    par1.set_ylabel('%sY Rolling quantile' % n)
    par1.axis['right'].major_ticklabels.set_visible(True)
    par1.axis['right'].label.set_visible(True)

    par2.set_ylabel('PE')
    new_axisline = par2.get_grid_helper().new_fixed_axis  # "_grid_helper"与"get_grid_helper()"等价，可以代替
    par2.axis['right2'] = new_axisline(loc='right', axes=par2, offset=(45, 0))

    fig.add_axes(host)

    df['date'] = pd.to_datetime(df['date'])
    df['date'] = df['date'].apply(lambda x: dates.date2num(x))
    p1, = host.plot(df['date'], df['close'], label="Close")
    p2, = par1.plot(df['date'], df['quantile'], label="Quantile")
    p3, = par2.plot(df['date'], df['pe'], label="PE")

    host.legend()
    # 轴名称，刻度值的颜色
    host.axis['left'].label.set_color(p1.get_color())
    host.xaxis.set_major_locator(ticker.MaxNLocator(math.floor(len(df) / 100)))
    host.xaxis.set_major_formatter(dates.DateFormatter('%Y-%m'))
    par1.axis['right'].label.set_color(p2.get_color())
    par2.axis['right2'].label.set_color(p3.get_color())
    par2.axis['right2'].major_ticklabels.set_color(p3.get_color())  # 刻度值颜色
    par2.axis['right2'].set_axisline_style('-|>', size=1.5)  # 轴的形状色
    par2.axis['right2'].line.set_color(p3.get_color())  # 轴的颜色
    # ax.xaxis.set_major_formatter(dates.DateFormatter('%Y-%m-%d'))

    # df[['quantile', 'close']].plot(secondary_y=['quantile'], figsize=(14, 10), alpha=.8)
    # plt.fill_between(df.index, y1=0.4, y2=0.6, color='blue', alpha=0.7)
    # plt.fill_between(df.index, y1=0.8, y2=1, color='red', alpha=0.7)
    # plt.fill_between(df.index, y1=0.0, y2=0.2, color='green', alpha=0.7)
    # plt.annotate('reasonable zone', (df.index[-1], 0.5))
    # 画出固定PE与收盘价的曲线
    plt.show()

if __name__ == "__main__":
    # 获取数据
    df = dp.load_bar_data('000300', 'XSHG', start_date=dt.datetime(2005, 1, 1),
                          end_data=dt.datetime(2020, 5, 1))
    df['dt'] = df['date'].copy()
    df['date'] = pd.to_datetime(df['date'])  # 转换时间类型
    df.set_index(['date'], inplace=True)
    # draw_pe_pb(df, '000300.XSHG')
    # model_1(df)
    forecast_start, forecast_end = forecast_bull_time()
    interval_show(df, forecast_start, forecast_end)
    regplot_trend(df)
    # draw_pe_pb('000300.XSHG', dt.datetime(2020, 1, 1), dt.datetime(2020, 4, 1))
    # draw_price_pe('000300', 'XSHG', dt.datetime(2014, 1, 2), dt.datetime(2020, 4, 1))
    # analyze_pe_pb_dis('000300', 'XSHG', dt.datetime(2014, 1, 2), dt.datetime(2020, 4, 1))
    # show_quantile('000300', 'XSHG', dt.datetime(2014, 1, 2), dt.datetime(2020, 4, 1))
    # model2('000300', 'XSHG', dt.datetime(2014, 1, 2), dt.datetime(2020, 4, 1))
    # dym_quantile(7.5)

max_stars_repo_name: CatTiger/vnpy

max_issues_repo_name: CatTiger/vnpy

max_forks_repo_name: CatTiger/vnpy


text:
# Copyright 2021 NVIDIA Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import numpy as np

from cunumeric.array import ndarray
from cunumeric.module import sqrt as _sqrt

def norm(x, ord=None, axis=None, keepdims=False, stacklevel=1):
    lg_array = ndarray.convert_to_cunumeric_ndarray(x)
    if (axis is None and lg_array.ndim == 1) or type(axis) == int:
        # Handle the weird norm cases
        if ord == np.inf:
            return abs(lg_array).max(
                axis=axis, keepdims=keepdims, stacklevel=(stacklevel + 1)
            )
        elif ord == -np.inf:
            return abs(lg_array).min(
                axis=axis, keepdims=keepdims, stacklevel=(stacklevel + 1)
            )
        elif ord == 0:
            # Check for where things are not zero and convert to integer
            # for sum
            temp = (lg_array != 0).astype(np.int64)
            return temp.sum(
                axis=axis, keepdims=keepdims, stacklevel=(stacklevel + 1)
            )
        elif ord == 1:
            return abs(lg_array).sum(
                axis=axis, keepdims=keepdims, stacklevel=stacklevel + 1
            )
        elif ord is None or ord == 2:
            s = (lg_array.conj() * lg_array).real
            return _sqrt(
                s.sum(axis=axis, keepdims=keepdims, stacklevel=stacklevel + 1)
            )
        elif isinstance(ord, str):
            raise ValueError(f"Invalid norm order '{ord}' for vectors")
        elif type(ord) == int:
            absx = abs(lg_array)
            absx **= ord
            ret = absx.sum(
                axis=axis, keepdims=keepdims, stacklevel=(stacklevel + 1)
            )
            ret **= 1 / ord
            return ret
        else:
            raise ValueError("Invalid 'ord' argument passed to norm")
    else:
        raise NotImplementedError(
            "cuNumeric needs support for other kinds of norms"
        )

max_stars_repo_name: ipdemes/cunumeric

max_issues_repo_name: ipdemes/cunumeric

max_forks_repo_name: ipdemes/cunumeric


text:
import torch
import torch.utils.data as data
from PIL import Image
import os
import math
import functools
import json
import copy
import cv2
import numpy as np
import pickle

from utils import load_value_file

import tarfile
import threading
import multiprocessing
import zipfile
from io import BytesIO
import io
import time

class _FileInFile(object):
    """A thin wrapper around an existing file object that
       provides a part of its data as an individual file
       object.
    """

    def __init__(self, fileobj, offset, size, blockinfo=None):
        self.fileobj = fileobj
        self.offset = offset
        self.size = size
        self.position = 0
        self.name = getattr(fileobj, "name", None)
        self.closed = False

        if blockinfo is None:
            blockinfo = [(0, size)]

        # Construct a map with data and zero blocks.
        self.map_index = 0
        self.map = []
        lastpos = 0
        realpos = self.offset
        for offset, size in blockinfo:
            if offset > lastpos:
                self.map.append((False, lastpos, offset, None))
            self.map.append((True, offset, offset + size, realpos))
            realpos += size
            lastpos = offset + size
        if lastpos < self.size:
            self.map.append((False, lastpos, self.size, None))

    def flush(self):
        pass

    def readable(self):
        return True

    def writable(self):
        return False

    def seekable(self):
        return self.fileobj.seekable()

    def tell(self):
        """Return the current file position.
        """
        return self.position

    def seek(self, position, whence=io.SEEK_SET):
        """Seek to a position in the file.
        """
        if whence == io.SEEK_SET:
            self.position = min(max(position, 0), self.size)
        elif whence == io.SEEK_CUR:
            if position < 0:
                self.position = max(self.position + position, 0)
            else:
                self.position = min(self.position + position, self.size)
        elif whence == io.SEEK_END:
            self.position = max(min(self.size + position, self.size), 0)
        else:
            raise ValueError("Invalid argument")
        return self.position

    def read(self, size=None):
        """Read data from the file.
        """
        if size is None:
            size = self.size - self.position
        else:
            size = min(size, self.size - self.position)

        buf = b""
        while size > 0:
            while True:
                data, start, stop, offset = self.map[self.map_index]
                if start <= self.position < stop:
                    break
                else:
                    self.map_index += 1
                    if self.map_index == len(self.map):
                        self.map_index = 0
            length = min(size, stop - self.position)
            if data:
                self.fileobj.seek(offset + (self.position - start))
                b = self.fileobj.read(length)
                if len(b) != length:
                    raise ReadError("unexpected end of data")
                buf += b
            else:
                buf += NUL * length
            size -= length
            self.position += length
        return buf

    def readinto(self, b):
        buf = self.read(len(b))
        b[:len(buf)] = buf
        return len(buf)

    def close(self):
        self.closed = True
#class _FileInFile

class ExFileObject(io.BufferedReader):

    def __init__(self, fileobj, tarinfo):
        fileobj = _FileInFile(fileobj, tarinfo.offset_data,
                tarinfo.size, tarinfo.sparse)
        super().__init__(fileobj)

def pil_loader(path):
    # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)
    with open(path, 'rb') as f:
        with Image.open(f) as img:
            return img.convert('RGB')

def accimage_loader(path):
    try:
        import accimage
        return accimage.Image(path)
    except IOError:
        # Potentially a decoding problem, fall back to PIL.Image
        return pil_loader(path)

def get_default_image_loader():
    from torchvision import get_image_backend
    if get_image_backend() == 'accimage':
        return accimage_loader
    else:
        return pil_loader

def video_loader(video_dir_path, frame_indices, image_loader):
    video = []
    for i in frame_indices:
        image_path = os.path.join(video_dir_path, 'image_{:05d}.jpg'.format(i))
        if os.path.exists(image_path):
            video.append(image_loader(image_path))
        else:
            return video

    return video

def get_default_video_loader():
    image_loader = get_default_image_loader()
    return functools.partial(video_loader, image_loader=image_loader)

def load_annotation_data(data_file_path):
    with open(data_file_path, 'r') as data_file:
        return json.load(data_file)

def get_class_labels(data):
    class_labels_map = {}
    index = 0
    for class_label in data['labels']:
        class_labels_map[class_label] = index
        index += 1
    return class_labels_map

def get_video_names_and_annotations(data, subset):
    video_names = []
    annotations = []

    for key, value in data['database'].items():
        this_subset = value['subset']
        if this_subset == subset:
            if subset == 'testing':
                video_names.append('test/{}'.format(key))
            else:
                label = value['annotations']['label']
                video_names.append('{}/{}'.format(label, key))
                annotations.append(value['annotations'])

    return video_names, annotations

def make_dataset_tar(root_path, subset, n_samples_for_each_video,
                 sample_duration, video_paths, audio_paths):

    video_names = list(video_paths.keys())

    dataset = []
    for i in range(len(video_names)):
        if i % 10000 == 0:
            print('dataset loading [{}/{}]'.format(i, len(video_names)))

        video_path = video_names[i]
        audio_path = video_path.replace('.jpg', '.npy')

        if not audio_path in audio_paths:
            continue

        #n_frames = video_paths[video_path]
        n_frames = 31

        begin_t = 1
        end_t = n_frames
        sample = {
            'video': video_path,
            'audio': audio_path,
            'segment': [begin_t, end_t],
            'n_frames': n_frames,
            #'video_id': video_names[i][:-14].split('/')[1]
        }
        sample['label'] = video_names[i][2]

        if n_samples_for_each_video == 1:
            sample['frame_indices'] = list(range(1, n_frames + 1))
            dataset.append(sample)
        else:
            if n_samples_for_each_video > 1:
                step = max(1,
                           math.ceil((n_frames - 1 - sample_duration) /
                                     (n_samples_for_each_video - 1)))
            else:
                step = sample_duration
            for j in range(1, n_frames, step):
                sample_j = copy.deepcopy(sample)
                sample_j['frame_indices'] = list(
                    range(j, min(n_frames + 1, j + sample_duration)))
                dataset.append(sample_j)

    return dataset

class TarReader(object):
    def __init__(self):
        super(TarReader, self).__init__()
        self.id_context = dict()
        self.name2member = dict()

    def read(self, tar_file, image_name):
        if tar_file in self.id_context:
            im = self.id_context[tar_file].extractfile(self.name2member[image_name])
            return im.read()
        else:
            file_handle = tarfile.open(tar_file)
            self.id_context[tar_file] = file_handle
            im = self.id_context[tar_file].extractfile(self.name2member[image_name])
            return im.read()

    def getnames(self, tar_file):
        if tar_file in self.id_context:
            im = self.id_context[tar_file].getnames()
        else:
            file_handle = tarfile.open(tar_file)
            pkl_file = tar_file.replace('.tar', '.pkl')
            if not os.path.exists(pkl_file):
                members = file_handle.getmembers()
                pickle.dump(members, open(pkl_file, 'wb'))
            else:
                members = pickle.load(open(pkl_file, 'rb'))
                file_handle.members = members
                file_handle._loaded = True

            for m in members:
                self.name2member[m.name] = m

            self.id_context[tar_file] = file_handle
            #self.lock[tar_file] = threading.RLock()
            im = self.id_context[tar_file].getnames()
        return im

class ZipReader(object):
    def __init__(self):
        super(ZipReader, self).__init__()
        self.id_context = dict()
        self.lock = multiprocessing.Lock()

    def read(self, zip_file, image_name):
        if zip_file in self.id_context:
            with self.lock:
                im = self.id_context[zip_file].open(image_name)
                res = im.read()
            return res
        else:
            file_handle = zipfile.ZipFile(zip_file)
            self.id_context[zip_file] = file_handle
            im = self.id_context[zip_file].open(image_name)
            return im.read()

    def getnames(self, zip_file):
        if zip_file in self.id_context:
            im = self.id_context[zip_file].namelist()
        else:
            file_handle = zipfile.ZipFile(zip_file)
            self.id_context[zip_file] = file_handle
            im = self.id_context[zip_file].namelist()
        return im

class Lrw_va_tar(data.Dataset):
    def __init__(self,
                 root_path,
                 subset,
                 n_samples_for_each_video=1,
                 spatial_transform=None,
                 temporal_transform=None,
                 target_transform=None,
                 sample_duration=16,
                 get_loader=get_default_video_loader):

        ziplist = []
        a = os.walk(root_path)
        for b, c, d in a:
            for item in d:
                if item.endswith('.zip'):
                    ziplist.append(b + '/' + item)

        self.zipreader = ZipReader()

        self.video_name_map = {}
        self.video_zip_map = {}
        self.video_paths = {}
        for i, zipname in enumerate(ziplist):
            ss = time.time()
            namelist = self.zipreader.getnames(zipname)
            ee = time.time()
            readtime = ee - ss
            ss = time.time()
            for n in namelist:
                #if ('/' + subset + '/') in n:
                if n.endswith('.jpg'):
                    tmp = n.split('/')
                    newn = '/'.join(tmp[1:])
                    video_name = newn

                    self.video_name_map[newn] = n
                    self.video_zip_map[n] = zipname
                    if not video_name in self.video_paths:
                        self.video_paths[video_name] = 0
                    self.video_paths[video_name] += 1
            ee = time.time()
            buildtime = ee - ss
            print('loading ', i, zipname, readtime, buildtime)

        self.audio_zipname = root_path.replace('/video', '') + '/audio/audio.zip'
        ss = time.time()
        namelist = self.zipreader.getnames(self.audio_zipname)
        ee = time.time()
        print('loading audio', ee - ss)
        self.audio_name_map = {}
        self.audio_paths = set()
        for n in namelist:
            if ('/' + subset + '/') in n:
                if n.endswith('.npy'):
                    tmp = n.split('/')
                    newn = '/'.join(tmp[1:])
                    audio_name = '/'.join(tmp[1:])
                    self.audio_name_map[newn] = n
                    self.audio_paths.add(audio_name)

        self.data = make_dataset_tar(
            root_path, subset, n_samples_for_each_video,
            sample_duration, self.video_paths, self.audio_paths)

        self.spatial_transform = spatial_transform
        self.temporal_transform = temporal_transform
        self.target_transform = target_transform

    def loader(self, path, frame_indices):
        #for i in frame_indices:
            #image_path = path + '/image_{:05d}.jpg'.format(i)

            #if image_path in self.video_name_map:
            #    video_name = self.video_name_map[image_path]
            #    tar_name = self.video_tar_map[video_name]
            #    im = self.tarreader.read(tar_name, video_name)
            #    im = Image.open(BytesIO(im))
            #    video.append(im)

        image_path = path
        video_name = self.video_name_map[image_path]
        zip_name = self.video_zip_map[video_name]
        im = self.zipreader.read(zip_name, video_name)
        im = Image.open(BytesIO(im))
        im = np.asarray(im)
        video = [Image.fromarray(im[:, i*240:(i+1)*240, :]) for i in frame_indices]
        return video

    def __getitem__(self, index):
        """
        Args:
            index (int): Index
        Returns:
            tuple: (image, target) where target is class_index of the target class.
        """
        path = self.data[index]['video']

        frame_indices = self.data[index]['frame_indices']
        video_tot = len(frame_indices)

        if self.temporal_transform is not None:
            frame_indices = self.temporal_transform(frame_indices)

        clip = self.loader(path, frame_indices)

        audio_path = self.data[index]['audio']
        audio = self.zipreader.read(self.audio_zipname, self.audio_name_map[audio_path])
        audio = np.load(BytesIO(audio), allow_pickle=True)
        audio_tot = audio.shape[0]

        tmp_indices = np.array(frame_indices).astype(np.float32)
        tmp_indices = tmp_indices / video_tot * audio_tot
        audio_indices = tmp_indices.astype(np.int32)

        audio_indices = np.clip(audio_indices, 0, audio_tot - 1)
        audio_clip = audio[audio_indices]

        if self.spatial_transform is not None:
            self.spatial_transform.randomize_parameters()
            clip = [self.spatial_transform(img) for img in clip]
        clip = torch.stack(clip, 0).permute(1, 0, 2, 3)

        audio_clip = torch.from_numpy(audio_clip)

        return clip, audio_clip

    def __len__(self):
        return len(self.data)

max_stars_repo_name: yunyikristy/CM-ACC

max_issues_repo_name: yunyikristy/CM-ACC

max_forks_repo_name: yunyikristy/CM-ACC


text:
import numpy as np
import scipy.io as sio
from pyplfv.data_structures import EEGData
from pyplfv.wavelet import waveleted_signal_with_farray
from pyplfv.tve import normalized_tve_with_farray
from pyplfv.utility import save_data
from pyplfv.utility import load_data
'''
eeg_data = EEGData('./SampleData/sample.mat')
sig = eeg_data.signals['Cz']
save_waveleted_signal(signal=sig, sampling_interval=0.002, f0=2.0, filename='./SampleData/wavelet_test0.pkl')
#save_waveleted_signal_with_farray(signal=sig, sampling_interval=0.002, farray=np.anange(1.0, 20.0, 1.0), filename='./SampleData/wavelet_test1.pkl')
#save_waveleted_eegdata_with_farray(eegdata=eeg_data, sampling_interval=0.002, farray=np.anange(1.0, 20.0, 1.0), filename='./SampleData/wavelet_test2.pkl')
'''

#Save

matdata = sio.loadmat('SampleData/simulation_data1.mat')
trial_num = len(matdata['seg'][0])
arr = np.array(matdata['seg'], dtype='float128')
arr = arr.T
sig = np.hstack([arr[i] for i in range(trial_num)])
time_interval = 0.002
farray = [1.0 * i for i in range(1,30)]
start_time_of_trials = [750 * i for i in range(trial_num)]
offset = 0
length = int(1.5 / 0.002)
import time

print(len(sig))
t = time.time()
w = waveleted_signal_with_farray(signal=sig, sampling_interval=time_interval, farray=farray)
ntve = normalized_tve_with_farray(w)
t = time.time() - t
print(t)

#Load
'''
_waveleted_signal_with_farray = load_intermediate_data('./SampleData/simulation_data1_wavelet_wfa.pkl')
print(_waveleted_signal_with_farray)
'''

reason: import numpy,import scipy


text:
import json
import numpy as np
import argparse
import os
from program_executor import program_executor

parser = argparse.ArgumentParser(description="Generating explanation based on reasoning process")
parser.add_argument("--data", type=str, default='./data', help="path to preprocessed data")
parser.add_argument("--question", type=str, default='./data', help="path to GQA questions and scene graphs")
parser.add_argument("--save", type=str, default='./', help="path for saving the data")
args = parser.parse_args()

template = json.load(open(os.path.join(args.data,'exp_template.json')))
for cur_split in ['train','val']:
    processed_data = json.load(open(os.path.join(args.data,'simplified_semantics_'+cur_split+'.json')))
    question = json.load(open(os.path.join(args.question,'question',cur_split+'_balanced_questions.json')))
    scene_graph = json.load(open(os.path.join(args.question,'sceneGraphs',cur_split+'_sceneGraphs.json')))
    attr_mapping = json.load(open(os.path.join(args.data,'attr_mapping.json')))
    gen_exp = dict()
    for qid in processed_data:

        cur_semantic = processed_data[qid]
        cur_scene_graph = scene_graph[question[qid]['imageId']]
        exp = program_executor(cur_scene_graph,cur_semantic,template,attr_mapping)
        # use hard coding to fix some boundary cases
        exp = exp.replace('there is no there is no','there is no').replace('(-)','')
        exp = exp.replace('there is neither there is no','there is neither')
        exp = exp.replace('nor there is no','nor')
        if '(obj*:direction)' in exp:
            continue

        gen_exp[qid] = exp
    with open(os.path.join(args.save,'processed_explanation_'+cur_split+'.json'),'w') as f:
        json.dump(gen_exp,f)

max_stars_repo_name: szzexpoi/regex

max_issues_repo_name: szzexpoi/rex

max_forks_repo_name: szzexpoi/rex


text:
import pytest
import numpy as np

from irmetrics.relevance import unilabel, multilabel
from contextlib import contextmanager

def arr(x):
    return np.array([x])

@pytest.mark.parametrize("y_true, y_pred, output", [
    (arr([1]), arr([1, 2, 3]), arr([True, False, False])),
    (arr([1]), arr([2, 2, 3]), arr([False, False, False])),
])
@pytest.mark.parametrize("relevance", [unilabel, multilabel])
def test_single_labels(y_true, y_pred, output, relevance):
    np.testing.assert_equal(relevance(y_true, y_pred), output)

@contextmanager
def does_not_raise():
    yield

def conditional_raises(relevance):
    if relevance == unilabel:
        return pytest.raises(ValueError)
    return does_not_raise()

@pytest.mark.parametrize("y_true, y_pred, output", [
    (arr([1, 2]), arr([1, 2, 3]), arr([True, True, False])),
    (arr([1, 2]), arr([2, 2, 3]), arr([True, True, False])),
])
@pytest.mark.parametrize("relevance", [unilabel, multilabel])
def test_multiple_labels(y_true, y_pred, output, relevance):
    with conditional_raises(relevance):
        np.testing.assert_equal(relevance(y_true, y_pred), output)

max_stars_repo_name: kqf/ir-metrics

max_issues_repo_name: kqf/ir-metrics

max_forks_repo_name: kqf/ir-metrics


text:
import numpy as np
from scipy.optimize import curve_fit

class Linear:

    def __init__(self):
        self.parameters = [0, 0]

    def fit(self, x, y):

        self.parameters = np.polyfit(x, y, deg = 1)

    def predict(self, x):

        y_hat = np.polyval(self.parameters, x)

        return y_hat

class Quadratic:
    def __init__(self):
        self.parameters = [0, 0, 0]

    def fit(self, x, y):

        self.parameters = np.polyfit(x, y, deg = 2)

    def predict(self, x, beta0 = None):

        y_hat = np.polyval(self.parameters, x)

        return y_hat

class Gaussian:

    def __init__(self, parameters = [0, 0, 0, 0]):
        self.parameters = parameters

    def model(self, x, amplitude = 1, position = 0, width = 0.1, level = 0):

        return amplitude * np.exp(-(x - position)**2 / (2*width) + level)

    def fit(self, x, y, beta0 = [0.7, 0.5, 0.1, 0]):

        self.parameters, _ = curve_fit(self.model, x, y, beta0)

    def predict(self, x):

        y_hat =  self.model(x, self.parameters[0], self.parameters[1], self.parameters[2], self.parameters[3])

        return y_hat

class Lorentz:

    def __init__(self, parameters = [0, 0, 0, 0]):

        self.parameters = parameters

    def model(self, x, amplitude = 1, position = 0, width = 0.1, level = 0):

        return amplitude * width ** 2 / (np.pi * width * ((x - position) ** 2) + width ** 2)

    def fit(self, x, y, beta0 = [0.7, 0.5, 0.1, 0]):

        self.parameters, _ = curve_fit(self.model, x, y, beta0)

    def predict(self, x):

        y_hat =  self.model(x, self.parameters[0], self.parameters[1], self.parameters[2], self.parameters[3])

        return y_hat

class PseudoVoigt:

    def __init__(self, parameters = [0, 0, 0, 0, 0.5]):
        self.parameters = parameters

    def model(self, x, amplitude = 1, position = 0, width = 0.1, level = 0, alpha = 0.5):

        return amplitude * ( (1-alpha)/(width * np.sqrt(2*np.pi)) * np.exp(-(x-position)**2 /(2*width**2)) +
                            alpha/np.pi * (width / ((x - position)**2 + width**2))   ) + level

    def fit(self, x, y, beta0 = [0.7, 0.5, 0.1, 0, 0.5]):

        self.parameters, _ = curve_fit(self.model, x, y, beta0)

    def predict(self, x):

        y_hat =  self.model(x, self.parameters[0], self.parameters[1], self.parameters[2], self.parameters[3], self.parameters[4])

        return y_hat


text:
import numpy as np
import torch
from mmcv.ops.nms import batched_nms
from mmcv.runner import force_fp32
from torch.nn import functional as F

from mmdet3d.core.bbox.structures import (DepthInstance3DBoxes,
                                          LiDARInstance3DBoxes,
                                          rotation_3d_in_axis)
from mmdet3d.models.builder import build_loss
from mmdet3d.models.losses import chamfer_distance
from mmdet.core import multi_apply
from mmdet.models import HEADS
from .vote_head import VoteHead

@HEADS.register_module()
class CAVoteHead(VoteHead):
    r"""
    Class agnostic vote head
    """
    def __init__(self,
                 num_classes,
                 bbox_coder,
                 train_cfg=None,
                 test_cfg=None,
                 vote_module_cfg=None,
                 vote_aggregation_cfg=None,
                 pred_layer_cfg=None,
                 conv_cfg=dict(type='Conv1d'),
                 norm_cfg=dict(type='BN1d'),
                 objectness_loss=None,
                 dir_class_loss=None,
                 dir_res_loss=None,
                 size_res_loss=None,
                 semantic_loss=None,
                 center_loss=None,
                 size_class_loss=None
                 ):
        super().__init__(
            num_classes,
            bbox_coder,
            train_cfg=train_cfg,
            test_cfg=test_cfg,
            vote_module_cfg=vote_module_cfg,
            vote_aggregation_cfg=vote_aggregation_cfg,
            pred_layer_cfg=pred_layer_cfg,
            conv_cfg=conv_cfg,
            norm_cfg=norm_cfg,
            objectness_loss=objectness_loss,
            center_loss=None,
            dir_class_loss=dir_class_loss,
            dir_res_loss=dir_res_loss,
            size_class_loss=None,
            size_res_loss=size_res_loss,
            semantic_loss=semantic_loss
        )

    def _get_cls_out_channels(self):
        if hasattr(self, 'semantic_loss'):
            return self.num_classes + 2
        else:
            return 2

    def _get_reg_out_channels(self):
        """Return the channel number of regression outputs."""
        # size regression (6)
        # heading class+residual (num_dir_bins*2)
        return 6 + self.num_dir_bins * 2

    @force_fp32(apply_to=('bbox_preds', ))
    def loss(self,
             bbox_preds,
             points,
             gt_bboxes_3d,
             gt_labels_3d,
             pts_semantic_mask=None,
             pts_instance_mask=None,
             img_metas=None,
             gt_bboxes_ignore=None,
             ret_target=False):
        targets = self.get_targets(points, gt_bboxes_3d, gt_labels_3d,
                                   pts_semantic_mask, pts_instance_mask, bbox_preds)

        (vote_targets, vote_target_masks, dir_class_targets,
        dir_res_targets, mask_targets, objectness_targets, objectness_weights,
        box_loss_weights, distance_targets, centerness_targets, dir_targets) = targets

        # calculate vote loss
        vote_loss = self.vote_module.get_loss(bbox_preds['seed_points'],
                                              bbox_preds['vote_points'],
                                              bbox_preds['seed_indices'],
                                              vote_target_masks, vote_targets)

        # calculate objectness loss
        objectness_loss = self.objectness_loss(
            bbox_preds['obj_scores'].transpose(2, 1),
            objectness_targets,
            weight=objectness_weights)

        # calculate distance loss
        size_res_loss = self.size_res_loss(
            bbox_preds['distance'],
            distance_targets,
            weight=box_loss_weights.unsqueeze(-1).repeat(1, 1, 6)
        )

        # calculate direction class loss
        dir_class_loss = self.dir_class_loss(
            bbox_preds['dir_class'].transpose(2, 1),
            dir_class_targets,
            weight=box_loss_weights)

        # calculate direction residual loss
        batch_size, proposal_num = dir_class_targets.shape[:2]
        heading_label_one_hot = vote_targets.new_zeros(
            (batch_size, proposal_num, self.num_dir_bins))
        heading_label_one_hot.scatter_(2, dir_class_targets.unsqueeze(-1), 1)
        dir_res_norm = torch.sum(
            bbox_preds['dir_res_norm'] * heading_label_one_hot, -1)
        dir_res_loss = self.dir_res_loss(
            dir_res_norm, dir_res_targets, weight=box_loss_weights)

        if hasattr(self, 'semantic_loss'):
            # calculate semantic loss
            semantic_loss = self.semantic_loss(
                bbox_preds['sem_scores'].transpose(2, 1),
                mask_targets,
                weight=box_loss_weights)

        losses = dict(
            vote_loss=vote_loss,
            objectness_loss=objectness_loss,
            dir_class_loss=dir_class_loss,
            dir_res_loss=dir_res_loss,
            size_res_loss=size_res_loss)

        if hasattr(self, 'semantic_loss'):
            losses.update(semantic_loss=semantic_loss)

        if ret_target:
            losses['targets'] = targets

        return losses

    def get_targets(self,
                    points,
                    gt_bboxes_3d,
                    gt_labels_3d,
                    pts_semantic_mask=None,
                    pts_instance_mask=None,
                    bbox_preds=None):
        # find empty example
        valid_gt_masks = list()
        gt_num = list()
        for index in range(len(gt_labels_3d)):
            if len(gt_labels_3d[index]) == 0:
                fake_box = gt_bboxes_3d[index].tensor.new_zeros(
                    1, gt_bboxes_3d[index].tensor.shape[-1])
                gt_bboxes_3d[index] = gt_bboxes_3d[index].new_box(fake_box)
                gt_labels_3d[index] = gt_labels_3d[index].new_zeros(1)
                valid_gt_masks.append(gt_labels_3d[index].new_zeros(1))
                gt_num.append(1)
            else:
                valid_gt_masks.append(gt_labels_3d[index].new_ones(
                    gt_labels_3d[index].shape))
                gt_num.append(gt_labels_3d[index].shape[0])
        max_gt_num = max(gt_num)

        if pts_semantic_mask is None:
            pts_semantic_mask = [None for i in range(len(gt_labels_3d))]
            pts_instance_mask = [None for i in range(len(gt_labels_3d))]

        aggregated_points = [
            bbox_preds['aggregated_points'][i]
            for i in range(len(gt_labels_3d))
        ]

        (vote_targets, vote_target_masks, size_res_targets,
         dir_class_targets, dir_res_targets, centerness_targets,
         mask_targets, objectness_targets, objectness_masks,
         distance_targets, centerness_targets, dir_targets) = multi_apply(
            self.get_targets_single, points, gt_bboxes_3d, gt_labels_3d,
            pts_semantic_mask, pts_instance_mask, aggregated_points
        )

        vote_targets = torch.stack(vote_targets)
        vote_target_masks = torch.stack(vote_target_masks)

        objectness_targets = torch.stack(objectness_targets)
        objectness_weights = torch.stack(objectness_masks)
        objectness_weights /= (torch.sum(objectness_weights) + 1e-6)
        box_loss_weights = objectness_targets.float() / (
            torch.sum(objectness_targets).float() + 1e-6)
        dir_class_targets = torch.stack(dir_class_targets)
        dir_res_targets = torch.stack(dir_res_targets)
        dir_targets = torch.stack(dir_targets)
        mask_targets = torch.stack(mask_targets)
        distance_targets = torch.stack(distance_targets)
        centerness_targets = torch.stack(centerness_targets)

        return (vote_targets, vote_target_masks, dir_class_targets,
                dir_res_targets, mask_targets, objectness_targets, objectness_weights,
                box_loss_weights, distance_targets, centerness_targets,
                dir_targets)

    def get_targets_single(self,
                           points,
                           gt_bboxes_3d,
                           gt_labels_3d,
                           pts_semantic_mask=None,
                           pts_instance_mask=None,
                           aggregated_points=None):
        gt_bboxes_3d = gt_bboxes_3d.to(points.device)

        # generate votes target
        num_points = points.shape[0]
        if self.bbox_coder.with_rot:
            vote_targets = points.new_zeros([num_points, 3 * self.gt_per_seed])
            vote_target_masks = points.new_zeros([num_points],
                                                 dtype=torch.long)
            vote_target_idx = points.new_zeros([num_points], dtype=torch.long)
            box_indices_all = gt_bboxes_3d.points_in_boxes(points)
            for i in range(gt_labels_3d.shape[0]):
                box_indices = box_indices_all[:, i]
                indices = torch.nonzero(
                    box_indices, as_tuple=False).squeeze(-1)
                selected_points = points[indices]
                vote_target_masks[indices] = 1
                vote_targets_tmp = vote_targets[indices]
                votes = gt_bboxes_3d.gravity_center[i].unsqueeze(
                    0) - selected_points[:, :3]

                for j in range(self.gt_per_seed):
                    column_indices = torch.nonzero(
                        vote_target_idx[indices] == j,
                        as_tuple=False).squeeze(-1)
                    vote_targets_tmp[column_indices,
                                     int(j * 3):int(j * 3 +
                                                    3)] = votes[column_indices]
                    if j == 0:
                        vote_targets_tmp[column_indices] = votes[
                            column_indices].repeat(1, self.gt_per_seed)

                vote_targets[indices] = vote_targets_tmp
                vote_target_idx[indices] = torch.clamp(
                    vote_target_idx[indices] + 1, max=2)
        elif pts_semantic_mask is not None:
            vote_targets = points.new_zeros([num_points, 3])
            vote_target_masks = points.new_zeros([num_points],
                                                 dtype=torch.long)

            for i in torch.unique(pts_instance_mask):
                indices = torch.nonzero(
                    pts_instance_mask == i, as_tuple=False).squeeze(-1)
                if pts_semantic_mask[indices[0]] < self.num_classes:
                    selected_points = points[indices, :3]
                    center = 0.5 * (
                        selected_points.min(0)[0] + selected_points.max(0)[0])
                    vote_targets[indices, :] = center - selected_points
                    vote_target_masks[indices] = 1
            vote_targets = vote_targets.repeat((1, self.gt_per_seed))
        else:
            raise NotImplementedError

        (center_targets, size_targets, dir_class_targets,
         dir_res_targets, dir_targets) = self.bbox_coder.encode(
            gt_bboxes_3d, gt_labels_3d, ret_dir_target=True)

        proposal_num = aggregated_points.shape[0]
        distance1, _, assignment, _ = chamfer_distance(
            aggregated_points.unsqueeze(0),
            center_targets.unsqueeze(0),
            reduction='none')
        assignment = assignment.squeeze(0)
        euclidean_distance1 = torch.sqrt(distance1.squeeze(0) + 1e-6)

        objectness_masks = points.new_zeros((proposal_num))
        objectness_masks[
            euclidean_distance1 < self.train_cfg['pos_distance_thr']] = 1.0
        objectness_masks[
            euclidean_distance1 > self.train_cfg['neg_distance_thr']] = 1.0

        center_targets = center_targets[assignment]
        dir_class_targets = dir_class_targets[assignment]
        dir_res_targets = dir_res_targets[assignment]
        dir_res_targets /= (np.pi / self.num_dir_bins)
        size_res_targets = size_targets[assignment]
        dir_targets = dir_targets[assignment]

        mask_targets = gt_labels_3d[assignment]

        # Centerness loss targets
        canonical_xyz = aggregated_points - center_targets
        # print(canonical_xyz.shape)
        # print(gt_bboxes_3d.yaw[assignment].shape)
        if self.bbox_coder.with_rot:
            canonical_xyz = rotation_3d_in_axis(
                canonical_xyz.unsqueeze(0).transpose(0, 1),
                -gt_bboxes_3d.yaw[assignment], 2).squeeze(1)

        distance_front  = size_res_targets[:, 0] - canonical_xyz[:, 0]
        distance_left   = size_res_targets[:, 1] - canonical_xyz[:, 1]
        distance_top    = size_res_targets[:, 2] - canonical_xyz[:, 2]
        distance_back   = size_res_targets[:, 0] + canonical_xyz[:, 0]
        distance_right  = size_res_targets[:, 1] + canonical_xyz[:, 1]
        distance_bottom = size_res_targets[:, 2] + canonical_xyz[:, 2]

        distance_targets = torch.cat(
            (distance_front.unsqueeze(-1),
             distance_left.unsqueeze(-1),
             distance_top.unsqueeze(-1),
             distance_back.unsqueeze(-1),
             distance_right.unsqueeze(-1),
             distance_bottom.unsqueeze(-1)),
            dim=-1
        )
        inside_mask = (distance_targets >= 0.).all(dim=-1)

        objectness_targets = points.new_zeros((proposal_num), dtype=torch.long)
        pos_mask = (euclidean_distance1 < self.train_cfg['pos_distance_thr']) & inside_mask
        objectness_targets[pos_mask] = 1

        distance_targets.clamp_(min=0)
        deltas = torch.cat(
            (distance_targets[:, 0:3, None], distance_targets[:, 3:6, None]),
            dim=-1
        )
        nominators = deltas.min(dim=-1).values.prod(dim=-1)
        denominators = deltas.max(dim=-1).values.prod(dim=-1) + 1e-6
        centerness_targets = (nominators / denominators + 1e-6) ** (1/3)
        centerness_targets = torch.clamp(centerness_targets, min=0, max=1)

        return (
            vote_targets, vote_target_masks, size_res_targets,
            dir_class_targets, dir_res_targets, centerness_targets, mask_targets.long(),
            objectness_targets, objectness_masks, distance_targets, centerness_targets,
            dir_targets
        )

    def get_bboxes(self,
                   points,
                   bbox_preds,
                   input_metas,
                   rescale=False,
                   use_nms=True):
        if hasattr(self, 'semantic_loss'):
            return super(CAVoteHead, self).get_bboxes(
                points, bbox_preds, input_metas, rescale=rescale, use_nms=use_nms
            )
        else:
            # decode boxes
            bbox3d = self.bbox_coder.decode(bbox_preds, mode='rpn')
            assert not use_nms
            return bbox3d

max_issues_repo_name: wf-hahaha/BRNet

max_forks_repo_name: wf-hahaha/BRNet


text:
"""NeuronUnit model class for reduced neuron models"""

import numpy as np
from neo.core import AnalogSignal
import quantities as pq

import neuronunit.capabilities as cap
import neuronunit.models as mod
import neuronunit.capabilities.spike_functions as sf
from neuronunit.models import backends
from generic_network import net_sim_runner, get_dummy_synapses

class NetworkModel(cap.ReceivesCurrent,
                   cap.ProducesMultiMembranePotentials,
                   cap.ProducesSpikeRasters,
                   ):

    """Base class for network models
    todo replace receives current with receives patterned input."""

    def __init__(self, name=None, backend=pyNN, synapses=None):
        """Instantiate a network model.
        name: Optional model name.
        """
        self.run_number = 0
        self.backend = backend
        self.tstop = None
        self.data = None
        self.vms = None
        self.binary_trains = None
        self.t_spike_axis = None
        self.synapses = get_dummy_synapses()
        try:
            self.sim = generic_network.sim
        except:
            pass
    def get_membrane_potentials(self):
        return self.vms

    def getSpikeRasters(self, **run_params):
        return self.binary_train

    def inject_noise_current(self, stim_current, syn_weights):
        import pyNN.neuron as sim
        noisee = sim.NoisyCurrentSource(mean=0.74/1000.0, stdev=4.00/1000.0, start=0.0, stop=2000.0, dt=1.0)
        noisei = sim.NoisyCurrentSource(mean=1.440/1000.0, stdev=4.00/1000.0, start=0.0, stop=2000.0, dt=1.0)
        stim_noise_currents = [noisee,noisei]
        self.data,self.vms,self.binary_trains,self.t_spike_axis = net_sim_runner(syn_weights,sim,self.synapses,stim_noise_currents)
        return (self.vms,self.binary_train,self.data)

max_stars_repo_name: russelljjarvis/NetworkUnit

max_issues_repo_name: russelljjarvis/NetworkUnit

max_forks_repo_name: russelljjarvis/NetworkUnit


text:
"""
Mama' age problem in cpmpy.

Mamma's Age from 'Amusements in Mathematics, Dudeney', number 40.
'''
Tommy: 'How old are you, mamma?'
Mamma: 'Our three ages add up to exactly seventy years.'
Tommy: 'And how old are you, papa?'
Papa: 'Just six times as old as you, my son.'
Tommy: 'Shall I ever be half as old as you, papa?'
Papa: 'Yes, Tommy; and when that happens our three ages will add up to
       exactly twice as much as today.'

Can you find the age of Mamma?
'''

This cpmpy model was written by Hakan Kjellerstrand (hakank@gmail.com)
See also my cpmpy page: http://hakank.org/cpmpy/

"""
from cpmpy import *
import cpmpy.solvers
import numpy as np
from cpmpy_hakank import *

def mamas_age():

  m = intvar(1,500,name="m") # mamma's age
  p = intvar(1,500,name="p") # papa's age
  t = intvar(1,500,name="t") # tommy's age
  i = intvar(1,500,name="i") # temp

  # the real ages
  m2 = intvar(1,100,name="m2") # mamma's age
  p2 = intvar(1,100,name="p2") # papa's age
  t2 = intvar(1,100,name="t2") # tommy's age

  model = Model([m + p + t == 70 * 12,
                 6 * t == p,
                 (t + i) * 2 == p + i,
                 m + i + p + i + t + i == 2 * (m + p + t),
                 m2 == m / 12,
                 p2 == p / 12,
                 t2 == t / 12
                 ])

  ss = CPM_ortools(model)
  while ss.solve():
    print("mamma's age:", m2.value())
    print("papa's age:", p2.value())
    print("tommy's age:", t2.value())
    get_different_solution(ss,[m2,p2,t2])

mamas_age()

max_stars_repo_name: tias/hakank

max_issues_repo_name: tias/hakank

max_forks_repo_name: tias/hakank


text:
"""
utils.py

This file provides simple functions that might be of use to people using the ChemPer package

"""

import os
import collections

def get_data_path(relative_path, package='chemper'):
    """
    Returns the absolute path to a specified relative path inside data directory of a given package
    For example,
        get_data_path('molecules', package='chemper')
    will find this path:
        [absolute path to Python packages]/chemper/data/molecules/

    If the final path does not exist it will raise an IOError

    Parameters
    ----------
    relative_path : str
                    filename or relative path to a file stored in the data/ directory
                    of a given package
    package : str
              name of the Python package that should contain the specified file
              default="chemper"

    Returns
    -------
    path : str
           absolute path to filename in specified package
    """
    from pkg_resources import resource_filename
    fn = resource_filename(package, os.path.join('data', relative_path))

    if not os.path.exists(fn):
        raise IOError("Sorry! The absolute path %s was not found, that is %s is not in the data directory for %s" \
                          % (fn, relative_path, package))

    return fn

def get_full_path(relative_path, package="chemper"):
    """
    This function checks to see if this relative path is available locally
    if it is it returns the absolute path to that file.
    Otherwise it checks if that relative path is located at [package location]/data/relative path
    it will raise an IOError if neither location exists.

    Parameters
    ----------
    relative_path : str
                    relative path to a file available locally or in package/data
    package : str
              package name with a data directory the file might be located in

    Returns
    -------
    abs_path : str
               The absolute path to a local file if available, otherwise the abolsute path
               to [package]/data/[relative_path]
    """
    if os.path.exists(relative_path):
        return os.path.abspath(relative_path)
    return get_data_path(relative_path, package)

# =======================================
# Check SMIRKS validity
# =======================================

def is_valid_smirks(smirks):
    """
    This function checks if a given smirks is valid.

    Parameters
    ----------
    smirks : SMIRKS (or SMARTS) string

    Returns
    -------
    is_valid : boolean
    """
    # ChemPer only works with SMIRKS that match the
    # SMIRNOFF parameter format that is it has to be a
    # valid substructure search pattern and meet the
    # following three restrictions:
    # 1) only for one molecule (no '.' character)
    if '.' in smirks:
        return False
    # 2) not for a reaction (no '>' character)
    if '>' in smirks:
        return False
    # 3) it must have at least one indexed (:n) atom
    import re
    if not re.findall(r'([:]\d+)', smirks):
        return False

    # now check this is a valid substructure search pattern
    from chemper.mol_toolkits.mol_toolkit import Mol
    mol = Mol.from_smiles('C')
    try:
        mol.smirks_search(smirks)
        return True
    except ValueError:
        return False

# ===================================================================
# custom classes and functions for matching sets of SMIRKS to molecules
# ===================================================================
"""
custom dictionaries

These custom dictionaries are taken from
openforcefield.typing.engines.smirnoff.forcefield.
They allow tracking of symmetrically equivalent parameters.
For example the bonds (1,2) and (2,1), and
the impropers (1,2,3,4) and (1,2,4,3) are equivalent
"""

class TransformedDict(collections.MutableMapping):
    """A dictionary that applies an arbitrary key-altering
       function before accessing the keys"""

    def __init__(self, *args, **kwargs):
        self.store = dict()
        self.update(dict(*args, **kwargs))  # use the free update to set keys

    def __getitem__(self, key):
        return self.store[self.__keytransform__(key)]

    def __setitem__(self, key, value):
        self.store[self.__keytransform__(key)] = value

    def __delitem__(self, key):
        del self.store[self.__keytransform__(key)]

    def __iter__(self):
        return iter(self.store)

    def __len__(self):
        return len(self.store)

    def __keytransform__(self, key):
        return key

    def __str__(self): return str(self.store)

class ValenceDict(TransformedDict):
    """Enforce uniqueness in atom indices"""
    def __keytransform__(self, key):
        """Reverse tuple if first element is larger than last element."""
        # Ensure key is a tuple.
        key = tuple(key)
        # Reverse the key if the first element is bigger than the last.
        if key[0] > key[-1]:
            key = tuple(reversed(key))
        return key

class ImproperDict(TransformedDict):
    """Symmetrize improper torsions"""
    def __keytransform__(self,key):
        """Reorder tuple in numerical order except for element[1] which is the central atom; it retains its position."""
        # Ensure key is a tuple
        key = tuple(key)
        # Retrieve connected atoms
        connectedatoms = [key[0], key[2], key[3]]
        # Sort connected atoms
        connectedatoms.sort()
        # Re-store connected atoms
        key = tuple( [connectedatoms[0], key[1], connectedatoms[1], connectedatoms[2]])
        return key

def get_typed_molecules(smirks_list, input_molecules):
    """
    Creates a dictionary assigning a typename
    for each set of atom indices in each molecule
    based on the provided smirks patterns.

    Parameters
    ----------
    smirks_list : list[smirks tuples]
                  SMIRKS tuples come in the form (label, smirks)
    input_molecules : list[Mol]
                      Molecules from any supported toolkit (ChemPer, OpenEye, RDKit)

    Returns
    -------
    typeDict : embedded dictionary
               { index of input_molecule:
                    {(tuple of indices): assigned parameter type } }
    """
    from chemper.mol_toolkits.mol_toolkit import Mol
    molecules = [Mol(m) for m in input_molecules]
    type_dict = dict()
    for mol_idx, mol in enumerate(molecules):
        type_dict[mol_idx] = {}
        for [label, smirks] in smirks_list:
            matches = get_smirks_matches(mol, smirks)
            for match in matches:
                type_dict[mol_idx][match] = label

    return type_dict

def create_tuples_for_clusters(smirks_list, molecules):
    """
    This function is used to get clusters of molecular fragments based
    on input SMIRKS.

    For example, let's assume you wanted to find all of the
    atoms that match this SMIRKS list
    'any', '[*:1]~[*:2]'
    'single', '[*:1]-[*:2]'

    In this case, the "any" bond would match all bonds, but then
    the "single" would match all single bonds.
    If you were typing Ethene (C=C) then you expect the double bond
    between atoms 0 and 1 to match any bond and all C-H bonds to match single.

    The output in this case would be:
    [ ('any', [[ (0, 1) ]] ),
      ('single', [[ (0, 2), (0, 3), (1,4), (1,5) ]] )
    ]

    Parameters
    ----------
    smirks_list : list[smirks tuples]
                  This is a list of tuples with the form (label, SMIRKS)
    molecules : list[Mols]
                This is a list of molecules you want to type with this list of SMIRKS
                They can be from any supported toolkit (ChemPer, OpenEye, RDKit)

    Returns
    -------
    type_list : list[typing tuples]
                These are a list of typing tuples that could be used as input for
                the SMIRKSifier with the form shown above:
                (label, [
                        [ (atom indices), ...], # atoms in mol0 with label
                        [ (atom indices), ... ], # atoms in mol1 with label
                        ... ] )

    """
    ordered_labels = [l for l, s in smirks_list]
    type_dict = get_typed_molecules(smirks_list, molecules)
    # start by getting things in the form
    # {label: {mol_idx: list of dictionary of indices} }
    label_dict = dict()
    for mol_idx, mol_dict in type_dict.items():
        for match, label in mol_dict.items():
            if label not in label_dict:
                label_dict[label] = [list() for i in range(len(molecules))]

            label_dict[label][mol_idx].append(match)

    # now we need to resort the mol_lists
    final_list = [ (l, label_dict[l]) for l in ordered_labels if l in label_dict]
    return final_list

def get_smirks_matches(mol, smirks):
    """
    Gets atom indices for a smirks string in a given molecule

    Parameters
    ----------
    mol : ChemPer Mol
    smirks : str
             SMIRKS pattern being matched to the molecule

    Returns
    --------
    matches : list[atom tuples]
              Atom indices where this smirks matches in this atom
    """
    from chemper.graphs.environment import ChemicalEnvironment

    env = ChemicalEnvironment(smirks)
    if env.get_type().lower() == 'impropertorsion':
        matches = ImproperDict()
    else:
        matches = ValenceDict()

    for match in mol.smirks_search(smirks):
        smirks_indices = sorted(list(match.keys()))
        atom_indices = tuple([match[s].get_index() for s in smirks_indices])
        matches[atom_indices] = ''

    return matches.keys()

# ===================================================================
# classes for evaluating lists of SMIRKS patterns
# ===================================================================

def score_match_reference(current_assignments, ref_assignments):
    """
    This function is intended to score two sets of SMIRKS patterns or other
    typing rules to see how similar they are.

    Typing assignments are provided in the form of embedded dictionaries:
        { molecule index:
            { (atom indices tuple): assigned label }
            }

    The assigned types do not (and should not) have the same name, rather
    this checks that the assignments types atoms into the same "groups."

    Parameters
    ----------
    current_assignments : dict
                          typing rules being assessed
    ref_assignments : dict
                      typing that is considered the reference or ideal

    Returns
    -------
    type_matches : list[match tuples]
                   match tuples have the form
                   (current_label, ref_label, count matched, count in reference label)
    score : float
            Score as a fraction of how well the current typing matches the reference.
            A score of 1.0 is a perfect match

    We would like to acknowledge Josh Fass <josh.fass@choderalab.org>
    who created this scoring algorithm using the bipartite graph for
    smarty (github.com/openforcefield/smarty and doi: 10.1021/acs.jctc.8b00821)
    """
    import networkx as nx
    total_counts = dict()
    total = 0
    for mol_idx, ref_dict in ref_assignments.items():
        for indices, label in ref_dict.items():
            if label not in total_counts:
                total_counts[label] = 0
            total += 1
            total_counts[label] +=1

    cur_labels = set()
    ref_labels = set()
    # check for missing tuples in dictionaries
    for mol_idx, current_dict in current_assignments.items():
        cur_keys = set(current_dict.keys())
        ref_keys = set(ref_assignments[mol_idx].keys())
        # check if there are atom index sets in references not in current
        if ref_keys - cur_keys:
            # TODO:
            return None, -1

        # store a set of labels
        c_labs = [e for k,e in current_dict.items()]
        r_labs = [e for k,e in ref_assignments[mol_idx].items()]
        cur_labels = cur_labels.union(set(c_labs))
        ref_labels = ref_labels.union(set(r_labs))

    # Create bipartite graph (U,V,E) matching current types U with
    # reference types V via edges E with weights equal to number of types in common.
    # create a dictionary for each possible pair of current and reference labels
    types_in_common = dict()
    for c_lab in cur_labels:
        for r_lab in ref_labels:
            types_in_common[(c_lab, r_lab)] = 0

    # up the count by +1 for each time a current and reference type matches the same set of atoms
    for mol_idx, index_dict in ref_assignments.items():
        for indices, r_lab in index_dict.items():
            c_lab = current_assignments[mol_idx][indices]
            types_in_common[(c_lab, r_lab)] += 1

    # Actually make the graph
    graph = nx.Graph()
    # Add current types
    for c_lab in cur_labels:
        graph.add_node(c_lab, bipartite=0)
    # add reference types
    for r_lab in ref_labels:
        graph.add_node(r_lab, bipartite=1)
    # add edges
    for c_lab in cur_labels:
        for r_lab in ref_labels:
            weight = types_in_common[(c_lab, r_lab)]
            graph.add_edge(c_lab, r_lab, weight=weight)

    mate = nx.algorithms.max_weight_matching(graph, maxcardinality=False)

    # Compute match dictionary and total number of matches.
    type_matches = list()
    total_type_matches = 0
    for lab1, lab2 in mate:
        counts = graph[lab1][lab2]['weight']
        total_type_matches += counts
        # labels come back in an arbitrary order, so determine if which is current/reference
        if lab1 in cur_labels:
            type_matches.append(( lab1, lab2, counts, total_counts[lab2]))
        else:
            type_matches.append( (lab2, lab1, counts, total_counts[lab1]))

    # compute fractional score:
    score = total_type_matches / total

    return type_matches, score

def match_reference(current_assignments, ref_assignments):
    """
    Determine if two sets of labels agree.
    This could be used to test if two sets of SMIRKS type a set of molecules
    in exactly the same way.

    This starts with two sets of "assignments" which are dictionaries
    like those created with get_typed_molecules.

    Let's imagine you wanted to type the molecule CF and CCl
    where most bonds are assigned X, but one bond in each molecule is
    assigned 'Y'. Then the reference_assignments would be:
    { 0: (0,1): 'X', (0,2): 'X', (0,3): 'X', (0,4): 'Y'},
      1: (0,1): 'X', (0,2): 'X', (0,3): 'X', (0,4): 'Y'},
    }

    Then you discover a set of SMIRKS patterns which match
    C-H bonds with one type and C-halogen have another.
    Then your current_assignments would be:
    { 0: (0,1): 'C-H', (0,2): 'C-H', (0,3): 'C-H', (0,4): 'C-hal.'},
      1: (0,1): 'C-H', (0,2): 'C-H', (0,3): 'C-H', (0,4): 'C-hal.'},
    }
    This would return a set tuples for the corresponding labels
    [('C-H', 'X'), ('C-hal.', 'Y')] and True for the two assigments matching.

    However, if your current set assigned different types to the two
    different carbon halogen bonds (C-F and C-Cl) then these two
    sets of assignments would not match and the result would be an empty set and False

    Parameters
    ----------
    current_assignments : dictionary of indices with current types
    ref_assignments : dictionary of indices with reference types

    Returns
    -------
    type_matches : set of tuples (current_label, reference_label)
                   pair of current and reference labels
                   None if the sets don't match
    is_match : boolean
               Does the current assignment exactly match the reference?
    """
    import networkx as nx

    graph = nx.Graph()
    matches = set()
    r_labs = set()
    c_labs = set()

    for mol_idx, index_dict in ref_assignments.items():
        # check that mol_idx is in current_assignments
        if mol_idx not in current_assignments:
            return set(), False

        for indices, r_label in index_dict.items():
            # check this set of indices is in the current set
            if indices not in current_assignments[mol_idx]:
                return set(), False

            c_label = current_assignments[mol_idx][indices]
            # check if nodes exist for the two labels (and make them)
            if c_label not in graph:
                graph.add_node(c_label)
                c_labs.add('current_'+c_label)

            if r_label not in graph:
                graph.add_node(r_label)
                r_labs.add(r_label)

            # create an edge connecting the reference and current type
            graph.add_edge('current_'+c_label, r_label)
            matches.add((c_label, r_label))

    for c_label in c_labs:
        if len(graph.edges(c_label)) != 1:
            return set(), False

    for r_label in r_labs:
        if len(graph.edges(r_label)) != 1:
            return set(), False

    return matches, True

def check_smirks_to_reference(current_types, reference_assignments, molecules):
    """
    Checks that the current SMIRKS assign types to this set of molecules
    to match the reference assignments.

    Parameters
    ----------
    current_types : list[smirks tuples]
                    SMIRKS types in the form (label, smirks)
    reference_assignments : embedded dictionary
                            This could be the output from get_typed_molecules
                            the dictionary has the form:
                            {mol_idx: {(atom indices tuple): label}, ... }
    molecules : list[Mol]
                These can be ChemPer Mols or
                Mols form a supported toolkit (currently OpenEye or RDKit)

    Returns
    -------
    agree : boolean
            Returns True if the SMIRKS type the molecules in the same way
            as the reference assignments
    """
    r_labs = [l for m,d in reference_assignments.items() for a,l in d.items()]
    if len(current_types) != len(set(r_labs)):
        return False

    current_assignments = get_typed_molecules(current_types, molecules)
    type_matches, matched = match_reference(current_assignments, reference_assignments)
    return matched

def check_smirks_agree(current_types, reference_types, molecules):
    """
    Checks if two lists of SMIRKS patterns type a list of molecules in the same way.

    Parameters
    ----------
    current_types : list[smirks tuples]
                    SMIRKS types in the form (label, smirks)
    reference_types : list[smirks tuples]
                      SMIRKS types in the form (label, smirks)
    molecules : list[Mol]
                ChemPer Mols or Mols form a supported toolkit (currently OpenEye or RDKit)

    Returns
    -------
    match : boolean
            True if the two sets of SMIRKS type
            the set of molecules in exactly the same way
    """
    if len(current_types) != len(reference_types):
        return False

    current_assignments = get_typed_molecules(current_types, molecules)
    reference_assignments = get_typed_molecules(reference_types, molecules)

    # check if they agree
    type_matches, matched = match_reference(current_assignments, reference_assignments)
    return matched

max_stars_repo_name: alexmalins/chemper

max_issues_repo_name: alexmalins/chemper

max_forks_repo_name: alexmalins/chemper

reason: import networkx


text:
# -*- coding: utf-8 -*-
import pyqtgraph as pg
import pyqtgraph.exporters
import numpy as np
plt = pg.plot(np.random.normal(size=100), title="Simplest possible plotting example")

if __name__ == '__main__':
    pg.exec()

max_stars_repo_name: secantsquared/pyqtgraph

max_issues_repo_name: secantsquared/pyqtgraph

max_forks_repo_name: secantsquared/pyqtgraph


text:
import numpy as np
import pandas as pd
from pathlib import Path
import os
import sys

RESULT_HOME = os.getenv('RESULT_HOME', f"{Path.home()}/results")

from argparse import ArgumentParser

parser = ArgumentParser()
parser.add_argument("-n", "--appName")
parser.add_argument("-a", "--adaptive")
parser.add_argument("-t", "--total_cores")
parser.add_argument("-f", "--topFile")
parser.add_argument("-o", "--outputFile")

args = parser.parse_args(sys.argv[1:])

appName = args.appName
adaptive = int(args.adaptive)
totalCores = args.total_cores
topFile = args.topFile
outputFile = args.outputFile

data = pd.read_csv(topFile)
filtered = (data[data['appName'] == appName] )
print(filtered)
output = filtered

output = output[["appName","stage", "duration"]].set_index("stage")
output["usedCores"] = 0
output["totalCores"] = totalCores
output["adaptive"] = adaptive
output["isIo"] = 0

ioStageDict = {
    "terasort" : [0, 1, 2],
    "pagerank": [0, 5],
    "join": [0, 1],
    "aggregation": [0]
}

filtered.set_index("appName")

dca_df = pd.read_csv(f"{RESULT_HOME}/combined.dca")
dca_grouped_df = dca_df.set_index("stage").sort_index().groupby("executorId")

# Add isIo column
for name, row in filtered.iterrows():
    stage = int(row["stage"])
    applicationName = row["appName"]
    print(f"stage {stage}")

    # Find the right number of cores used
    totalSelectedCore = 0
    for executorId, group in dca_grouped_df:
        group = group[group.index == stage]
        selectedCore = group["selection"].max()
        print(f"selectedCore for stage {stage} and exec {executorId}: {selectedCore}")
        totalSelectedCore += selectedCore
        print(f"totalSelectedCore: {totalSelectedCore}")

    for appName, ioStages in ioStageDict.items():
        if appName in applicationName:
            if (stage in ioStages):
                # Only set isIo for non-default runs
                print (totalCores == totalSelectedCore)
                if int(totalSelectedCore) != int(totalCores):
                    output.at[stage,'isIo'] = 1
                output.at[stage,'usedCores'] = totalSelectedCore
                break
            else:
                output.at[stage,'usedCores'] = totalCores
    print(output)

# Move appName to the start
output = output.reset_index()
output = output.set_index('appName').reset_index()

def writeToOutput(data):
    global fileNameWithoutExtension
#     f"{RESULT_HOME}/resultDca.csv"
    fileName = outputFile
    file_exists = os.path.isfile(fileName)

    with open(fileName, mode='a') as f:
        needHeader = f.tell() == 0
        data.to_csv(f, index=False, header=needHeader)

writeToOutput(output)

max_stars_repo_name: SobhanOmranian/spark-dca

max_issues_repo_name: SobhanOmranian/spark-dca

max_forks_repo_name: SobhanOmranian/spark-dca