text:
# -*- coding: utf-8 -*-

"""
      Written by : Arien Sligar (arien.sligar@ansys.com)
      Last updated : 12.09.2019

    incldues all functions that are related to
"""

import numpy as np
import os
import Lib.Utillities as utils

class NearField_Utils():
    '''
    Various function used to extract near field from HFSS. Including creating
    a near field setup, extracting size of near field setup, and export fields
    '''
    def __init__(self,aedtapp):
        self.aedtapp= aedtapp

    def create_cs_sheet_center(self,sheet_name,cs_name="nearfield_cs",overwrite_existing=True):

        """
        This uses the built in function in HFSS to create an object CS
        I am creating this so it is centered on the rectangle and oriented so
        X and Y direction are in teh plane of the rectangle based on edge lengths

        Parameters
        ----------
        sheet_name : str
            The name of the user defined sheed object that near fields should be based on.
        cs_name : str, optional
            The name to use, this doesn't really matter and any name can be used.
            The default is "nearfield_cs".
        overwrite_existing : bool, optional
            Mainly for testing, but if a user has already exported the fields,
            we can just use the existing fields if they want to change something
            only in the post processing. The default is True.

        Returns
        -------
        str
            return the name of the CS used.

        add the properties for cs_name,length,width,norm_unitvec,units
        """
        oDesign = self.aedtapp.odesign
        oEditor = oDesign.SetActiveEditor("3D Modeler")

        object_ids = oEditor.GetMatchedObjectName(sheet_name)
        if len(object_ids)==0:
            print("ERROR: Surface " + sheet_name + " does not exist")
            return False

        face_id = oEditor.GetFaceIDsOfSheet(sheet_name)
        face_id =face_id[0] #should only be 1 face

        self.aedtapp.modeler.set_working_coordinate_system('Global')

        units = oEditor.GetModelUnits()
        edge_IDs = oEditor.GetEdgeIDsFromObject(sheet_name)

        face_center = oEditor.GetFaceCenter(face_id)
        #face_center = [ ConvertUnits(float(x),model_units,"meter") for x in face_center ]

        edge_lengths =[]
        for edge in edge_IDs:
            edge_lengths.append(oEditor.GetEdgeLength(edge))
        edge_ids_and_length = zip(edge_lengths,edge_IDs)

        if len(set(edge_lengths)) <= 1: #this means it is a square
            short_edge_id = edge_IDs[0] #assigning first edge to just be the short edge
            short_edge_length= oEditor.GetEdgeLength(short_edge_id )
            width = short_edge_length
            short_edge_vert_ids = oEditor.GetVertexIDsFromEdge(short_edge_id)
            id_index=1
            adjacent_edge=False
            while adjacent_edge==False:
                vert_ids = oEditor.GetVertexIDsFromEdge(edge_IDs[id_index])
                #check to see if an edge shares a vertex id with another edge
                #if it does, it is an adjacent edge
                adjacent_edge = any(item in short_edge_vert_ids for item in vert_ids)
                if adjacent_edge:
                    long_edge_id = edge
                    long_edge_length= oEditor.GetEdgeLength(long_edge_id )
                    length = long_edge_length
                id_index+=1
        else:
            short_edge_length = np.min(edge_lengths)
            short_edge_id = edge_IDs[np.argmin(edge_lengths)]
            long_edge_length = np.max(edge_lengths)
            long_edge_id = edge_IDs[np.argmax(edge_lengths)]
            width = short_edge_length
            length = long_edge_length

        #get normal vector
        vert_ids_short = oEditor.GetVertexIDsFromEdge(short_edge_id)
        vert_ids_long = oEditor.GetVertexIDsFromEdge(long_edge_id)
        origin_index = list(set(vert_ids_short).intersection(vert_ids_long))

        short_endpoint_id = [x for x in vert_ids_short if x != origin_index[0] ]
        long_endpoint_id = [x for x in vert_ids_long if x != origin_index[0] ]

        origin_pos = oEditor.GetVertexPosition(origin_index[0])
        origin_pos = [float(i) for i in origin_pos]
        short_endpoint_pos = oEditor.GetVertexPosition(short_endpoint_id[0])
        short_endpoint_pos = [float(i) for i in short_endpoint_pos]
        long_endpoint_pos = oEditor.GetVertexPosition(long_endpoint_id[0])
        long_endpoint_pos= [float(i) for i in long_endpoint_pos]

        short_vec = np.array([
            short_endpoint_pos[0]-origin_pos[0],
            short_endpoint_pos[1]-origin_pos[1],
            short_endpoint_pos[2]-origin_pos[2]])

        short_vec_mag = np.linalg.norm(short_vec)
        short_unitvector = short_vec/short_vec_mag
        x_vector = short_unitvector #just using short axis of surface as x and long as y

        long_vec = np.array([
            long_endpoint_pos[0]-origin_pos[0],
            long_endpoint_pos[1]-origin_pos[1],
            long_endpoint_pos[2]-origin_pos[2]])

        long_vec_mag = np.linalg.norm(long_vec)
        long_unitvector = long_vec/long_vec_mag
        y_vector = long_unitvector #just using short axis of surface as x and long as y

        #calculate surface normal unit vector
        norm_unitvec_1 = np.cross(short_unitvector,long_unitvector)
        norm_unitvec_2 = np.cross(long_unitvector,short_unitvector)
        face_center_vec = np.array(face_center,dtype='float')

        #this unit vector may point in or outward,
        #this should always  point away from global origin
        #checking which face center vector + normal vector is larger
        #larger distance means that is the direction pointing away from origin
        dist_norm1 = np.linalg.norm(face_center_vec+norm_unitvec_1)
        dist_norm2 = np.linalg.norm(face_center_vec+norm_unitvec_2)
        if dist_norm1>dist_norm2:
            norm_unitvec = norm_unitvec_1
        else:
            norm_unitvec = norm_unitvec_2

        exisiting_cs = oEditor.GetCoordinateSystems()

        if (overwrite_existing and cs_name in exisiting_cs):
            oEditor.ChangeProperty(
                [
                    "NAME:AllTabs",
                [
                    "NAME:Geometry3DCSTab",
                [
                    "NAME:PropServers",
                    cs_name
                ],
                [
                    "NAME:ChangedProps",
                [
                    "NAME:Origin",
                    "X:="            , str(face_center[0]) + units,
                    "Y:="            , str(face_center[1]) + units,
                    "Z:="            , str(face_center[2]) + units
                ],
                [
                    "NAME:X Axis",
                    "X:="            ,str(x_vector[0])+ units,
                    "Y:="            , str(x_vector[1])+ units,
                    "Z:="            , str(x_vector[2])+ units
                ],
                [
                    "NAME:Y Point",
                    "X:="            , str(y_vector[0])+ units,
                    "Y:="            , str(y_vector[1])+ units,
                    "Z:="            , str(y_vector[2])+ units
                ]]]])
        else: #creates new CS with name increment
            original_cs_name = cs_name
            n=1
            while cs_name in exisiting_cs:
                cs_name = original_cs_name + "_" + str(n)
                n+=1

            oEditor.CreateRelativeCS(
                [
                    "NAME:RelativeCSParameters",
                    "Mode:="        , "Axis/Position",
                    "OriginX:="        , str(face_center[0]) + units,
                    "OriginY:="        , str(face_center[1]) + units,
                    "OriginZ:="        , str(face_center[2]) + units,
                    "XAxisXvec:="        , str(x_vector[0])+ units,
                    "XAxisYvec:="        , str(x_vector[1])+ units,
                    "XAxisZvec:="        , str(x_vector[2])+ units,
                    "YAxisXvec:="        , str(y_vector[0])+ units,
                    "YAxisYvec:="        , str(y_vector[1])+ units,
                    "YAxisZvec:="        , str(y_vector[2])+ units
                ],
                [
                    "NAME:Attributes",
                    "Name:="        , cs_name
                ])

        self.aedtapp.modeler.set_working_coordinate_system(cs_name)
        self.cs_name = cs_name
        self.norm_unitvec = norm_unitvec
        self.y_vector = y_vector
        self.x_vector = x_vector
        self.length = length
        self.length_meter = utils.convert_units(length,units,'meter')
        self.width = width
        self.width_meter = utils.convert_units(width,units,'meter')
        self.units = units
        return self.cs_name

    def generate_nearfield_setup(self,nf_setup_name,cs_name,
                                 grid_size=1e-3,
                                 update_if_exists=True,
                                 use_true_size=True):

        '''
        Generate a near field setup on a rectangle defined by geo_name
        This will be the output of the near fields

        """

        Parameters
        ----------
        nf_setup_name : str
            DESCRIPTION.
        cs_name : str
            DESCRIPTION.
        grid_size : int, optional
            size of grid to be created. The default is 1e-3.
        update_if_exists : bool, optional
            If a near field setup already exist, just update it. If False, a
            new setup wil be created and the name will be incremented.
            The default is True.
        use_true_size : bool, optional
            The original user defined rectangle may not align with an even number
            of grid points. Use this option to force the grid points to not extend
            beyone the user defined rectable if True. If False, the grid may extend
            beyond the edge of the rectangle, forcing the number grid size to be
            as user specified, but the overall size of the sample area may be slighty
            larger than the user defined rectangle. The default is True.

        Returns
        -------
        TYPE
            DESCRIPTION.

        '''
        length_n = int(np.ceil(self.length_meter/grid_size)+1)
        width_n = int(np.ceil(self.width_meter/grid_size)+1)
        if use_true_size:
            actual_width = self.width_meter
            actual_length = self.length_meter
        else:
            actual_width = grid_size*(width_n-1)
            actual_length = grid_size*(length_n-1)

        if (("length" not in dir(self)) or ("width" not in dir(self))):
            print("create_cs_sheet_center() needs to be run before generate_nearfield_setup() can be run")
            return False
        oDesign = self.aedtapp.odesign

        oModule = oDesign.GetModule("BoundarySetup")
        does_rad_exist = oModule.GetBoundariesOfType('radiation')
        if len(does_rad_exist)==0:
            return False
        oModule = oDesign.GetModule("RadField")

        self.length_n = length_n
        self.width_n = width_n
        self.actual_width = actual_width
        self.actual_length = actual_length

        existing_setups = oModule.GetSetupNames("Rectangle")

        if nf_setup_name in existing_setups:
            if update_if_exists==False:
                x=1
                orig_name = nf_setup_name
                while nf_setup_name in existing_setups:
                    nf_setup_name = orig_name +str(x)
                    x = x+1

                oModule.InsertNearFieldRectangleSetup(
                    [
                        "NAME:"+nf_setup_name,
                        "UseCustomRadiationSurface:=", False,
                        "Length:="        , str(self.actual_width) ,
                        "Width:="        , str(self.actual_length),
                        "LengthSamples:="    , int(width_n),
                        "WidthSamples:="    , int(length_n),
                        "CoordSystem:="        , cs_name
                    ])
            else:
                oModule.EditNearFieldRectangleSetup(nf_setup_name,
                    [
                        "NAME:"+nf_setup_name,
                        "UseCustomRadiationSurface:=", False,
                        "Length:="        , str(self.actual_width),
                        "Width:="        , str(self.actual_length),
                        "LengthSamples:="    , int(width_n),
                        "WidthSamples:="    , int(length_n),
                        "CoordSystem:="        , cs_name
                    ])
        else:
            oModule.InsertNearFieldRectangleSetup(
                [
                    "NAME:"+nf_setup_name,
                    "UseCustomRadiationSurface:=", False,
                    "Length:="        , str(self.actual_width),
                    "Width:="        , str(self.actual_length),
                    "LengthSamples:="    , int(width_n),
                    "WidthSamples:="    , int(length_n),
                    "CoordSystem:="        , cs_name
                ])
        self.nf_setup_name = nf_setup_name
        return nf_setup_name

    def build_edit_sources_array(self,name_array,mag_array,phase_array):
        '''
        This is a repeated function from Codebook.py,
        Builds an array that is used as in input into HFSS edit sources. The
        input if baseically an array of value for each port and corropsonding
        mag and phase for each beam id

        Parameters
        ----------
        name_array : list of string
            all port names in desgin.
        mag_array : TYPE
            all magnitudes for each port (can be an expression).
        phase_array : TYPE
            all magnitudes for each port (can be an expression)..

        Returns
        -------
        TYPE
            array in format required by HFSS.

        '''
        # ToDo, add port terminal name to port name mapping so you can use
        # the same port name for codebooks in driven terminal and modal
        solution_type = self.aedtapp.odesign.GetSolutionType()
        sources =[]
        if solution_type == 'DrivenTerminal':
            sources.append(["UseIncidentVoltage:=", True,
                "IncludePortPostProcessing:=", False,"SpecifySystemPower:=", False])
        else:
            sources.append(["IncludePortPostProcessing:=", False,"SpecifySystemPower:=", False])

        for count, excite in enumerate(name_array):
            sources.append(["Name:=", excite,"Magnitude:=",
                str(mag_array[count]),"Phase:=", str(phase_array[count]) ])
        edit_sources_array =sources
        return edit_sources_array

    def edit_sources(self,edit_sources_array):
        '''
        This is a repeated function from Codebook.py,
        Sets the edit sources values in HFSS based on the array
        Parameters
        ----------
        edit_sources_array : Array
            Array in the format expected by HFSS for edit sources.

        Returns
        -------
        None. Sets edit source in HFSS

        '''
        # ToDo, add port terminal name to port name mapping so you can use
        # the same port name for codebooks in driven terminal and modal
        oModule = self.aedtapp.odesign.GetModule("Solutions")
        oModule.EditSources(edit_sources_array)

    def export_all_nfd(self,nf_setup_name,ports,freq=None,setup_name="Setup1:LastAdaptive",export_path='./',overwrite=True):
        '''

        Parameters
        ----------
        nf_setup_name : str
            DESCRIPTION.
        ports : list
            DESCRIPTION.
        freq : str, optional
            DESCRIPTION. The default is None.
        setup_name : str, optional
            DESCRIPTION. The default is "Setup1:LastAdaptive".
        export_path : str, optional
            DESCRIPTION. The default is './'.
        overwrite : bool, optional
            DESCRIPTION. The default is True.

        Returns
        -------
        dict or bool
            returns a dictionary with port names as keys for filename and path
            to the location of the nfd. If no fields exists it will return False

        '''
        oModule = self.aedtapp.odesign.GetModule("Solutions")
        has_fields = oModule.HasFields(setup_name,freq)
        nfd_files_dict = {}
        if has_fields:
            num_ports = len(ports)
            nfd_files = []
            for n, port in enumerate(ports):
                freq_str = str(freq/1e9) + "GHz"
                setup_name_str = setup_name.replace(':',"_")
                file_path = export_path+port +'_'+ freq_str + '_' + setup_name_str + '.nfd'
                does_file_exist= os.path.isfile(file_path)
                if (not does_file_exist or overwrite):
                    mag_array=[0]*num_ports
                    mag_array[n] = 1
                    phase_array=[0]*num_ports
                    name_array = ports
                    edit_sources_array = self.build_edit_sources_array(name_array,mag_array,phase_array)
                    self.edit_sources(edit_sources_array)
                    file_name = self.export_nfd(nf_setup_name,freq,setup_name,file_path)
                else:
                    file_name = file_path
                    print("Using existing nfd: "+ file_name)
                nfd_files_dict[port]=file_name
            return nfd_files_dict
        else:
            print("no fields exists")
            return False

    def export_nfd(self,nf_setup_name,freq,setup_name,file_name='./out.nfd'):
        '''
        exports NFD file on near field setup and returns the path name

        Parameters
        ----------
        nf_setup_name : str
            name of near field setup to extract data from.
        freq : str
            frequency to be exported. This frequency must be solved with fields
            saved within the setup to be exported. Typically a string like "28GHz"
        setup_name : str
            name in HFSS of the setup that is to be exported. For example this
            shoudl be "Setup1:LastAdpative" or "Setup1:Sweep1" or whatever they
            are named in HFSS
        file_name : str, optional
            The name of the exported nfd file, this should ideally be set to
            easily identify which port and other setup information that the
            fields belong to. The default is './out.nfd'.

        Returns
        -------
        str
            path to file.

        '''

        if freq==None:
            print("setup requires a valid frequency point")
            return False

        path_directory = os.path.dirname(file_name)
        if not os.path.exists(path_directory):
            os.makedirs(path_directory)

        export_options =         [
            "ExportFileName:="    , file_name,
            "SetupName:="        , nf_setup_name
            ]
        variation_key1 = "IntrinsicVariationKey:="
        variation_key2 ="Freq=\'" + str(freq) + "\'"
        export_options.append(variation_key1)
        export_options.append(variation_key2)
        export_options.append("DesignVariationKey:=")
        export_options.append("")
        export_options.append("SolutionName:=")
        export_options.append(setup_name)

        oModule = self.aedtapp.odesign.GetModule("RadField")
        oModule.ExportRadiationFieldsToFile(export_options)

        print('NFD Exported: '+file_name)
        return file_name


text:
from vgg_net import Vgg16Conv
import argparse
import torch
from dataloader import *
from torch.utils.data import DataLoader
import cv2
import json
import os
import numpy as np

parser = argparse.ArgumentParser(description='PyTorch AlphaPose Training')

"------------------------------dataloader------------------------------"
parser.add_argument('--videodir',dest='inputpath',type = str,help='video-directory',default="")
parser.add_argument('--time',dest = 'time',default=4500,type=int,help='choose the unify video length to train and test')
parser.add_argument('--inpdim',dest = 'inp_dim',default=224,type=int,help = 'input dim for vgg19')
parser.add_argument('--efps',dest = 'efps',default=20,type=int,help='the count of extract frame per second')
parser.add_argument('--indim',dest = 'indim',default=224,type=int,help='imgsize input vgg')
parser.add_argument('--outdir',dest = 'outdir',default="",type=str,help='save the feature generate by vgg')
opt = parser.parse_args()

rootdir = os.getcwd()
if not os.path.exists(opt.outdir):
    os.mkdir(opt.outdir)

if __name__ == '__main__':

    #load video data
    video_dir = opt.inputpath
    vggdataset = RawVideoDataLoader(video_dir)
    vggdataloader = DataLoader(vggdataset,pin_memory = True)
    model = Vgg16Conv().cuda()
    for video_data ,video_name in tqdm(vggdataloader):
        if ((video_data == False)or (video_name == False)):
            continue
        vggfeature_map = []
        time = []
        for i in range(len(video_data['time_list'])):
            timerecord = video_data['time_list'][i].numpy().tolist()
            time.append(timerecord)
        for perframe in tqdm(video_data['crop_frame']):
            perframe = perframe.cuda()
            output = model(perframe)
            output = output.view(1000,1) #numclass = 1000
            output = output.cpu()
            output = output.detach().numpy().tolist()
            vggfeature_map.append(output)
        vggfeature_map = np.array(vggfeature_map)
        vggfeature_map = vggfeature_map.transpose(1,0,2).tolist()
        dict_result = {'vggfeaturemap':vggfeature_map,'time':time}
        json_result = json.dumps(dict_result)
        print(os.path.join(opt.outdir,str(video_name[0][:-4])+'_vggfeaturemap.json'))
        with open (os.path.join(opt.outdir,str(video_name[0][:-4])+'_vggfeaturemap.json'),'w') as f:
            f.write(json_result)

ext: py

max_stars_repo_name: wanghaiyangMIT/rhythm-detection

reason: import numpy


text:
import pandas as pd
import numpy as np
import statsmodels.api as sm
from statsmodels import regression
import matplotlib.pyplot as plt
import requests
import re
import bs4
import akshare as ak

def cal_rsrs(data2, data1, N=16):
    """
    计算data2中标的所在日期的RSRS斜率指标和拟合优度
    """
    data11 = data1[data1['sec_code'].isin(data2['sec_code'])].reset_index()
    loc = data11[data11['tradeday'].isin(data2['tradeday'])].index.tolist()
    if loc[0] >= N:
        x = sm.add_constant(data11['low_slice'][(loc[0] - N):loc[0]]).astype('float64')
        y = data11['high_slice'][(loc[0] - N):loc[0]].astype('float64')
        est = sm.OLS(y, x)
        res = est.fit()
        params = res.params.low_slice
        r_squared = res.rsquared
    else:
        params = None
        r_squared = None
    data2['rsrs'] = params
    data2['r_squared'] = r_squared
    return data2

def rsrs(data, N=16):
    """
    由日频数据计算RSRS斜率指标和拟合优度
    """
    data = data.dropna()
    data['tradeday'] = pd.to_datetime(data['tradeday'], format='%Y/%m/%d %H:%M:%S')
    data.sort_values(by=['sec_code', 'tradeday'], inplace=True)
    data1 = data.copy()
    data1 = data1.groupby(['sec_code', 'tradeday']).apply(lambda x, data1=data1, N=N: cal_rsrs(x, data1, N))
    print("RSRS斜率指标和拟合优度已计算")

    return data1

def cal_rsrs_std(data2, data1, M=300, N=16):
    """
    计算data2中标的所在日期的RSRS标准分
    """
    data11 = data1[data1['sec_code'].isin(data2['sec_code'])].reset_index()
    loc = data11[data11['tradeday'].isin(data2['tradeday'])].index.tolist()
    # print(loc[0])
    rsrs = data2['rsrs']
    if loc[0] < M + N:
        rsrs_std = None
    else:
        rsrs_mean = np.mean(data11['rsrs'][(loc[0] - M):loc[0]])
        rsrs_s = np.std(data11['rsrs'][(loc[0] - M):loc[0]])
        rsrs_std = (rsrs - rsrs_mean) / rsrs_s
    data2['rsrs_std'] = rsrs_std
    return data2

def rsrs_std(data1, M=300, N=16):
    """
    由RSRS斜率计算RSRS标准分
    """
    data2 = data1.groupby(['sec_code', 'tradeday']).apply(lambda x, data1=data1, M=M, N=N: cal_rsrs_std(x, data1, M, N))
    print("RSRS标准分已计算")
    return data2

def rsrs_std_cor(data2):
    """
    由RSRS标准分计算RSRS修正标准分
    """
    data2['rsrs_std_cor'] = data2['r_squared'] * data2['rsrs_std']
    print('RSRS修正标准分已计算')
    return data2

def rsrs_std_cor_right(data2):
    """
    由RSRS修正标准分计算RSRS右偏修正标准分
    """
    data2['rsrs_std_cor_right'] = data2['rsrs_std_cor'] * data2['rsrs']
    print('RSRS右偏修正标准分已计算')
    return data2

def rsrs_std_cor_right_mean(data2, ndays=5):
    """
    计算RSRS右偏修正标准分的ndays均线
    """
    data2['rsrs_std_cor_right_mean']=data2.groupby('sec_code')['rsrs_std_cor_right'].rolling(window=ndays).mean()

    # data2['rsrs_std_cor_right_mean'] = data2.groupby('sec_code')['rsrs_std_cor_right'].apply(pd.rolling_mean, ndays)
    print('RSRS右偏修正标准分均线已计算')
    return data2

def get_rsrs(data, N=16, M=300, ndays=5):
    """
    根据日频数据计算RSRS相关指标
    data 数据，包括：标的代码sec_code,交易日tradeday,最高价high_slice、最低价low_slice、收盘价close_slice
    N，回归时间窗口，默认取16，不足16赋值为None
    M，标准分计算窗口，默认取300，不足300赋值为None
    """
    data1 = rsrs(data, N)
    data2 = rsrs_std(data1, M, N)
    data2 = rsrs_std_cor(data2)
    data2 = rsrs_std_cor_right(data2)
    data2 = rsrs_std_cor_right_mean(data2, ndays)
    return data2

def get_signal(data2, S):
    """
    根据RSRS指标和阈值S判断是否有交易信号（最简单的情况）,trade_dir为0代表买入，为1代表卖出，为-1代表无信号
    """
    data3 = data2.copy()
    data3.loc[:,'trade_dir']=-1
    data3.loc[(data3['rsrs_std_cor_right'] > S) & (data3['trade_dir'] == -1), 'trade_dir'] = 0
    data3.loc[(data3['rsrs_std_cor_right'] > S) & (data3['trade_dir'] == 1), 'trade_dir'] = -1

    data3.loc[(data3['rsrs_std_cor_right'] < -S) & (data3['trade_dir'] == -1), 'trade_dir'] = 1
    data3.loc[(data3['rsrs_std_cor_right'] < -S) & (data3['trade_dir'] == 0), 'trade_dir'] = -1
    # data3.drop(['rsrs','r_squared','rsrs_std','rsrs_std_cor','rsrs_std_cor_right'], axis = 1, inplace = True)
    return data3

def RSRS(data, N=16, M=300, S=0.7, ndays=5):
    """
    根据日频数据计算RSRS相关指标，并判断交易信号，更新trade_dir,删除中间变量
    """
    data2 = get_rsrs(data, N, M, ndays)
    data3 = get_signal(data2, S)
    return data3

if __name__=='__main__':
    stock_zh_index_daily_df = ak.stock_zh_index_daily(symbol="sh000300")
    stock_zh_index_daily_df['tradeday']=stock_zh_index_daily_df.index
    stock_zh_index_daily_df['high_slice']=stock_zh_index_daily_df['high']
    stock_zh_index_daily_df['low_slice'] = stock_zh_index_daily_df['low']
    stock_zh_index_daily_df['close_slice'] = stock_zh_index_daily_df['close']
    stock_zh_index_daily_df['sec_code'] = 'sz399552'
    rsrs_=RSRS(stock_zh_index_daily_df, N=16, M=300, S=0.7, ndays=5)
    data2 = get_rsrs(stock_zh_index_daily_df, N=16, M=300,  ndays=5)
    data2.loc[:,'trade_dir']=-1
    S=0.7
    data2.loc[(data2['rsrs_std_cor_right'] > S) & (data2['trade_dir'] == -1), 'trade_dir'] = 0

reason: import numpy,import statsmodels,from statsmodels


text:
import numpy as np
from pkg_resources import resource_filename
from pyraysum import prs, Model

dipfile = resource_filename('pyraysum',
                           'examples/models/model_Porter2011_dip.txt')
anisofile = resource_filename('pyraysum',
                           'examples/models/model_Porter2011_aniso.txt')

def test_read_model_dip():
    model = prs.read_model(dipfile)
    assert isinstance(model, Model)
    return model

def test_read_model_aniso():
    model = prs.read_model(anisofile)
    assert isinstance(model, Model)
    return model

def test_def_model():
    thick = [20000., 10000., 0.]
    rho = [2800., 2950., 3300.]
    vp = [4600., 5000., 6000.]
    vs = [2600., 3000., 3600.]
    flag = [1, 0, 1]
    ani = [0, 5, 0]
    model = Model(thick, rho, vp, vs, flag=flag, ani=ani)
    assert isinstance(model, Model)
    return model

def test_plot_model():
    model = test_def_model()
    model.plot()


text:
import os
import sys
import glob
import urllib2

import pandas as pa
import numpy as np
import simplejson as js

from sklearn.feature_selection import SelectKBest, f_classif

def getuuid(fname):
    return os.path.basename(fname).split('.')[2]

def istumor(legend, fname):
    return int(legend[getuuid(fname)].split('-')[3][:-1]) < 10

def build_legend(path, globstring):
    uuids = []
    for fname in glob.glob(os.path.join(path,globstring)):
        uuids.append(os.path.basename(fname).split('.')[2])
    url = 'https://tcga-data.nci.nih.gov/uuid/uuidws/mapping/json/uuid/batch'
    req = urllib2.Request(url=url, headers={'Content-Type': 'text/plain'},data=','.join(uuids))
    data = js.loads(urllib2.urlopen(req).read())
    legend = {}
    for d in data['uuidMapping']:
        legend[d['uuid']] = d['barcode']
    return legend

def load_df(path, dtype):
    if dtype == 'norm':
        colname = 'normalized_count'
        globstring = '*_results'
    elif dtype == 'raw':
        colname = 'raw_count'
        globstring = '*.results'
    else:
        raise Exception("Invalid data type requested")
    legend = build_legend(path,globstring)
    accum = []
    for fname in glob.glob(os.path.join(path,globstring)):
        if istumor(legend, fname):
            df = pa.read_csv(fname, sep='\t', index_col=0, usecols=['gene_id',colname])
            df.rename(columns={colname: getuuid(fname)}, inplace=True)
            accum.append(df)
    return pa.concat(accum, axis=1)

#### WRITE ####
#store = pa.HDFStore('store.h5', complib='blosc', complevel=6)

#store['lusc_norm'] = load_df('tcga-lusc','norm')
#store['lusc_raw'] = load_df('tcga-lusc','raw')

#store['luad_norm'] = load_df('tcga-luad','norm')
#store['luad_raw'] = load_df('tcga-luad','raw')

#store.close()
#sys.exit()
#### WRITE ####

#### READ ####
store = pa.HDFStore('store.h5')
#brca = store['brca_norm']
#paad = store['paad_norm']
luad = store['luad_norm']
lusc = store['lusc_norm']

#alldata = np.hstack(( paad_res, brca_all  )).T
#alllabels = np.hstack(( np.ones(paad_res.shape[1]), np.zeros(brca_all.shape[1]) ))

#somedata = np.hstack(( paad_res, brca_some  )).T
#somelabels = np.hstack(( np.ones(paad_res.shape[1]), np.zeros(brca_some.shape[1]) ))

#selector = SelectKBest(f_classif, k=4)
#selector.fit(somedata,somelabels)

##td = selector.transform(somedata)
#inds = selector.pvalues_.argsort()
#start = 8004 + np.random.randint(0,1000)
#td = alldata[:, inds[start:start+2]]

#import pylab as p
#p.figure()

#p.plot(td[40:,0], td[40:,1], 'r.')
#p.plot(td[:40,0], td[:40,1], 'g.')
#p.show()
#### READ ####

max_stars_repo_name: binarybana/samcnet

max_issues_repo_name: binarybana/samcnet

max_forks_repo_name: binarybana/samcnet


text:
"""
Helper functions for het.py; primarily counting help.
"""

import numpy as np
from numba.typed import List
from numba import njit

@njit
def edge_core(
    cell_data: np.ndarray,
    cell_type_index: int,
    neighbourhood_bcs: List,
    neighbourhood_indices: List,
    spot_indices: np.array = None,
    neigh_bool: np.array = None,
    cutoff: float = 0.2,
) -> np.array:
    """Gets the edges which connect inputted spots to neighbours of a given cell type.

        Parameters
        ----------
        cell_data: np.ndarray          Spots*CellTypes; value indicates \
                                        proportion of spot due to a specific \
                                        cell type. Rows sum to 1; pure spots \
                                        or spot annotations have a single 1 \
                                        per row.

        cell_type_index: int            Column of cell_data that contains the \
                                        cell type of interest.

        neighbourhood_bcs: List         List of lists, inner list for each \
                                        spot. First element of inner list is \
                                        spot barcode, second element is array \
                                        of neighbourhood spot barcodes.

        neighbourhood_indices: List     Same structure as neighbourhood_bcs, \
                                        but first inner list element is index \
                                        of the spot, and second is array of \
                                        neighbour indices.

        spot_indices: np.array          Array of indices indicating which spots \
                                        we want edges associated with. Can be \
                                        used to subset to significant spots.

        neigh_bool: np.array            Array of booleans of length n-spots, \
                                        True indicates the spot is an allowed \
                                        neighbour. This is useful where we only \
                                        want edges to neighbours which express \
                                        an indicated ligand or receptor.

        cutoff: float                   Cutoff above which cell is considered \
                                        to be present within the spot, is \
                                        applied on cell_data, and thereby allows \
                                        a spot to be counted as having multiple \
                                        cell types.

        Returns
        -------
        edges: List   List of 2-tuples containing spot barcodes, indicating \
                        an edge between a spot in spot_indices and a neighbour \
                        where neigh_bool is True, and either the
    """

    # Subsetting to relevant cell types #
    cell_data = cell_data[:, cell_type_index]

    # Initialising the edge list #
    edge_list = init_edge_list(neighbourhood_bcs)

    # If spots have no neighbours, no counting !
    if len(edge_list) == 0:
        return edge_list
    elif len(spot_indices) == 0:
        return edge_list[1:]

    ### Within-spot mode
    # within-spot, will have only itself as a neighbour in this mode
    within_mode = edge_list[0][0] == edge_list[0][1]
    if within_mode:
        # Numba implimentation #
        for i in spot_indices:
            if neigh_bool[i] and cell_data[i] > cutoff:
                edge_list.append((neighbourhood_bcs[i][0], neighbourhood_bcs[i][1][0]))

    ### Between-spot mode
    else:
        # Subsetting the neighbourhoods to relevant spots #
        neighbourhood_bcs_sub = List()
        neighbourhood_indices_sub = List()
        for spot_i in spot_indices:
            neighbourhood_bcs_sub.append(neighbourhood_bcs[spot_i])
            neighbourhood_indices_sub.append(neighbourhood_indices[spot_i])

        # Number of unique edges will be the count of interactions.
        get_between_spot_edge_array(
            edge_list,
            neighbourhood_bcs_sub,
            neighbourhood_indices_sub,
            neigh_bool,
            cell_data,
            cutoff=cutoff,
        )

    return edge_list[1:]  # Removing the initial edge added for typing #

@njit
def init_edge_list(neighbourhood_bcs):
    """ Initialises the edge-list in a way which ensures consistent typing to \
        prevent errors with Numba.
    """

    # Initialising the edge list #
    edge_list = List()
    # This ensures consistent typing #
    # spots_have_neighbours = False
    # first_spot = -1
    for i in range(len(neighbourhood_bcs)):
        if len(neighbourhood_bcs[i][1]) > 0:
            edge_list.append((neighbourhood_bcs[i][0], neighbourhood_bcs[i][1][0]))
            # spots_have_neighbours = True
            # first_spot = i
            break
    return edge_list

@njit
def get_between_spot_edge_array(
    edge_list: List,
    neighbourhood_bcs: List,
    neighbourhood_indices: List,
    neigh_bool: np.array,
    cell_data: np.array,
    cutoff: float = 0,
):
    """ Populates edge_list with edges linking spots with a valid neighbour \
        of a given cell type. Validity of neighbour determined by neigh_bool, \
        which can indicate whether the neighbour expresses a certain ligand \
        or receptor. See edge_core for parameter information.
    """
    edge_starts = List()
    edge_ends = List()
    for i in range(len(neighbourhood_bcs)):
        bcs, indices = neighbourhood_bcs[i], neighbourhood_indices[i]
        spot_bc, neigh_bcs = bcs
        neigh_indices = indices[1]
        # Subset the neighbours to only those fitting indicated criteria #
        neigh_bcs = neigh_bcs[neigh_bool[neigh_indices]]
        neigh_indices = neigh_indices[neigh_bool[neigh_indices]]

        if len(neigh_indices) == 0:  # No cases where neighbours meet criteria
            continue  # Don't add any interactions for this neighbourhood

        # Note that can keep all by inputting cell_data with all 1's #
        interact_neigh_bool = cell_data[neigh_indices] > cutoff

        # Retrieving the barcodes of the interacting neighbours #
        interact_neigh_bcs = neigh_bcs[interact_neigh_bool]
        for interact_neigh_bc in interact_neigh_bcs:
            edge_starts.append(spot_bc)
            edge_ends.append(interact_neigh_bc)

    # Getting the unique edges #
    if len(edge_starts) > 0:
        add_unique_edges(edge_list, edge_starts, edge_ends)

@njit
def add_unique_edges(edge_list, edge_starts, edge_ends):
    """ Adds the unique edges to the given edge list. \
    Complicated in order to satisfy Numba compilation in no-python mode.
    """
    n_edges = len(edge_starts)

    # Adding the unique edges #
    edge_added = np.zeros((1, len(edge_starts)))[0, :] == 1
    for i in range(n_edges):
        if not edge_added[i]:
            edge_start, edge_end = edge_starts[i], edge_ends[i]
            edge_list.append((edge_start, edge_end))
            for j in range(i, n_edges):
                edge_startj, edge_endj = edge_starts[j], edge_ends[j]
                # Direction doesn't matter #
                if (edge_start == edge_startj and edge_end == edge_endj) or (
                    edge_end == edge_startj and edge_start == edge_endj
                ):
                    edge_added[j] = True

def get_data_for_counting(adata, use_label, mix_mode, all_set):
    """Retrieves the minimal information necessary to perform edge counting."""
    # First determining how the edge counting needs to be performed #
    # Ensuring compatibility with current way of adding label_transfer to object
    if use_label == "label_transfer" or use_label == "predictions":
        obs_key, uns_key = "predictions", "label_transfer"
    else:
        obs_key, uns_key = use_label, use_label

    # Getting the neighbourhoods #
    neighbours, neighbourhood_bcs, neighbourhood_indices = get_neighbourhoods(adata)

    # Getting the cell type information; if not mixtures then populate
    # matrix with one's indicating pure spots.
    if mix_mode:
        cell_props = adata.uns[uns_key]
        cols = cell_props.columns.values.astype(str)
        col_order = [
            np.where([cell_type in col for col in cols])[0][0] for cell_type in all_set
        ]
        cell_data = adata.uns[uns_key].iloc[:, col_order].values.astype(np.float64)
    else:
        cell_labels = adata.obs.loc[:, obs_key].values
        cell_data = np.zeros((len(cell_labels), len(all_set)), dtype=np.float64)
        for i, cell_type in enumerate(all_set):
            cell_data[:, i] = (
                (cell_labels == cell_type).astype(np.int_).astype(np.float64)
            )

    spot_bcs = adata.obs_names.values.astype(str)
    return spot_bcs, cell_data, neighbourhood_bcs, neighbourhood_indices

def get_neighbourhoods(adata):
    """Gets the neighbourhood information."""

    # Old stlearn version where didn't store neighbourhood barcodes, not good
    #   for anndata subsetting!!
    if not "spot_neigh_bcs" in adata.obsm:
        # Determining the neighbour spots used for significance testing #
        neighbours = List()
        for i in range(adata.obsm["spot_neighbours"].shape[0]):
            neighs = np.array(adata.obsm["spot_neighbours"].values[i, :][0].split(","))
            neighs = neighs[neighs != ""].astype(int)
            # DONE: store neigh_bcs to get below to work reliably
            #       after subsetting anndata #
            # neighs = neighs[neighs<adata.shape[0]] # Removing subsetted spots..
            neighbours.append(neighs)

        # Getting the neighbourhood information #
        neighbourhood_bcs = List()
        neighbourhood_indices = List()
        spot_bcs = adata.obs_names.values.astype(str)
        for spot_i in range(len(spot_bcs)):
            neighbourhood_indices.append((spot_i, neighbours[spot_i]))
            neighbourhood_bcs.append((spot_bcs[spot_i], spot_bcs[neighbours[spot_i]]))
    else:  # Newer version
        # Determining the neighbour spots used for significance testing #
        neighbours = List()
        neighbourhood_bcs = List()
        neighbourhood_indices = List()
        spot_bcs = adata.obs_names.values.astype(str)
        str_dtype = f"<U{max([len(bc) for bc in spot_bcs])}"  # ensures correct typing
        for i in range(adata.shape[0]):
            neigh_bcs = np.array(
                adata.obsm["spot_neigh_bcs"].values[i, :][0].split(",")
            )
            neigh_bcs = neigh_bcs[neigh_bcs != ""]
            neigh_bcs = np.array(
                [neigh_bc for neigh_bc in neigh_bcs if neigh_bc in spot_bcs],
                dtype=str_dtype,
            )
            neigh_indices = np.array(
                [np.where(spot_bcs == neigh_bc)[0][0] for neigh_bc in neigh_bcs],
                dtype=np.int64,
            )
            neighbours.append(neigh_indices)
            neighbourhood_indices.append((i, neigh_indices))
            neighbourhood_bcs.append((spot_bcs[i], neigh_bcs))

    return neighbours, neighbourhood_bcs, neighbourhood_indices

reason: import numpy,from numba


text:
# Author: Nathan Trouvain at 11/03/2022 <nathan.trouvain@inria.fr>
# Licence: MIT License
# Copyright: Xavier Hinaut (2018) <xavier.hinaut@inria.fr>
import matplotlib.pyplot as plt
import numpy as np

from reservoirpy.datasets import narma

steps = 1000
X = narma(steps)
X = (X - X.min()) / (X.ptp())
sigma = 0.1
from reservoirpy.nodes import IPReservoir

reservoir = IPReservoir(100, mu=0.0, sigma=sigma, sr=0.95, activation="tanh", epochs=10)
reservoir.fit(X, warmup=100).run(X[:100])
states = reservoir.run(X[100:])
from scipy.stats import norm

def heavyside(x):
    return 1.0 if x >= 0 else 0.0

def bounded(dist, x, mu, sigma, a, b):
    num = dist.pdf(x, loc=mu, scale=sigma) * heavyside(x - a) * heavyside(b - x)
    den = dist.cdf(b, loc=mu, scale=sigma) - dist.cdf(a, loc=mu, scale=sigma)
    return num / den

fig, (ax1) = plt.subplots(1, 1, figsize=(10, 7))
ax1.set_xlim(-1.0, 1.0)
ax1.set_ylim(0, 16)
for s in range(states.shape[1]):
    hist, edges = np.histogram(states[:, s], density=True, bins=200)
    points = [np.mean([edges[i], edges[i + 1]]) for i in range(len(edges) - 1)]
    ax1.scatter(points, hist, s=0.2, color="gray", alpha=0.25)
ax1.hist(
    states.flatten(),
    density=True,
    bins=200,
    histtype="step",
    label="Global activation",
    lw=3.0,
)
x = np.linspace(-1.0, 1.0, 200)
pdf = [bounded(norm, xi, 0.0, sigma, -1.0, 1.0) for xi in x]
ax1.plot(x, pdf, label="Target distribution", linestyle="--", lw=3.0)
ax1.set_xlabel("Reservoir activations")
ax1.set_ylabel("Probability density")
plt.legend()
plt.show()

reason: import numpy,from scipy


text:
import numpy as np
import tensorflow as tf
from SpatialConnectionAwareNetwork_arch import SpatialConnectionAwareNetwork
import h5py
import time
import logging
import argparse

parser = argparse.ArgumentParser(description='Tensorflow DeepVolume Test')
parser.add_argument('--gpu', type=int, default=0, help='gpu device id')
parser.add_argument('--datafile', default='../datafile/', type=str, help='path to datafile folder')
parser.add_argument('--savepath', default='../output/', type=str, help='path to output folder')
parser.add_argument('--modelpath', default='../models/', type=str, help='path to model save folder')
parser.add_argument('--samplingpath', default='../preprocessing/SamplingForSpatialConnectionAwareNetwork/', type=str, help='path to model save folder')
parser.add_argument('-b', '--batch_size', type=int, default=2, help='The batch size of the training')
parser.add_argument('--start_epoch', type=int, default=0, help='Set the starting epoch of the training')
parser.add_argument('--epoch', type=int, default=30, help='Number of training epoch')
parser.add_argument("-c", "--continue_training", help="use this if you want to continue a training", action="store_true")

args = parser.parse_args()

def train():

  batch_size = args.batch_size
  Samplingpath = args.samplingpath
  start_epoch = args.start_epoch
  epoch = args.epoch
  filespath = args.datafile

  # make a template for reuse
  network_template = tf.make_template('network', SpatialConnectionAwareNetwork)

  logname = 'TrainSpatialConnectionAwareNetwork_log.log'

  logging.basicConfig(level=logging.INFO,
                      format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                      datefmt='%a, %d %b %Y %H:%M:%S',
                      filename=logname,
                      filemode='a')

  config = tf.ConfigProto()
  config.gpu_options.visible_device_list = str(args.gpu)
  sess = tf.InteractiveSession(config=config)

  with tf.name_scope('input'):
    LR = tf.placeholder(tf.float32, shape=[batch_size, 2, 360, 432, 3], name='Lowresolute_image')
    HR = tf.placeholder(tf.float32, shape=[batch_size, 360, 432, 1], name='Highresolute_image')

  # conv network
  hidden = None
  x_1, hidden = network_template(LR[:, 0, :, :, :], hidden, batch_size)
  x_2, hidden = network_template(LR[:, 1, :, :, :], hidden, batch_size)

  with tf.name_scope('loss_function'):
    loss1 = tf.losses.mean_squared_error(HR, x_1)
    loss2 = tf.losses.mean_squared_error(HR, x_2)
    loss = loss1 + loss2

  with tf.name_scope('trainer'):
    trainer = tf.train.AdamOptimizer(learning_rate=1e-5).minimize(loss)

  sess.run(tf.global_variables_initializer())

  if args.continue_training == True:
    saver = tf.train.Saver()
    savepath = args.modelpath + 'SpatialConnectionAwareNetwork/Model'
    saver.restore(sess, savepath + str(start_epoch) + '.ckpt')
    print("Model loaded")

  def feed_dict(j, k):
    pointer = j
    xs = np.zeros((batch_size, 2, 360, 432, 3))
    ys = np.zeros((batch_size, 360, 432, 1))
    xs[:, 0, :, :, 0] = datainput1[batch_size * k:batch_size * (k + 1), pointer, 0:360, 0:432]
    xs[:, 0, :, :, 1] = datainput0[batch_size * k:batch_size * (k + 1), pointer, 0:360, 0:432]
    xs[:, 0, :, :, 2] = datainputsag[batch_size * k:batch_size * (k + 1), 2 * pointer, 0:360, 0:432]
    xs[:, 1, :, :, 0] = datainput1[batch_size * k:batch_size * (k + 1), pointer, 0:360, 0:432]
    xs[:, 1, :, :, 1] = datainput0[batch_size * k:batch_size * (k + 1), pointer, 0:360, 0:432]
    xs[:, 1, :, :, 2] = datainputsag[batch_size * k:batch_size * (k + 1), 2 * pointer + 1, 0:360, 0:432]
    ys[:, :, :, 0] = dataoutput[batch_size * k:batch_size * (k + 1), pointer, 0:360, 0:432]
    return {LR: xs, HR: ys}

  for i in range(start_epoch, epoch):
    time_start = time.time()

    logging.info('++++++++++++++++++++++++++++++++++++ Start training at epoch ' + str(i + 1) + ' ++++++++++++++++++++++++++++++++++++')
    print('++++++++++++++++++++++++++++++++++++ Start training at epoch ' + str(i + 1) + ' ++++++++++++++++++++++++++++++++++++')

    train_axialThickpath = filespath + 'axialThick-train.txt'
    train_axialfile = open(train_axialThickpath)
    train_axialread = train_axialfile.read().splitlines()
    ntrain = len(train_axialread)

    lossall = np.zeros(ntrain * 140)
    ncounti = 0

    matinput0 = Samplingpath + 'train-1-total_Axial2.mat'
    file0 = h5py.File(matinput0)
    datainput0 = file0['total_Axial2'][()]
    logging.info('Data Axial2 loaded')
    print('Data Axial2 loaded')

    matinput1 = Samplingpath + 'train-1-total_Reconstruction.mat'
    file1 = h5py.File(matinput1)
    datainput1 = file1['total_Reconstruction'][()]
    logging.info('Data Reconstruction loaded')
    print('Data Reconstruction loaded')

    matinput_sag = Samplingpath + 'train-1-totalSag.mat'
    file2 = h5py.File(matinput_sag)
    datainputsag = file2['total_Sag'][()]
    logging.info('Data Sagittal loaded')
    print('Data Sagittal loaded')

    matoutput = Samplingpath + 'train-1-total_Axial3.mat'
    file3 = h5py.File(matoutput)
    dataoutput = file3['total_Axial3'][()]
    logging.info('Data GT loaded')
    print('Data GT loaded')

    totalnum = datainput1.shape[1]
    nbatch = np.ceil(datainput1.shape[0] / batch_size)
    nbatch = nbatch.astype(np.int16)

    losss = np.zeros(nbatch*totalnum)

    for k in range(0,nbatch):
      for j in range(0, np.int16(totalnum)):
        _, lossp = sess.run([trainer, loss], feed_dict=feed_dict(j, k))
        lossall[ncounti] = lossp
        ncounti = ncounti + 1
        losss[k * totalnum + j] = lossp

    time_end = time.time()
    logging.info('Loss mean of epoch ' + str(i + 1) + ' is ' + str(np.mean(losss)))
    print('Loss mean of epoch ' + str(i + 1) + ' is ' + str(np.mean(losss)))
    logging.info('Loss std of epoch ' + str(i + 1) + ' is ' + str(np.std(losss)))
    print('Loss std of epoch ' + str(i + 1) + ' is ' + str(np.std(losss)))
    logging.info('Time cost of training at epoch ' + str(i + 1) + ' has been ' + str(time_end - time_start) + ' s')
    print('Time cost of training at epoch ' + str(i + 1) + ' has been ' + str(time_end - time_start) + ' s')

    saver = tf.train.Saver()
    savepath = args.modelpath + 'SpatialConnectionAwareNetwork/Model'
    model_path = '%s%s%s' % (savepath, i + 1, '.ckpt')
    saver.save(sess, model_path)
    logging.info('Models saved for epoch ' + str(i + 1))
    print('Models saved for epoch ' + str(i + 1))

if __name__ == '__main__':
  train()

max_stars_repo_name: ZerojumpLine/DeepVolume

max_issues_repo_name: ZerojumpLine/DeepVolume

max_forks_repo_name: ZerojumpLine/DeepVolume


text:
import os
import math
import random
import multiprocessing
import numpy as np
import librosa

from .datasets import Dataset

def normalise_amplitude(y, amplitude_factor):
    return 2**(amplitude_factor-1) * y / max(np.max(np.abs(y)), 1e-16)

def pitch_shift(y, shift_factor, sr):
    bins_per_octave = 12
    n_steps = int(shift_factor * bins_per_octave)
    return librosa.effects.pitch_shift(y, sr, n_steps)

def time_stretch(y, stretch_factor):
    return librosa.effects.time_stretch(y, 2**stretch_factor)

def transform(y, sr, **kwargs):
    amplitude_factor = kwargs.get('amplitude_factor', 0)
    shift_factor = kwargs.get('shift_factor', 0)
    stretch_factor = kwargs.get('stretch_factor', 0)
    na = lambda y: normalise_amplitude(y, amplitude_factor)
    ts = lambda y: time_stretch(y, stretch_factor)
    ps = lambda y: pitch_shift(y, shift_factor, sr)
    return ts(ps(na(y)))

class Sampler:
    def __init__(self):
        self.duration = 2.
        self.samplerate = 44100
        self.amplitude_factor = (0, 0)
        self.stretch_factor = (0, 0)
        self.shift_factor = (0, 0)

    @property
    def duration(self):
        return self._duration
    @duration.setter
    def duration(self, value):
        self._duration = float(value)

    @property
    def samplerate(self):
        return self._samplerate
    @samplerate.setter
    def samplerate(self, value):
        self._samplerate = int(value)

    @property
    def amplitude_factor(self):
        return self._amplitude_factor
    @amplitude_factor.setter
    def amplitude_factor(self, value):
        try:
            value = float(value)
            value = (value, value)
        except TypeError:
            if not hasattr(value, '__len__') or not 0 < len(value) <= 2:
                raise TypeError
            if len(value) == 1:
                value = next(iter(value))
                value = (value, value)
        self._amplitude_factor = value

    @property
    def stretch_factor(self):
        return self._stretch_factor
    @stretch_factor.setter
    def stretch_factor(self, value):
        try:
            value = float(value)
            value = (value, value)
        except TypeError:
            if not hasattr(value, '__len__') or not 0 < len(value) <= 2:
                raise TypeError
            if len(value) == 1:
                value = next(iter(value))
                value = (value, value)
        self._stretch_factor = value

    @property
    def shift_factor(self):
        return self._shift_factor
    @shift_factor.setter
    def shift_factor(self, value):
        try:
            value = float(value)
            value = (value, value)
        except TypeError:
            if not hasattr(value, '__len__') or not 0 < len(value) <= 2:
                raise TypeError
            if len(value) == 1:
                value = next(iter(value))
                value = (value, value)
        self._shift_factor = value

    def dataset_size(self):
        return 0

    def sample(self, n_samples=1, n_jobs=1):
        return [None] * n_samples

    def generate(self, batch_size=1, n_batch_per_round=1, n_jobs=1):
        while True:
            rand_idx = np.arange(batch_size * n_batch_per_round)
            np.random.shuffle(rand_idx)
            samples = self.sample(
                batch_size * n_batch_per_round, n_jobs)[rand_idx]
            for i in range(n_batch):
                yield samples[i*batch_size:(i+1)*batch_size]

class DatasetSampler(Sampler):
    def __init__(self, dataset):
        super().__init__()
        self.dataset = dataset

    def dataset_size(self):
        return len(self.dataset)

    def load(self, index, **kwargs):
        offset = kwargs.get('offset', random.uniform(0, 1))
        amplitude_factor = kwargs.get(
            'amplitude_factor', random.uniform(*self.amplitude_factor))
        shift_factor = kwargs.get(
            'shift_factor', random.uniform(*self.shift_factor))
        stretch_factor = kwargs.get(
            'stretch_factor', random.uniform(*self.stretch_factor))

        load_duration = self.duration * (2**stretch_factor)
        y = self.dataset.load(
            index, self.samplerate, offset=offset, duration=load_duration)
        y = transform(
            y, self.samplerate,
            amplitude_factor=amplitude_factor,
            shift_factor=shift_factor,
            stretch_factor=stretch_factor
        )
        samples = librosa.time_to_samples(self.duration, sr=self.samplerate)
        if y.size < samples:
            y = np.hstack((y, np.zeros((samples - y.size,))))
        elif y.size > samples:
            y = y[:samples]
        return y[None, :]

    def sample(self, n_samples=1, n_jobs=1):
        m = self.dataset_size()

        # generate random index
        idx = np.arange(m)
        np.random.shuffle(idx)
        idx = np.hstack((
            np.arange(m).repeat(int(math.ceil(n_samples // m))).flatten(),
            idx[:n_samples%m]
        ))
        np.random.shuffle(idx)

        # load
        if n_jobs <= 0:
            n_jobs = os.cpu_count()
        if n_jobs > 1:
            p = multiprocessing.Pool(n_jobs, maxtasksperchild=1)
            samples = np.array(list(p.map(self.load, idx)))
            p.close()
            p.join()
        else:
            samples = np.array(list(map(self.load, idx)))
        return samples

class AggregateSampler(Sampler):
    def __init__(self, *samplers):
        self.samplers = []
        for s in samplers:
            if isinstance(s, Dataset):
                s = DatasetSampler(s)
            if not isinstance(s, Sampler):
                raise TypeError
            self.samplers.append(s)
        super().__init__()

    @property
    def duration(self):
        durations = tuple(s.duration for s in self.samplers)
        duration = durations[0]
        if any(duration != d for d in durations):
            raise ValueError
        return duration
    @duration.setter
    def duration(self, value):
        for s in self.samplers:
            s.duration = value

    @property
    def samplerate(self):
        samplerates = tuple(s.samplerate for s in self.samplers)
        samplerate = samplerates[0]
        if any(samplerate != s for s in samplerates):
            raise ValueError
        return samplerate
    @samplerate.setter
    def samplerate(self, value):
        for s in self.samplers:
            s.samplerate = value

    @property
    def amplitude_factor(self):
        return None
    @amplitude_factor.setter
    def amplitude_factor(self, value):
        pass

    @property
    def stretch_factor(self):
        return None
    @stretch_factor.setter
    def stretch_factor(self, value):
        pass

    @property
    def shift_factor(self):
        return None
    @shift_factor.setter
    def shift_factor(self, value):
        pass

    def __getitem__(self, key):
        return self.samplers[key]

class AsyncSampler(AggregateSampler):
    def dataset_size(self):
        return max(s.dataset_size() for s in self.samplers)
    def sample(self, n_samples=1, n_jobs=1):
        return np.hstack([s.sample(n_samples, n_jobs) for s in self.samplers])

class SyncSampler(AggregateSampler):
    @property
    def amplitude_factor(self):
        i = iter(self.samplers)
        min_factor, max_factor = next(i).amplitude_factor
        for s in i:
            f = s.amplitude_factor
            if min_factor < f[0]:
                min_factor = f[0]
            if max_factor > f[1]:
                max_factor = f[1]
        if min_factor > max_factor:
            raise RuntimeError('no overlap among sub-samplers')
        return min_factor, max_factor
    @amplitude_factor.setter
    def amplitude_factor(self, value):
        pass

    @property
    def stretch_factor(self):
        i = iter(self.samplers)
        min_factor, max_factor = next(i).stretch_factor
        for s in i:
            f = s.stretch_factor
            if min_factor < f[0]:
                min_factor = f[0]
            if max_factor > f[1]:
                max_factor = f[1]
        if min_factor > max_factor:
            raise RuntimeError('no overlap among sub-samplers')
        return min_factor, max_factor
    @stretch_factor.setter
    def stretch_factor(self, value):
        pass

    @property
    def shift_factor(self):
        i = iter(self.samplers)
        min_factor, max_factor = next(i).shift_factor
        for s in i:
            f = s.shift_factor
            if min_factor < f[0]:
                min_factor = f[0]
            if max_factor > f[1]:
                max_factor = f[1]
        if min_factor > max_factor:
            raise RuntimeError('no overlap among sub-samplers')
        return min_factor, max_factor
    @shift_factor.setter
    def shift_factor(self, value):
        pass

    def load(self, index, **kwargs):
        offset = kwargs.get('offset', random.uniform(0, 1))
        amplitude_factor = kwargs.get(
            'amplitude_factor', random.uniform(*self.amplitude_factor))
        shift_factor = kwargs.get(
            'shift_factor', random.uniform(*self.shift_factor))
        stretch_factor = kwargs.get(
            'stretch_factor', random.uniform(*self.stretch_factor))

        ys = np.array([
            s.load(
                index, offset=offset,
                amplitude_factor=amplitude_factor,
                shift_factor=shift_factor,
                stretch_factor=stretch_factor
            )
            for s in self.samplers
        ])
        return ys

    def dataset_size(self):
        return min(s.dataset_size() for s in self.samplers)

    def sample(self, n_samples=1, n_jobs=1):
        m = self.dataset_size()

        # generate random index
        idx = np.arange(m)
        np.random.shuffle(idx)
        idx = np.hstack((
            np.arange(m).repeat(int(math.ceil(n_samples // m))).flatten(),
            idx[:n_samples%m]
        ))
        np.random.shuffle(idx)

        # load
        if n_jobs <= 0:
            n_jobs = os.cpu_count()
        if n_jobs > 1:
            p = multiprocessing.Pool(n_jobs, maxtasksperchild=1)
            samples = np.hstack(list(p.map(self.load, idx)))
            p.close()
            p.join()
        else:
            samples = np.hstack(list(map(self.load, idx)))
        return samples.transpose((1, 0, 2))

max_stars_repo_name: leichtrhino/ChimeraNet

max_issues_repo_name: arity-r/ChimeraNet

max_forks_repo_name: leichtrhino/ChimeraNet


text:
# Copyright (c) Microsoft. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for full license information.

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

from HelperClass2.NeuralNet_2_2 import *
from HelperClass2.Visualizer_1_1 import *

train_data_name = "../../Data/ch11.train.npz"
test_data_name = "../../Data/ch11.test.npz"

def Show3D(net, dr):
    X,Y = dr.GetTestSet()
    net.inference(X)

    colors = ['b', 'r', 'g']
    shapes = ['o', 'x', 's']

    fig = plt.figure(figsize=(6,6))
    ax = Axes3D(fig)
    count = Y.shape[0]
    for i in range(count):
        for j in range(Y.shape[1]):
            if Y[i,j] == 1:
                ax.scatter(net.Z1[i,0],net.Z1[i,1],net.Z1[i,2], color=colors[j], marker=shapes[j])
    plt.show()

    fig = plt.figure(figsize=(6,6))
    ax = Axes3D(fig)
    count = Y.shape[0]
    for i in range(count):
        for j in range(Y.shape[1]):
            if Y[i,j] == 1:
                ax.scatter(net.A1[i,0],net.A1[i,1],net.A1[i,2], color=colors[j], marker=shapes[j])
    plt.show()

if __name__ == '__main__':
    dataReader = DataReader_2_0(train_data_name, test_data_name)
    dataReader.ReadData()
    dataReader.NormalizeY(NetType.MultipleClassifier, base=1)

    dataReader.NormalizeX()
    dataReader.Shuffle()
    dataReader.GenerateValidationSet()

    n_input = dataReader.num_feature
    n_hidden = 3
    n_output = dataReader.num_category
    eta, batch_size, max_epoch = 0.1, 10, 5000
    eps = 0.1

    hp = HyperParameters_2_0(n_input, n_hidden, n_output, eta, max_epoch, batch_size, eps, NetType.MultipleClassifier, InitialMethod.Xavier)
    net = NeuralNet_2_2(hp, "Bank_233_2")

    #net.LoadResult()
    net.train(dataReader, 100, True)
    net.ShowTrainingHistory()

    Show3D(net, dataReader)

max_stars_repo_name: microsoft/ai-edu

max_issues_repo_name: microsoft/ai-edu

max_forks_repo_name: microsoft/ai-edu


text:
# Visualization
import matplotlib.pyplot as plt
import cv2
from PIL import Image

import numpy as np
import random
import os
import time

# Model
from keras.models import Sequential
from keras.layers import Dropout, Flatten, Lambda
from keras.layers import Conv2D, MaxPooling2D

# Constructing the conv-net
def get_conv(input_shape=(64, 64, 3), filename=None):
    model = Sequential()
    model.add(Lambda(lambda x: x / 127.5 - 1., input_shape=input_shape, output_shape=input_shape))
    model.add(Conv2D(32, (3, 3), activation='relu', name='conv1', input_shape=input_shape, padding="same"))
    model.add(Conv2D(64, (3, 3), activation='relu', name='conv2', padding="same"))
    model.add(MaxPooling2D(pool_size=(3, 3)))
    model.add(Dropout(0.25))
    model.add(Conv2D(128, (8, 8), activation="relu", name="dense1"))
    model.add(Dropout(0.5))
    model.add(Conv2D(1, (14, 14), name="dense2", activation="sigmoid"))

    # for layer in model.layers:
    #     print(layer.input_shape, layer.output_shape)
    if filename:
        model.load_weights(filename)
    return model

model = get_conv()
model.add(Flatten())
model.compile(loss='mse', optimizer='adadelta', metrics=['accuracy'])

heatmodel = get_conv(input_shape=(None, None, 3), filename="Models/localize7.h5")

# Store a 64x64 image (hopefully) without Waldo in it
def store_image(path, name, x, y):
    image = cv2.imread(path)[x:x + 64, y:y + 64]
    image_PIL = Image.open(path).crop((x, y, x + 64, y + 64))
    image_PIL.save('Data/Predictions/' + name + str(x) + '-' + str(y) + '.png')

# Locate waldo in image
def locate(img, filepath="Data/Raw/Train/"):
    num_sub_img = 100
    data = cv2.cvtColor(cv2.imread(filepath + img), cv2.COLOR_BGR2RGB)
    gray = cv2.cvtColor(cv2.imread(filepath + img), cv2.COLOR_BGR2GRAY)
    coloured_heat = cv2.cvtColor(gray, cv2.COLOR_GRAY2RGB)

    heatmap = heatmodel.predict(data.reshape(1, data.shape[0], data.shape[1], data.shape[2]))
    plt.imshow(heatmap[0, :, :, 0])
    plt.title("Heatmap")
    plt.show()

    # Show 95 certainty heatmap and grayscale->color
    # plt.imshow(heatmap[0, :, :, 0] > 0.95, cmap="gray")
    # plt.title("Heatmap")
    # plt.show()
    # coloured_heat = color_gray(data, heatmap, gray).astype(np.uint8)
    # Image.fromarray(coloured_heat).show()

    xx, yy = np.meshgrid(np.arange(heatmap.shape[2]), np.arange(heatmap.shape[1]))
    x = (xx[heatmap[0, :, :, 0] > 0.99])
    y = (yy[heatmap[0, :, :, 0] > 0.99])

    # Store a subset of 64x64 images of this image to train next model on
    # chosen_sub_images = random.sample([[x[num], y[num]] for num in range(len(x))], min(num_sub_img, len(x)))
    # for i, j in chosen_sub_images:
    #     store_image('Data/Raw/Train/' + img, img, int(i*3),int(j*3)))

    for i, j in zip(x, y):
        y_pos = j * 3
        x_pos = i * 3
        cv2.rectangle(data, (x_pos, y_pos), (x_pos + 64, y_pos + 64), (0, 0, 255), 5)

    coloured_heat = coloured_heat.astype(np.uint8)
    # Image.fromarray(coloured_heat).show()
    # if random.randint(0, 10) < 1:
    #     store_image('Data/Raw/Train/'+img, img, i, j)
    return data, heatmap

def color_gray(image, heatmap, gray):
    height, width = image.shape[:2]
    heat_height, heat_width = heatmap.shape[1:3]
    return np.array([[[int(
        int(gray[y, x]) + heatmap[0, int(y * heat_height / height), int(x * heat_width / width), 0] * (
                int(image[y, x, c]) - gray[y, x])) for c in range(3)] for x in range(width)] for y in
        range(height)])

# Predict all test images
for img in os.listdir("Data/Raw/Test/"):
    print(img)
    try:
        annotated, heatmap = locate(img, filepath="Data/Raw/Test/")
        # Image.fromarray(annotated).show()
        Image.fromarray(annotated).save("README/" + img)
        Image.fromarray(heatmap).save("README/heat" + img)
        # plt.title("Augmented")
        # plt.imshow(annotated)
        # plt.show()
    except Exception as e:
        print('exception', e)

# Predict a specific image
annotated, heatmap = locate('16t.jpg', filepath="Data/Raw/Test/")
Image.fromarray(annotated).show()
print(annotated.shape[:2])
plt.title("Augmented")
plt.imshow(annotated)
plt.show()


text:
import numpy as np
import scipy.linalg

class rls(object):
    """docstring for ClassName"""

    def __init__(self, lbd, theta, nn_dim, output_dim):

        self.lbd = lbd
        self.nn_dim = nn_dim
        self.y_dim = output_dim
        self.draw = []

        self.theta = theta * np.ones([self.nn_dim, self.y_dim])

        self.initialize()

    def initialize(self):
        self.rls_state = []

        self.F = 1000 * np.eye(self.nn_dim)

        self.F_M = self.F

        for i in range(self.y_dim - 1):
            self.F_M = scipy.linalg.block_diag(self.F_M, self.F)

    # rls.update(hidden_vec, obs_Y[i,:])
    def update(self, hidden_vec, obs_Y):
        hidden_vec = np.concatenate([hidden_vec, np.ones([1, 1])], axis=1)
        # print("original pred:")
        # print(hidden_vec @ self.theta)
        # print("ground truth:")
        # print(obs_Y)
        for j in range(self.y_dim):
            self.F = self.F_M[self.nn_dim * j:self.nn_dim * (j + 1),
                     self.nn_dim * j:self.nn_dim * (j + 1)]
            # print(self.F)
            k = self.lbd + hidden_vec @ self.F @ hidden_vec.T
            # 65 * 1
            k = self.F @ hidden_vec.T / k
            # print("rls debug info:")
            # print(k, hidden_vec @ k)
            # print("original theta:")
            # print(self.theta)
            # print("theta has updated:")
            # print(k @ (obs_Y[:, j] - hidden_vec @ self.theta[:, j]))
            self.theta[:, j] = self.theta[:, j] + k @ (obs_Y[:, j] - hidden_vec @ self.theta[:, j])
            # print("updated theta:")
            # print(self.theta)

            # self.F = (self.F - k @ hidden_vec @ self.F) / self.lbd

            self.F_M[self.nn_dim * j:self.nn_dim * (j + 1),
            self.nn_dim * j:self.nn_dim * (j + 1)] = self.F

        pred = hidden_vec @ self.theta
        # print("updated pred:")
        # print(pred)

        self.rls_state.append(pred)

    def predict(self, hidden_vec):
        hidden_vec = np.concatenate([hidden_vec, np.ones([1, 1])], axis=1)
        # print("Roll out action:")
        # print(hidden_vec, self.theta)
        # print(hidden_vec @ self.theta)
        return hidden_vec @ self.theta

max_stars_repo_name: intelligent-control-lab/Autoenv

max_issues_repo_name: intelligent-control-lab/Autoenv

max_forks_repo_name: intelligent-control-lab/Autoenv

reason: import numpy,import scipy


text:
"""
Using meshio:
    Mesh writing subroutines to create
    input files for the finite element
    code WAVES.
@autor Juan Gomez
"""
from __future__ import division
import numpy as np

def face_recognition(cells , cell_data , phy_sur , phy_lin , nini):
    """
    For the incoming elements required to create a plane wave
    this function extracts the set according to the physical
    surface and identifies the element face in contact with
    th scatterer.
    INPUT PARAMTERS:
    ---------------
        cell and cell_data: Are the dictionaries creatd by meshio
        phy_sur: Integer defining the physical surface for the strip.
        phy_lin: Integer defining the physical line for the strip-scatterer interface
        nini   : Integer defining the element id for the first element in the set.
    OUTPUT PARAMTERS:
    ----------------
        nf        : Integer defining the element id for the last element in the set
        els1_array: Integer array with the elemental data according to WAVES.
    """

    elements1 = cells["quad9"] + 1
    phy_surface = cell_data["quad9"]['physical']
    strip_id = [cont for cont, _ in enumerate(phy_surface[:]) if phy_surface[cont] == phy_sur]
    els1_array = np.zeros([len(strip_id) , 14], dtype=int)
    elcon      = np.zeros([len(strip_id) , 9 ], dtype=int)
    els1_array[: , 0] = range(1 + nini , len(strip_id) + 1 + nini)
#
    els1_array[: , 3] = 5
    els1_array[: , 1] = 3
#
    els1_array[: , 2] = 18
    els1_array[: , 4] = 9
    els1_array[: , 5::] = elements1[strip_id, :]
    elcon[:] = elements1[strip_id , :]
    #
    # Identify the lines along the scatterer
    #
    line_inco = cells["line3"] + 1
    phy_lines = cell_data['line3']['physical']
    sca_id = [cont for cont, _ in enumerate(phy_lines[:]) if phy_lines[cont] == phy_lin]
    els2_array = np.zeros([len(sca_id), 3], dtype=int)
    els2_array =  line_inco[sca_id, :]
    #
    # Extracts the element faces lying along the scatterer
    #
    nlines = len(sca_id)
    for i in range(nlines):
        id_n = els2_array[i , 2]
        ir , ic = np.where(elcon == id_n)
        els1_array[ir , 3] = ic-3
        els1_array[ir , 1] = 5

    nf = nini + len(strip_id)
    return nf , els1_array

def ele_writer(cells , cell_data , ele_tag , phy_sur , ele_type , mat_tag , ndof , nnode , nini):
    """
    Extracts a subset of elements from a complete mesh according to the physical surface
    phy_sur and writes down the proper fields into an elements array.
    INPUT PARAMTERS:
    ---------------
        cell and cell_data: Are the dictionaries creatd by meshio.
        ele_tag : String defining the element type according to meshio (e.g., quad9 , line3, etc).
        phy_sur : Integer defining the physical surface for the subset.
        ele_type: Integer defining the element type according to WAVES.
        mat_tag : Integer defining the material profile for the subset.
        ndof    : Integer defining the number of degrees of freedom for the elements.
        nnode   : Integer defining the number of nodes for the element.
        nini   : Integer defining the element id for the first element in the set.
    OUTPUT PARAMTERS:
    ----------------
        nf        : Integer defining the element id for the last element in the set
        els_array : Integer array with the elemental data according to WAVES.
    """

    elements = cells[ele_tag]+1           # Element connectivities (adds 1 to fortranize)
    phy_surface = cell_data[ele_tag]['physical']
    ele_id = [cont for cont, _ in enumerate(phy_surface[:]) if phy_surface[cont] == phy_sur]
    els_array = np.zeros([len(ele_id) , 5 + nnode], dtype=int)
    els_array[: , 0] = range(1 + nini , len(ele_id) + 1 + nini )
    els_array[: , 1] = ele_type
    els_array[: , 2] = ndof
    els_array[: , 3] = mat_tag
    els_array[: , 4] = nnode
    els_array[: , 5::] = elements[ele_id, :]
    nf = nini + len(ele_id)

    return nf , els_array

def boundary_writer(cells , cell_data , phy_lin , mat_tag , nini ):
    """
    Extracts a subset of line elements from a complete mesh and corresponding
    to Lysmer absorbing  boundaries. The elements are identified according to
    the physical line phy_lin.
    INPUT PARAMTERS:
    ---------------
        cell and cell_data: Are the dictionaries created by meshio.
        ele_tag : String defining the element type according to meshio (e.g., quad9 , line3, etc).
        phy_lin: Integer defining thw physical line for the strp-scatterer interface
        mat_tag : Integer defining the material profile for the subset.
        nini   : Integer defining the element id for the first element in the set.
    OUTPUT PARAMTERS:
    ----------------
        nf        : Integer defining the element id for the last element in the set
        els_array : Integer array with the elemental data according to WAVES.
    """

    elements = cells["line3"] + 1
    line_id   = cell_data["line3"]["physical"]
    abs_id = [cont for cont, _ in enumerate(line_id[:]) if line_id[cont] == phy_lin]
    els_array = np.zeros([len(abs_id), 8], dtype=int)
    els_array[: , 0] = range(1 + nini , len(abs_id) + 1 + nini)
    els_array[: , 1] = 7
    els_array[: , 2] = 6
    els_array[: , 3] = mat_tag
    els_array[: , 4] = 3
    els_array[: , 5] = elements[abs_id, 1]
    els_array[: , 6] = elements[abs_id, 2]
    els_array[: , 7] = elements[abs_id, 0]
    nf = nini + len(abs_id)

    return nf , els_array

def node_writer(points , point_data):
    """
    Writes down the nodal data as required by WAVES.
        INPUT PARAMTERS:
    ---------------
        points and point_data: Are the dictionaries creatd by meshio.
    OUTPUT PARAMTERS:
    ----------------
        nodes_array : Integer array with the nodal data according to WAVES.
    """

    npoints = points.shape[0]
    nodes_array = np.zeros((npoints , 6))
    nodes_array[: , 0] = range(1, npoints+1)
    nodes_array[:, 4:6] = points[:, :2]/10.0
    nodes_array[nodes_array[:, 1]==0, 1] = 2

    return nodes_array


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import pycocotools.coco as coco
import numpy as np
import torch
import json
import cv2
import os
import math

from ..video_dataset import VideoDataset

class PDTracking(VideoDataset):
  num_categories = 5
  dataset_folder = 'pd'
  default_resolution = [384, 960]
  class_name = ['Pedestrian', 'Car', 'Cyclist', 'Caravan/RV', 'Truck']
  # negative id is for "not as negative sample for abs(id)".
  # 0 for ignore losses for all categories in the bounding box region
  # ['Pedestrian', 'Car', 'Bicyclist', 'Bus', 'Caravan/RV',  'OtherMovable',
  # 'Motorcycle', 'Motorcyclist', 'OtherRider', 'Train', 'Truck', 'Dontcare']
  cat_ids = {1:1, 2:2, 3:3, 4:-9999, 5:4, 6:-2, 7:-9999, 8:-1, 9:-1, 10:-9999, 11:5}
  max_objs = 500
  def __init__(self, opt, split, rank=None):
    data_dir = os.path.join(opt.data_dir, self.dataset_folder)
    split_ = 'train' if opt.dataset_version != 'test' else 'test' #'test'
    img_dir = data_dir
    if split == 'train':
      ann_file_ = "train"
    else:
      ann_file_ = 'val'
    ann_path = os.path.join(
      data_dir, 'annotations', 'tracking_{}.json'.format(
        ann_file_))
    self.images = None
    super(PDTracking, self).__init__(opt, split, ann_path, img_dir)

    self.box_size_thresh = [300, 500, 300, 500, 500]

    if opt.only_ped:
      self.num_categories = 1
      self.class_name = ['person']
      self.cat_ids = {1:1, 2:-9999, 3:-1, 4:-9999, 5:-9999, 6:-9999, 7:-9999, 8:-1, 9:-1, 10:-9999, 11:-9999}
      self.box_size_thresh = [300]

    if opt.nu:
      self.num_categories = 8
      self.class_name = ['Car', 'Truck', 'Bus', 'Trailer', 'construction_vehicle', 'Pedestrian', 'Motorcycle', 'Bicycle']
      self.cat_ids =  {1:6, 2:1, 3:0, 4:3, 5:1, 6:-1, 7:-7, 8:0, 9:0, 10:-9999, 11:2, 12:5, 13:-8}
      self.box_size_thresh = [500, 500, 500, 500, 500, 300, 500, 500]

    self.alpha_in_degree = False
    self.depth_scale = 1
    self.dep_mask = 0
    self.dim_mask = 1
    self.rot_mask = 0
    self.amodel_offset_mask = 0
    self.ignore_amodal = True
    self.num_samples = len(self.images)
    self.exp_id = opt.exp_id
    if opt.const_v_over_occl:
      self.const_v_over_occl = True

    print('Loaded {} {} samples'.format(split, self.num_samples))

  def save_results_ioueval(self, results, save_dir):
    formattted_results = []
    if not os.path.exists(save_dir):
      os.mkdir(save_dir)

    for video in self.coco.dataset['videos']:
      video_id = video['id']
      images = self.video_to_images[video_id]

      for image_info in images:
        img_id = image_info['id']
        if not (img_id in results):
          continue
        frame_id = image_info['frame_id']
        for i in range(len(results[img_id])):
          item = results[img_id][i]
          if item['age'] != 1:
            continue
          if 'visibility' in item and not item['visibility']:
            continue
          category_id = item['class']
          track_id = item['tracking_id'] if 'tracking_id' in item else -1
          bbox = [item['bbox'][0].item(), item['bbox'][1].item(), item['bbox'][2].item() - item['bbox'][0].item(), item['bbox'][3].item() - item['bbox'][1].item()]

          entry = {'video_id': video_id, 'image_id': img_id, 'category_id': category_id, 'track_id': track_id, 'bbox': bbox, 'score': item['score'].item()}
          formattted_results.append(entry)

    print(save_dir + '/iou_eval.json')
    json.dump(formattted_results, open(save_dir + '/iou_eval.json', 'w'))

  def run_eval(self, results, save_dir, write_to_file=False, dataset_version="val"):
    self.save_results_ioueval(results, save_dir)
    os.chdir("../tao")
    command = 'python scripts/evaluation/evaluate.py ' + \
              '../data/%s/annotations/tracking_%s_tao.json ' % (self.dataset_folder, dataset_version) + \
              '{}/iou_eval.json'.format(save_dir) + ' --config-updates CATEGORIES 1,2'

    if write_to_file:
      print("Writing to file")
      command += ' > ../exp/tracking/{}/eval_out.txt'.format(self.exp_id)
    os.system(command)

  def __len__(self):
    return self.num_samples

  def _to_float(self, x):
    return float("{:.2f}".format(x))


text:
import unittest
import numpy as np

import os
os.environ['TEST'] = '1'

import sys
from pathlib import Path
TEST_DIR = str(Path(__file__).parent.resolve())
BASE_DIR = str(Path(__file__).parent.parent.resolve())
sys.path.append(BASE_DIR)

from core.search import Searcher, VectorIndexSearcher
from core.indexes import IndexesDirectory
from config.config import indexes_dir

class TestSearcherClass(unittest.TestCase):
	pass

class TestVectorIndexSearcher(unittest.TestCase):

	def setUp(self):
		self.indexes = IndexesDirectory(indexes_dir)
		self.searcher = VectorIndexSearcher()
		self.unitvec = np.ones(self.indexes.dims)

	def test_can_search_in_one_index(self):
		results = self.search(self.unitvec, 'Y02T.abs', 10)
		self.assertGreater(len(results), 0)

	def test_can_search_in_multiple_indexes(self):
		results = self.search(self.unitvec, 'Y02T', 10)
		self.assertGreaterEqual(len(results), 10)

	def test_ask_for_zero_results(self):
		results = self.search(self.unitvec, 'Y02T', 0)
		self.assertCount(0, results)

	def test_ask_for_negative_results(self):
		results = self.search(self.unitvec, 'Y02T', -1)
		self.assertCount(0, results)

	def search(self, needle, haystack, n):
		indexes = self.indexes.get(haystack)
		results = self.searcher.search(needle, indexes, n)
		return results

	def assertCount(self, n, results):
		self.assertIsInstance(results, list)
		self.assertEqual(n, len(results))

if __name__ == '__main__':
    unittest.main()


text:
# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/03_Train Flyvec.ipynb (unless otherwise specified).

__all__ = ['BIN', 'init', 'main']

# Cell
import os
from pathlib import Path
import ctypes
from ctypes import *
import array as arr
import sys
import argparse
import time
import numpy
import flyvec.path_fixes as pf
import subprocess as sp

# Cell
# Load binaries
BIN = pf.CU_BIN

# Cell
from fastcore.script import *

@call_parse
def init():
    sp.call(["sh", str(BIN / "short_make")], cwd=str(BIN))

# Cell
@call_parse
def main(
    encodings_source:Param("""Path to the tokens encoded as an ID. (eltype should be np.int32)""", str),
    offsets_source:Param("""Path to the offsets array indicating where each value of the array indicates where a chunk of text should start. (eltype should be np.uint64, first value should be 0)""",str),
    output_dir:Param("""Directory in which to save the checkpoints. Created if it does not exist""", str)=".",
    save_every:Param("""How many epochs to run before saving a checkpoint""", int)=1,
    ckpt_prefix:Param("""Prefix to name each checkpoint. Additional parameter choices are inserted into the checkpoint name.""", str)="flyvec_model",
    starting_checkpoint:Param("""Path to .npy file of saved checkpoint""", str)=None,
    W:Param("Size of the W-gram sliding window used to train the word vectors", int)=11,
    hid:Param("Number of hidden units (neurons). Do not change", int)=400,
    initial_learning_rate:Param("Initial learning rate", float)=0.002,
    delta:Param("From equation", float)=0.,
    mu:Param("If no checkpoint provided, use this as mean for random normal initialization of synapses", float)=0.,
    sigma:Param("If no checkpoint provided, use this as stdev for random normal initialization of synapses", float)=0.02,
    Nep:Param("Maximum number of epochs, fewer if starting from a checkpoint", int)=15,
    batch_size:Param("Minibatch size", int)=10000,
    prec:Param("Precision, avoid dividing by 0", float)=1.0E-30,
):

    # Obsolete Parameters
    stride = int(1)
    IM_HEIGHT = int(1)
    IM_WIDTH = int(11)
    Nchannels = int(1)

    # Unexposed Parameters
    frequency_scaling = 1 # 1 or 0, true or false
    Lmid = 1.0
    Lbase = 1.0
    sparse_input = 1 # 1 or 0, true or false, make input sparse and only store column

    # Load object files
    model_descriptor = ctypes.CDLL(str(BIN / 'model_descriptor.so'))
    model_arrays = ctypes.CDLL(str(BIN / 'model_arrays.so'))
    cuda_helpers = ctypes.CDLL(str(BIN / 'cuda_helpers.so'))
    model = ctypes.CDLL(str(BIN / 'cuda_funcs.so'))
    prune_input = ctypes.CDLL(str(BIN / 'prune_input.so'))

    # m = 2
    # p = 2
    # parser.add_argument("--stride", default=1, type=int, help="Stride. Do not change.")
    # parser.add_argument("--IM_HEIGHT", default=1, type=int, help="Height of the image data. Obsolete.")
    # parser.add_argument("--IM_WIDTH", default=11, type=int, help="Width of the image data. Obsolete.")
    # parser.add_argument("--Nchannels", default=1, type=int, help="Number of channels in the image data. Obsolete.")
    # parser.add_argument("--m", default=2, type=int, help="Parameter from equation. Obsolete.")
    # parser.add_argument("--p", default=2, type=int, help="Parameter from equation. Obsolete.")

    # Assign values to memory
    py_prune_input_data = prune_input.prune_input_data
    py_prune_input_data.rgtypes=[c_void_p,c_uint64,c_uint64]

    py_getnum_samples = prune_input.getnum_samples
    py_getnum_samples.rgtypes=[c_void_p,c_uint64,c_uint64]
    py_getnum_samples.restype = c_uint64

    py_compute_offset_phrases = prune_input.compute_offset_phrases
    py_compute_offset_phrases.rgtypes=[c_void_p, c_void_p, c_uint64, c_uint64]

    py_model_create_descriptor = model_descriptor.model_create_descriptor
    py_model_create_descriptor.rgtypes=[c_int32]
    py_model_create_descriptor.restype = ctypes.c_void_p

    py_set_model_param_int = model_descriptor.set_model_param_int
    py_set_model_param_float = model_descriptor.set_model_param_float
    py_compute_model_derived_parameters = model_descriptor.compute_model_derived_parameters

    py_set_model_param_int.rgtypes=[c_uint64,c_char_p,c_void_p]
    py_set_model_param_float.rgtypes=[c_float,c_char_p,c_void_p]

    py_compute_model_derived_parameters.rgtypes=[c_void_p]

    py_copy_model_params = model_descriptor.copy_model_params
    py_copy_model_params.rgtypes=[c_void_p,c_int32,c_int32]

    py_model_create_arrays = model_arrays.model_create_arrays
    py_model_create_arrays.restype = ctypes.c_void_p
    py_model_create_arrays.rgtypes=[c_int32]

    py_model_arrays_allocate_memory = model_arrays.model_arrays_allocate_memory
    py_model_arrays_allocate_memory.rgtypes=[c_void_p,c_void_p,c_int32]

    py_model_arrays_reshuffle_indices = model_arrays.do_reshuffle_indices
    py_model_arrays_reshuffle_indices.rgtypes=[c_void_p,c_void_p,c_int32]

    py_compute_inverse_word_frequency = model_arrays.compute_inverse_word_frequency
    py_compute_inverse_word_frequency.rgtypes=[c_void_p,c_void_p,c_void_p,c_uint64,c_int32]

    python_get_cuda_pinned_memory = cuda_helpers.do_gpu_cudaHostAlloc
    python_get_cuda_pinned_memory.restype = ctypes.c_void_p

    python_get_cuda_managed_memory = cuda_helpers.do_gpu_cudaMallocManaged
    python_get_cuda_managed_memory.restype = ctypes.c_void_p

    py_run_epoch_INPUT_AS_IMAGE = model.launch_epoch_INPUT_AS_IMAGE
    py_run_epoch_INPUT_AS_IMAGE.rgtypes=[c_void_p,c_void_p,c_int32,c_uint64,c_uint64]

    py_run_epoch_INPUT_AS_FLOAT = model.launch_epoch_INPUT_AS_FLOAT
    py_run_epoch_INPUT_AS_FLOAT.rgtypes=[c_void_p,c_void_p,c_int32,c_uint64,c_uint64]

    py_run_epoch_INPUT_AS_INT = model.launch_epoch_INPUT_AS_INT
    py_run_epoch_INPUT_AS_INT.rgtypes=[c_void_p,c_void_p,c_int32,c_uint64,c_uint64]

    py_model_arrays_get_data_pointer = model_arrays.get_data_pointer
    py_model_arrays_get_data_pointer.restype = ctypes.c_void_p
    py_model_arrays_get_data_pointer.rgtypes = [c_char_p, c_void_p, c_int32]

    num_gpus = cuda_helpers.get_cuda_num_devices()
    print('num_gpus=', num_gpus)

    print("py_model_create_arrays and descriptor...")
    MODEL_DSCR = py_model_create_descriptor( c_int32(num_gpus) )
    MODEL_DATA = py_model_create_arrays( c_int32(num_gpus) )

    # Create output directory
    output_dir = Path(output_dir)
    if not output_dir.exists(): output_dir.mkdir(parents=True)
    OUTPUT_NAME=f"{ckpt_prefix}_H_{hid}_W_{W}_LR_{initial_learning_rate}_"
    OUTPUT = str(output_dir / OUTPUT_NAME)

    # Allocate memory for initial encodings
    input_data_on_disk_encoding = numpy.load(encodings_source, mmap_mode='r')
    Ns_1 = input_data_on_disk_encoding.shape[0]

    # Cannot push to GPU
    INPUT = python_get_cuda_pinned_memory(ctypes.c_uint64(Ns_1*ctypes.sizeof(ctypes.c_int32)))

    #create numpy array to store input data, use memory allocated for INPUT
    INPUT_data_pointer = ctypes.cast(INPUT,ctypes.POINTER(ctypes.c_int32))
    INPUT_np_array = numpy.ctypeslib.as_array(INPUT_data_pointer,shape=(Ns_1,))

    print('Copying input...')
    numpy.copyto(INPUT_np_array,input_data_on_disk_encoding)

    vocabulary_size = numpy.uint64(numpy.max(INPUT_np_array)+1)
    N = numpy.uint64(vocabulary_size*2)
    print(f"Determined a vocabulary size of {vocabulary_size}")

    input_data_on_disk_offsets = numpy.load(offsets_source,mmap_mode='r')
    Number_of_sentences = input_data_on_disk_offsets.shape[0] - 1

    Number_of_phrases = prune_input.getnum_samples(ctypes.c_void_p(input_data_on_disk_offsets.__array_interface__['data'][0]),
            c_uint64(Number_of_sentences), c_uint64(W) )
    print('Number of phrases: ', Number_of_phrases)

    #allocate memory for offsets for phrases of size W
    INPUT_phrases_offsets = python_get_cuda_pinned_memory(ctypes.c_uint64((Number_of_phrases+1)*ctypes.sizeof(ctypes.c_int64) ))

    #compute offsets
    py_compute_offset_phrases(c_void_p(INPUT_phrases_offsets),
            c_void_p(input_data_on_disk_offsets.__array_interface__['data'][0]), c_uint64(Number_of_sentences),
            c_uint64(W))

    Ns_1 = Number_of_phrases

    model_descriptor.print_model_params(ctypes.c_void_p(MODEL_DSCR))

    py_set_model_param_int(IM_HEIGHT,b'IM_HEIGHT',ctypes.c_void_p(MODEL_DSCR))

    py_set_model_param_int(IM_WIDTH,b'IM_WIDTH',ctypes.c_void_p(MODEL_DSCR))
    py_set_model_param_int(Nchannels,b'Nchannels',ctypes.c_void_p(MODEL_DSCR))
    py_set_model_param_int(ctypes.c_uint64(Ns_1),b'Ns_1',ctypes.c_void_p(MODEL_DSCR))
    #
    py_set_model_param_int(W, b'W', ctypes.c_void_p(MODEL_DSCR))
    py_set_model_param_int(stride, b'ST', ctypes.c_void_p(MODEL_DSCR))
    py_set_model_param_int(hid, b'hid', ctypes.c_void_p(MODEL_DSCR))
    py_set_model_param_int(batch_size, b'Num', ctypes.c_void_p(MODEL_DSCR))

    py_set_model_param_int(c_uint64(vocabulary_size), b'vocabulary_size',ctypes.c_void_p(MODEL_DSCR))
    py_set_model_param_int(sparse_input, b'sparse_input',ctypes.c_void_p(MODEL_DSCR))
    py_set_model_param_int(frequency_scaling, b'frequency_scaling',ctypes.c_void_p(MODEL_DSCR))

    py_set_model_param_float(c_float(initial_learning_rate), b'initial_learning_rate', ctypes.c_void_p(MODEL_DSCR))
    py_set_model_param_float(c_float(delta), b'delta', ctypes.c_void_p(MODEL_DSCR))
    py_set_model_param_float(c_float(prec), b'prec', ctypes.c_void_p(MODEL_DSCR))
    py_set_model_param_float(c_float(mu), b'mu', ctypes.c_void_p(MODEL_DSCR))
    py_set_model_param_float(c_float(sigma), b'sigma', ctypes.c_void_p(MODEL_DSCR))
    py_set_model_param_float(c_float(Lmid), b'Lmid', ctypes.c_void_p(MODEL_DSCR))
    py_set_model_param_float(c_float(Lbase), b'Lbase', ctypes.c_void_p(MODEL_DSCR))
    py_compute_model_derived_parameters(ctypes.c_void_p(MODEL_DSCR))

    for gpu in range(1,num_gpus,1):
        py_copy_model_params( ctypes.c_void_p(MODEL_DSCR), c_int32(0), c_int32(gpu) )

    model_descriptor.print_model_params(ctypes.c_void_p(MODEL_DSCR))

    py_model_arrays_allocate_memory(ctypes.c_void_p(MODEL_DSCR), ctypes.c_void_p(MODEL_DATA),c_int32(num_gpus))

    print('setting INPUT pointer')
    for gpu in range(0,num_gpus,1):
        model_arrays.set_up_INPUT_pointer(b'INPUT', ctypes.c_void_p(MODEL_DATA), ctypes.c_void_p(MODEL_DSCR), ctypes.c_void_p(INPUT),ctypes.c_void_p(INPUT_phrases_offsets), b'i4',c_int32(gpu))

    if frequency_scaling==1 :
        print('Computing inverse word frequency...')
        for gpu in range(0,num_gpus,1):
            py_compute_inverse_word_frequency(ctypes.c_void_p(MODEL_DATA), ctypes.c_void_p(MODEL_DSCR), ctypes.c_void_p(INPUT), c_uint64(input_data_on_disk_encoding.shape[0]),  c_int32(gpu) )

    #model_arrays.push_INPUT_memory_to_GPU(ctypes.c_void_p(MODEL_DATA), ctypes.c_void_p(MODEL_DSCR),c_int32(0), b'i4')

    #initialize SYNAPSES
    print('Setting initial weights...')
    epoch_start = 0
    if starting_checkpoint is not None:
        R=numpy.load(starting_checkpoint,mmap_mode='r')
        # Assume that the epoch_starting_number is the last value of the checkpoint name
        epoch_start = int(starting_checkpoint.split("_")[-1])
    else:
        R = numpy.float32(numpy.random.normal(mu, sigma, (hid,N)))

    #push the same initial model data into all GPUs
    for gpu in range(0,num_gpus,1):
        SYNAPSES_data_pointer = ctypes.cast(py_model_arrays_get_data_pointer(b'synapses',ctypes.c_void_p(MODEL_DATA),c_int32(gpu)),ctypes.POINTER(ctypes.c_float))
        SYNAPSES_np_array = numpy.ctypeslib.as_array(SYNAPSES_data_pointer,shape=(hid,N))
        numpy.copyto(SYNAPSES_np_array,R)

    SYNAPSES_data_pointer = ctypes.cast(py_model_arrays_get_data_pointer(b'synapses',ctypes.c_void_p(MODEL_DATA),c_int32(0)),ctypes.POINTER(ctypes.c_float))
    SYNAPSES_np_array = numpy.ctypeslib.as_array(SYNAPSES_data_pointer,shape=(hid,N))

    #push INPUT data to the GPU (depending on available memory)
    for i, ep in enumerate(range(epoch_start,Nep)):
        print('epoch ID = ',ep)
        t1 = time.time()
        py_model_arrays_reshuffle_indices(ctypes.c_void_p(MODEL_DSCR), ctypes.c_void_p(MODEL_DATA),c_int32(0))
        t11 = time.time()
        py_run_epoch_INPUT_AS_INT(ctypes.c_void_p(MODEL_DSCR), ctypes.c_void_p(MODEL_DATA), c_int32(num_gpus),
                c_uint64(ep), c_uint64(Nep) )
        t2 = time.time()
        print('time per epoch = ',t2-t1,'[s]', '  py_model_arrays_reshuffle_indices time = ',t11-t1,'[s]')
        if ((i+1) % save_every) == 0 or i == (Nep - 1):
            numpy.save(OUTPUT+str(ep),SYNAPSES_np_array)

max_stars_repo_name: bhoov/flyvec

max_issues_repo_name: bhoov/flyvec

max_forks_repo_name: bhoov/flyvec