text:
import numpy as np
import os
import pandas as pd
from setup.load_paths import load_paths

try:
    print(Location)
except NameError:
    if os.name == "posix": Location ="NUCLUSTER"
    else: Location ="Local"

datapath, wdir, exe_dir, sim_dir = load_paths(Location=Location)

def load_sim_data(exp_name, region_suffix ='_All', input_wdir=None, fname=None,
                  input_sim_output_path =None, column_list=None, add_incidence=True,
                  select_traces=True, traces_to_keep_ratio=4, traces_to_keep_min=100) :
    input_wdir = input_wdir or wdir
    sim_output_path_base = os.path.join(input_wdir, '../simulation_output', exp_name)
    sim_output_path = input_sim_output_path or sim_output_path_base

    if region_suffix is not None:
        ems_nr = region_suffix.replace("_EMS-", "")
        if region_suffix == "_All": ems_nr = 0

        if fname is None:
            fname = f'trajectoriesDat_region_{str(ems_nr)}.csv'
            if os.path.exists(os.path.join(sim_output_path, fname)) == False:
                fname = f'trajectoriesDat_region_{str(ems_nr)}_trim.csv'
            if os.path.exists(os.path.join(sim_output_path, fname)) == False:
                fname = 'trajectoriesDat_trim.csv'
            if os.path.exists(os.path.join(sim_output_path, fname)) == False:
                fname = 'trajectoriesDat.csv'

        print(f'Using {fname}')
        if column_list is not None:
            column_list = list(set(['run_num', 'sample_num', 'scen_num','startdate', 'time'] + column_list))
        df = pd.read_csv(os.path.join(sim_output_path, fname), usecols=column_list)
        reg_cols = [col for col in df.columns if region_suffix in col]
        reg_cols = list(set(reg_cols + [col for col in df.columns if region_suffix.replace('-','_') in col]))
        if region_suffix =='_EMS-1':
            reg_cols = [col for col in reg_cols if not 'EMS-10' in col]
            reg_cols = [col for col in reg_cols if not 'EMS-11' in col]
        #df = df[['run_num', 'sample_num', 'scen_num','startdate', 'time']+reg_cols]
        df.columns = df.columns.str.replace(region_suffix, '')

        """ If trajectoriesDat_region_0 was re-generated from single region runs, it already includes selected traces only"""
        if "_combined" in exp_name and ems_nr == 0:
            select_traces = False

        if select_traces:
            if os.path.exists(os.path.join(sim_output_path, f'traces_ranked_region_{str(ems_nr)}.csv')):
                rank_export_df = pd.read_csv(os.path.join(sim_output_path, f'traces_ranked_region_{str(ems_nr)}.csv'))

                n_traces_to_keep = int(len(rank_export_df) / traces_to_keep_ratio)
                if n_traces_to_keep < traces_to_keep_min :
                    n_traces_to_keep = traces_to_keep_min
                if len(rank_export_df) < traces_to_keep_min :
                    n_traces_to_keep = len(rank_export_df)

                rank_export_df_sub = rank_export_df[0:n_traces_to_keep]
                df = df[df['sample_num'].isin(rank_export_df_sub.sample_num.unique())]
    else :
        fname = 'trajectoriesDat.csv'
        if os.path.exists(os.path.join(sim_output_path, fname)) == False:
            fname = 'trajectoriesDat_trim.csv'
        print(f'Using {fname}')
        df = pd.read_csv(os.path.join(sim_output_path, fname), usecols=column_list)  ## engine='python'

    df = df.dropna()
    first_day = pd.Timestamp(df['startdate'].unique()[0])
    df['date'] = df['time'].apply(lambda x: first_day + pd.Timedelta(int(x),'days'))

    if add_incidence:
        #if 'recovered' in df.columns:
        #    df['infected_cumul'] = df['infected'] + df['recovered'] + df['deaths']
        df = calculate_incidence(df)
    return df

def merge_county_covidregions(df_x, key_x='region', key_y='County', add_pop =True):
    """ Add COVID-19 regions to counties
    to a file that only includes counties. Country names are changes to lowercase before the merge.
    Keeps all rows from df_x and only those that match from df_y (left join).
    """

    df_y = pd.read_csv(os.path.join(datapath, 'covid_IDPH', 'EMS Population','covidregion_county.csv'))
    df_x[key_x] = df_x[key_x] .str.lower()
    df_y[key_y] = df_y[key_y] .str.lower()
    df = pd.merge(how='left', left=df_x, left_on=key_x, right=df_y, right_on=key_y)

    if add_pop:
        del df_y
        df_y = pd.read_csv(os.path.join(datapath, 'covid_IDPH', 'population', 'co-est2019-annres-17.csv'))
        df_y['population'] = df_y['2019_adj'] # adjusted including Chicago as subset from Cook County
        df_y[key_y] = df_y[key_y].str.lower()
        df_y = df_y[[key_y,'population']]
        df = pd.merge(how='left', left=df, left_on=key_y, right=df_y, right_on=key_y)

    return df

def get_latest_LLfiledate(file_path, split_string ='_jg_' , file_pattern='aggregated_covidregion.csv'):

    files= os.listdir(file_path)
    filedates = [x.split(split_string)[0] for x in files if file_pattern in x]
    latest_filedate = max([int(x) for x in filedates])

    return latest_filedate

def get_vents(crit_det_array):

	vent_frac_global = 0.660

	return crit_det_array * vent_frac_global

def count_new(df, curr_ch) :

    ch_list = list(df[curr_ch].values)
    diff = [0] + [ch_list[x] - ch_list[x - 1] for x in range(1, len(df))]
    return diff

def CI_5(x) :

    return np.percentile(x, 5)

def CI_95(x) :

    return np.percentile(x, 95)

def CI_25(x) :

    return np.percentile(x, 25)

def CI_75(x) :

    return np.percentile(x, 75)

def CI_2pt5(x) :

    return np.percentile(x, 2.5)

def CI_97pt5(x) :

    return np.percentile(x, 97.5)

def CI_50(x) :

    return np.percentile(x, 50)

def load_ref_df(ems_nr):
    ref_df_emr = pd.read_csv(os.path.join(datapath, 'covid_IDPH', 'Corona virus reports', 'emresource_by_region.csv'))
    ref_df_emr['suspected_and_confirmed_covid_icu'] = ref_df_emr['suspected_covid_icu'] + ref_df_emr['confirmed_covid_icu']
    data_channel_names_emr = ['confirmed_covid_deaths_prev_24h', 'confirmed_covid_icu', 'covid_non_icu','suspected_covid_icu','suspected_and_confirmed_covid_icu']
    ref_df_emr = ref_df_emr.groupby(['date_of_extract','covid_region'])[data_channel_names_emr].agg(np.sum).reset_index()
    ref_df_emr['date'] = pd.to_datetime(ref_df_emr['date_of_extract'])

    LL_file_date = get_latest_LLfiledate(file_path=os.path.join(datapath, 'covid_IDPH', 'Cleaned Data'))
    ref_df_ll = pd.read_csv(os.path.join(datapath, 'covid_IDPH', 'Cleaned Data', f'{LL_file_date}_jg_aggregated_covidregion.csv'))
    ref_df_ll['date'] = pd.to_datetime(ref_df_ll['date'])

    ref_df_cli = pd.read_csv(os.path.join(datapath, 'covid_IDPH', 'Corona virus reports', 'CLI_admissions.csv'))
    ref_df_cli = merge_county_covidregions(df_x=ref_df_cli, key_x='region', key_y='County', add_pop=False)
    ref_df_cli = ref_df_cli.groupby(['date','covid_region'])['inpatient'].agg(np.sum).reset_index()
    ref_df_cli['date'] = pd.to_datetime(ref_df_cli['date'])

    ref_df_public = pd.read_csv(os.path.join(datapath, 'covid_IDPH', 'Corona virus reports', 'IDPH_public_county.csv'))
    ref_df_public = merge_county_covidregions(df_x=ref_df_public, key_x='county', key_y='County', add_pop=False)
    ref_df_public = ref_df_public.groupby(['test_date','covid_region'])['confirmed_cases'].agg(np.sum).reset_index()
    ref_df_public['test_date'] = pd.to_datetime(ref_df_public['test_date'])
    ref_df_public.rename(columns={"test_date" : "date"}, inplace=True)

    if not isinstance(ems_nr, list):
        if ems_nr > 0:
            ref_df_emr = ref_df_emr[ref_df_emr['covid_region'] == ems_nr]
            ref_df_ll = ref_df_ll[ref_df_ll['covid_region'] == ems_nr]
            ref_df_cli = ref_df_cli[ref_df_cli['covid_region'] == ems_nr]
            ref_df_public = ref_df_public[ref_df_public['covid_region'] == ems_nr]
        if ems_nr == 0:
            ref_df_emr['covid_region'] = 0
            ref_df_ll['covid_region'] = 0
            ref_df_cli['covid_region'] = 0
            ref_df_public['covid_region'] = 0

            ref_df_emr = ref_df_emr.groupby('date').agg(np.sum).reset_index()
            ref_df_ll = ref_df_ll.groupby('date').agg(np.sum).reset_index()
            ref_df_cli = ref_df_cli.groupby('date').agg(np.sum).reset_index()
            ref_df_public = ref_df_public.groupby('date').agg(np.sum).reset_index()

        ref_df_public = ref_df_public.sort_values('date')
        ref_df_public['new_confirmed_cases'] = count_new(ref_df_public, 'confirmed_cases')

    if isinstance(ems_nr, list):
        inc_df = pd.DataFrame()
        for region, df in ref_df_public.groupby('covid_region'):
            df = df.sort_values('date')
            sdf = pd.DataFrame({'date': df['date'], 'new_confirmed_cases': count_new(df, 'confirmed_cases')})
            sdf['covid_region'] = region
            inc_df = pd.concat([inc_df, sdf])
        ref_df_public_ext = pd.merge(left=ref_df_public, right=inc_df, on=['date', 'covid_region'])
        ref_df_public = ref_df_public_ext

    merge_keys = ['date', 'covid_region']
    ref_df = pd.merge(how='outer', left=ref_df_ll,  right=ref_df_emr, on=merge_keys)
    ref_df = pd.merge(how='outer', left=ref_df, right=ref_df_cli, on=merge_keys)
    ref_df = pd.merge(how='outer', left=ref_df, right=ref_df_public, on=merge_keys)

    ref_df = ref_df.sort_values(['covid_region', 'date'])
    ref_df['date'] = pd.to_datetime(ref_df['date'])
    ref_df = ref_df[ref_df['date'].between(pd.Timestamp('2020-01-01'), pd.Timestamp.today())]

    if isinstance(ems_nr, list):
        if 0 in ems_nr:
            ref_df_IL = ref_df.copy()
            ref_df_IL['covid_region'] = 0
            ref_df_IL = ref_df_IL.groupby(['date']).agg(np.nansum).reset_index()
            ref_df = ref_df_IL.append(ref_df)
            #ref_df.covid_region.unique()
        ref_df = ref_df[ref_df['covid_region'].isin(ems_nr)]

    return ref_df

def calculate_prevalence(df):

    df['N'] = df['N'] - df['deaths']
    df['IFR'] = df['deaths'] / (df['recovered'] + df['deaths'])
    df['IFR_t'] = df['new_deaths'] / (df['new_recovered'] + df['new_deaths'])
    df['prevalence'] = df['infected'] / df['N']
    df['seroprevalence_current'] = df['recovered'] / df['N']
    df['seroprevalence'] = df.groupby(['scen_num', 'sample_num'])['seroprevalence_current'].transform('shift', 14)

    if 'infected_det' in df.columns:
        df['N_det'] = df['N'] - df['deaths_det']
        df['IFR_det'] = df['deaths_det'] / (df['recovered_det'] + df['deaths_det'])
        df['IFR_det_t'] = df['new_deaths_det'] / (df['new_recovered_det'] + df['new_deaths_det'])
        df['prevalence_det'] = df['infected_det'] / df['N_det']
        df['seroprevalence_current_det'] = df['recovered_det'] / df['N_det']
        df['seroprevalence_det'] = df.groupby(['scen_num', 'sample_num'])['seroprevalence_current_det'].transform('shift', 14)
    return df

def calculate_incidence(adf, output_filename=None) :

    inc_df = pd.DataFrame()
    channel_cumul = [col for col in adf.columns if 'cumul' in col]
    channel_cumul = channel_cumul + [col for col in adf.columns if 'deaths' in col]
    channel_cumul = channel_cumul + [col for col in adf.columns if 'recovered' in col]
    channel_cumul_new = ['new_'+ col.replace('_cumul','') for col in channel_cumul]

    if 'susceptible' in adf.columns:
        channel_cumul = channel_cumul + ['susceptible']
        channel_cumul_new = channel_cumul_new + ['new_exposures']

    for (run, samp, scen), df in adf.groupby(['run_num','sample_num', 'scen_num']) :

        sdf = pd.DataFrame({'date' : df['date']})
        for i, ch in enumerate(channel_cumul):
            if ch =='susceptible':
                sdf[channel_cumul_new[i]] = [-1 * x for x in count_new(df, 'susceptible')]
            else:
                sdf[channel_cumul_new[i]] = count_new(df, ch)

        sdf['run_num'] = run
        sdf['sample_num'] = samp
        sdf['scen_num'] = scen
        inc_df = pd.concat([inc_df, sdf])

    adf = pd.merge(left=adf, right=inc_df, on=['run_num','sample_num', 'scen_num', 'date'])
    if output_filename :
        adf.to_csv(output_filename, index=False)
    return adf

def load_capacity(ems):
    ### note, names need to match, simulations and capacity data already include outputs for all illinois

    file_path = os.path.join(datapath, 'covid_IDPH', 'Corona virus reports', 'hospital_capacity_thresholds')
    files = os.listdir(file_path)
    files = [name for name in files if not 'extra_thresholds' in name]
    filedates = [item.replace('capacity_weekday_average_', '') for item in files]
    filedates = [item.replace('.csv', '') for item in filedates]
    latest_filedate = max([int(x) for x in filedates])

    fname = 'capacity_weekday_average_' + str(latest_filedate) + '.csv'
    ems_fname = os.path.join(datapath, 'covid_IDPH/Corona virus reports/hospital_capacity_thresholds/', fname)
    df = pd.read_csv(ems_fname)
    df = df.drop_duplicates()

    df = df[df['overflow_threshold_percent'] == 1]
    df['ems'] = df['geography_modeled']
    df['ems'] = df['geography_modeled'].replace("covidregion_", "", regex=True)
    df = df[['ems', 'resource_type', 'avg_resource_available']]
    df = df.drop_duplicates()

    ## if conflicting numbers, take the lower ones!
    dups = df.groupby(["ems", "resource_type"])["avg_resource_available"].nunique()
    if int(dups.nunique()) >1 :
        print(f'{ems_fname} contains multiple capacity values, selecting the lower ones.')
        df= df.loc[df.groupby(["ems", "resource_type"])["avg_resource_available"].idxmax()]

    df = df.pivot(index='ems', columns='resource_type', values='avg_resource_available')
    df.index.name = 'ems'
    df.reset_index(inplace=True)

    if ems == 'illinois' or ems == 0:
        df['ems'] = 'illinois'
        df = df.groupby('ems')[['hb_availforcovid', 'icu_availforcovid', 'vent_availforcovid']].agg(np.sum).reset_index()
    else :
        df = df[df['ems'] == str(ems)]

    capacity = {'hosp_det': int(df['hb_availforcovid']),
                'total_hosp_census': int(df['hb_availforcovid']),
                'crit_det': int(df['icu_availforcovid']),
                'ventilators': int(df['vent_availforcovid'])}
    return capacity

def get_datacomparison_channels():
    outcome_channels = ['hosp_det_cumul', 'hosp_cumul', 'hosp_det', 'hospitalized',
                    'crit_det_cumul', 'crit_cumul', 'crit_det', 'critical',
                    'deaths_det_cumul', 'deaths']
    channels = ['new_deaths_det', 'crit_det', 'hosp_det', 'new_hosp_det', 'new_hosp_det']
    data_channel_names = ['deaths','confirmed_covid_icu', 'covid_non_icu', 'inpatient', 'admissions']
    titles = ['New Detected\nDeaths (LL)', 'Critical Detected (EMR)', 'Inpatient non-ICU\nCensus (EMR)',
              'Covid-like illness\nadmissions (IDPH)', 'New Detected\nHospitalizations (LL)']
    return outcome_channels, channels, data_channel_names, titles

def get_parameter_names(include_new=True):

    sample_params_core = ['time_to_infectious',
                          'time_to_symptoms',
                          'time_to_hospitalization',
                          'time_to_critical',
                          'time_to_death',
                          'time_to_detection',
                          'time_to_detection_As',
                          'time_to_detection_Sym',
                          'time_to_detection_Sys',
                          'recovery_time_asymp',
                          'recovery_time_mild',
                          'recovery_time_hosp',
                          'recovery_time_crit',
                          'fraction_symptomatic',
                          'fraction_severe',
                          'fraction_critical',
                          'reduced_inf_of_det_cases',
                          'd_Sys',
                          'd_As',
                          'd_P']

    IL_specific_param = ['d_Sys_incr1',
                         'd_Sys_incr2',
                         'd_Sys_incr3',
                         'd_Sys_incr4',
                         'd_Sys_incr5',
                         'd_Sys_incr6',
                         'd_Sys_incr7',
                         'd_Sys_incr8',
                         'd_Sym_change1',
                         'd_Sym_change2',
                         'd_Sym_change3',
                         'd_Sym_change4',
                         'd_Sym_change5',
                         'd_Sym_change6',
                         'd_Sym_change7',
                         'd_Sym_change8',
                         'fraction_dead_change1',
                         'fraction_dead_change2',
                         'fraction_dead_change3',
                         'fraction_dead_change4',
                         'fraction_dead_change5',
                         'fraction_dead_change6',
                         'fraction_dead_change7',
                         'fraction_dead_change8',
                         'fraction_dead_change9',
                         'fraction_critical_change1',
                         'fraction_critical_change2',
                         'fraction_critical_change3',
                         'fraction_critical_change4',
                         'fraction_critical_change5',
                         'fraction_critical_change6',
                         'fraction_critical_change7',
                         'fraction_critical_change8',
                         'fraction_critical_change9',
                         'fraction_critical_change10']

    IL_locale_param_stem = ['ki_multiplier_3a','ki_multiplier_3b','ki_multiplier_3c',
                            'ki_multiplier_4','ki_multiplier_5','ki_multiplier_6',
                            'ki_multiplier_7','ki_multiplier_8','ki_multiplier_9',
                            'ki_multiplier_10','ki_multiplier_11','ki_multiplier_12',
                            'ki_multiplier_13','Ki','time_infection_import']

    sample_params = sample_params_core + IL_specific_param

    if include_new:
        sample_params_new = ['reduced_infectious_As', 'time_to_loose_immunity', 'fraction_lost_immunity']
        sample_params = sample_params.append(sample_params_new)

    return sample_params, sample_params_core, IL_specific_param, IL_locale_param_stem

def get_grp_list(exp_name,  input_wdir=None, input_sim_output_path =None):
    """Note, the N parameter in the sampled_parameters might also include regions that were not run,
     when specyfing fewer regions in the emodl file (i.e. in testing)"""
    input_wdir = input_wdir or wdir
    sim_output_path_base = os.path.join(input_wdir, '../simulation_output', exp_name)
    sim_output_path = input_sim_output_path or sim_output_path_base

    df_samples = pd.read_csv(os.path.join(sim_output_path, 'sampled_parameters.csv'))
    N_cols = [col for col in df_samples.columns if 'N_' in col]
    if len(N_cols) != 0:
        grp_list = [col.replace('N_', '') for col in N_cols]
    else:
        grp_list = None
    return grp_list

def get_group_names(exp_path, uniquechannel ='Ki_t', fname="trajectoriesDat.csv"):
    """Similar to get_grp_list, but uses trajectoriesDat column names"""

    if "_combined" in exp_path:
        pattern = 'trajectoriesDat_region'
        grp_suffix = 'EMS'
        files = os.listdir(exp_path)
        trajectories = [x.replace(f'{pattern}_','') for x in files if pattern in x]
        grp_numbers = [int(x.replace('.csv','')) for x in trajectories]
        grp_numbers.sort()

        grp_list =  [f'{grp_suffix}-{str(x)}' for x in grp_numbers if x > 0]
        if 0 in grp_numbers:
            grp_list =  grp_list + ['All']

    else:
        trajectories_cols = pd.read_csv(os.path.join(exp_path, fname), index_col=0,
                                        nrows=0).columns.tolist()
        cols = [col for col in trajectories_cols if uniquechannel in col]
        if len(cols) != 0:
            grp_list = [col.replace(f'{uniquechannel}_', '') for col in cols]
            grp_suffix = grp_list[0][:3]
            grp_numbers =  [int(grp.replace('EMS-', '')) for grp in grp_list]
            if len(cols) > 1:
                grp_list = grp_list + ['All']
                grp_numbers = grp_numbers + [0]
        else:
            grp_list = None
            grp_suffix=None
            grp_numbers = None
    return grp_list, grp_suffix, grp_numbers


text:
import numpy as np
from pandas import DataFrame, Series, Index, DatetimeIndex, Timestamp

class Constructors(object):

    goal_time = 0.2

    def setup(self):
        N = 10**2
        np.random.seed(1234)
        self.arr = np.random.randn(N, N)
        self.arr_str = np.array(['foo', 'bar', 'baz'], dtype=object)

        self.data = np.random.randn(N)
        self.index = Index(np.arange(N))

        self.s = Series([Timestamp('20110101'), Timestamp('20120101'),
                         Timestamp('20130101')] * N * 10)

    def time_frame_from_ndarray(self):
        DataFrame(self.arr)

    def time_series_from_ndarray(self):
        Series(self.data, index=self.index)

    def time_index_from_array_string(self):
        Index(self.arr_str)

    def time_dtindex_from_series(self):
        DatetimeIndex(self.s)

    def time_dtindex_from_index_with_series(self):
        Index(self.s)

ext: py

max_stars_repo_name: mwaskom/pandas

max_issues_repo_name: mwaskom/pandas

max_forks_repo_name: mwaskom/pandas

reason: import numpy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Jan 26 16:23:44 2019

@author: Macbook
"""

#-*- coding: utf-8 -*-
from __future__ import print_function
from math import *
import pandas as pd
# from apriori import *  # 导入自行编写的apriori函数
import time  # 导入时间库用来计算用时'
import numpy as np
inputfile = '3.xlsx'  # 输入事务集文件
data = pd.read_excel(inputfile, dtype=object)

import numpy as np
import pandas as pd
from sklearn.datasets import load_iris
from sklearn.model_selection import  train_test_split
import matplotlib.pyplot as plt
# input Lat_A 纬度A
# input Lng_A 经度A
# input Lat_B 纬度B
# input Lng_B 经度B
# output distance 距离(km)
def calcDistance(Lat_A, Lng_A, Lat_B, Lng_B):
    ra = 6378.140  # 赤道半径 (km)
    rb = 6356.755  # 极半径 (km)
#     print(Lat_A, Lng_A, Lat_B, Lng_B)
    flatten = (ra - rb) / ra  # 地球扁率
    rad_lat_A = radians(Lat_A)
    rad_lng_A = radians(Lng_A)
    rad_lat_B = radians(Lat_B)
    rad_lng_B = radians(Lng_B)
    pA = atan(rb / ra * tan(rad_lat_A))
    pB = atan(rb / ra * tan(rad_lat_B))
    xx = acos(sin(pA) * sin(pB) + cos(pA) * cos(pB) * cos(rad_lng_A - rad_lng_B))
    c1 = (sin(xx) - xx) * (sin(pA) + sin(pB)) ** 2 / cos(xx / 2) ** 2
#     print(xx,pA,pB)
    c2 = (sin(xx) + xx) * (sin(pA) - sin(pB)) ** 2 / sin(xx / 2) ** 2
    dr = flatten / 8 * (c1 - c2)
    distance = ra * (xx + dr)
    return distance
# data=pd.DataFrame(data)
print(data.values)
jl=np.zeros([461,461])
print(jl.shape)
print(data.iat[1,2])
i=np.linespace(0:460)
FIPS_id=zip(i,)
for i in range(461):
    for j in range(461):
        if data.iat[i,1]!=data.iat[j,1] and data.iat[i,2]!=data.iat[j,2]:
            jl[i][j]=calcDistance(data.iat[i,1],data.iat[i,2],data.iat[j,1],data.iat[j,2])


text:
import numpy as np
import torch
from torch import nn
from torch.nn.utils import weight_norm

from ..layers.wavegrad import DBlock, FiLM, UBlock, Conv1d

class Wavegrad(nn.Module):
    # pylint: disable=dangerous-default-value
    def __init__(self,
                 in_channels=80,
                 out_channels=1,
                 use_weight_norm=False,
                 y_conv_channels=32,
                 x_conv_channels=768,
                 dblock_out_channels=[128, 128, 256, 512],
                 ublock_out_channels=[512, 512, 256, 128, 128],
                 upsample_factors=[5, 5, 3, 2, 2],
                 upsample_dilations=[[1, 2, 1, 2], [1, 2, 1, 2], [1, 2, 4, 8],
                                     [1, 2, 4, 8], [1, 2, 4, 8]]):
        super().__init__()

        self.use_weight_norm = use_weight_norm
        self.hop_len = np.prod(upsample_factors)
        self.noise_level = None
        self.num_steps = None
        self.beta = None
        self.alpha = None
        self.alpha_hat = None
        self.noise_level = None
        self.c1 = None
        self.c2 = None
        self.sigma = None

        # dblocks
        self.y_conv = Conv1d(1, y_conv_channels, 5, padding=2)
        self.dblocks = nn.ModuleList([])
        ic = y_conv_channels
        for oc, df in zip(dblock_out_channels, reversed(upsample_factors)):
            self.dblocks.append(DBlock(ic, oc, df))
            ic = oc

        # film
        self.film = nn.ModuleList([])
        ic = y_conv_channels
        for oc in reversed(ublock_out_channels):
            self.film.append(FiLM(ic, oc))
            ic = oc

        # ublocks
        self.ublocks = nn.ModuleList([])
        ic = x_conv_channels
        for oc, uf, ud in zip(ublock_out_channels, upsample_factors, upsample_dilations):
            self.ublocks.append(UBlock(ic, oc, uf, ud))
            ic = oc

        self.x_conv = Conv1d(in_channels, x_conv_channels, 3, padding=1)
        self.out_conv = Conv1d(oc, out_channels, 3, padding=1)

        if use_weight_norm:
            self.apply_weight_norm()

    def forward(self, x, spectrogram, noise_scale):
        shift_and_scale = []

        x = self.y_conv(x)
        shift_and_scale.append(self.film[0](x, noise_scale))

        for film, layer in zip(self.film[1:], self.dblocks):
            x = layer(x)
            shift_and_scale.append(film(x, noise_scale))

        x = self.x_conv(spectrogram)
        for layer, (film_shift, film_scale) in zip(self.ublocks,
                                                   reversed(shift_and_scale)):
            x = layer(x, film_shift, film_scale)
        x = self.out_conv(x)
        return x

    def load_noise_schedule(self, path):
        beta = np.load(path, allow_pickle=True).item()['beta']
        self.compute_noise_level(beta)

    @torch.no_grad()
    def inference(self, x, y_n=None):
        """ x: B x D X T """
        if y_n is None:
            y_n = torch.randn(x.shape[0], 1, self.hop_len * x.shape[-1], dtype=torch.float32).to(x)
        else:
            y_n = torch.FloatTensor(y_n).unsqueeze(0).unsqueeze(0).to(x)
        sqrt_alpha_hat = self.noise_level.to(x)
        for n in range(len(self.alpha) - 1, -1, -1):
            y_n = self.c1[n] * (y_n -
                        self.c2[n] * self.forward(y_n, x, sqrt_alpha_hat[n].repeat(x.shape[0])))
            if n > 0:
                z = torch.randn_like(y_n)
                y_n += self.sigma[n - 1] * z
            y_n.clamp_(-1.0, 1.0)
        return y_n

    def compute_y_n(self, y_0):
        """Compute noisy audio based on noise schedule"""
        self.noise_level = self.noise_level.to(y_0)
        if len(y_0.shape) == 3:
            y_0 = y_0.squeeze(1)
        s = torch.randint(0, self.num_steps - 1, [y_0.shape[0]])
        l_a, l_b = self.noise_level[s], self.noise_level[s+1]
        noise_scale = l_a + torch.rand(y_0.shape[0]).to(y_0) * (l_b - l_a)
        noise_scale = noise_scale.unsqueeze(1)
        noise = torch.randn_like(y_0)
        noisy_audio = noise_scale * y_0 + (1.0 - noise_scale**2)**0.5 * noise
        return noise.unsqueeze(1), noisy_audio.unsqueeze(1), noise_scale[:, 0]

    def compute_noise_level(self, beta):
        """Compute noise schedule parameters"""
        self.num_steps = len(beta)
        alpha = 1 - beta
        alpha_hat = np.cumprod(alpha)
        noise_level = np.concatenate([[1.0], alpha_hat ** 0.5], axis=0)
        noise_level = alpha_hat ** 0.5

        # pylint: disable=not-callable
        self.beta = torch.tensor(beta.astype(np.float32))
        self.alpha = torch.tensor(alpha.astype(np.float32))
        self.alpha_hat = torch.tensor(alpha_hat.astype(np.float32))
        self.noise_level = torch.tensor(noise_level.astype(np.float32))

        self.c1 = 1 / self.alpha**0.5
        self.c2 = (1 - self.alpha) / (1 - self.alpha_hat)**0.5
        self.sigma = ((1.0 - self.alpha_hat[:-1]) / (1.0 - self.alpha_hat[1:]) * self.beta[1:])**0.5

    def remove_weight_norm(self):
        for _, layer in enumerate(self.dblocks):
            if len(layer.state_dict()) != 0:
                try:
                    nn.utils.remove_weight_norm(layer)
                except ValueError:
                    layer.remove_weight_norm()

        for _, layer in enumerate(self.film):
            if len(layer.state_dict()) != 0:
                try:
                    nn.utils.remove_weight_norm(layer)
                except ValueError:
                    layer.remove_weight_norm()

        for _, layer in enumerate(self.ublocks):
            if len(layer.state_dict()) != 0:
                try:
                    nn.utils.remove_weight_norm(layer)
                except ValueError:
                    layer.remove_weight_norm()

        nn.utils.remove_weight_norm(self.x_conv)
        nn.utils.remove_weight_norm(self.out_conv)
        nn.utils.remove_weight_norm(self.y_conv)

    def apply_weight_norm(self):
        for _, layer in enumerate(self.dblocks):
            if len(layer.state_dict()) != 0:
                layer.apply_weight_norm()

        for _, layer in enumerate(self.film):
            if len(layer.state_dict()) != 0:
                layer.apply_weight_norm()

        for _, layer in enumerate(self.ublocks):
            if len(layer.state_dict()) != 0:
                layer.apply_weight_norm()

        self.x_conv = weight_norm(self.x_conv)
        self.out_conv = weight_norm(self.out_conv)
        self.y_conv = weight_norm(self.y_conv)

    def load_checkpoint(self, config, checkpoint_path, eval=False):  # pylint: disable=unused-argument, redefined-builtin
        state = torch.load(checkpoint_path, map_location=torch.device('cpu'))
        self.load_state_dict(state['model'])
        if eval:
            self.eval()
            assert not self.training
            if self.use_weight_norm:
                self.remove_weight_norm()
            betas = np.linspace(config['test_noise_schedule']['min_val'],
                                config['test_noise_schedule']['max_val'],
                                config['test_noise_schedule']['num_steps'])
            self.compute_noise_level(betas)
        else:
            betas = np.linspace(config['train_noise_schedule']['min_val'],
                                config['train_noise_schedule']['max_val'],
                                config['train_noise_schedule']['num_steps'])
            self.compute_noise_level(betas)

max_stars_repo_name: mightmay/Mien-TTS

max_issues_repo_name: mightmay/Mien-TTS

max_forks_repo_name: mightmay/Mien-TTS


text:
import os
import numpy as np
from PIL import Image
from gym_pcgrl.envs.probs.problem import Problem
from gym_pcgrl.envs.helper import get_range_reward, get_tile_locations, calc_num_regions, calc_longest_path

"""
Generate a fully connected top down layout where the longest path is greater than a certain threshold
"""
class BinaryProblem(Problem):
    """
    The constructor is responsible of initializing all the game parameters
    """
    def __init__(self):
        super().__init__()
        self._width = 16
        self._height = 16
        self._prob = {"empty": 0.5, "solid":0.5}
        self._border_tile = "solid"

        self._target_path = 20
        self._random_probs = True

        self._rewards = {
            "regions": 5,
            "path-length": 1
        }

        self._max_path_length = np.ceil(self._width / 2) * (self._height) + np.floor(self._height/2)
        self.render_path = False
        self.path_coords = []
        self.path_length = None

    """
    Get a list of all the different tile names

    Returns:`
        string[]: that contains all the tile names
    """
    def get_tile_types(self):
        return ["empty", "solid"]

    """
    Adjust the parameters for the current problem

    Parameters:
        width (int): change the width of the problem level
        height (int): change the height of the problem level
        probs (dict(string, float)): change the probability of each tile
        intiialization, the names are "empty", "solid"
        target_path (int): the current path length that the episode turn when it reaches
        rewards (dict(string,float)): the weights of each reward change between the new_stats and old_stats
    """
    def adjust_param(self, **kwargs):
        self.render_path = kwargs.get('render', self.render_path) or kwargs.get('render_path', self.render_path)
        super().adjust_param(**kwargs)

        self._target_path = kwargs.get('target_path', self._target_path)
        self._random_probs = kwargs.get('random_probs', self._random_probs)

        rewards = kwargs.get('rewards')
        if rewards is not None:
            for t in rewards:
                if t in self._rewards:
                    self._rewards[t] = rewards[t]

    """
    Resets the problem to the initial state and save the start_stats from the starting map.
    Also, it can be used to change values between different environment resets

    Parameters:
        start_stats (dict(string,any)): the first stats of the map
    """
    def reset(self, start_stats):
        super().reset(start_stats)
        if self._random_probs:
            self._prob["empty"] = self._random.random()
            self._prob["solid"] = 1 - self._prob["empty"]

    """
    Get the current stats of the map

    Returns:
        dict(string,any): stats of the current map to be used in the reward, episode_over, debug_info calculations.
        The used status are "reigons": number of connected empty tiles, "path-length": the longest path across the map
    """
    def get_stats(self, map, lenient_paths=False):
        map_locations = get_tile_locations(map, self.get_tile_types())
        self.path_length, self.path_coords = calc_longest_path(map, map_locations, ["empty"], get_path=self.render_path)
        return {
            "regions": calc_num_regions(map, map_locations, ["empty"]),
            "path-length": self.path_length,
        }

    """
    Get the current game reward between two stats

    Parameters:
        new_stats (dict(string,any)): the new stats after taking an action
        old_stats (dict(string,any)): the old stats before taking an action

    Returns:
        float: the current reward due to the change between the old map stats and the new map stats
    """
    def get_reward(self, new_stats, old_stats):
        #longer path is rewarded and less number of regions is rewarded
        rewards = {
            "regions": get_range_reward(new_stats["regions"], old_stats["regions"], 1, 1),
            "path-length": get_range_reward(new_stats["path-length"],old_stats["path-length"], 125, 125)
        }
        #calculate the total reward
        return rewards["regions"] * self._rewards["regions"] +\
            rewards["path-length"] * self._rewards["path-length"]

    """
    Uses the stats to check if the problem ended (episode_over) which means reached
    a satisfying quality based on the stats

    Parameters:
        new_stats (dict(string,any)): the new stats after taking an action
        old_stats (dict(string,any)): the old stats before taking an action

    Returns:
        boolean: True if the level reached satisfying quality based on the stats and False otherwise
    """
    def get_episode_over(self, new_stats, old_stats):
#       return new_stats["regions"] == 1 and new_stats["path-length"] - self._start_stats["path-length"] >= self._target_path
        return new_stats["regions"] == 1 and new_stats["path-length"] == self._max_path_length

    """
    Get any debug information need to be printed

    Parameters:
        new_stats (dict(string,any)): the new stats after taking an action
        old_stats (dict(string,any)): the old stats before taking an action

    Returns:
        dict(any,any): is a debug information that can be used to debug what is
        happening in the problem
    """
    def get_debug_info(self, new_stats, old_stats):
        return {
            "regions": new_stats["regions"],
            "path-length": new_stats["path-length"],
            "path-imp": new_stats["path-length"] - self._start_stats["path-length"]
        }

    """
    Get an image on how the map will look like for a specific map

    Parameters:
        map (string[][]): the current game map

    Returns:
        Image: a pillow image on how the map will look like using the binary graphics
    """
    def render(self, map):
        if self._graphics == None:
            if self.GVGAI_SPRITES:
                self._graphics = {
                    "empty": Image.open(os.path.dirname(__file__) + "/sprites/oryx/floor3.png").convert('RGBA'),
                    "solid": Image.open(os.path.dirname(__file__) + "/sprites/oryx/wall3.png").convert('RGBA'),
                    "path" : Image.open(os.path.dirname(__file__) + "/sprites/newset/snowmanchest.png").convert('RGBA'),
                }
            else:
                self._graphics = {
                    "empty": Image.open(os.path.dirname(__file__) + "/binary/empty.png").convert('RGBA'),
                    "solid": Image.open(os.path.dirname(__file__) + "/binary/solid.png").convert('RGBA'),
                    "path" : Image.open(os.path.dirname(__file__) + "/binary/path_g.png").convert('RGBA'),
                }
        return super().render(map, render_path=self.path_coords)

max_stars_repo_name: JiangZehua/control-pcgrl

max_issues_repo_name: JiangZehua/control-pcgrl

max_forks_repo_name: JiangZehua/control-pcgrl


text:
import os
import cv2
import numpy as np
from tqdm import tqdm
import json

class Augumentor():
    def __init__(self, img_root, ld_root, save_dir, rot=False, flip=False, gray=False):
        self.images_root_path = img_root
        self.ld_root_path = ld_root
        self.landmarks_file = open(os.path.join(self.ld_root_path, 'train.txt'), 'r')
        self.rot = rot
        self.flip = flip
        self.gray = gray
        self.save_dir = save_dir

    def transfer(self):
        file_names = self.landmarks_file.readlines()
        for i in tqdm(range(len(file_names))):
            line = file_names[i].strip()

            name = line.split('/')[0]
            img_name = name + '.jpg'
            img_path = os.path.join(self.images_root_path, img_name)
            landmark_name = name + '.json'
            landmarks_path = os.path.join(self.ld_root_path, landmark_name)

            if self.gray:
                img = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE)
            else:
                img = cv2.imread(img_path, cv2.IMREAD_COLOR)
            mark = np.array(json.load(open(landmarks_path, 'r')))

            if self.gray:
                gray_img = self.ToGary(img)
                save_name = self.save_dir + "/" + name + "_gray"
                print(save_name)
                cv2.imwrite(save_name + ".jpg", gray_img)
                points_to_save = mark.flatten()
                with open(save_name + ".json", mode='w') as file:
                    json.dump(list(points_to_save), file)
            if self.rot:
                for angle in range(-30, 31, 5):
                    rot_img, rot_mark = self.Rotate(img, mark, angle)
                    save_name = self.save_dir + "/" + name + "_rot" + str(angle)
                    print(save_name)
                    cv2.imwrite(save_name + ".jpg", rot_img)
                    points_to_save = rot_mark.flatten()
                    with open(save_name + ".json", mode='w') as file:
                        json.dump(list(points_to_save), file)

            if self.flip:
                flip_img, flip_mark = self.HorizontalFlip(img, mark)
                save_name = self.save_dir + "/" + name + "_flip"
                print(save_name)
                cv2.imwrite(save_name + ".jpg" ,flip_img)
                points_to_save = flip_mark.flatten()
                with open(save_name + ".json", mode='w') as file:
                    json.dump(list(points_to_save), file)

    def Rotate(self, image, landmarks, angle):

        h, w = image.shape[:2]
        rot_mat = cv2.getRotationMatrix2D((w*0.5, h*0.5), angle, 1.)
        image = cv2.warpAffine(image, rot_mat, (w, h), flags=cv2.INTER_LANCZOS4)
        rot_mat_l = cv2.getRotationMatrix2D((0.5, 0.5), angle, 1.)
        landmarks = cv2.transform(landmarks.reshape(1, 68, 2), rot_mat_l).reshape(68, 2)
        return image, landmarks

    def HorizontalFlip(self, image, landmarks):

        image = cv2.flip(image, 1)
        landmarks = landmarks.reshape(68, 2)
        landmarks[:, 0] = 1. - landmarks[:, 0]
        return image, landmarks

    def ToGary(self, image):

        image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)

        return image

    def draw_landmark_point(self, image, points):
        h, w = image.shape[:2]
        pts = np.reshape(points, (-1, 2))
        pts = pts * h
        for point in pts:
            cv2.circle(image, (int(point[0]), int(
                point[1])), 3, (0, 255, 0), -1, cv2.LINE_AA)

def test():
    num = [72, 73, 90, 91]
    path = "/home/share/landmark/image/helentrainset-22864555_1_flip.jpg"
    mark = "/home/share/landmark/mark/helentrainset-22864555_1_flip.json"
    img = cv2.imread(path, cv2.IMREAD_COLOR)
    h = img.shape[0]
    print(img.shape)
    mark = np.array(json.load(open(mark, 'r')))
    # pts = np.reshape(mark, (-1, 2))
    # pts = [mark[i] for i in num]
    mark = [mark[i] for i in num]
    pts = np.reshape(np.array(mark), (-1, 2))
    pts = pts * h
    for point in pts:
        cv2.circle(img, (int(point[0]), int(
        point[1])), 3, (0, 255, 0), -1, cv2.LINE_AA)
    cv2.imshow("img", img)
    cv2.waitKey()
def main():
    img_root = "/home/share/landmark/image"
    ld_root = "/home/share/landmark/mark"
    save_dir = "/home/share/landmark/aug"
    aug = Augumentor(img_root, ld_root, save_dir, rot=True)
    aug.transfer()

if __name__ == "__main__":
    test()


text:
# -*- coding: utf-8 -*-
"""
Created on Fri Aug 10 13:44:31 2018

@author: zyv57124
"""

import numpy
import PIL
from PIL import Image

flowers = numpy.asarray(Image.open('flowers.png'))

background = Image.open('flowers.png')
#im.save('./blah2.png')
im.show()

max_stars_repo_name: vais-ral/CCPi-ML

max_issues_repo_name: vais-ral/CCPi-ML

max_forks_repo_name: vais-ral/CCPi-ML


text:
# Exploring the latent space with PCA
import numpy as np
import torch
import matplotlib.pyplot as plt
import matplotlib.patches as patches

from data_load import get_data
from eval_skrnn import load_pretrained_congen, draw_image
from model import skrnn_sample

from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from matplotlib.path import Path

# Code inspired from http://louistiao.me/notes/exploring-the-google-quickdraw-dataset-with-sketchrnn-part-3/

class SketchPath(Path):
    def __init__(self, data, factor=.2, *args, **kwargs):
        print(data.shape)
        vertices = np.cumsum(data[::,:-1], axis=0) / factor
        print(vertices.shape)
        codes = np.roll(self.to_code(data[::,-1].astype(int)),
                        shift=1)
        codes[0] = Path.MOVETO

        super(SketchPath, self).__init__(vertices, codes, *args, **kwargs)

    @staticmethod
    def to_code(cmd):
        # if cmd == 0, the code is LINETO
        # if cmd == 1, the code is MOVETO (which is LINETO - 1)
        return Path.LINETO - cmd

def PCA_plot(data_type, n_points, bi_mode):
    encoder, decoder, hid_dim, latent_dim, t_step, cond_gen, mode, device = load_pretrained_congen(data_type)
    data_enc, _ , _ = get_data(data_type=data_type, part="test")
    # Compute the representations of random images in the test set
    data_points_indices = np.random.choice(range(len(data_enc)),n_points,replace=False)
    data_enc=np.array(data_enc)
    data_points = data_enc[data_points_indices]
    sketches = data_points[:,:,[0,1,3]]
    z = torch.zeros((n_points,1,latent_dim), device=device)
    hidden_dec = [(torch.zeros((1,latent_dim),device=device),torch.zeros((1,latent_dim),device=device)) for i in range(n_points)]
    for i in range(n_points):
      hidden_enc = (torch.zeros(bi_mode, 1, hid_dim, device=device), torch.zeros(bi_mode, 1, hid_dim, device=device))
      z[i], hidden_dec[i], _ , _ = encoder(torch.tensor(data_points[i],device=device,dtype=torch.float).unsqueeze(0),hidden_enc)
    # Perform PCA and T-SNE on the resulting z vectors
    z = z.squeeze().detach().cpu().numpy()
    pca = PCA(n_components=2)
    pca.fit(z)
    z_pca = pca.transform(z)
    fig, ax = plt.subplots(figsize=(10, 10))
    ((pc1_min, pc2_min), (pc1_max, pc2_max)) = np.percentile(z_pca, q=[5, 95], axis=0)
    ax.set_xlim(pc1_min, pc1_max)
    ax.set_ylim(pc2_min, pc2_max)

    for i, sketch in enumerate(sketches):
        sketch_path = SketchPath(sketch, factor=7e+1)
        sketch_path.vertices[::,1] *= -1
        sketch_path.vertices += z_pca[i]
        patch = patches.PathPatch(sketch_path, facecolor='none')
        ax.add_patch(patch)

    ax.set_xlabel('$pc_1$')
    ax.set_ylabel('$pc_2$')
    plt.show()

PCA_plot("cat",100,2)


text:
import numpy as np
import pandas as pd
import xarray as xr
import glob, os, io
import gzip
import boto3, s3fs
from boto3 import client as boto_client
from datetime import date, time, datetime, timedelta

to_rad = np.pi / 180.0
to_deg = 180.0 / np.pi

def regionrad(region):
    return [r * to_rad for r in region]

def sec2Z(t):
    return "{}Z".format(datetime.utcfromtimestamp(t).isoformat())

def DateTime(Sec):
    return (Sec.astype('timedelta64[s]') + np.datetime64('1970-01-01'))

def mkfolder(folder):
    if (not os.path.exists(folder)):
        try:
            os.makedirs(folder)
            print('Success to create folder %s' % folder)
        except OSError:
            print('Failed to create folder %s' % folder)
            quit()
    else:
        print('%s already exists' % folder)

def add24hr(hr):
    """Correction of time in CRS for going over the next day in UTC"""
    b = np.where(hr < hr[0])
    hr[b] = hr[b] + 24
    return hr

def CRSaccess(fname, s3bucket=False, Verb=False):
    """
    Access the CRS file
    Return CRS filename with path (absolute path) for "local" access
    Return CRS data as object for "cloud access"
    Either way, the return value can be open by Xarray as netcdf file object
    """

    print("\%% Accessing data from Cloud. This may take a little time...\n")
    s3 = boto_client('s3', region_name=os.environ['AWS_REGION'])
    fileobj = s3.get_object(Bucket=s3bucket, Key=fname)
    fileCRS = fileobj['Body'].read()

    return fileCRS

def get_CRS(fdate, s3bucket):
    """ Get CRS data
    call the following functions:
     CRSaccess()
     add24hr()
    """
    fname = 'fieldcampaign/goesrplt/CRS/data/GOESR_CRS_L1B_' + fdate.replace('-', '') + '_v0.nc'
    fileCRS = CRSaccess(fname, s3bucket=s3bucket)
    with xr.open_dataset(fileCRS, decode_cf=False) as ds:
        CRSlat = ds['lat'].values
        CRSlon = ds['lon'].values
        hr = add24hr(ds['time'].values)
        Time = (hr * 3600).astype('timedelta64[s]') + np.datetime64(fdate)
        CRStime = (Time - np.datetime64('1970-01-01')).astype('timedelta64[s]').astype(np.int64)
    return CRSlat, CRSlon, CRStime, Time

def S3list(s3bucket, fdate, instrm, network='OKLMA'):
    """
    get list of files in a s3 bucket for a specific fdate and instrument (prefix)
    fdate: e.g. '2017-05-17'
    instrm: e.g. 'GLM'
    """
    prefix = {'GLM': 'fieldcampaign/goesrplt/GLM/data/L2/' + fdate + '/OR_GLM-L2-LCFA_G16',
              'LIS': 'fieldcampaign/goesrplt/ISS_LIS/data/' + fdate + '/ISS_LIS_SC_V1.0_',
              # 'FEGS': 'fieldcampaign/goesrplt/FEGS/data/goesr_plt_FEGS_' + fdate.replace('-', '') + '_Flash',
              'CRS': 'fieldcampaign/goesrplt/CRS/data/GOESR_CRS_L1B_' + fdate.replace('-', ''),
              'NAV': 'fieldcampaign/goesrplt/NAV_ER2/data/goesrplt_naver2_IWG1_' + fdate.replace('-', ''),
              'LMA': 'fieldcampaign/goesrplt/LMA/' + network + '/data/' + fdate + '/goesr_plt_' + network + '_' + fdate.replace(
                  '-', '')}

    print("S3list searching for ", prefix[instrm])

    s3 = boto3.resource('s3')
    bucket = s3.Bucket(s3bucket)
    keys = []
    for obj in bucket.objects.filter(Prefix=prefix[instrm]):
        keys.append(obj.key)
    return keys

def s3FileObj(s3bucket, fname, verb=False):
    """
    Return S3 file object to be accessed using xarray or hdf5/netcdf4/txt/csv.
    """
    if (verb): print(f"\%% Accessing {fname.split('/')[-1]} from Cloud...")

    file = s3bucket + '/' + fname
    fs = s3fs.S3FileSystem()  # (anon=True)--> access public buckets
    fileObj = fs.open(file)

    return fileObj

def LMAfiles(bucket, fdate, tstart, Trange, network='OKLMA'):
    """
    get LMA filename list within Trange [sec] starting from tstart on fdate
    Note that LMA files are every 10min/1hr
    """
    filesLMA = []

    files = S3list(bucket, fdate, 'LMA', network=network)
    print('No. of LMA files for ', fdate, len(files))
    if (len(files) == 0):
        return filesLMA

    for t in range(0, Trange, 600):  # 1):
        ss = (tstart + timedelta(seconds=t)).strftime("%Y%m%d_%H%M%S")
        if (network == 'NALMA'): ss = (tstart + timedelta(seconds=t)).strftime("%Y%m%d_%H")
        for file in files:
            if (ss in file and '.dat' in file and file not in filesLMA):
                filesLMA.append(file)

    print("no. of LMA found for selected period: ", len(filesLMA))
    return filesLMA

def get_LMAheader(bucket, file, slabel='*** data ***'):
    if (bucket == 'local'):
        lines = open(file, "r")
    else:
        s3 = boto3.resource("s3")
        obj = s3.Object(bucket, file)
        with gzip.GzipFile(fileobj=obj.get()["Body"]) as gzipfile:
            content = gzipfile.read().decode("utf-8")  # <--convert bytes to string
            lines = content.split('\n')

    n = 1;
    nheader = 0
    for line in lines:
        if (slabel in line):
            nheader = n
            break
        n = n + 1

    if (nheader == 0):
        print("%%Can't find where data starts.",
              "\n%%GO Check start indicator! Is it '%s'?" % slabel)
        return -1

    if (bucket == 'local'): lines.close()
    return nheader

def get_LMA(bucket, file, stns_min=7, nheader=None):
    """
    1. Read LMA data with header excluded
    2. Add column of no. of stations that detected lightning
    3. Cleanse data by filtering out noise
    Note that
     a. nheader: header rows to skip (upon meeting w/ slabel)
     b. column "mask" is hexadecimal, converted to binary for the stations that detected lightning
     c. column "Nstns" added to count no. of stations that detected lightning
    """
    if (not nheader):
        nheader = get_LMAheader(bucket, file, slabel='*** data ***')

    if (bucket == 'local'):
        DF = pd.read_csv(file, names=['Time', 'Lat', 'Lon', 'Alt', 'chi^2', 'dBW', 'mask'],
                         sep=r"\s+", index_col=None, skiprows=nheader, header=None,
                         compression='gzip')

    else:
        s3 = boto3.client('s3')
        obj = s3.get_object(Bucket=bucket, Key=file)
        DF = pd.read_csv(io.BytesIO(obj['Body'].read()),
                         names=['Time', 'Lat', 'Lon', 'Alt', 'chi^2', 'dBW', 'mask'],
                         sep=r"\s+", index_col=None, skiprows=nheader, header=None,
                         compression='gzip')

    ba = [bin(int(a, 16))[2:] for a in DF['mask']]  # <--hexa to bin in str
    DF['Nstns'] = [(sum([int(i) for i in a])) for a in ba]

    # Noise reduction: chi^2 < 1, minimum no. of stations detected lightning
    # print('No. of raw data:',len(DF))
    DF = DF[(DF['chi^2'] < 1) & (DF['Nstns'] >= stns_min)]
    DF.index = range(len(DF))
    DF = DF.drop(['chi^2', 'mask'], 1)
    # print('No. of filtered data:',len(DF))

    return DF, nheader

def LISfiles(s3bucket, fdate, bigbox, start, end, Verb=False):
    """
    Get LIS filename list within range [sec] starting from start to end on fdate
    and within the bigbox domain
    """
    lonW, latS, lonE, latN = bigbox[0:4]
    filesLIS = []

    print("This will take some time....")
    filesALL = S3list(s3bucket, fdate, 'LIS')
    files = [file for file in filesALL if file.split('.')[-1] == 'nc']
    if (Verb): print('No. of all LIS .nc files: ', len(files))

    if (Verb): print("Searching between {} and {}".format(start, end))
    nf = 0
    for file in files:
        fileobj = s3FileObj(s3bucket, file, verb=False)

        ds = xr.open_dataset(fileobj, engine='h5netcdf')
        try:
            nflash = len(ds['flash_dim'].values)
        except KeyError:
            print('%% %s has NO lightning data %%' % file.split('/')[-1])
            continue

        lat, lon, Time = get_LIS('area', ds, bigbox)
        ds.close()

        mask1 = np.where((Time > start) & (Time < end))
        if (len(mask1[0]) > 0):
            mask2 = np.where(
                (lon > lonW) & (lon < lonE) & (lat > latS) & (lat < latN) & (Time > start) & (Time < end))
            if (len(mask2[0]) > 0):
                print("no. of flashes: ", len(mask2[0]))
                nf = nf + 1
                filesLIS.append(file)
                if (Verb): print(file)
        else:
            if (Verb): print("{}-{} not in bounds".format(Time[0], Time[-1]))
            if (nf > 0): break  # <--if prev file in range and this one is not, later ones won't be

    print("No. of LIS files found in domain/range: ", len(filesLIS), nf)
    return filesLIS

def get_LIS(ltype, ds, bigbox):
    """
    get GLM data for point cloud
    Note that Secs is minisecond offset from the file time in the .nc file
    *Using xarray, TAI is automatically converted to np.datetime64[ns]
    *Do not use xr.open_mfdataset. Dimensions in LIS files are not fixed.
    """
    Lat = ds['lightning_' + ltype + '_lat'].values
    Lon = ds['lightning_' + ltype + '_lon'].values
    TAI = ds['lightning_' + ltype + '_TAI93_time'].values  # <--as datetime64[ns]
    Time = (TAI - np.datetime64('1970-01-01')).astype('timedelta64[s]').astype(np.int64)
    if (ltype == 'area'): return Lat, Lon, Time

    Rad = ds['lightning_' + ltype + '_radiance'].values  # <--"area" doesn't have _radiance

    print('Size of Lon, Time:', len(Lon), len(Time))
    return Lat, Lon, Time, Rad

def matchPatt(Pattern, files, nfile=1):
    """
    Find a file(s) with matching pattern (of date/time) among files
    Return:
    1. pattern matching file(s)
    2. shortened file list starting from the file found
       (so search can start from here next time around)
    """
    result = [];
    ii = None
    for i, fn in enumerate(files):
        if (Pattern in fn):
            if (nfile == 1):  # <--return ONLY the 1st file found
                return fn, files[i:]
            result.append(fn)
            ii = i
    return result, files[ii:]

def GLMfiles(s3bucket, fdate, tstart, Trange, files=None):
    """
    get GLM filename list within Trange [sec] starting from tstart on fdate
    Note that GLM files are every 20 sec
    """
    filesGLM = []

    print("This will take some time....")
    if (files is None): files = S3list(s3bucket, fdate, 'GLM')

    for t in range(0, Trange, 20):  # 1):
        ss = (tstart + timedelta(seconds=t)).strftime("s%Y%j%H%M%S")
        result, newlist = matchPatt(ss, files, 1)
        if (result != []):
            filesGLM.append(result)
            files = newlist

    print("no. of GLM found and the rest", len(filesGLM), len(files))
    return filesGLM, files

def get_GLM(ltype, ds):
    """
    get GLM data for point cloud
    Note that Secs is minisecond offset from the file time in the .nc file
    *Using xarray, Secs is automatically converted to np.datetime64[ns]
    *Do not use xr.open_mfdataset. Dimensions in GLM files are not fixed.
    """

    if (ltype == 'event'):
        Secs = ds['event_time_offset'].values
    elif (ltype == 'group'):
        Secs = ds['group_time_offset'].values
    elif (ltype == 'flash'):
        Secs = ds['flash_time_offset_of_first_event'].values
    else:
        print('Duh!')

    # Time=np.array([sec.tolist()+sec00 for sec in Sec2])
    Time = (Secs - np.datetime64('1970-01-01')).astype('timedelta64[s]').astype(np.int64)
    Lat = ds[ltype + '_lat'].values
    Lon = ds[ltype + '_lon'].values
    Rad = ds[ltype + '_energy'].values
    return Lat, Lon, Time, Rad

max_stars_repo_name: nasa/GHRC-FieldCampaign-eXplorer-core

max_issues_repo_name: nasa/GHRC-FieldCampaign-eXplorer-core

max_forks_repo_name: nasa/GHRC-FieldCampaign-eXplorer-core