text:
import re

import json
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import metrics
from sklearn.preprocessing import normalize
from matplotlib import pyplot as plt

from wikipedia import query_wikipedia
from sentence_preprocessing import split_sentences, filter_sentences, clean_sentence
from pretty_print import print_with_emphasis, print_summary

with open('stopwords.json', 'r') as fin:
    STOP_WORDS = json.load(fin)

def sentence_similarity(sentences, stop_word_language=None):
    """Compute similarity between sentences

    Using:
    - cosine similarity on
    - character n-grams
    """
    stop_words = STOP_WORDS.get(stop_word_language, None)
    vectorizer = TfidfVectorizer(analyzer='char_wb', min_df=0.05, max_df=0.5, ngram_range=(3, 10), stop_words=stop_words)
    vectors = vectorizer.fit_transform(sentences)
    similarity = metrics.pairwise.cosine_similarity(vectors, vectors)
    return similarity

def edge_weights(similarity, d=0.85):
    """Transform sentence similarities to proper normalized weights

    :param d: damping factor, percentage of jumps to random vertex
    """
    weights = similarity
    np.fill_diagonal(weights, 0)
    weights = normalize(weights, 'l1', axis=0)
    weights = (1 - d) / weights.shape[0] + d * weights
    return weights

def compute_rank_naive(weights, steps=100):
    """Compute rank by iterative updating

    Similar to first eigenvector if the number of steps is large enough
    """
    rank = np.ones(weights.shape[0])
    for _ in range(steps):
        rank = np.array([weights[:,i].dot(vertices) for i in range(rank.shape[0])])
    return rank

def compute_rank_eigenvector(weights):
    """Compute rank by eigenvector decomposition"""
    eigenvalue, eigenvector = np.linalg.eig(weights)
    ind = eigenvalue.argmax()
    largest_vector = np.abs(eigenvector[:, ind])
    return (largest_vector - largest_vector.min()) / (largest_vector.max() - largest_vector.min())

def local_connections(n, m, width):
    """Local connections filter"""
    index = np.eye(n, m)
    for i in range(width):
        index += np.eye(n, m, i+1) + np.eye(n, m, -i-1)
    return index

def summarize_wiki(page, language='nl', n_input_sentences=None, sort=False,
                   min_rank=None, max_rank=None, n_rank_sentences=None):
    """Summarize wikipedia page using TextRank

    :param language: wikipedia language to use
    :param n_input_sentences: if not None, uses only the n largest sentences from the wikipedia page
    :param sort: if the output should be sorted on rank
    :param min_rank: filter sentences with minimum rank
    :param max_rank: filter sentences with maximum rank
    :param n_rank_sentences: if not None, show only the n highest ranked sentences
    """
    text = query_wikipedia(page, language)
    sentences = filter_sentences(split_sentences(text), n_input_sentences)
    clean_sentences = list(map(clean_sentence, sentences))

    similarities = sentence_similarity(clean_sentences, stop_word_language=language)
    weights = edge_weights(similarities, d=0.85)
    ranks = compute_rank_eigenvector(weights)

    print_summary(sentences, ranks, sort, min_rank, max_rank, n_rank_sentences)

max_stars_repo_name: pietermarsman/me-understanding-textrank

max_issues_repo_name: pietermarsman/me-understanding-textrank

max_forks_repo_name: pietermarsman/me-understanding-textrank


text:
# # # # # # #
# CALCULATE ANNUAL Summer Warmth Index from SNAP standard 2km AKCAN outputs
# # # # # # #

def list_files( base_dir, wildcard='tas_*.tif' ):
	import glob
	out = []
	for root, subs, files in os.walk( base_dir ):
		if len( [i for i in files if 'tas_' in i ] ) > 0:
			out = out + glob.glob(os.path.join( root, wildcard ) )
	return out

def make_grouper( x ):
	group = os.path.basename( x ).split('.')[0].split('_')
	year = group[-1]
	return year

def swi( files, output_path ):
	import rasterio
	import numpy as np

	months = [ fn.split('_')[-2] for fn in files ]
	files = [ fn for fn, month in zip( files, months ) if month in ['05', '06', '07', '08', '09'] ]

	arr = np.array( [ rasterio.open(i).read(1) for i in files ] )
	arr[ arr < 0 ] = 0
	arr = np.sum( arr, axis=0 )

	variable, metric, units, project, model, \
		scenario, month, year = \
			os.path.basename( files[0] ).split( '.' )[0].split( '_' )

	# make sure the year is a decadal year for grouping if annual...
	# this works with both decadal and annual monthlies.
	rst = rasterio.open( files[0] )
	mask = rst.read_masks( 1 )
	meta = rst.meta

	meta.update( compress='lzw' )
	output_filename = os.path.join( output_path, model, scenario, 'swi', '_'.join(['swi', 'cumulative', units, project, model, scenario, year])+'.tif' )

	try:
		dirname = os.path.dirname( output_filename )
		if not os.path.exists( dirname ):
			os.makedirs( dirname )
	except:
		pass

	with rasterio.open( output_filename, 'w', **meta ) as out:
		arr[ mask == 0 ] = meta[ 'nodata' ]
		out.write( np.round( arr, 1 ), 1 )
	return output_filename

if __name__ == '__main__':
	import os
	import pandas as pd
	import multiprocessing as mp
	from functools import partial
	import argparse

	parser = argparse.ArgumentParser( description='downscale the AR5-CMIP5 data to the AKCAN extent required by SNAP' )
	parser.add_argument( "-b", "--base_path", action='store', dest='base_path', type=str, help="path to the directory where the downscaled modeled data are stored" )
	parser.add_argument( "-o", "--output_path", action='store', dest='output_path', type=str, help="path to the output directory" )
	parser.add_argument( "-m", "--model", action='store', dest='model', type=str, help="model name (exact)" )
	parser.add_argument( "-s", "--scenario", action='store', dest='scenario', type=str, help="scenario name (exact)" )
	parser.add_argument( "-p", "--project", action='store', dest='project', type=str, help="project name (exact)" )
	parser.add_argument( "-v", "--variable", action='store', dest='variable', type=str, help="cmip5 variable name (exact)" )

	# unpack args for cleaner access
	args = parser.parse_args()
	base_path = args.base_path
	output_path = args.output_path
	model = args.model
	scenario = args.scenario
	project = args.project
	variable = 'tas' # swi only on tas data

	# list data
	l = pd.Series( list_files( os.path.join( base_path, model, scenario, variable ) ) )

	# group it by year
	groups = l.apply( make_grouper )
	grouped = l.groupby( groups )
	data_grouped = [ sorted(group) for name, group in grouped ]

	# parallel
	swi_f = partial( swi, output_path=output_path )
	pool = mp.Pool(32)
	done = pool.map( swi_f, data_grouped )
	pool.close()
	pool.join()

# # # # # NOTES:

# # run example
# import os, itertools, subprocess

# base_path = '/workspace/Shared/Tech_Projects/DeltaDownscaling/project_data/downscaled'
# output_path = '/workspace/Shared/Tech_Projects/DeltaDownscaling/project_data/derived'
# models = [ 'GFDL-CM3', 'IPSL-CM5A-LR', 'MRI-CGCM3', 'GISS-E2-R', 'NCAR-CCSM4', '5ModelAvg' ]
# scenarios = [ 'historical', 'rcp45', 'rcp60', 'rcp85' ]
# project = 'ar5'

# os.chdir('/workspace/UA/malindgren/repos/downscale/snap_scripts/data_requests')
# for model, scenario in itertools.product( models, scenarios ):
# 	_ = subprocess.call(['python', 'summer_warmth_index_annual.py','-b', base_path, '-o', output_path, '-m', model, '-s', scenario, '-p', project, '-v', 'tas' ])

# # # # # # # # #
# # TESTING STUFF
# base_path = '/workspace/Shared/Tech_Projects/DeltaDownscaling/project_data/downscaled'
# model = 'GFDL-CM3'
# scenario = 'rcp60'
# variable = 'tas'
# begin = 2006
# end = 2100
# ncpus = 32
# metric = 'mean'
# project = 'ar5'
# output_path = '/workspace/Shared/Tech_Projects/DeltaDownscaling/project_data/derived'
# # # # # # # # # #

ext: py

max_stars_repo_name: ua-snap/downscale

max_issues_repo_name: ua-snap/downscale

max_forks_repo_name: ua-snap/downscale

reason: import numpy


text:
import numpy as np
from utils.AwkwardOps import get_nth_object

class LHEPartAssigner(object):
    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)

    def event(self, event):
        if event.config.dataset.parent not in self.old_parents:
            return True

        pdg = event.LHEPart.pdgId
        event.LeptonDecay = get_nth_object(np.abs(
            pdg[(np.abs(pdg)==11) | (np.abs(pdg)==13) | (np.abs(pdg)==15)]
        ), 0, event.size)
        event.LeptonIsElectron = (event.LeptonDecay == 11)
        event.LeptonIsMuon = (event.LeptonDecay == 13)
        event.LeptonIsTau = (event.LeptonDecay == 15)
        event.delete_branches(["LHEPart_pdgId"])

class GenPartAssigner(object):
    def __init__(self, **kwargs):
        self.__dict__.update(**kwargs)

    def event(self, event):
        if event.config.dataset.parent not in self.old_parents:
            return True

        flag = event.GenPart.statusFlags
        pdgs = event.GenPart.pdgId

        event.nGenTauL = (
            (flag&(1<<10)==(1<<10)) & ((np.abs(pdgs)==11) | (np.abs(pdgs)==13))
        ).sum()

        event.delete_branches(["GenPart_pdgId", "GenPart_statusFlags"])

max_stars_repo_name: albertdow/zinv-analysis

max_issues_repo_name: albertdow/zinv-analysis

max_forks_repo_name: albertdow/zinv-analysis


text:
import nose
from Perceptron.functions.activationFunctions.sgn import Sign
from Perceptron.functions.activationFunctions.heaviside import Heaviside
from Perceptron.functions.activationFunctions.indentity import Identity
from Perceptron.functions.activationFunctions.sigmoid import Sigmoid
from Perceptron.functions.activationFunctions.relu import ReLU
from Perceptron.functions.activationFunctions.softmax import SoftMax
from Perceptron.functions.activationFunctions.tanh import Tanh
from Perceptron.functions.activationFunctions.leaky_relu import LeakyReLU
from Perceptron.functions.activationFunctions.smooth_relu import SmoothReLU
from Perceptron.functions.function import Function
import numpy as np

def test_sing_fn():
    """Test sign function"""
    fn = Sign()
    nose.tools.assert_is_instance(fn, Function)
    nose.tools.assert_is_instance(fn, Sign)
    nose.tools.assert_false(fn.is_diff)
    nose.tools.assert_equal(fn.compute(10), 1)
    nose.tools.assert_equal(fn.compute(0), 0)
    nose.tools.assert_equal(fn.compute(-10), -1)

def test_heavside_fn():
    """Test Heaviside function"""
    fn = Heaviside()
    nose.tools.assert_is_instance(fn, Function)
    nose.tools.assert_is_instance(fn, Heaviside)
    nose.tools.assert_false(fn.is_diff)
    nose.tools.assert_equal(fn.compute(10), 1)
    nose.tools.assert_equal(fn.compute(0), 1)
    nose.tools.assert_equal(fn.compute(-10), 0)

def test_identity_fn():
    """Test Identity function"""
    fn = Identity()
    nose.tools.assert_is_instance(fn, Function)
    nose.tools.assert_is_instance(fn, Identity)
    nose.tools.assert_true(fn.is_diff)
    nose.tools.assert_equal(fn.compute(10), 10)
    nose.tools.assert_equal(fn.compute(0), 0)
    nose.tools.assert_equal(fn.compute(-10), -10)
    nose.tools.assert_equal(fn.compute_derivative(), 1)

def test_sigmoid_fn():
    """Test the sigmoid function"""
    fn = Sigmoid()
    nose.tools.assert_is_instance(fn, Function)
    nose.tools.assert_is_instance(fn, Sigmoid)
    nose.tools.assert_equal(fn.compute(0), 0.5)
    nose.tools.assert_equal(fn.compute_derivative(0), 0.25)

def test_relu():
    """Test ReLU function"""
    fn = ReLU()
    nose.tools.assert_is_instance(fn, Function)
    nose.tools.assert_is_instance(fn, ReLU)
    nose.tools.assert_equal(fn.compute(sum([1, 2, 3, 4, 5])), 15)
    nose.tools.assert_equal(fn.compute(-1), 0)
    nose.tools.assert_equal(fn.compute_derivative(sum([1, 2, 3, 4, 5])), 1)
    nose.tools.assert_equal(fn.compute_derivative(-1), 0)

def test_softmax():
    """Test the softmax function"""
    fn = SoftMax()
    nose.tools.assert_is_instance(fn, Function)
    nose.tools.assert_is_instance(fn, SoftMax)
    np.allclose(
        fn.compute(np.array([1, 2, 3, 4, 1, 2, 3])),
        np.array([0.024, 0.064, 0.175, 0.475, 0.024, 0.064, 0.175]),
    )

def test_tanh():
    """Test the tanh function"""
    fn = Tanh()
    nose.tools.assert_is_instance(fn, Function)
    nose.tools.assert_is_instance(fn, Tanh)
    nose.tools.assert_equals(fn.compute(0), 0)
    nose.tools.assert_equals(fn.compute_derivative(0), 1)

def test_leaky_relu():
    """Test the leaky ReLU function"""
    fn = LeakyReLU()
    nose.tools.assert_is_instance(fn, LeakyReLU)
    nose.tools.assert_is_instance(fn, Function)
    nose.tools.assert_equals(fn.compute(0), 0)
    nose.tools.assert_equals(fn.compute(1), 1)
    nose.tools.assert_equals(fn.compute(-1), -0.01)
    nose.tools.assert_equals(fn.compute_derivative(1), 1)
    nose.tools.assert_equals(fn.compute_derivative(-2), 0.01)

def test_smooth_relu():
    """Test the leaky ReLU function"""
    fn = SmoothReLU()
    nose.tools.assert_is_instance(fn, SmoothReLU)
    nose.tools.assert_is_instance(fn, Function)
    nose.tools.assert_equals(fn.compute(0), 0.6931471805599453)
    nose.tools.assert_equals(fn.compute_derivative(0), 0.5)


text:
"""
Copyright 2020 The OneFlow Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import unittest
from random import randint
from random import choice

import numpy as np

from oneflow.test_utils.automated_test_util import *

import oneflow as flow
import oneflow.unittest

class TestGridSample(flow.unittest.TestCase):
    def test_grid_sample_4d(test_case):
        input = flow.tensor(
            np.arange(1.0, 11).reshape((1, 1, 2, 5)), dtype=flow.float32
        )
        np_grid = np.array(
            [
                [[-0.9, -4.1], [0, 0.2000], [1, -1], [-0.333, 1e-6], [0.5, 1.0]],
                [[-1.0, -0.5], [0, 0.3333], [1, -1], [-0.200, 1e-6], [1.5, 0.5]],
            ]
        ).reshape(1, 2, 5, 2)
        grid = flow.tensor(np_grid, dtype=flow.float32)
        groundtruth = np.reshape(
            np.array([[0.0, 8.0, 5.0, 7.0, 9.0], [1.0, 8.0, 5.0, 8.0, 0.0]]),
            (1, 1, 2, 5),
        )
        output = flow.nn.functional.grid_sample(
            input, grid, mode="nearest", padding_mode="zeros", align_corners=True
        )
        test_case.assertTrue(
            np.allclose(output.numpy(), groundtruth, rtol=1e-3, atol=1e-4)
        )

    @unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
    @autotest(rtol=1e-03, atol=1e-04, check_graph=False)
    def test_flow_grid_sample_cudnn_with_random_data(test_case):
        # cudnn only support 4D input, with mode = 'bilinear' && padding_mode = 'zeros' && align_corners
        N = randint(1, 8)
        C = randint(1, 8)
        in_H = randint(1, 8)
        in_W = randint(1, 8)
        out_H = randint(1, 8)
        out_W = randint(1, 8)
        device = "cuda"
        mode = "bilinear"
        padding_mode = "zeros"
        align_corners = True
        theta = random_tensor(ndim=3, dim0=N, dim1=2, dim2=3).to(device)
        grid = torch.nn.functional.affine_grid(
            theta, (N, C, out_H, out_W), align_corners=align_corners
        ).to(device)
        input = random_tensor(ndim=4, dim0=N, dim1=C, dim2=in_H, dim3=in_W).to(device)
        output = torch.nn.functional.grid_sample(
            input,
            grid,
            mode=mode,
            padding_mode=padding_mode,
            align_corners=align_corners,
        )
        return output

    # This test may fail due to using ::floor in backward
    # floor(1.99999988) = 1 和 floor(2.000000) = 2, then select differente images pixel
    @autotest(
        auto_backward=False,
        rtol=1e-03,
        atol=1e-04,
        check_graph=False,
        check_allclose=False,
    )
    def test_flow_grid_sample_4d_with_random_data(test_case):
        N = randint(1, 8)
        C = randint(1, 8)
        in_H = randint(1, 8)
        in_W = randint(1, 8)
        out_H = randint(1, 8)
        out_W = randint(1, 8)
        device = random_device()
        mode = choice(["bilinear", "nearest", "bicubic"])
        padding_mode = choice(["zeros", "border", "reflection"])
        align_corners = choice([True, False])
        theta = random_tensor(ndim=3, dim0=N, dim1=2, dim2=3).to(device)
        grid = torch.nn.functional.affine_grid(
            theta, (N, C, out_H, out_W), align_corners=align_corners
        ).to(device)
        input = random_tensor(ndim=4, dim0=N, dim1=C, dim2=in_H, dim3=in_W).to(device)
        output = torch.nn.functional.grid_sample(
            input,
            grid,
            mode=mode,
            padding_mode=padding_mode,
            align_corners=align_corners,
        )
        return output

    @autotest(auto_backward=False, rtol=1e-03, atol=1e-03, check_graph=False)
    def test_flow_grid_sample_5d_with_random_data(test_case):
        N = randint(1, 8)
        C = randint(1, 8)
        in_D = randint(1, 8)
        in_H = randint(1, 8)
        in_W = randint(1, 8)
        out_D = randint(1, 8)
        out_H = randint(1, 8)
        out_W = randint(1, 8)
        device = random_device()
        mode = choice(["bilinear", "nearest"])
        padding_mode = choice(["zeros", "border", "reflection"])
        align_corners = choice([True, False])
        theta = random_tensor(ndim=3, dim0=N, dim1=3, dim2=4).to(device)
        grid = torch.nn.functional.affine_grid(
            theta, (N, C, out_D, out_H, out_W), align_corners=align_corners
        ).to(device)
        input = random_tensor(
            ndim=5, dim0=N, dim1=C, dim2=in_D, dim3=in_H, dim4=in_W
        ).to(device)
        output = torch.nn.functional.grid_sample(
            input,
            grid,
            mode=mode,
            padding_mode=padding_mode,
            align_corners=align_corners,
        )
        return output

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: LiuHaolan/oneflow

max_issues_repo_name: LiuHaolan/oneflow

max_forks_repo_name: LiuHaolan/oneflow


text:
# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/03_read_data.ipynb (unless otherwise specified).

__all__ = ['AttrDict', 'cData', 'cCLS_CNES', 'cESA_SWE', 'cESA_SOLMAG', 'cNOAA45', 'fget_external_forecasts']

# Cell
import os
import re
import io
import sys
import numpy as np
import pandas as pd
import datetime as dt
from pathlib import Path
import shutil
import urllib.request as request
from urllib.error import URLError
from contextlib import closing
import requests
from requests.exceptions import HTTPError
from functools import reduce

# Cell
from fastai import *
from fastai.vision.all import *

# Cell
from .stats_utils import cStationary

# Cell
class AttrDict(dict):
    """
    access dictionary keys as attributes

    obj = AttrDict()
    obj.update(dict)
    """
    def __init__(self, *args, **kwargs):
        super(AttrDict, self).__init__(*args, **kwargs)
        self.__dict__ = self

# Cell
class cData:
    """
    parent data class
    download functions
    """

    def __init__(self):
        pass

    def fdownload_url_ftp(self, url, dest, overwrite=False):
        """
        download url from ftp server

        cannot use fastai as this uses requests library
        requests cannot handle ftp (ned to use urllib)

        url  = (ftp) url
        dest = destination filename (pathlib object)
        """
        if dest.exists() and not overwrite:
            if self.verb: print("{} already downloaded".format(dest))
            return
        else:
            if self.verb: print("Downloading data to {}...".format(dest))
            with closing(request.urlopen(url)) as r:
                with open(dest, 'wb') as f:
                    shutil.copyfileobj(r, f)

    def fdownload_url_http(self, url, dest, overwrite=False):
        """
        download url from http server

        Throw exception if response not 200
        r.status_code == 200: success
        r.status_code == 404: not found
        """
        try:
            response = requests.get(url)
            response.raise_for_status()
        except HTTPError as http_err:
            print(f'HTTP error occurred: {http_err}')  # Python 3.6
        except Exception as err:
            print(f'Other error occurred: {err}')  # Python 3.6

        # if file exists pass, else download
        if dest.exists() and not overwrite:
            if self.verb: print("{} already downloaded".format(dest))
            return
        else:
            if self.verb: print("Downloading data to {}...".format(dest))
            with open(dest, "wb") as f:
                f.write(response.content)

    def fdownload_url(self, url, dest, overwrite=False):
        """
        Fastai download data: https://github.com/fastai/fastai2/blob/master/fastai2/data/external.py
        """
        try:
            # use standard fastai download or custom http function
            download_data(url, fname=dest, force_download=overwrite)
            #self.fdownload_url_http(url, dest, overwrite)
        except requests.exceptions.InvalidSchema:
            # custom for ftp
            self.fdownload_url_ftp(url, dest, overwrite)

        #return os.listdir()

    def fget_fnames_url_ftp(self, url, ext="txt"):
        """
        return list of filenames displayed in ftp webpage
        """
        # ftp: cannot use requests .get
        resource = request.urlopen(url)
        content =  resource.read().decode('utf-8')

        fnames = [i for i in content.split() if ext in i]

        # print out if no new data
        #if self.verb: if (path/fnames[-1]).exists():
        #    print("No New Data")

        return fnames

    def fget_fnames_dir(self, datadir, ext="txt"):
        """
        obtain list of filenames from local directory
        """
        filenames = []
        for filename in os.listdir(datadir):
            if filename.endswith(ext):
                filenames.append("{}/{}".format(datadir,filename))
            else:
                pass

        return filenames

    def fcopy_local_files(self, src, dst):
        """
        copy data files from local server (e.g. those that have been downloaded with cron)
        """
        #from shutil import copyfile #or use copy, copy2 (check different properties)
        #copyfile(src, dst)
        pass

    def fget_forecast_comp(self, df_forecast, df_archive, cname="y"):
        """
        generate dataframe containing forecasts alongside truth values for metric calculations
        assumes df_forecast and df_archive have same interpolation
        https://pandas.pydata.org/pandas-docs/stable/user_guide/merging.html

        df_forecast = dataframe with columns [index, gendate, ds, cname]
        df_archive  = dataframe with columns [ds, cname] (ds as index)
        cname       = column name of index interested in in archive df
        """
        # merge on matching ds
        df_unsorted = pd.merge(df_forecast, df_archive, on=['ds'])

        # sort by gendate then by ds
        df = df_unsorted.sort_values(by=['gendate', 'ds'])

        # rename columns
        df.columns = ['gendate' , 'ds', cname, '{}_true'.format(cname)]

        return df.reset_index(drop=True)

    def fget_persistence(self, df, ind, colname="persistence"):
        """
        take truth value from the day before gendate and use as forecast
        if cannot access "yesterday" truth: sets persistence as NaN

        ind = column name of truth value to be used as persistence
        """
        # get indices where gendate is the same as ds -> these are truth values for a given gendate==ds
        truth = df[df.gendate == df.ds]
        x1 = np.array(truth.index)
        # need to also append final index, as x2 is taing differences so will be one index short
        x1 = np.append(x1, len(df))

        # get length of each forecast for each gendate (for which persistence has to be repeated)
        x2 = np.diff(x1)

        # for a given gendate, take the truth value from the day before and repeat it x2 times
        x3 = []
        for i in range(len(x2)):
            try:
                yesterday_idx = x1[i-1]
                yesterday_truth = df.iloc[yesterday_idx][ind]
            except IndexError:
                yesterday_truth = np.nan
            x3.append([yesterday_truth]*x2[i])

        # set column in df as persistence
        df[colname] = [item for sublist in x3 for item in sublist]

        return df

    def fget_average(self, df):
        """
        take average of truth values over last x days and use as forecast
        """
        # get indices where gendate is the same as ds -> these are truth values for a given gendate==ds
        #truth = df[df.gendate == df.ds]
        pass

    def finterpolate(self, df, timestep, method="time"):
        """
        interpolate dataframe to regular time spacing

        timestep = interpolation frequency (hours)
        """
        statsobj = cStationary()
        df_interp = statsobj.fmake_regular_freq(df, timestep=dt.timedelta(hours=timestep),
                                                method=method)
        df_interp = df_interp.dropna()

        return df_interp

    def fget_daily(self, df, method="20:00"):
        """
        get single data value per day
        (until 1996 only one value per day, subsequently 3 per day)

        options:
        - "20:00"  : take value at 20:00
        - "interp" : interpolate to midnight
        - False    : take no action

        (may have issues accessing single date string from index despite converting to DatetimeIndex)
        (have to access as df4["1996-03-15":"1996-03-22"] or df4.loc["1996-03-17"] but not df4["1996-03-17"])
        (use dff.sample(1) for manual checking against original dataset)
        """
        if method == "20:00":
            if self.verb: print("Take daily value as at 20:00...")

            # keep observations at midnight and 20:00, drop times
            # drop duplicates (some dates in 1996 have both, in all of these cases they are the same)
            df1 = df.at_time("00:00").reset_index()
            df2 = df.at_time("20:00").reset_index()
            df1["ds"] = df1["ds"].dt.date
            df2["ds"] = df2["ds"].dt.date
            dff = pd.concat([df1, df2], axis=0).drop_duplicates(subset='ds', keep='first')
            dff = dff.set_index('ds')
            dff.index = pd.DatetimeIndex(pd.to_datetime(dff.index))

            return dff

        elif method == "interp":
            if self.verb: print("Interpolating to get daily value...")
            # interpolation frequency [HOURS] (to ensure regularly spaced data)
            interp_freq = 24
            dff = self.finterpolate(df, interp_freq)

            return dff

        else:
            return df

    def fcheck_missing(self, df):
        """
        check if missing data using dates of dataframe (assumes indices are dates)
        """
        data_i = pd.DatetimeIndex(df.index)
        true_i = pd.date_range(start=df.index[0], end=df.index[-1])
        if not data_i.equals(true_i):
            print("MISSING DATA: {} missing entries".format(len(true_i) - len(data_i)))
        else:
            print("No missing data!")

    def fmissing_data(self, df, method="interp_linear"):
        """
        deal with missing data:
        add NaNs to missing dates and use built in interpolation functions

        options:
        - "interp_linear" : linearly interpolate missing value
        - "pad"           : take missing value as last know value
        - False           : take no action
        """
        if method is False:
            if self.verb: print("Ignore missing data...")
            return df

        else:
            # check & print if missing data
            if self.verb: self.fcheck_missing(df)

            # generate dataframe with flag as to which days there is missing data
            dates, counts = [], []
            for name, group in df.groupby(pd.Grouper(freq='D')):
                dates.append(name)
                # (can also use this with len(group)==4 to ensure that 20:00==00:00 for fget_daily)
                if len(group) == 0:
                    counts.append(1)
                else:
                    counts.append(0)
            df_missing = pd.DataFrame({'ds': dates, 'missing': counts})
            df_missing.set_index('ds', inplace=True)

            # merge with input dataframe: set NaN where data is missing
            dff = pd.merge(df, df_missing, on='ds', how='outer', sort=True)
            dff = dff[dff.index.duplicated() == False]

            # interpolate missing data
            # linear interpolation
            if method == "interp_linear":
                if self.verb: print("Linearly interpolating missing data...")
                return dff.interpolate().drop(columns=['missing'])

            # use last known value (for max of 2 consecutive NaNs)
            elif method == "pad":
                if self.verb: print("Padding missing data...")
                return dff.interpolate(method='pad', limit=2).drop(columns=['missing'])

    def fsave():
        """
        save and load pandas dataframes to/from pickle
        from_pickle option in data classes? e.g. if cls data takes too long to load
        https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_pickle.html
        """
        pass

# Cell
class cCLS_CNES(cData):
    """
    https://spaceweather.cls.fr/services/radioflux/

    Forecasts archived in multiple files:
    * need to download based on dates
    * read in each and stack into single df with additional gendate column

    #      f<lambda>   : observed values with interpolated data gaps. Missing values are indicated as -1
    #      f<lambda>_c : observed or forecasted values with interpolated data gaps and flare correction
    #      f<lambda>_p : precision (after processing) for observed values, or root mean square error for forecasts
    #      f<lambda>_f : flag indicating the processing applied, calculated as the sum of the following numbers :
    #                       1 if values are provisional
    #                       2 if values are interpolated
    #                       4 if values are corrected for flares
    #                       8 if values are forecasted
    #                    As a result, a flag of 0 means values are final observations, not interpolated and with no flares detected
    """

    def __init__(self, verb=False):
        # verbose
        self.verb = verb

        self.archive_url  = "ftp://ftpsedr.cls.fr/pub/previsol/solarflux/observation/radio_flux_{}_observation.txt"
        # archive_url.format("absolute") absolute or adjusted
        self.forecast_url = "ftp://ftpsedr.cls.fr/pub/previsol/solarflux/forecast/{}/{}/{:02d}/radio_flux_{}_forecast_{}{:02d}{:02d}.txt"
        # forecast_url.format("absolute", year, month, "absolute", year, month, day) absolute or adjusted
        self.forecast_url2 = "ftp://ftpsedr.cls.fr/pub/previsol/solarflux/forecast/{}/radio_flux_{}_forecast_{}{:02d}{:02d}.txt"
        # sometimes new forecasts not added immediately to year directory and placed in main dir

    def fread_data(self, fname):
        """
        convert single file to pandas dataframe

        ??pd.read_csv
        """
        # get header
        names_row = [i for i in open(fname).readlines() if i.startswith("#")][-1]
        names = names_row.split()[1:]

        df = pd.read_csv(fname, comment="#", delimiter="\s+", names=names)

        df.insert(0, "ds", pd.to_datetime(df[['year', 'month', 'day']]))
        df = df.drop(columns=["year", "month", "day", "cont_day"])

        return df

    def fread_forecasts(self, gendates, fnames):
        """
        stack fnames into single dataframe including gendate column
        """
        df = pd.DataFrame()

        for i in range(len(fnames)):
            df_i = self.fread_data(fnames[i])
            df_i.insert(0, "gendate", gendates[i])

            # only want when _f == 8 (forecast)
            df_for = df_i[df_i["f10.7_f"] == 8]

            # stack
            df = pd.concat([df, df_for])

        return df.reset_index(drop=True)

    def fget_archive_data(self, destdir, ldate, udate, adj=False):
        """
        download (if required) archive file and read data within within given date range
        """
        url = self.archive_url

        # create directory if does not exist
        path = Path(destdir)
        path.mkdir(parents=True, exist_ok=True)

        adj = "adjusted" if adj else "absolute"

        # generate url and destination
        url  = url.format(adj)
        dest = path/"radio_flux_{}_observation.txt".format(adj)

        # download data
        # TO DO: MAY NEED TO OVERWRITE IN FUTURE AS FNAME NOT DATESTAMPED
        self.fdownload_url(url, dest, overwrite=False)

        # read fnames into df
        df = self.fread_data(dest)

        # limit between ldate and udate
        df = df[df.ds >= ldate][df.ds <= udate]

        return df.reset_index(drop=True)

    def fget_forecast_data(self, destdir, ldate, udate, adj=False):
        """
        download (if required) and read in forecasts generated within given date range

        dest   = data directory [str]
        ldate  = lower date range [str] e.g. "2019-12-30"
        udate  = upper date range [str] e.g. "2020-01-02"
        adj    = absolute or adjusted data [bool]
        """
        #url = self.forecast_url

        # create directory if does not exist
        path = Path(destdir)
        path.mkdir(parents=True, exist_ok=True)

        # generate daterange
        dates = pd.date_range(start=ldate,end=udate)
        adj = "adjusted" if adj else "absolute"
        # list of destination filenames
        fnames = []

        # download data if does not already exist
        for i in dates:
            try:
                furl  = self.forecast_url.format(adj, i.year, i.month, adj, i.year, i.month, i.day)
                fname = path/"radio_flux_{}_forecast_{}{:02d}{:02d}.txt".format(adj, i.year, i.month, i.day)
                if self.verb: print(furl, fname)
                self.fdownload_url(furl, fname, overwrite=False)
                fnames.append(fname)
            except URLError:
                furl  = self.forecast_url2.format(adj, adj, i.year, i.month, i.day)
                fname = path/"radio_flux_{}_forecast_{}{:02d}{:02d}.txt".format(adj, i.year, i.month, i.day)
                if self.verb: print(furl, fname)
                self.fdownload_url(furl, fname, overwrite=False)
                fnames.append(fname)

        # read fnames into df
        df = self.fread_forecasts(dates, fnames)

        return df

# Cell
class cESA_SWE(cData):
    """
    http://swe.ssa.esa.int/

    Archived forecasts are available but cannot be access via API (need to download manually from url)
    """

    def __init__(self, verb=False):
        # verbose
        self.verb = verb

    def fget_archive_data(self, filename):
        """
        read archive csv data (date, param)
        """
        data = pd.read_csv(filename, comment="#", sep="[;,\s+]", engine='python',
                           parse_dates=[[0,1]], infer_datetime_format=True,
                           names=["yyyy_mm_dd", "HH_MM", "y"])

        data.rename(columns={"yyyy_mm_dd_HH_MM": "ds"}, inplace=True)

        return data

    def fget_forecast_data(self, filename):
        """
        read forecast csv data (generation date, datestamp, param)
        """
        data = pd.read_csv(filename, comment="#", engine='python',
                           infer_datetime_format=True, names=["gendate", "ds", "y"])

        data["gendate"] = pd.to_datetime(data["gendate"])
        data["ds"] = pd.to_datetime(data["ds"])

        return data

    def fget_data(self, filenames=None, datadir=None):
        """
        read csv data into dictionary
        """
        if filenames == None:
            filenames = self.fget_fnames_dir(datadir)
        if type(filenames) == str:
            filenames = [filenames]

        datadict = {}
        for filename in filenames:

            header = open(filename).readlines()[:20]
            meta = {}
            for i in header:
                if (i.startswith("#")) and (":" in i):
                    meta[i[1:].split(":")[0].strip()] =  i.split(":")[1].strip()

            param = meta["parameter"]

            if ("Error" in "".join(header)) or ("error" in "".join(header)):
                datadict[param] = "ERROR"
            else:
                cols = meta["columns"].replace(' ', '').replace(';',' ').replace(',',' ').split()

                if len(cols) == 2:
                    datadict[param] = self.fget_archive_data(filename)
                elif len(cols) == 3:
                    datadict[param] = self.fget_forecast_data(filename)

        return datadict

# Cell
class cESA_SOLMAG(cData):
    """
    https://sdup.esoc.esa.int/solmag/
    https://static.sdo.esoc.esa.int/SOLMAG/fap_day.dat
    https://static.sdo.esoc.esa.int/SOLMAG/fap_mon.dat

    Archived forecasts are not available (these are being downloaded by me via cron)
    """

    def __init__(self):
        pass

    def fread_fapday(self, filename):
        """
        read in fap_day.dat
        Daily SOLMAG forecast
        """
        numcols = 6
        headers = open(filename).readline().rstrip().strip("#").split()[:6]

        data = pd.read_csv(filename, sep=" ", usecols=list(range(numcols)), comment="#",
                             names=headers, parse_dates=[0], infer_datetime_format=True)

        data.rename(columns={"d/mm/yyyy": "epoch"}, inplace=True)

        return data

    def fread_fapmon(self, filename):
        """
        read in fap_mon.dat
        Monthly SOLMAG: long-term activity forecast (predicted values from 04 2019)
        """
        headers = ["mm", "yyyy", "SSN_obs", "SSN_pred", "F10_obs", "F10_pred", "F10_err",
                  "AA_obs", "AA_pred", "Ap_obs", "Ap_pred", "Ap_err"]

        data = pd.read_csv(filename, comment="#", delim_whitespace=True, names=headers,
                           parse_dates=[[0,1]], infer_datetime_format=True)

        data.rename(columns={"mm_yyyy": "epoch"}, inplace=True)

        return data

# Cell
class cNOAA45(cData):
    """
    https://www.swpc.noaa.gov/products/usaf-45-day-ap-and-f107cm-flux-forecast

    latest: "https://services.swpc.noaa.gov/text/45-day-ap-forecast.txt"
    last 20 days: "ftp://ftp.swpc.noaa.gov/pub/forecasts/45DF"

    Archived forecasts are not available (these are being downloaded by me via cron)
    """

    def __init__(self):
        # verbose
        self.verb = True

    def fget_archive_data(self, destdir, adj=False):
        """
        download (if required) archive file and read data within within given date range

        DATASET ONLY AVAILABLE UNTIL 2018
        """
        adj = "adjusted" if adj else "observed"

        if adj == "adjusted":
            dest = Path(destdir/"listing_drao_noontime-flux-adjusted_daily.txt")
            url = "ftp://ftp.ngdc.noaa.gov/STP/space-weather/solar-data/solar-features/solar-radio/noontime-flux/penticton/penticton_adjusted/listings/listing_drao_noontime-flux-adjusted_daily.txt"
        elif adj == "observed":
            dest = Path(destdir/"listing_drao_noontime-flux-observed_daily.txt")
            url = "ftp://ftp.ngdc.noaa.gov/STP/space-weather/solar-data/solar-features/solar-radio/noontime-flux/penticton/penticton_observed/listings/listing_drao_noontime-flux-observed_daily.txt"
        self.fdownload_url_ftp(url, dest)

        # need to fill missing values with na
        with open(dest) as f:
            lines = f.readlines()
        new_lines = []
        for line in lines:
            splt = line.split()
            if len(splt) == 1:
                new_lines.append("{},na".format(splt[0]))
            elif len(splt) == 2:
                new_lines.append(','.join(splt))
        fstr = '\n'.join(new_lines)

        # read in as dataframe
        dfa_noaa = pd.read_csv(io.StringIO(fstr), sep=",", parse_dates=[0], infer_datetime_format=True, na_values=['na', '.'])
        dfa_noaa.columns = ['ds', 'y_noaa_true']

        # str to float
        dfa_noaa["y_noaa_true"] = pd.to_numeric(dfa_noaa["y_noaa_true"], downcast="float")

        return dfa_noaa

    def fget_sub_forecast(self, forecast):
        """
        convert sub-section of 45 day NOAA forecasts (Ap or F10.7) to df
        """
        # flatten list
        a = [i[:-1].split() for i in forecast[1:]]
        b = [item for sublist in a for item in sublist]
        # b[::2], b[1::2] as even and odd

        col_name = forecast[0].split()[1]
        t        = np.array([dt.datetime.strptime(tt,'%d%b%y') for tt in b[::2]])
        x        = np.array(b[1::2]).astype(int)

        df = pd.DataFrame(np.array([t,x]).T, columns=["ds", col_name])

        return df.set_index("ds")

    def fread_data(self, fname):
        """
        read in single forecast file as df

        to do: read meta data (issue date of prediction)
        """
        # read in lines as list
        lines = open(fname).readlines()

        # sublists containing ap and flux forecasts
        ap_forecast = lines[8:18]
        flux_forecast = lines[18:28]

        # convert lists to pd
        df_ap   = self.fget_forecast_df(ap_forecast)
        df_flux = self.fget_forecast_df(flux_forecast)

        # combine pds
        df = pd.concat([df_ap, df_flux], axis=1)
        df.reset_index()

        return df

# Cell
def fget_external_forecasts(config):
    """
    generate dataframe containing forecasts and "truths" for external sources
    get persistence
    """
    data_frames = []

    # ESA BGS
    if "esa" in config.data_comp:
        dataobj_esa = cESA_SWE()
        # read in archive data
        df = dataobj_esa.fget_data(filenames=config.esa_archive_fname)[config.esa_archive_key]
        df.set_index('ds', inplace=True)
        df_lim = df[config.date_llim:config.date_ulim]
        #dfa_esa = dataobj_esa.finterpolate(df_lim, config.interp_freq) # TO DO
        df_daily = dataobj_esa.fget_daily(df_lim, config.get_daily_method)
        dfa_esa  = dataobj_esa.fmissing_data(df_daily, config.missing_data_method)
        # read in esa forecast data
        dff_esa = dataobj_esa.fget_data(filenames=config.esa_forecast_fname)[config.esa_forecast_key]
        # add "truth" from archive to forecast
        dff_comp_esa = dataobj_esa.fget_forecast_comp(dff_esa, dfa_esa, cname="y")

        # rename columns
        dff_comp_esa.columns = ['gendate' , 'ds', 'y_pred_bgs', 'y_esa_true']
        # Calculate persistence
        dff_comp_esa = dataobj_esa.fget_persistence(dff_comp_esa, 'y_esa_true', "persistence_esa")

        data_frames.append(dff_comp_esa)

    # CLS-CNES
    if "cls" in config.data_comp:
        dataobj_cls = cCLS_CNES()
        # read in archive data and restrict to ds and key variable
        # need to ensure upper date for archive is 30 days ahead of upper date of forecast gendate
        udate_a = (dt.datetime.strptime(config.cls_forecast_udate, "%Y-%m-%d") + dt.timedelta(days=30)).strftime("%Y-%m-%d")
        dfa_cls = dataobj_cls.fget_archive_data(config.cls_datadir, config.cls_forecast_ldate, udate_a)
        dfa_cls = dfa_cls[['ds',config.cls_key]]
        dfa_cls = dfa_cls.set_index("ds")
        # read in forecast data and restrict to key variable
        dff_cls = dataobj_cls.fget_forecast_data(config.cls_datadir, config.cls_forecast_ldate, config.cls_forecast_udate)

        dff_cls = dff_cls[['gendate', 'ds', "{}_c".format(config.cls_key)]]
        # add "truth" from archive to forecast
        dff_comp_cls = dataobj_cls.fget_forecast_comp(dff_cls, dfa_cls, cname=config.cls_key)

        # rename columns
        dff_comp_cls.columns = ['gendate' , 'ds', 'y_pred_cls', 'y_cls_true']
        # Calculate persistence
        dff_comp_cls = dataobj_cls.fget_persistence(dff_comp_cls, 'y_cls_true', "persistence_cls")

        data_frames.append(dff_comp_cls)

    # NOAA USAF
    if "usaf" in config.data_comp:
        dataobj_esa = cESA_SWE()
        dff_usaf = dataobj_esa.fget_data(filenames=config.usaf_forecast_fname)[config.usaf_forecast_key]
        dff_usaf["gendate"] = dff_usaf["gendate"].dt.normalize() #.dt.date # only keep date part
        dff_usaf.columns = ['gendate' , 'ds', 'y_pred_usaf']

        data_frames.append(dff_usaf)

    # NOAA SWPC
    if "swpc" in config.data_comp:
        dataobj_esa = cESA_SWE()
        dff_swpc = dataobj_esa.fget_data(filenames=config.swpc_forecast_fname)[config.swpc_forecast_key]
        dff_swpc["gendate"] = dff_swpc["gendate"].dt.normalize() #.dt.date # only keep date part
        dff_swpc.columns = ['gendate' , 'ds', 'y_pred_swpc']

        data_frames.append(dff_swpc)

    # COMBINE AND RETURN
    df_merged = reduce(lambda  left,right: pd.merge(left,right,on=['gendate', 'ds']), data_frames)

    return df_merged.dropna().reset_index(drop=True)


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Apr  3 15:14:40 2017

@author: mariapanteli
"""

import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelBinarizer
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA
from sklearn.decomposition.pca import PCA
from sklearn.decomposition import NMF
from sklearn.preprocessing import scale
from sklearn.neighbors import KNeighborsClassifier
from sklearn import svm
from sklearn import metrics
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import normalize
from numpy.linalg import pinv

import nmftools

class Transformer:
    def __init__(self):
        self.pca_transformer = None
        self.lda_transformer = None
        self.nmf_transformer = None
        self.ssnmf_H = None
        self.modelKNN = None
        self.modelLDA = None
        self.modelSVM = None
        self.modelRF = None

    def ssnmf_fit(self, data, labels, npc=None):
        binarizer = LabelBinarizer()
        F_class = binarizer.fit_transform(labels)
        F, G, W, H, cost = nmftools.ssnmf(data, R=npc, F=F_class, n_iter=200)
        ssWH = np.dot(F, G) + np.dot(W, H)
        rec_err = np.linalg.norm(data - ssWH)
        return G, W, H, rec_err

    def fit_lda_data(self, X_train, Y_train, n_components=None, pca_only=False):
        X_train = scale(X_train, axis=0)
        # then pca
        print "training with PCA transform..."
        self.pca_transformer = PCA(n_components=n_components).fit(X_train)
        print "variance explained " + str(np.sum(self.pca_transformer.explained_variance_ratio_))
        if pca_only:
            # return pca transformer only
            return
        # then lda
        print "training with LDA transform..."
        self.lda_transformer = LDA(n_components=n_components).fit(X_train, Y_train)
        print "variance explained " + str(np.sum(self.lda_transformer.explained_variance_ratio_))

    def transform_lda_data(self, X_test):
        X_test = scale(X_test, axis=0)
        print "transform test data..."
        pca_testdata = self.pca_transformer.transform(X_test)
        lda_testdata = self.lda_transformer.transform(X_test)
        transformed_data = {'none': X_test, 'pca': pca_testdata,
                                            'lda': lda_testdata,
                                            'nmf': [],
                                            'ssnmf': []}
        return transformed_data

    def fit_data(self, X_train, Y_train, n_components=None, pca_only=False):
        if n_components is None:
            n_components = X_train.shape[1]
        X_train = scale(X_train, axis=0)
        # then pca
        print "training with PCA transform..."
        self.pca_transformer = PCA(n_components=n_components).fit(X_train)
        print "variance explained " + str(np.sum(self.pca_transformer.explained_variance_ratio_))
        if pca_only:
            # return pca transformer only
            return
        # then lda
        print "training with LDA transform..."
        self.lda_transformer = LDA(n_components=n_components).fit(X_train, Y_train)
        print "variance explained " + str(np.sum(self.lda_transformer.explained_variance_ratio_))
        # then nmf
        print "training with NMF transform..."
        norm_traindata = normalize(X_train - np.min(X_train))
        self.nmf_transformer = NMF(n_components=n_components).fit(norm_traindata)
        print "reconstruction error " + str(np.sum(self.nmf_transformer.reconstruction_err_))
        # then ssnmf
        print "training with SSNMF transform..."
        G, W, self.ssnmf_H, rec_err = self.ssnmf_fit(norm_traindata, Y_train, npc=n_components)
        print "reconstruction error " + str(rec_err)

    def transform_data(self, X_test):
        X_test = scale(X_test, axis=0)
        print "transform test data..."
        pca_testdata = self.pca_transformer.transform(X_test)
        lda_testdata = self.lda_transformer.transform(X_test)
        norm_testdata = normalize(X_test - np.min(X_test))
        nmf_testdata = self.nmf_transformer.transform(norm_testdata)
        ssnmf_testdata = np.dot(norm_testdata, pinv(self.ssnmf_H))
        transformed_data = {'none': X_test, 'pca': pca_testdata,
                                            'lda': lda_testdata,
                                            'nmf': nmf_testdata,
                                            'ssnmf': ssnmf_testdata}
        return transformed_data

    def classification_accuracy(self, X_train, Y_train, X_test, Y_test, model=None):
        if model is None:
            model = LDA()
        model.fit(X_train, Y_train)
        predictions = model.predict(X_test)
        accuracy = metrics.f1_score(Y_test, predictions, average='weighted')  # for imbalanced classes
        return accuracy, predictions

    def classify(self, X_train, Y_train, X_test, Y_test, transform_label=" "):
        self.modelKNN = KNeighborsClassifier(n_neighbors=3, metric='euclidean')
        self.modelLDA = LDA()
        self.modelSVM = svm.SVC(kernel='rbf', gamma=0.1)
        self.modelRF = RandomForestClassifier()
        model_labels = ['KNN', 'LDA', 'SVM', 'RF']
        models = [self.modelKNN, self.modelLDA, self.modelSVM, self.modelRF]
        df_results = pd.DataFrame()
        for model, model_label in zip(models, model_labels):
            acc, _ = self.classification_accuracy(X_train, Y_train, X_test, Y_test, model=model)
            print model_label + " " + transform_label + " " + str(acc)
            df_results = df_results.append(pd.DataFrame([[transform_label, model_label, acc]]))
        return df_results

if __name__ == '__main__':
    Transformer()

max_stars_repo_name: hideodaikoku/music-outliers

max_issues_repo_name: hideodaikoku/music-outliers

max_forks_repo_name: hideodaikoku/music-outliers

reason: import numpy,from numpy


text:
from os import listdir
from os.path import isfile, join

import cv2
import numpy as np
import tensorflow as tf

class FramesDataGenerator(tf.keras.utils.Sequence):
    """
    Custom Keras generator for frames
    """

    def __init__(self, dir, batch_size=32, frames_to_use=12, is_test=False, shuffle=True, height=320, width=480):
        self.frames_to_use = frames_to_use
        self.height = height
        self.width = width
        self.batch_size = batch_size
        self.is_test = is_test
        self.dir = dir
        if is_test:
            self.shuffle = False
        else:
            self.shuffle = shuffle
        self.classes = self.find_classes()
        self.video_names, self.video_map, self.video_to_class, self.num_samples, self.min_frames = self.find_samples()
        self.on_epoch_end()
        if self.is_test:
            print(f"Found {self.num_samples} frames belonging to {len(self.video_names)} videos (test-mode).")
        else:
            print(
                f"Found {self.num_samples} frames belonging to {len(self.video_names)} videos belonging to {len(self.classes)} classes.")
        print(f"Min frames determined to be {self.min_frames}")

    def find_classes(self):
        if self.is_test:
            return []
        else:
            category_folders = [f for f in listdir(self.dir) if not isfile(join(self.dir, f))]
            return sorted(list(set(category_folders)))

    def find_samples(self):
        num_samples = 0
        min_frames = -1
        video_map = {}
        vid_to_cat = {}
        if self.is_test:
            category_folders = [self.dir]
        else:
            category_folders = [f for f in listdir(self.dir) if not isfile(join(self.dir, f))]
        for category_folder in category_folders:
            if self.is_test:
                cat_path = category_folder
            else:
                cat_path = join(self.dir, category_folder)
            frames = [f for f in listdir(cat_path) if isfile(join(cat_path, f))]
            for frame in frames:
                # frame = frame_101_7.mp4_8.jpg
                frame_arr = frame.split(".mp4_")
                vid_name = frame_arr[0]
                if vid_name not in video_map:
                    video_map[vid_name] = []
                    vid_to_cat[vid_name] = category_folder
                video_map[vid_name].append(frame)

            for k in video_map.keys():
                # make sure the frames for each video are in sorted order
                video_map[k] = sorted(video_map[k],
                                      key=lambda x: x.split(".mp4_")[0] + x.split(".mp4_")[1].split(".jpg")[0].zfill(3))
                if min_frames == -1 or len(video_map[k]) < min_frames:
                    min_frames = len(video_map[k])

        return list(video_map.keys()), video_map, vid_to_cat, len(vid_to_cat), min_frames

    def __len__(self):
        """
        Denotes the number of batches per epoch
        """
        return int(np.ceil(self.num_samples / self.batch_size))

    def __getitem__(self, index):
        """
        Generate one batch of data
        """
        video_names = self.video_names[index * self.batch_size:(index + 1) * self.batch_size]
        num_frames = self.min_frames if self.frames_to_use == -1 else self.frames_to_use
        X = np.zeros((len(video_names), num_frames, self.height, self.width, 3), dtype=np.uint8)
        y = []
        i = 0
        for vid in video_names:
            j = 0
            for frame in self.video_map[vid]:
                if self.is_test:
                    frame_path = join(self.dir, frame)
                else:
                    frame_path = join(join(self.dir, self.video_to_class[vid]), frame)

                img = cv2.imread(frame_path)
                img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
                X[i, j, :, :, :] = img
                j += 1
                if j >= num_frames:
                    break

            if self.is_test:
                y.append(0)
            else:
                y.append(int(self.video_to_class[vid]) - 1)
            i += 1
        y = np.array(y)
        return X, tf.keras.utils.to_categorical(y, num_classes=len(self.classes))

    def on_epoch_end(self):
        if self.shuffle == True:
            np.random.shuffle(self.video_names)

max_issues_repo_name: fusical/emotiw

max_forks_repo_name: fusical/emotiw


text:
import matlab.engine
import os
import logging
import argparse
import numpy as np
from train_and_evaluate import evaluate, train
from net import Generator
import utils
import torch

# start matlab engine
eng = matlab.engine.start_matlab()
# RCWA path
eng.addpath(eng.genpath('/home/users/jiangjq/Desktop/reticolo_allege'));
eng.addpath(eng.genpath('solvers'));

# parser
parser = argparse.ArgumentParser()
parser.add_argument('--output_dir', default='results',
                                        help="Results folder")
parser.add_argument('--wavelength', default=None)
parser.add_argument('--angle', default=None)
parser.add_argument('--restore_from', default=None,
                                        help="Optional, directory or file containing weights to reload before training")

if __name__ == '__main__':
    # Load the directory from commend line
    args = parser.parse_args()

    # Set the logger
    utils.set_logger(os.path.join(args.output_dir, 'train.log'))

    # Load parameters from json file
    json_path = os.path.join(args.output_dir,'Params.json')
    assert os.path.isfile(json_path), "No json file found at {}".format(json_path)
    params = utils.Params(json_path)

    # Add attributes to params
    params.output_dir = args.output_dir
    params.cuda = torch.cuda.is_available()
    params.restore_from = args.restore_from
    params.numIter = int(params.numIter)
    params.noise_dims = int(params.noise_dims)
    params.gkernlen = int(params.gkernlen)
    params.step_size = int(params.step_size)

    if args.wavelength is not None:
        params.wavelength = int(args.wavelength)
    if args.angle is not None:
        params.angle = int(args.angle)

    # make directory
    os.makedirs(args.output_dir + '/outputs', exist_ok = True)
    os.makedirs(args.output_dir + '/model', exist_ok = True)
    os.makedirs(args.output_dir + '/figures/histogram', exist_ok = True)
    os.makedirs(args.output_dir + '/figures/deviceSamples', exist_ok = True)

    # Define the models
    generator = Generator(params)

    # Move to gpu if possible
    if params.cuda:
        generator.cuda()

    # Define the optimizer
    optimizer = torch.optim.Adam(generator.parameters(), lr=params.lr, betas=(params.beta1, params.beta2))

    # Define the scheduler
    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=params.step_size, gamma = params.gamma)

    # Load model data
    if args.restore_from is not None :
        params.checkpoint = utils.load_checkpoint(restore_from, generator, optimizer, scheduler)
        logging.info('Model data loaded')

    # Train the model and save
    if params.numIter != 0 :
        logging.info('Start training')
        train(generator, optimizer, scheduler, eng, params)

    # Generate images and save
    logging.info('Start generating devices')
    evaluate(generator, eng, numImgs=500, params=params)

max_stars_repo_name: jonfanlab/GLOnet


text:
import dash
import dash_core_components as dcc
import dash_html_components as html
from datetime import datetime as dt
import dash_table
import dash_daq as daq
from dash.dependencies import Input, Output, State
import copy
from elasticsearch import Elasticsearch
import numpy as np
import datetime
import unittest
import os
from mlbridge_ui.src.ui import *
from faker import Faker
from unittest import mock

class TestMaliciousDomainNameAnalysis(unittest.TestCase):

    def setUp(self):
        self.es = Elasticsearch()

    # Historical Analysis

    # Control Messages

    def test_input_message(self):
        message_none = input_message(1, None)
        message_not_existed = input_message(1, 'not_exist')
        self.assertEqual(message_none, 'Please enter a Domain Name')
        self.assertEqual(message_not_existed, 'Domain Name does not exist in '
                                              'Database')

    def test_date_message(self):
        message_none = date_message(1, '', None, None)

        check = False
        random_option = ['Minute', 'Hour', 'Day']
        option = random_option[np.random.randint(0, 3)]
        fake = Faker()
        start_date = datetime.date(year=2020, month=1, day=1)
        end_date = datetime.datetime.now().date()
        date_start = fake.date_between(start_date=start_date, end_date=end_date)
        date_end = fake.date_between(start_date=date_start, end_date=end_date)
        message_option = date_message(1, option, str(date_start), str(date_end))

        if 'Data from' in message_option or 'please enter' in message_option:
            check = True

        self.assertEqual(message_none, 'Please enter the date range')
        self.assertTrue(check)

    def test_radio_button_message(self):
        message_none = radio_button_message(1, None)
        message_option = radio_button_message(1, 'Test_Option')

        self.assertEqual(message_none, 'Please select an option')
        self.assertEqual(message_option, 'You have selected: Test_Option')

    def test_hour_range_message(self):
        message_none = hour_range_message('', None, None)

        random_option = ['Minute', 'Hour']
        random_choice = random_option[np.random.randint(0, 2)]
        start = np.random.randint(0, 24)
        end = np.random.randint(start, 25)
        message_option = hour_range_message(random_choice, start, end)
        check = False

        if 'Hour range' in message_option or 'Please enter' in message_option \
                or 'The difference' in message_option:
            check = True

        self.assertEqual(message_none, 'Enter an integer hour range (0 to 24)')
        self.assertTrue(check)

    # Graph and Div Updates

    def test_display_hour_range(self):
        display_none = display_hour_range('Day')
        display_option = display_hour_range('')

        self.assertEqual(display_none['display'], 'none')
        self.assertEqual(display_option['display'], 'unset')

    def test_update_pie_graph(self):
        figure_none = update_pie_graph(1, None)
        figure_benign = update_pie_graph(1, 'google.com')
        figure_mal = update_pie_graph(1, '1-remont.com')

        self.assertTrue(figure_none['data'][0]['values'][0] == 0.5)

    def test_update_line_graph(self):
        figure_none = update_line_graph(1, None, None, None, None, None, None)
        check = False
        if 'data' in figure_none.keys() and 'layout' in figure_none.keys():
            check = True

        self.assertTrue(check)

    def test_update_ip_table(self):
        data_none = update_ip_table(1, None)

        self.assertEqual(data_none, [])

    def test_display_mal_list(self):
        display_false = display_mal_list(False)
        display_true = display_mal_list(True)

        self.assertEqual(display_false['display'], 'none')
        self.assertEqual(display_true['display'], 'unset')

    def test_display_mal_graph(self):
        display_false = display_mal_graph(False)
        display_true = display_mal_graph(True)

        self.assertEqual(display_false['display'], 'unset')
        self.assertEqual(display_true['display'], 'none')

    def test_update_mal_dns_table(self):
        data = update_mal_dns_table(1, '')

        self.assertTrue(len(data) >= 0)

    def test_update_mal_bar_graph(self):
        figure = update_mal_bar_graph(1, '')
        check = False
        if 'data' in figure.keys() and 'layout' in figure.keys():
            check = True

        self.assertTrue(check)

    def test_display_benign_list(self):
        display_false = display_benign_list(False)
        display_true = display_benign_list(True)

        self.assertEqual(display_false['display'], 'none')
        self.assertEqual(display_true['display'], 'unset')

    def test_display_benign_graph(self):
        display_false = display_benign_graph(False)
        display_true = display_benign_graph(True)

        self.assertEqual(display_false['display'], 'unset')
        self.assertEqual(display_true['display'], 'none')

    def test_update_benign_dns_table(self):
        data = update_benign_dns_table(1, '')

        self.assertTrue(len(data) >= 0)

    def test_update_benign_bar_graph(self):
        figure = update_benign_bar_graph(1, '')
        check = False
        if 'data' in figure.keys() and 'layout' in figure.keys():
            check = True

        self.assertTrue(check)

    # Manual Vetting

    def test_update_and_input_vet_message_vet_tables(self):

        message_none, _, _, _, _, _, = update_and_input_vet_message_vet_tables(
            None, None, None, None, None, None)
        check = False
        if 'Please select' in message_none:
            check = True
        self.assertTrue(check)

    def test_update_not_vetted_table(self):
        data = update_not_vetted_table('')

        self.assertTrue(len(data) >= 0)

    def test_update_benign_vet_table(self):
        data = update_benign_vet_table('')

        self.assertTrue(len(data) >= 0)

    def test_update_honeypot_vet_table(self):
        data = update_honeypot_vet_table('')

        self.assertTrue(len(data) >= 0)

    def test_update_blacklist_vet_table(self):
        data = update_blacklist_vet_table('')

        self.assertTrue(len(data) >= 0)

if '__name__' == '__main__':
    unittest.main()


text:
import matplotlib

matplotlib.use('Agg')
from utils.data_reader import Personas_CVAE
from model.common_layer import NoamOpt, evaluate
from utils import config
from model.CVAE.util.config import Model_Config
import torch
import torch.nn as nn
import numpy as np
from random import shuffle
from copy import deepcopy
import math
from tensorboardX import SummaryWriter
import os

os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
from model.CVAE.model import Model

def make_infinite(dataloader):
    while True:
        for x in dataloader:
            yield x

def make_infinite_list(personas):
    while True:
        print("New epoch(shuffle all personas)")
        shuffle(personas)
        for x in personas:
            yield x

def do_learning(model, train_iter, iterations):
    p, l = [], []
    for i in range(iterations):
        # print(train_iter.__next__())
        loss, ppl, _ = model.train_one_batch(train_iter.__next__())
        l.append(loss)
        p.append(ppl)
    return loss

def do_learning_early_stop(model, train_iter, val_iter, iterations, strict=1):
    # b_loss, b_ppl = do_evaluation(model, val_iter)
    b_loss, b_ppl = 100000, 100000
    best = deepcopy(model.state_dict())
    cnt = 0
    idx = 0
    for _, _ in enumerate(range(iterations)):
        train_l, train_p = [], []
        for d in train_iter:
            t_loss, t_ppl, _ = model.train_one_batch(d)
            train_l.append(t_loss)
            train_p.append(t_ppl)

        n_loss, n_ppl = do_evaluation(model, val_iter)
        ## early stopping
        if (n_ppl <= b_ppl):
            b_ppl = n_ppl
            b_loss = n_loss
            cnt = 0
            idx += 1
            best = deepcopy(model.state_dict())  ## save best weights
        else:
            cnt += 1
        if (cnt > strict): break

    ## load the best model
    model.load_state_dict({name: best[name] for name in best})

    return (np.mean(train_l), np.mean(train_p), b_loss, b_ppl), idx

def do_learning_fix_step(model, train_iter, val_iter, iterations, test=False):
    # fine-tuning and calculate loss
    ppl_list = []
    nll_list = []
    kld_list = []
    weight_list = []
    loss_sum = 0

    for _ in range(iterations):
        for batch in train_iter:
            model.train_one_batch(batch)

    i = 0
    for batch in val_iter:
        _, ppl, loss, nll_loss, kld_loss, kld_weight = model.train_one_batch(batch, train=False)
        loss_sum += loss
        ppl_list.append(ppl)
        nll_list.append(nll_loss)
        kld_list.append(kld_loss)
        weight_list.append(kld_weight)
        i = i + 1
    return loss_sum / i, np.mean(ppl_list), np.mean(nll_list), np.mean(kld_list), np.mean(weight_list)

def do_evaluation(model, test_iter):
    p, l, nll, kl, w = [], [], [], [], []
    for batch in test_iter:
        loss, ppl, _, nll_loss, kl_loss, weight = model.train_one_batch(batch, train=False)
        l.append(loss)
        p.append(ppl)
        nll.append(nll_loss)
        kl.append(kl_loss)
        w.append(weight)
    return np.mean(l), np.mean(p), np.mean(nll), np.mean(kl), np.mean(w)

# =================================main=================================

# Model config for CVAE-dialog
model_config = Model_Config()

# p = Personas()
p = Personas_CVAE()
writer = SummaryWriter(log_dir=config.save_path)
# Build model, optimizer, and set states
if not (config.load_frompretrain == 'None'):
    # meta_net = Transformer(p.vocab, model_file_path=config.load_frompretrain, is_eval=False)
    meta_net = Model(model_config, p.vocab)
else:
    # meta_net = Transformer(p.vocab)
    meta_net = Model(model_config, p.vocab)
    meta_net.print_parameters()  # 输出模型参数个数
    if config.USE_CUDA:
        meta_net.to('cuda')
if config.meta_optimizer == 'sgd':
    meta_optimizer = torch.optim.SGD(meta_net.parameters(), lr=config.meta_lr)
elif config.meta_optimizer == 'adam':
    meta_optimizer = torch.optim.Adam(meta_net.parameters(), lr=config.meta_lr)
elif config.meta_optimizer == 'noam':
    meta_optimizer = NoamOpt(config.hidden_dim, 1, 4000,
                             torch.optim.Adam(meta_net.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))
else:
    raise ValueError

global_step = 0
meta_batch_size = config.meta_batch_size
tasks = p.get_personas('train')
# tasks_loader = {t: p.get_data_loader(persona=t,batch_size=config.batch_size, split='train') for t in tasks}
tasks_iter = make_infinite_list(tasks)

# meta early stop
patience = 300
if config.fix_dialnum_train:
    patience = 300
# best_loss = 10000000
best_loss = 100.0
stop_count = 0
# Main loop
for meta_iteration in range(config.epochs):

    ## save original weights to make the update
    weights_original = deepcopy(meta_net.state_dict())
    train_loss_before = []
    train_loss_meta = []
    # loss accumulate from a batch of tasks
    batch_loss = 0
    batch_loss_before = 0
    for _ in range(meta_batch_size):
        # Get task
        if config.fix_dialnum_train:
            train_iter, val_iter = p.get_balanced_loader(persona=tasks_iter.__next__(), batch_size=config.batch_size,
                                                         split='train')
        else:
            train_iter, val_iter = p.get_data_loader(persona=tasks_iter.__next__(), batch_size=config.batch_size,
                                                     split='train')
        # before first update
        v_loss, v_ppl, v_nll, v_kl, v_w = do_evaluation(meta_net, val_iter)
        train_loss_before.append(v_ppl)
        batch_loss_before += v_loss
        # Update fast nets
        val_loss, val_ppl, _, _, _ = do_learning_fix_step(meta_net, train_iter, val_iter,
                                                          iterations=config.meta_iteration)
        train_loss_meta.append(val_ppl)
        batch_loss += val_loss
        # log

        # reset
        meta_net.load_state_dict({name: weights_original[name] for name in weights_original})

    writer.add_scalars('loss_before', {'train_loss_before': np.mean(train_loss_before)}, meta_iteration)
    writer.add_scalars('loss_meta', {'train_loss_meta': np.mean(train_loss_meta)}, meta_iteration)
    # print(' loss_before: {}'.format(np.mean(train_loss_before)))
    # print(' loss_meta: {}'.format(np.mean(np.mean(train_loss_meta))))

    # meta_net.optim.update_lr(global_step)  # 每个epoch更新学习率

    # meta Update
    if (config.meta_optimizer == 'noam'):
        meta_optimizer.optimizer.zero_grad()
    else:
        meta_optimizer.zero_grad()

    batch_loss_before /= meta_batch_size
    batch_loss /= meta_batch_size
    batch_loss.backward()
    meta_net.global_step += 1
    print('train_loss: {:.3f} ===> {:.3f}'.format(batch_loss_before, batch_loss))

    # clip gradient
    nn.utils.clip_grad_norm_(meta_net.parameters(), config.max_grad_norm)
    meta_optimizer.step()

    ## Meta-Evaluation
    if meta_iteration % 10 == 0:
        print('--------- Validation ---------')
        print('meta_iteration:', meta_iteration)
        val_loss_before = []
        val_loss_meta = []
        val_batch_loss_before = 0
        val_batch_loss_after = 0
        val_nll_loss_before = 0
        val_nll_loss_after = 0
        val_kld_loss_before = 0
        val_kld_loss_after = 0
        val_kld_weight_before = 0
        val_kld_weight_after = 0
        weights_original = deepcopy(meta_net.state_dict())
        for idx, per in enumerate(p.get_personas('valid')):
            # num_of_dialog = p.get_num_of_dialog(persona=per, split='valid')
            # for dial_i in range(num_of_dialog):
            if config.fix_dialnum_train:
                train_iter, val_iter = p.get_balanced_loader(persona=per, batch_size=config.batch_size, split='valid',
                                                             fold=0)

            else:
                train_iter, val_iter = p.get_data_loader(persona=per, batch_size=config.batch_size, split='valid',
                                                         fold=0)
            # zero shot result
            loss, ppl, nll, kld, w = do_evaluation(meta_net, val_iter)
            val_loss_before.append(math.exp(loss))
            val_batch_loss_before += loss
            val_nll_loss_before += nll
            val_kld_loss_before += kld
            val_kld_weight_before += w
            # mate tuning
            val_loss, val_ppl, val_nll, val_kld, val_weight = do_learning_fix_step(meta_net, train_iter, val_iter,
                                                                                iterations=config.meta_iteration)
            val_loss_meta.append(math.exp(val_loss.item()))

            val_batch_loss_after += val_loss
            val_nll_loss_after += val_nll
            val_kld_loss_after += val_kld
            val_kld_weight_after += val_weight
            # updated result

            meta_net.load_state_dict({name: weights_original[name] for name in weights_original})

        writer.add_scalars('loss_before', {'val_loss_before': np.mean(val_loss_before)}, meta_iteration)
        writer.add_scalars('loss_meta', {'val_loss_meta': np.mean(val_loss_meta)}, meta_iteration)

        val_batch_loss_before /= 100
        val_nll_loss_before /= 100
        val_kld_loss_before /= 100
        val_kld_weight_before /= 100

        val_batch_loss_after /= 100
        val_nll_loss_after /= 100
        val_kld_loss_after /= 100
        val_kld_weight_after /= 100

        print('val_loss: {:.3f} ===> {:.3f}'.format(val_batch_loss_before, val_batch_loss_after))
        print('zero-shot: {:.3f} = {:.3f} + {:.3f} * {:.3f}'.format(
            val_batch_loss_before,
            val_nll_loss_before,
            val_kld_weight_before,
            val_kld_loss_before))
        print('few-shot: {:.3f} = {:.3f} + {:.3f} * {:.3f}'.format(
            val_batch_loss_after,
            val_nll_loss_after,
            val_kld_weight_after,
            val_kld_loss_after))

        # check early stop
        # if np.mean(val_loss_meta) < best_loss:
        if val_batch_loss_after < best_loss:
            # best_loss = np.mean(val_loss_meta)
            best_loss = val_batch_loss_after
            stop_count = 0
            meta_net.save_model(best_loss, meta_iteration, global_step)
        else:
            stop_count += 1
            print('remain patience: {}'.format(patience - stop_count))
            if stop_count > patience:
                break
        print('------------------------------')

max_stars_repo_name: Guaguago/PAML

max_issues_repo_name: Guaguago/PAML

max_forks_repo_name: Guaguago/PAML


text:
'''
    Face Detector based on CAFFE Model ( DNN )

    Author  : Viki (a) Vignesh Natarajan
    Contact : vikiworks.io
'''

import cv2
import time
from imutils.video import VideoStream
import numpy as np
import imutils

neural_network = cv2.dnn.readNetFromCaffe("deploy.prototxt.txt", "res10_300x300_ssd_iter_140000.caffemodel")

def get_video_from_webcam():
    return VideoStream(src=0).start()

def get_image_frame(video_stream):
    return video_stream.read()

def resize_image(image, width):
    return imutils.resize(image, width=width)

def image2blob(image):
    return cv2.dnn.blobFromImage(cv2.resize(image, (300, 300)), 1.0,(300, 300), (104.0, 177.0, 123.0))

def detect_faces(image_blob):
    neural_network.setInput(image_blob)
    return neural_network.forward()

def add_label(image, label, x, y):
    cv2.putText(image, label, (x, y), cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 0, 255), 2)

def draw_bounding_box(image_frame, faces):
    (h, w) = image_frame.shape[:2]
    for i in range(0, faces.shape[2]):
        accuracy = faces[0, 0, i, 2]

        if accuracy < 0.5:
            continue

        box = faces[0, 0, i, 3:7] * np.array([w, h, w, h])
        (startX, startY, endX, endY) = box.astype("int")

        y = startY - 10
        if y < 10:
            y = startY + 10

        cv2.rectangle(image_frame, (startX, startY), (endX, endY),(0, 0, 255), 2)

        label = "{:.2f}%".format(accuracy * 100)
        add_label(image_frame, label, startX, y)

    return image_frame

def is_exit_key_pressed():
    key = cv2.waitKey(30) & 0xff
    if key == 27:
        return True
    else:
        return False

def display_image(image):
    cv2.imshow('Frame', image)

def main():
    video_stream = get_video_from_webcam()
    time.sleep(1.0)

    while True:
        image_frame = get_image_frame(video_stream)
        image_frame = resize_image(image_frame, 400)

        blob = image2blob(image_frame)

        faces = detect_faces(blob)

        image_frame = draw_bounding_box(image_frame, faces)

        display_image(image_frame)

        if is_exit_key_pressed() == True:
            cv2.destroyAllWindows()
            video_stream.stop()

main()


text:
from typing import Any

import numpy as np
from gym import spaces
from habitat.config import Config
from habitat.core.registry import registry
from habitat.core.simulator import Sensor, SensorTypes, Simulator
from habitat.sims.habitat_simulator.actions import HabitatSimActions
from habitat.tasks.nav.shortest_path_follower import ShortestPathFollower

from habitat_extensions.shortest_path_follower import ShortestPathFollowerCompat

@registry.register_sensor(name="GlobalGPSSensor")
class GlobalGPSSensor(Sensor):
    r"""The agents current location in the global coordinate frame

    Args:
        sim: reference to the simulator for calculating task observations.
        config: Contains the DIMENSIONALITY field for the number of dimensions
                to express the agents position
    Attributes:
        _dimensionality: number of dimensions used to specify the agents position
    """

    def __init__(self, sim: Simulator, config: Config, *args: Any, **kwargs: Any):
        self._sim = sim
        self._dimensionality = getattr(config, "DIMENSIONALITY", 2)
        assert self._dimensionality in [2, 3]
        super().__init__(config=config)

    def _get_uuid(self, *args: Any, **kwargs: Any):
        return "globalgps"

    def _get_sensor_type(self, *args: Any, **kwargs: Any):
        return SensorTypes.POSITION

    def _get_observation_space(self, *args: Any, **kwargs: Any):
        return spaces.Box(
            low=np.finfo(np.float32).min,
            high=np.finfo(np.float32).max,
            shape=(self._dimensionality,),
            dtype=np.float32,
        )

    def get_observation(self, *args: Any, observations, episode, **kwargs: Any):
        agent_position = self._sim.get_agent_state().position
        return agent_position.astype(np.float32)

@registry.register_sensor
class VLNOracleActionSensor(Sensor):
    r"""Sensor for observing the optimal action to take. The assumption this
    sensor currently makes is that the shortest path to the goal is the
    optimal path.

    Args:
        sim: reference to the simulator for calculating task observations.
        config: config for the sensor.
    """

    def __init__(self, sim: Simulator, config: Config, *args: Any, **kwargs: Any):
        super().__init__(config=config)

        # all goals can be navigated to within 0.5m.
        goal_radius = getattr(config, "GOAL_RADIUS", 0.5)
        if config.USE_ORIGINAL_FOLLOWER:
            self.follower = ShortestPathFollowerCompat(
                sim, goal_radius, return_one_hot=False
            )
            self.follower.mode = "geodesic_path"
        else:
            self.follower = ShortestPathFollower(sim, goal_radius, return_one_hot=False)

    def _get_uuid(self, *args: Any, **kwargs: Any):
        return "vln_oracle_action_sensor"

    def _get_sensor_type(self, *args: Any, **kwargs: Any):
        return SensorTypes.TACTILE

    def _get_observation_space(self, *args: Any, **kwargs: Any):
        return spaces.Box(low=0.0, high=100, shape=(1,), dtype=np.float)

    def get_observation(self, observations, *args: Any, episode, **kwargs: Any):
        best_action = self.follower.get_next_action(episode.goals[0].position)
        return np.array(
            [best_action if best_action is not None else HabitatSimActions.STOP]
        )

@registry.register_sensor
class VLNOracleProgressSensor(Sensor):
    r"""Sensor for observing how much progress has been made towards the goal.

    Args:
        sim: reference to the simulator for calculating task observations.
        config: config for the sensor.
    """

    def __init__(self, sim: Simulator, config: Config, *args: Any, **kwargs: Any):
        self._sim = sim
        super().__init__(config=config)

    def _get_uuid(self, *args: Any, **kwargs: Any):
        return "progress"

    def _get_sensor_type(self, *args: Any, **kwargs: Any):
        # TODO: what is the correct sensor type?
        return SensorTypes.MEASUREMENT

    def _get_observation_space(self, *args: Any, **kwargs: Any):
        return spaces.Box(low=0.0, high=1.0, shape=(1,), dtype=np.float)

    def get_observation(self, observations, *args: Any, episode, **kwargs: Any):
        current_position = self._sim.get_agent_state().position.tolist()

        distance_to_target = self._sim.geodesic_distance(
            current_position, episode.goals[0].position
        )

        distance_from_start = episode.info["geodesic_distance"]

        return (distance_from_start - distance_to_target) / distance_from_start

max_stars_repo_name: awesomericky/VLN-CE

max_issues_repo_name: awesomericky/VLN-CE

max_forks_repo_name: awesomericky/VLN-CE


text:
import numpy as np
from matplotlib import pyplot as plt
import pickle

def read():
    file = open('../../data/real/garch_cost', 'rb')
    data = np.array(pickle.load(file))
    dataf = [df.seconds + df.microseconds / 1000000 for df in data.flatten()]
    return np.array(dataf).reshape(data.shape)

def main():
    data = read().sum(axis=0)
    print(data)
    group = [[0], [1, 2, 3], [1, 2, 4], [1, 2, 5], [1, 2, 6, 7], [1, 2, 6, 8]]
    data = np.array([data[:, gp].sum(axis=1) for gp in group])
    labels = ['IIS', 'NIS', 'MIS$^*$', 'MIS', 'RIS', 'MLE']
    scenarios = ['(1, 0.05)', '(1, 0.01)', '(2, 0.05)', '(2, 0.01)', '(5, 0.05)', '(5, 0.01)']
    plt.style.use('ggplot')
    fig, ax = plt.subplots(figsize=[5, 3])
    for i, dat in enumerate(data):
        ax.plot(scenarios, dat, label=labels[i])

    ax.legend(loc=2)
    fig.tight_layout()
    fig.show()
    with open('../../data/real/time', 'wb') as file:
        pickle.dump(data, file)

if __name__ == '__main__':
    main()

max_stars_repo_name: IanFla/Importance-Sampling

max_issues_repo_name: IanFla/Importance-Sampling

max_forks_repo_name: IanFla/Importance-Sampling


text:
import numpy as np

import SpoutGL
from OpenGL import GL
from itertools import repeat
import array

SENDER_NAME = "SpoutGL-test"

with SpoutGL.SpoutReceiver() as receiver:
    receiver.setReceiverName(SENDER_NAME)

    buffer = None
    shared_memory = None

    while True:
        result = receiver.receiveImage(buffer, GL.GL_RGBA, False, 0)

        if receiver.isUpdated():
            width = receiver.getSenderWidth()
            height = receiver.getSenderHeight()
            buffer = array.array('B', repeat(0, width * height * 4))

        if buffer and result and not SpoutGL.helpers.isBufferEmpty(buffer):
            print("Got bytes", bytes(buffer[0:64]), "...")

        # Wait until the next frame is ready
        # Wait time is in milliseconds; note that 0 will return immediately
        receiver.waitFrameSync(SENDER_NAME, 10000)
        print("sync received")

        if shared_memory is None:
            memory_length = receiver.getMemoryBufferSize(SENDER_NAME)
            shared_memory = array.array('B', repeat(0, memory_length))

        result = receiver.readMemoryBuffer(SENDER_NAME, shared_memory, len(shared_memory))
        if result:
            message = bytes(shared_memory).decode().split("\x00")[0]
            print(message)

max_stars_repo_name: jlai/Python-SpoutGL

max_issues_repo_name: jlai/Python-SpoutGL

max_forks_repo_name: jlai/Python-SpoutGL


text:
# View more 3_python 2_tensorflow_old on my Youtube and Youku channel!!!

# Youtube video tutorial: https://www.youtube.com/channel/UCdyjiB5H8Pu7aDTNVXTTpcg
# Youku video tutorial: http://i.youku.com/pythontutorial

"""
Please note, this code is only for 3_python 3+. If you are using 3_python 2+, please modify the code accordingly.
"""
from __future__ import print_function
import pandas as pd
import numpy as np

# concatenating
# ignore index
df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d'])
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['a','b','c','d'])
df3 = pd.DataFrame(np.ones((3,4))*2, columns=['a','b','c','d'])
res = pd.concat([df1, df2, df3], axis=0, ignore_index=True)

# join, ('inner', 'outer')
df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d'], index=[1,2,3])
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['b','c','d', 'e'], index=[2,3,4])
res = pd.concat([df1, df2], axis=1, join='outer')
res = pd.concat([df1, df2], axis=1, join='inner')

# join_axes
res = pd.concat([df1, df2], axis=1, join_axes=[df1.index])

# append
df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d'])
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['a','b','c','d'])
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['b','c','d', 'e'], index=[2,3,4])
res = df1.append(df2, ignore_index=True)
res = df1.append([df2, df3])

s1 = pd.Series([1,2,3,4], index=['a','b','c','d'])
res = df1.append(s1, ignore_index=True)

print(res)


text:
import os
from collections import Counter, defaultdict, deque

import numpy as np
import matplotlib.pyplot as plt

sym_splitter = " "
ref_splitter = "_"
alphabet = "abcdefghijklmnopqrstuvwxyzåäö"
alphabet = alphabet + alphabet.upper()

if 1:
    enc_alphabet = "abcdefghijklmnopqrstuvwxyz"
    enc_alphabet = enc_alphabet + enc_alphabet.upper()
    enc_alphabet = enc_alphabet + ".:,;!%&/()=@${[]}^~'*<>|-`\\"
    enc_alphabet = "0123456789" + enc_alphabet
else:
    enc_alphabet = "0123456789"
b = len(enc_alphabet)

root = "0"
symbol_set = set([])

class WZ_Model:
    def __init__(self):
        self.S = {}
        self.G = defaultdict(dict)
        self.s = []
        self.rewind = 0
        self.top = 0
        self.symbol_set = set([]) #symbol_set
        self.prime_with_symbol_set()

    def prime_with_symbol_set(self):
        for a in self.symbol_set:
            self.update(root, a, sym_splitter)

    def get_s(self):
        return self.s[len(self.symbol_set):]

    def get(self, seq, a):
        return self.G[seq][a]

    def update(self, seq, a, splitter):
        self.top += 1
        top = alph(self.top)
        self.S[top] = (seq, a)
        self.G[seq][a] = top

        for _ in range(self.rewind):
            self.s.pop() # remove last elements
        self.rewind = 0

        s = list(self.climb_to_root(seq))
        if splitter == ref_splitter:
            self.rewind = self.get_rewind("".join(s))
        self.s.extend(s)
        self.s.append(a)

        return seq, a, splitter

    def climb_to_root(self, seq):
        stack = []
        while seq != root:
            seq, a = self.S[seq]
            stack.append(a)
        return reversed(stack)

    def get_rewind(self, s):
        if all([ch in alphabet for ch in s]):
            return 0

        m = s.rstrip(alphabet)
        return len(s) - len(m) + 1

    def can_be_stripped(self, seq):
        s = "".join(self.climb_to_root(seq))
        rewind = self.get_rewind(s)
        return rewind

    def longest_match(self, s):
        """Finds the longest match in the phrase tree
        of a string
        """
        match = []
        seq = root
        for a in s:
            try:
                seq = self.G[seq][a]
            except KeyError:
                break
            match.append(a)
        return "".join(match)

    def get_seq(self, s):
        """
        Get the phrase tree sequence ID that corresponds
        to a certain string
        Raises a KeyError if s is not in phrase tree
        """
        seq = root
        for a in s:
            seq = self.G[seq][a]
        return seq

def encode(reader):
    global symbol_set

    seq = root
    seq_old = root
    s = reader.read()
    symbol_set = set(s)
    model = WZ_Model()
    N = len(s)
    idx = 0
    match = []

    while idx < N:
        a = s[idx]
        seq_old = seq

        try:
            seq = model.get(seq, a)
            match.append(a)
        except KeyError:
            rewind = model.can_be_stripped(seq)
            if rewind > 0:
                splitter = ref_splitter
                idx -= rewind
                #print("match:", match)
                #print("match[:-rewind]:", match[:-rewind])
                seq_len = len(model.get_seq(match[:-rewind]))
            else:
                splitter = sym_splitter
                seq_len = len(seq)
            yield seq, a, splitter, seq_len
            model.update(seq, a, splitter)
            seq = root
            match = []

        idx += 1

    if seq != root:
        yield seq_old, a, sym_splitter, len(seq_old)
        model.update(seq_old, a, sym_splitter)

    #poll_phrase_tree(model)
    #print("end of encoding")
    #print("G encode:", model.G)
    #print("S encode:", model.S)
    #print(model.s)

def decode(reader):
    model = WZ_Model()
    reader = iter(reader.read())

    queue = deque(maxlen=100)

    while True:
        try:
            seq, splitter = get_id(reader)
        except StopIteration:
            break
        a = next(reader)
        model.update(seq, a, splitter)
        match = "".join(model.climb_to_root(seq)) + a
        queue.append(match)

    yield from model.get_s()

    #for m in queue:
    #    print(m)
    #print("G decode:", model.G)
    #print("S encode:", model.S)
    #print(model.s)

def get_id(reader):
    id_num = []
    while True:
        a = next(reader)
        #print("digit", a)
        if a not in enc_alphabet:
            return "".join(id_num), a
        else:
            id_num.append(a)

def alph(seq):
    """
    Convert an int in base-10 to
    base-whatever length the alphabet is
    Assumes 0 is the 0
    """
    if not seq:
        return root
    s = []
    quot = seq
    while quot:
        quot, rest = quot // b, quot % b
        s.append(enc_alphabet[rest])
    return "".join(reversed(s))

def test_alph():
    for seq in range(11):
        print("alph({}): {}".format(seq, alph(seq)))

default_dbg = False
def debug(*s, dbg=False):
    if default_dbg or dbg:
        print(*s)

def poll_phrase_tree(model):
    strings = ["Nils", "Holgersson", "Akka",
               "Yksi", "Kaksi", "Kolme",
               "Neljä", "Viisi", "Kuusi",
               "vildgåsflock"]

    for s in strings:
        match = model.longest_match(s)
        print()
        print("s:", s)
        print("m:", match)

def main():
    fn = "/home/jdw/garageofcode/data/compression/big.txt"
    #fn = "/home/jdw/garageofcode/data/compression/words.txt"
    #fn = "/home/jdw/garageofcode/data/compression/nilsholg2.txt"
    #fn = "/home/jdw/garageofcode/data/compression/nilsholg.txt"
    #fn = "/home/jdw/garageofcode/data/compression/medium.txt"
    #fn = "short.txt"
    #fn = "veryshort.txt"
    fn_compressed = fn.split(".")[0] + ".wzip"
    fn_reconstructed = fn.split(".")[0] + "_rec.txt"
    fn_zip = fn.split(".")[0] + ".zip"
    # encoding step
    reduced_len = 0
    with open(fn, "r") as r:
        with open(fn_compressed, "w") as f:
            for idx, (seq, a, spl, seq_len) in enumerate(encode(r)):
                f.write("{}{}{}".format(seq, spl, a))
                reduced_len += (seq_len + 1)

    num_seqs = idx
    print("Before ", os.stat(fn).st_size)
    print("After  ", os.stat(fn_compressed).st_size)
    print("LZW opt  ", os.stat(fn_compressed).st_size - num_seqs)
    print("Impl. opt  ", reduced_len)
    print(".zip ", os.stat(fn_zip).st_size)

    #  exit()

    with open(fn_compressed, "r") as r:
        with open(fn_reconstructed, "w") as f:
            for a in decode(r):
                #if a == "\\n":
                #    f.write("\n")
                #else:
                f.write(a)

    #exit()
    res = os.system("diff {} {}".format(fn, fn_reconstructed))
    if res:
        print("Reconstruction: failed")
    else:
        print("Reconstruction: OK")

if __name__ == '__main__':
    main()

    #test_alph()

max_issues_repo_name: jonatanwestholm/garageofcode

max_forks_repo_name: jonatanwestholm/garageofcode