text:
import random

import cv2  # https://www.jianshu.com/p/f2e88197e81d
import numpy as np
import torch
import torch.utils.data
from skimage import color
from skimage.io import imread
from torchvision import datasets, models, transforms

class Dataset(torch.utils.data.Dataset):

    def __init__(self, img_paths, mask_paths, transform=None):
        self.img_paths = img_paths
        self.mask_paths = mask_paths
        self.transform = transform

    def __len__(self):
        return len(self.img_paths)

    def __getitem__(self, idx):
        img_path = self.img_paths[idx]
        mask_path = self.mask_paths[idx]
        #读numpy数据(npy)的代码
        npimage = np.load(img_path)
        npmask = np.load(mask_path)

        npimage = npimage.transpose((2, 0, 1))

        liver_label = npmask.copy()
        liver_label[npmask == 2] = 1
        liver_label[npmask == 1] = 1

        tumor_label = npmask.copy()
        tumor_label[npmask == 1] = 0
        tumor_label[npmask == 2] = 1

        nplabel = np.empty((448,448,2))
        nplabel[:, :, 0] = liver_label
        nplabel[:, :, 1] = tumor_label
        nplabel = nplabel.transpose((2, 0, 1))

        nplabel = nplabel.astype("float32")
        npimage = npimage.astype("float32")

        return npimage,nplabel

max_stars_repo_name: hanyang-sh/LITS-liver-tumor

max_issues_repo_name: hanyang-sh/LITS-liver-tumor

max_forks_repo_name: hanyang-sh/LITS-liver-tumor


text:
from nose.tools import *
import networkx as nx

class TestAtlas(object):
    def setUp(self):
        self.GAG=nx.graph_atlas_g()

    def test_sizes(self):
        G=self.GAG[0]
        assert_equal(G.number_of_nodes(),0)
        assert_equal(G.number_of_edges(),0)

        G=self.GAG[7]
        assert_equal(G.number_of_nodes(),3)
        assert_equal(G.number_of_edges(),3)

    def test_names(self):
        i=0
        for g in self.GAG:
            name=g.name
            assert_equal(int(name[1:]),i)
            i+=1

    def test_monotone_nodes(self):
        # check for monotone increasing number of nodes
        previous=self.GAG[0]
        for g in self.GAG:
            assert_false(len(g)-len(previous) > 1)
            previous=g.copy()

    def test_monotone_nodes(self):
        # check for monotone increasing number of edges
        # (for fixed number of nodes)
        previous=self.GAG[0]
        for g in self.GAG:
            if len(g)==len(previous):
                assert_false(g.size()-previous.size() > 1)
            previous=g.copy()

    def test_monotone_degree_sequence(self):
        # check for monotone increasing degree sequence
        # (for fixed number f nodes and edges)
        # note that 111223 < 112222
        previous=self.GAG[0]
        for g in self.GAG:
            if len(g)==0:
                continue
            if len(g)==len(previous) & g.size()==previous.size():
                deg_seq=sorted(g.degree().values())
                previous_deg_seq=sorted(previous.degree().values())
                assert_true(previous_deg_seq < deg_seq)
                previous=g.copy()

ext: py

max_stars_repo_name: kushgrover/apt-vs-dift

max_issues_repo_name: kushgrover/apt-vs-dift

max_forks_repo_name: kushgrover/apt-vs-dift

reason: import networkx


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
nc2ascii.py

Extract synthetics from a NetCDF waveform database created by AxiSEM3D
(named axisem3d_synthetics.nc by the solver) and save them in ascii format.

To see usage, type
python nc2ascii.py -h
'''

################### PARSER ###################

aim = '''Extract synthetics from a NetCDF waveform database created by AxiSEM3D
(named axisem3d_synthetics.nc by the solver) and save them in ascii format.'''

notes = '''String replacement rules for filename, header and footer:
  @NW@ -> network name
  @ST@ -> station name
  @CH@ -> channel
  @NS@ -> number of steps
  @DT@ -> time step
  @SR@ -> sampling rate
  @T0@ -> start time w.r.t. source origin
  @T1@ -> end time w.r.t. source origin

'''

import argparse
from argparse import RawTextHelpFormatter
parser = argparse.ArgumentParser(description=aim, epilog=notes,
                                 formatter_class=RawTextHelpFormatter)
parser.add_argument('-i', '--input', dest='in_nc_file', action='store',
                    type=str, required=True,
                    help='NetCDF waveform database created by AxiSEM3D\n' +
                         '<required>')
parser.add_argument('-o', '--output', dest='out_ascii_dir', action='store',
                    type=str, required=True,
                    help='directory to store the ascii output files;\n' +
                         '<required>')
parser.add_argument('-s', '--stations', dest='stations', action='store',
                    nargs='+', type=str, default=['*.*'],
                    help='stations to be extracted, given as a\n' +
                         'list of regexes of "Network.Station";\n' +
                         'default = *.* (all stations)')
parser.add_argument('-c', '--channels', dest='channels', action='store',
                    type=str, default='ENZRTSP',
                    help='channels to be extracted, depending on\n' +
                         'OUT_STATIONS_COMPONENTS in inparam.time_src_recv;\n' +
                         'default = ENZRTSP (all channels)')
parser.add_argument('-e', '--numfmt', dest='number_format', action='store',
                    type=str, default='%.6e',
                    help='output number format;\n' +
                         'default = %%.6e')
parser.add_argument('-n', '--fnamefmt', dest='fname_format', action='store',
                    type=str, default='@NW@.@ST@..BH@CH@.ascii',
                    help='output filename format;\n' +
                         'default = @NW@.@ST@..BH@CH@.ascii')
parser.add_argument('-H', '--headerfmt', dest='header_format', action='store',
                    type=str, default='',
                    help='format of the header to be placed\n' +
                         'at the beginning of each ascii file;\n' +
                         'default = ""')
parser.add_argument('-F', '--footerfmt', dest='footer_format', action='store',
                    type=str, default='',
                    help='format of the footer to be placed\n' +
                         'at the end of each ascii file;\n' +
                         'default = ""')
parser.add_argument('-p', '--nproc', dest='nproc', action='store',
                    type=int, default=1,
                    help='number of processors; default = 1')
parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
                    help='verbose mode')
args = parser.parse_args()

################### PARSER ###################

import numpy as np
from netCDF4 import Dataset
import fnmatch, os, shutil
from multiprocessing import Pool

# open file
ncdf = Dataset(args.in_nc_file, 'r')

# time info
vartime = ncdf.variables['time_points'][:]
nsteps = len(vartime)
strnsteps = str(nsteps)
assert nsteps > 0, 'Zero time steps'
if nsteps == 1:
    dt = 'n.a.'
    sampling_rate = 'n.a.'
else:
    dt = str(vartime[1] - vartime[0])
    sampling_rate = str(1. / (vartime[1] - vartime[0]))
tstart = str(vartime[0])
tend = str(vartime[-1])

if args.verbose:
    print('--- Time info ---')
    print('Number of steps: ' + strnsteps)
    print('Time step: ' + dt)
    print('Sampling rate: ' + sampling_rate)
    print('Start time: ' + tstart)
    print('End time: ' + tend)
    print()

# create output directory
try:
    os.makedirs(args.out_ascii_dir)
except OSError:
    pass

# names
fname = args.fname_format.replace('@NS@', strnsteps)
fname = fname.replace('@DT@', dt)
fname = fname.replace('@SR@', sampling_rate)
fname = fname.replace('@T0@', tstart)
fname = fname.replace('@T1@', tend)

header = args.header_format.replace('@NS@', strnsteps)
header = header.replace('@DT@', dt)
header = header.replace('@SR@', sampling_rate)
header = header.replace('@T0@', tstart)
header = header.replace('@T1@', tend)

footer = args.footer_format.replace('@NS@', strnsteps)
footer = footer.replace('@DT@', dt)
footer = footer.replace('@SR@', sampling_rate)
footer = footer.replace('@T0@', tstart)
footer = footer.replace('@T1@', tend)

# stations
stations = []
for var in ncdf.variables:
    if var == 'time_points':
        continue

    # station info
    varlist = var.split('.')
    assert len(varlist) == 3
    network = var.split('.')[0]
    station = var.split('.')[1]
    synchannels = var.split('.')[2]

    # match station regexes
    key = network + '.' + station
    for regex in args.stations:
        if fnmatch.fnmatch(key, regex):
            stations.append([network, station, var])
            break

# close
ncdf.close()

# extract waveforms
nstations = len(stations)
if args.verbose:
    print('--- Extracting Waveforms at %d Stations ---' % (nstations))

def write_ascii(iproc):
    if args.nproc == 1:
        ncdf = Dataset(args.in_nc_file, 'r')
        iproc = 0
    else:
        # copy netcdf file for parallel access
        tempnc = args.out_ascii_dir + '/nc_temp.nc' + str(iproc)
        shutil.copy(args.in_nc_file, tempnc)
        ncdf = Dataset(tempnc, 'r')

    for ist, st in enumerate(stations):
        if ist % args.nproc != iproc:
            continue

        # station info
        network = st[0]
        station = st[1]
        var = st[2]

        # names
        fname_st = fname.replace('@NW@', network)
        fname_st = fname_st.replace('@ST@', station)
        header_st = header.replace('@NW@', network)
        header_st = header_st.replace('@ST@', station)
        footer_st = footer.replace('@NW@', network)
        footer_st = footer_st.replace('@ST@', station)

        # data
        wave = ncdf.variables[var][:, :]

        # waveform
        for channel in args.channels:
            dim = synchannels.find(channel)
            if dim < 0:
                continue
            # names
            fname_stc = fname_st.replace('@CH@', channel)
            header_stc = header_st.replace('@CH@', channel)
            footer_stc = footer_st.replace('@CH@', channel)

            # save to ascii
            fname_stc = args.out_ascii_dir + '/' + fname_stc
            np.savetxt(fname_stc, wave[:, dim],
                fmt=args.number_format,
                header=header_stc, footer=footer_stc, comments='')
            if args.verbose:
                print('Done with ascii file = %s; no. station = %d / %d; iproc = %d' \
                    % (fname_stc, ist + 1, nstations, iproc))

    # close
    ncdf.close()
    # remove temp nc
    if args.nproc > 1:
        os.remove(tempnc)

# write_ascii in parallel
args.nproc = max(args.nproc, 1)
if args.nproc == 1:
    write_ascii(0)
else:
    with Pool(args.nproc) as p:
        p.map(write_ascii, range(0, args.nproc))

reason: import numpy


text:
from __future__ import print_function, division, absolute_import

__author__ = 'Tatiana Likhomanenko'

from rep import utils
import numpy
import pandas

def test_calc():
    prediction = numpy.random.random(10000)
    iron = utils.Flattener(prediction)
    assert numpy.allclose(numpy.histogram(iron(prediction), normed=True, bins=30)[0], numpy.ones(30), rtol=1e-02)

    x, y, yerr, xerr = utils.calc_hist_with_errors(iron(prediction), bins=30, x_range=(0, 1))
    assert numpy.allclose(y, numpy.ones(len(y)), rtol=1e-02)
    width = 1. / 60
    means = numpy.linspace(width, 1 - width, 30)
    assert numpy.allclose(x, means)
    assert numpy.allclose(xerr, numpy.zeros(len(xerr)) + width)
    assert numpy.allclose(yerr, numpy.zeros(len(yerr)) + yerr[0], rtol=1e-2)

    random_labels = numpy.random.choice(2, size=10000)
    (tpr, tnr), _, _ = utils.calc_ROC(prediction, random_labels)

    # checking for random classifier
    assert numpy.max(abs(1 - tpr - tnr)) < 0.05

    # checking efficiencies for random mass, random prediction
    mass = numpy.random.random(10000)
    result = utils.get_efficiencies(prediction, mass)
    for threshold, (xval, yval) in result.items():
        assert ((yval + threshold - 1) ** 2).mean() < 0.1

def test_train_test_split_group():
    data = list(range(50)) * 2
    group_column = list(range(50)) * 2
    train, test = utils.train_test_split_group(group_column, data)
    assert len(set.intersection(set(test), set(train))) == 0

def test_corr_coeff_with_weights(n_samples=1000):
    """
    testing that corrcoeff with equal weights works as default.
    """
    weights = numpy.ones(n_samples)
    df = pandas.DataFrame(data=numpy.random.random([n_samples, 10]))
    z1 = numpy.corrcoef(df.values.T)
    z2 = utils.calc_feature_correlation_matrix(df)
    z3 = utils.calc_feature_correlation_matrix(df, weights=weights)
    assert numpy.allclose(z1, z2)
    assert numpy.allclose(z1, z3)

def test_get_columns(n_samples=10000):
    x = numpy.random.random([n_samples, 3])
    df = pandas.DataFrame(x, columns=['a', 'b', 'c'])

    result = utils.get_columns_in_df(df, ['a: a-b+b', 'b: b + 0 * c** 2.', 'c: c + 1 + c * (b - b)'])
    result['c'] -= 1
    assert not numpy.allclose(result, df + 1e-3), 'test not working'

    assert numpy.allclose(result, df), 'result of evaluation is incorrect'

def test_weighted_quantile(size=10000):
    x = numpy.random.normal(size=size)
    weights = numpy.random.random(size=size)
    quantile_level = numpy.random.random()

    quantile_value = utils.weighted_quantile(x, quantile_level, sample_weight=weights)

    passed_weight = numpy.sum((x < quantile_value) * weights)
    expected_weight = quantile_level * numpy.sum(weights)
    assert numpy.abs(passed_weight - expected_weight) < 1.1, 'wrong cut'

max_stars_repo_name: vkuznet/rep

max_issues_repo_name: vkuznet/rep

max_forks_repo_name: vkuznet/rep


text:
import tensorflow as tf
import dyn.model_attention as M
import network as N
import numpy as np

class AgingNet():
	def __init__(self,training=True):
		N.bn_training = training
		self.inpholder = tf.placeholder(tf.float32,[None,128,128,3])
		self.feat = N.feat_encoder(self.inpholder)

		self.age_features = []

		self.recon_features = []
		self.recon_imgs = []

		self.dis_scores = []
		self.dis_feat_scores = []
		self.dis_scores_real = []
		self.dis_feat_scores_real = []

		for i in range(10):
			age_feat = N.age_encoder(self.feat,i)
			self.age_features.append(age_feat)

			recon_img = N.generator(age_feat)
			self.recon_imgs.append(recon_img)

			recon_feature = N.feat_encoder(recon_img)
			self.recon_features.append(recon_feature)

			dis_scr = N.discriminator(recon_img)
			self.dis_scores.append(dis_scr)

			feat_dis_scr = N.discriminator_feature(recon_feature)
			self.dis_feat_scores.append(feat_dis_scr)

			dis_scr_real = N.discriminator(self.inpholder)
			self.dis_scores_real.append(dis_scr_real)

			feat_dis_scr_real = N.discriminator_feature(self.feat)
			self.dis_feat_scores_real.append(feat_dis_scr_real)

		self.age_features = tf.stack(self.age_features,1)
		self.img_feature = N.attention_blk(self.age_features)

	def load_model(self,path='./model/'):
		self.sess = tf.Session()
		M.loadSess(path,self.sess)

	def eval(self,data,normalize=True):
		if normalize:
			data = np.float32(data) / 127.5 - 1.
		else:
			data = np.float32(data)
		if len(data.shape)==3:
			data = [data]
		if not self.sess:
			self.load_model()
		res = self.sess.run(self.img_feature,feed_dict={self.inpholder:data})
		return res

def siamese_aging_net():
	def __init__(self):
		# build nets
		left_net = AgingNet()
		right_net = AgingNet()

		# build losses
		left_dis = self.dis_loss(left_net.dis_scores,False)
		left_dis_feat = self.dis_loss(left_net.dis_feat_scores,False)
		left_dis2 = self.dis_loss(left_net.dis_scores,True)
		left_dis_feat2 = self.dis_loss(left_net.dis_feat_scores,True)
		left_dis_real = self.dis_loss(left_net.dis_scores_real,True)
		left_dis_feat_real = self.dis_loss(left_net.dis_feat_scores_real,True)

		right_dis = self.dis_loss(right_net.dis_scores,False)
		right_dis_feat = self.dis_loss(right_net.dis_feat_scores,False)
		right_dis2 = self.dis_loss(right_net.dis_scores,True)
		right_dis_feat2 = self.dis_loss(right_net.dis_feat_scores,True)
		right_dis_real = self.dis_loss(right_net.dis_scores_real,True)
		right_dis_feat_real = self.dis_loss(right_net.dis_feat_scores_real,True)

		labelholder = tf.placeholder(tf.float32,[None])
		contra_loss = self.contra_loss(left_net.img_feature,right_net.img_feature,0.3, labelholder)

		target_left = tf.placeholder(tf.float32,[None,128,128,3])
		target_right = tf.placeholder(tf.float32,[None,128,128,3])

		recon_loss_left = self.recon_loss(left_net.recon_imgs, target_left)
		recon_loss_right = self.recon_loss(left_net.recon_imgs, target_right)

		# establish train_op
		self.train_gan_left = self.gan_optimizer(left_dis, left_dis_feat, left_dis_real, left_dis_feat_real, left_dis2, left_dis_feat2, recon_loss_left)
		self.train_gan_right = self.gan_optimizer(right_dis, right_dis_feat, right_dis_real, right_dis_feat_real, right_dis2, right_dis_feat2, recon_loss_right)

		self.train_contra = self.contra_optimizer(contra)

		# set class variables
		self.labelholder = labelholder
		self.gan_loss = left_dis + left_dis_feat + left_dis2 + left_dis_feat2 + left_dis_real + left_dis_feat_real + right_dis + right_dis_feat + right_dis2 + right_dis_feat2 + right_dis_real + right_dis_feat_real
		self.contra_loss = contra_loss
		self.recon_loss = recon_loss_left + recon_loss_right
		self.left_net = left_net
		self.right_net = right_net
		self.target_left = target_left
		self.target_right = target_right

	def dis_loss(self,scr,is_real=True):
		res = []
		with tf.variable_scope('dis_loss'):
			for i in range(10):
				if is_real:
					lab = tf.ones_like(scr[i])
				else:
					lab = tf.zeros_like(scr[i])
				loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=scr[i],labels=lab))
				res.append(loss)
		return res

	def contra_loss(self,feat1,feat2,margin, label):
		with tf.variable_scope('contra_loss'):
			d = tf.reduce_sum(tf.square(feat1 - feat2),1)
			d_ = tf.sqrt(d)
			loss = label * tf.square(tf.maximum(0., margin - d_)) + (1. - label) * d
			loss = tf.reduce_mean(loss)
		return loss

	def recon_loss(self,recon,target):
		res = []
		with tf.variable_scope('recon_loss'):
			for i in range(10):
				# add some noise here
				ls = recon[i] - target
				ls = tf.reduce_sum(tf.reduce_mean(tf.abs(ls),0))
				res.append(ls)
		return res

	def gan_optimizer(self, dis, dis_feat, dis_real, dis_feat_real, dis2, dis_feat2, recon):
		res = []
		for i in range(10):
			train_op = tf.train.AdamOptimizer(0.0001).minimize(dis[i]+dis_feat[i]+dis_real[i]+dis_feat_real[i],var_list=M.get_trainable_vars('discriminator')+M.get_trainable_vars('discriminator_feature'))
			train_op2 = tf.train.AdamOptimizer(0.0001).minimize(dis2[i]+dis_feat2[i]+recon[i],var_list=M.get_trainable_vars('generator')+M.get_trainable_vars('decoder'+str(i)))
			with tf.control_denpendencies(M.get_update_ops('discriminator')+M.get_update_ops('discriminator_feature')+M.get_update_ops('generator')+M.get_update_ops('decoder'+str(i))+[train_op2, train_op]):
				train_op = tf.no_op()
			res.append(train_op)
		return res

	def contra_optimizer(self, contra):
		decoder_vars = []
		for i in range(10):
			decoder_vars += M.get_trainable_vars('decoder'+str(i))
		train_op = tf.train.AdamOptimizer(0.0001).minimize(contra,var_list=decoder_vars+M.get_trainable_vars('attention_blk'))
		train_op2 = tf.train.AdamOptimizer(1e-6).minimize(contra,var_list=M.get_trainable_vars('encoder'))
		with tf.control_denpendencies([train_op,train_op2]):
			train_op = tf.no_op()
		return train_op

	def load_model(self,path='./model/'):
		self.sess = tf.Session()
		M.loadSess(path,self.sess)

	def train_gan(self,left, left_target, left_index, right, right_target, right_index, normalize=True, return_loss=False):
		if normalize:
			left = np.float32(left) / 127.5 - 1.
			right = np.float32(right) / 127.5 - 1.
			left_target = np.float32(left_target) / 127.5 - 1.
			right_target = np.float32(right_target) / 127.5 - 1.
		else:
			left = np.float32(left)
			right = np.float32(right)
			left_target = np.float32(left_target)
			right_target = np.float32(right_target)
		feed_dict = {self.left_net.inpholder: left, self.target_left: left_target, self.right_net.inpholder: right, self.target_right: right_target}
		self.sess.run(self.train_gan_left[left_index], feed_dict = feed_dict)
		self.sess.run(self.train_gan_right[right_index], feed_dict = feed_dict)
		if return_loss:
			gan_loss, recon_loss = self.sess.run([self.gan_loss, self.recon_loss],feed_dict = feed_dict)
			return gan_loss, recon_loss
		return 0,0

	def train_contra(self, left, right, label, normalize=True, return_loss=False):
		if normalize:
			left = np.float32(left) / 127.5 - 1.
			right = np.float32(right) / 127.5 - 1.
		else:
			left = np.float32(left)
			right = np.float32(right)
		feed_dict = {self.left_net.inpholder: left, self.right_net.inpholder: right, self.labelholder: label}
		self.sess.run(self.train_contra,feed_dict=feed_dict)
		if return_loss:
			contra_loss = self.sess.run(self.contra_loss ,feed_dict=feed_dict)
			return contra_loss
		return 0

	def generate(self, data, normalize=True):
		if normalize:
			data = np.float32(data) / 127.5 - 1.
		else:
			data = np.float32(data)
		if len(data.shape)==3:
			data = [data]
		recon = self.sess.run(self.left_net.recon_imgs, feed_dict={self.left_net.inpholder:data})
		recon = [(i+1.)*127.5 for i in recon]
		recon = [np.uint8(i) for i in recon]
		return recon

	def eval(self, data, normalize=True):
		if normalize:
			data = np.float32(data) / 127.5 - 1.
		else:
			data = np.float32(data)
		if len(data.shape)==3:
			data = [data]
		feat = self.sess.run(self.left_net.img_feature, feed_dict={self.left_net.inpholder:data})
		return feat


text:
import music21
import numpy as np

def getChordArrayFromPrediction(pred, chordlen):
    #pred will have the form of a 12-elem np array with vals between 0 and 1.
    # we get the 3 highest ranking notes (but can be 2 or 5, based on CHORDLEN)
    returnchord = np.repeat(0,12)
    indices = np.argpartition(-pred[0], chordlen - 1)[:chordlen] #this gets the indices of the top n values in the arr (n=chordlen)
    #print(pred)
    for i in indices:
        returnchord[i] = 1
    return returnchord #type np ndarray

#this is to attempt to predict without any seed chords:
#"seed" the first SUBSEQLEN chords using the melody notes themselves, no sugar on top
def seedFirstChords(melodylist, chordslist, subseqlen):
    for i in range(subseqlen):
        chordslist[i] = melodylist[i]

#iterate over subsequences of the melody and the chords. Update the chord list on the fly with the newly predicted chord. Then predict on the next subseq.
def predictOnArrays(kerasmodel, melodylist, chordslist, subseqlen, chordlen):
    melodyarray = np.array(melodylist)
    chordsarray = np.array(chordslist)
    print(chordsarray.shape)
    for i in range(len(melodyarray) - subseqlen + 1):
        ipred = i + subseqlen - 1 #index of the chord to be predicted in the fullchords list
        if np.any(chordsarray[ipred]) == False:    #if there is no chord in the fullchords list at that position, (all 0 values)
            inputchordsubseq = np.expand_dims(chordsarray[i:(ipred + 1)], axis=0) #the shape is now (1,subseqlen, 12)
            inputmelodysubseq = np.expand_dims(melodyarray[i:(ipred + 1)], axis=0)
            nextchord = kerasmodel.predict([inputchordsubseq, inputmelodysubseq])
            #(there's a +1 because the splice end index isn't inclusive).
            #there's expand dims (that wraps the one subseq in a []) because the model expects a list of subseqs, even if we only have a single subseq.)
            chordsarray[ipred] = getChordArrayFromPrediction(nextchord, chordlen) #let that position be the chord predicted using this subsequence's melody and chords
        else: #if the user has already set a chord for that melody note,
            continue #then keep going until we hit a blank chord
    chordslist[:] = list(chordsarray)

def getMusicObjectFrom12ToneDegrees(degs, scorekey):
    if len(degs) == 1:
        objInC = music21.pitch.Pitch(degs[0])
    else:
        objInC = music21.chord.Chord(degs)
    #music21 Chord and Pitch classes both support instantiation with a single/a list of 12tone pitch class numbers, with C=0

    #we just need to now transpose the Pitch or Chord in C=0 to our key
    fromCtoKey = music21.interval.Interval(noteStart = music21.pitch.Pitch("c4"), noteEnd = scorekey.tonic)
    objInKey = objInC.transpose(fromCtoKey)
    return objInKey

def findOnes(mylist): #this returns list of pitch class nums between 0 and 11 from a one-hot note or chord representation
    return [i for i in range(len(mylist)) if mylist[i]==1]

def getMusicObjectsFromOneHots(melody_or_chord_list, scorekey):
    returnlist = []

    for noc in melody_or_chord_list: #noc is a note-or-chord
        twelvetonedegs = findOnes(noc) #list of nums between 0 and 11 indicating which indices has a 1
        returnlist.append(getMusicObjectFrom12ToneDegrees(twelvetonedegs, scorekey))

    return returnlist

#write to midi once we've predicted a chord for all melody notes
def writeToMidi(outputfile, timestamps, melodylist, chordslist, scorekey):
    myScore = music21.stream.Score()
    melodyPart = music21.stream.Voice()
    chordsPart = music21.stream.Voice()
    durations = [j-i for i, j in zip(timestamps[:-1], timestamps[1:])]
    durations.append(2) #default length of 2 quarter-notes for the last note

    melodyPitches = getMusicObjectsFromOneHots(melodylist, scorekey)
    chordsChords = getMusicObjectsFromOneHots(chordslist, scorekey)

    for t in zip(durations, melodyPitches, chordsChords):
        melodyNote = music21.note.Note(t[1]) #turn Pitch object into Note object with duration
        melodyNote.quarterLength = t[0]
        melodyNote.octave = 6
        melodyPart.append(melodyNote)

        chordChord = music21.chord.Chord([t[2]]) #since some Chord predictions may end up being just a single note, we make sure all are Chord objs.
        if len(chordChord.pitches) == 0:
            chordChord = music21.note.Rest()
        chordChord.quarterLength = t[0]
        chordsPart.append(chordChord)

    myScore.insert(timestamps[0], melodyPart)
    myScore.insert(timestamps[0], chordsPart)

    #myScore.show("text")

    myScore.write("midi", outputfile)


text:
"""
SpatioTemporal_descriptormodels
-------------------------------
Main class to the class of model descriptors. This class contains the main
functions and indications to compute the local descriptors.

"""

import numpy as np
from spatial_descriptormodels import SpatialDescriptorModel

class SpatioTemporalDescriptorModel:
    """The spatio-temporal descriptor model is an interface to compute
    spatio-temporal descriptors for a descriptor model processer mainly.
    It contains the utility classes of:
        * Spatial descriptor models: extract the descriptors from spatial data.
    Its main function in the process of computing descriptors from points is to
    manage the dealing with perturbation of the system for the sake of testing
    predictors and models.

    TODO:
    -----
    - Unification of input unique ids

    """

    def _initialization(self):
        ## Main classes
        self.spdesc_models = []
        ## Parameters useful
        self.n_inputs = [0]

    def __init__(self, spdesc_models):
        """Spatial descriptor model initialization.

        Parameters
        ----------
        spdesc_models: list
            the spatial descriptor models that instance stores.

        """
        self._initialization()
        self._format_spdesc(spdesc_models)

    ############################### Formatters ################################
    ###########################################################################
    def _format_spdesc(self, spdesc_models):
        """Format spatial descriptors.

        Parameters
        ----------
        spdesc_models: list
            the spatial descriptor models that instance stores.

        """
        ## 0. Parsing inputs
        if type(spdesc_models) != list:
            spdesc_models = [spdesc_models]

        ## 1. Creating spdesc
        spdescs = []
        for i in range(len(spdesc_models)):
            if type(spdesc_models[i]) == dict:
                spdescs.append(SpatialDescriptorModel(**spdesc_models[i]))
            elif type(spdesc_models[i]) == tuple:
                assert(len(spdesc_models[i]))
                spdesc_i = SpatialDescriptorModel(spdesc_models[i][0],
                                                  spdesc_models[i][1],
                                                  **spdesc_models[i][2])
                spdesc_models.append(spdesc_i)
            elif isinstance(spdesc_models[i], SpatialDescriptorModel):
                spdescs.append(spdesc_models[i])
        ## 2. Storing in instance
        self.spdesc_models = spdescs
        self.n_inputs = [self.spdesc_models[i].n_inputs
                         for i in range(len(self.spdesc_models))]

    def _format_inputs(self, indices, y=None):
        """Format input indices.

        Parameters
        ----------
        indices: np.ndarray or list
            the indices of the samples used to compute the model.
        y: np.ndarray or list
            the target we want to predict.

        Returns
        -------
        indices: np.ndarray or list
            the indices of the samples used to compute the model.
        y: np.ndarray or list
            the target we want to predict.

        """

        ## 0. Create indices formatting
        assert(np.max(indices) <= np.sum(self.n_inputs))
        ranges = np.cumsum([0]+self.n_inputs)
        spdesc_i, spdesc_k = -1*np.ones(len(indices)), -1*np.ones(len(indices))
        spdesc_i, spdesc_k = spdesc_i.astype(int), spdesc_k.astype(int)
        new_indices, new_y = [], []
        print indices
        print ranges
        for i in range(len(self.n_inputs)):
            logi = np.logical_and(indices >= ranges[i], indices <= ranges[i+1])
            spdesc_k[logi] = i
            spdesc_i[logi] = indices-ranges[i]
            new_indices.append(list(indices[logi]))
            if y is not None:
                new_y.append(y[logi])
        self._spdesc_i = spdesc_i.astype(int)
        self._spdesc_k = spdesc_k.astype(int)
        ## 1. Output format
        if y is None:
            return new_indices
        else:
            return new_indices, new_y

    def _format_output(self, y_pred):
        """Re-format the output in the same way the input is given.

        Parameters
        ----------
        y_pred: list
            the list of prediction targets for each possible spatial model.

        Return
        ------
        y_pred: np.ndarray
            the target predictions in the format is given the input.

        """
        n = sum([len(e) for e in y_pred])
        new_y_pred = np.zeros(n)
        for i in range(len(y_pred)):
            logi = self._spdesc_k == i
            assert(len(y_pred[i]) == np.sum(logi))
            new_y_pred[logi] = y_pred[i]
        return new_y_pred

    ########################## Main model functions ###########################
    ###########################################################################
    def fit(self, indices, y):
        """Use the SpatioTemporalDescriptorModel as a model.

        Parameters
        ----------
        indices: np.ndarray or list
            the indices of the samples used to compute the model.
        y: np.ndarray or list
            the target we want to predict.

        Returns
        -------
        self : returns an instance of self.

        """
        ## 0. Input check
        if type(indices) == list:
            assert(type(y) == list)
            assert(len(indices) == len(self.spdesc_models))
            assert(len(y) == len(self.spdesc_models))
        else:
            assert(type(indices) == np.ndarray)
            assert(type(y) == np.ndarray)
            assert(len(indices) == len(y))
        indices, y = self._format_inputs(indices, y)

        ## 1. Fit model
        for i in range(len(self.spdesc_models)):
            self.spdesc_models[i] = self.spdesc_models[i].fit(indices[i], y[i])
        return self

    def predict(self, indices):
        """Use the SpatioTemporalDescriptorModel as a model to predict targets
        from spatial descriptors.

        Parameters
        ----------
        indices: np.ndarray or list
            the indices of the samples used to compute the target predicted.

        Returns
        -------
        y_pred : np.ndarray
            the predicted target.

        """
        ## 0. Parse inputs
        if type(indices) == list:
            assert(len(indices) == len(self.spdesc_models))
        else:
            assert(type(indices) == np.ndarray)
            indices = self._format_inputs(indices)
            assert(len(indices) == len(self.spdesc_models))

        ## 1. Fit model
        y_pred = []
        for i in range(len(self.spdesc_models)):
            y_pred.append(self.spdesc_models[i].predict(indices[i]))

        ## 2. Format output
        y_pred = self._format_output(y_pred)

        return y_pred

    ########################## Interaction with data ##########################
    ###########################################################################
    def apply_aggregation(self, regs, agg_info, selectors):
        """Apply aggregations.

        Parameters
        ----------
        regs: np.ndarray
            the regions in which we want to aggregate information.
        agg_info: tuple
            the information to aggregate the information.
        selectors: int, tuple, np.ndarray
            how to select which retriever.

        """
        for i in range(regs):
            self.spdesc_models[i].apply_aggregation(regs[i], agg_info[i],
                                                    selectors[i])

max_stars_repo_name: tgquintela/pySpatialTools

max_issues_repo_name: tgquintela/pySpatialTools

max_forks_repo_name: tgquintela/pySpatialTools


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import cv2
import numpy as np

from utils import train_utils
import model
import data

import tensorflow as tf

slim = tf.contrib.slim

flags = tf.app.flags

FLAGS = flags.FLAGS

# temporary constant
MODELNET10_TRAIN_DATA_SIZE = 5293
MODELNET10_VALIDATE_DATA_SIZE = 1000

flags.DEFINE_string('train_logdir', './models',
                    'Where the checkpoint and logs are stored.')
flags.DEFINE_string('ckpt_name_to_save', 'resnet_v2.ckpt',
                    'Name to save checkpoint file')
flags.DEFINE_integer('log_steps', 10,
                     'Display logging information at every log_steps.')
flags.DEFINE_integer('save_interval_secs', 1200,
                     'How often, in seconds, we save the model to disk.')
flags.DEFINE_boolean('save_summaries_images', False,
                     'Save sample inputs, labels, and semantic predictions as '
                     'images to summary.')
flags.DEFINE_string('summaries_dir', './models/train_logs',
                     'Where to save summary logs for TensorBoard.')

flags.DEFINE_enum('learning_policy', 'poly', ['poly', 'step'],
                  'Learning rate policy for training.')
flags.DEFINE_float('base_learning_rate', .0005,
                   'The base learning rate for model training.')
flags.DEFINE_float('learning_rate_decay_factor', 1e-3,
                   'The rate to decay the base learning rate.')
flags.DEFINE_float('learning_rate_decay_step', .2000,
                   'Decay the base learning rate at a fixed step.')
flags.DEFINE_float('learning_power', 0.9,
                   'The power value used in the poly learning policy.')
flags.DEFINE_float('training_number_of_steps', 300000,
                   'The number of steps used for training.')
flags.DEFINE_float('momentum', 0.9, 'The momentum value to use')
flags.DEFINE_integer('slow_start_step', MODELNET10_TRAIN_DATA_SIZE * 2,
                     'Training model with small learning rate for few steps.')
flags.DEFINE_float('slow_start_learning_rate', .00005,
                   'Learning rate employed during slow start.')

# Settings for fine-tuning the network.
flags.DEFINE_string('pre_trained_checkpoint',
                    # './pre-trained/resnet_v2_50.ckpt',
                    None,
                    'The pre-trained checkpoint in tensorflow format.')
flags.DEFINE_string('checkpoint_exclude_scopes',
                    # 'resnet_v2_50/logits,resnet_v2_50/SpatialSqueeze,resnet_v2_50/predictions',
                    None,
                    'Comma-separated list of scopes of variables to exclude '
                    'when restoring from a checkpoint.')
flags.DEFINE_string('trainable_scopes',
                    # 'ssd_300_vgg/block4_box, ssd_300_vgg/block7_box, \
                    #  ssd_300_vgg/block8_box, ssd_300_vgg/block9_box, \
                    #  ssd_300_vgg/block10_box, ssd_300_vgg/block11_box',
                    None,
                    'Comma-separated list of scopes to filter the set of variables '
                    'to train. By default, None would train all the variables.')
flags.DEFINE_string('checkpoint_model_scope',
                    None,
                    'Model scope in the checkpoint. None if the same as the trained model.')
flags.DEFINE_string('model_name',
                    'resnet_v2_50',
                    'The name of the architecture to train.')
flags.DEFINE_boolean('ignore_missing_vars',
                     False,
                     'When restoring a checkpoint would ignore missing variables.')

flags.DEFINE_string('dataset_dir',
                    '/home/ace19/dl_data/modelnet10',
                    'Where the dataset reside.')

flags.DEFINE_integer('how_many_training_epochs', 60,
                     'How many training loops to run')
flags.DEFINE_integer('batch_size', 4, 'batch size')
flags.DEFINE_integer('num_views', 12, 'number of views')
flags.DEFINE_integer('height', 224, 'height')
flags.DEFINE_integer('width', 224, 'width')
flags.DEFINE_string('labels',
                    'airplane,bed,bookshelf,bottle,chair,monitor,sofa,table,toilet,vase',
                    'Labels to use')

def main(unused_argv):
    tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO)

    labels = FLAGS.labels.split(',')
    num_classes = len(labels)

    # tf.compat.v1.logging.info('Creating train logdir: %s', FLAGS.train_logdir)

    with tf.Graph().as_default() as graph:
        global_step = tf.compat.v1.train.get_or_create_global_step()

        X = tf.compat.v1.placeholder(tf.float32,
                           [None, FLAGS.num_views, FLAGS.height, FLAGS.width, 3],
                           name='X')
        ground_truth = tf.compat.v1.placeholder(tf.int64, [None], name='ground_truth')
        is_training = tf.compat.v1.placeholder(tf.bool, name='is_training')
        dropout_keep_prob = tf.compat.v1.placeholder(tf.float32, name='dropout_keep_prob')
        # learning_rate = tf.placeholder(tf.float32, name='lr')

        # metric learning
        logits, features = \
            model.mvcnn_with_deep_cosine_metric_learning(X,
                                                         num_classes,
                                                         is_training=is_training,
                                                         keep_prob=dropout_keep_prob,
                                                         attention_module='se_block')
        # logits, features = mvcnn.mvcnn(X, num_classes)

        cross_entropy = tf.compat.v1.losses.sparse_softmax_cross_entropy(labels=ground_truth, logits=logits)
        tf.compat.v1.summary.scalar("cross_entropy_loss", cross_entropy)

        # Gather update ops. These contain, for example, the updates for the
        # batch_norm variables created by model.
        update_ops = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.UPDATE_OPS)

        # Gather initial summaries.
        summaries = set(tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.SUMMARIES))

        predition = tf.argmax(logits, 1, name='prediction')
        correct_predition = tf.equal(predition, ground_truth)
        confusion_matrix = tf.math.confusion_matrix(ground_truth,
                                               predition,
                                               num_classes=num_classes)
        # accuracy = tf.reduce_mean(tf.cast(correct_predition, tf.float32))
        # summaries.add(tf.summary.scalar('accuracy', accuracy))
        accuracy = slim.metrics.accuracy(tf.cast(predition, tf.int64),
                                             ground_truth)
        tf.compat.v1.summary.scalar("accuracy", accuracy)

        # Add summaries for model variables.
        for model_var in slim.get_model_variables():
            summaries.add(tf.compat.v1.summary.histogram(model_var.op.name, model_var))

        # Add summaries for losses.
        for loss in tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.LOSSES):
            summaries.add(tf.compat.v1.summary.scalar('losses/%s' % loss.op.name, loss))

        learning_rate = train_utils.get_model_learning_rate(
            FLAGS.learning_policy, FLAGS.base_learning_rate,
            FLAGS.learning_rate_decay_step, FLAGS.learning_rate_decay_factor,
            FLAGS.training_number_of_steps, FLAGS.learning_power,
            FLAGS.slow_start_step, FLAGS.slow_start_learning_rate)
        # optimizer = tf.train.MomentumOptimizer(learning_rate, FLAGS.momentum)
        optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate)
        summaries.add(tf.compat.v1.summary.scalar('learning_rate', learning_rate))

        total_loss, grads_and_vars = train_utils.optimize(optimizer)
        total_loss = tf.compat.v1.check_numerics(total_loss, 'Loss is inf or nan')
        summaries.add(tf.compat.v1.summary.scalar('total_loss', total_loss))

        # TensorBoard: How to plot histogram for gradients
        # grad_summ_op = tf.compat.v1.summary.merge([tf.compat.v1.summary.histogram("%s-grad" % g[1].name, g[0]) for g in grads_and_vars])

        # Create gradient update op.
        grad_updates = optimizer.apply_gradients(grads_and_vars,
                                                 global_step=global_step)
        update_ops.append(grad_updates)
        update_op = tf.group(*update_ops)
        with tf.control_dependencies([update_op]):
            train_op = tf.identity(total_loss, name='train_op')

        # Add the summaries. These contain the summaries created by model
        # and either optimize() or _gather_loss()
        summaries |= set(tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.SUMMARIES))

        # Merge all summaries together.
        summary_op = tf.compat.v1.summary.merge(list(summaries))
        train_writer = tf.compat.v1.summary.FileWriter(FLAGS.summaries_dir, graph)
        validation_writer = tf.compat.v1.summary.FileWriter(FLAGS.summaries_dir + '/validation', graph)

        #####################
        # prepare data
        #####################
        tfrecord_names = tf.compat.v1.placeholder(tf.string, shape=[])
        _dataset = data.Dataset(tfrecord_names,
                                FLAGS.num_views,
                                FLAGS.height,
                                FLAGS.width,
                                FLAGS.batch_size)
        iterator = _dataset.dataset.make_initializable_iterator()
        next_batch = iterator.get_next()

        sess_config = tf.compat.v1.ConfigProto(gpu_options=tf.compat.v1.GPUOptions(allow_growth=True))
        with tf.compat.v1.Session(config=sess_config) as sess:
            sess.run(tf.compat.v1.global_variables_initializer())

            saver = tf.compat.v1.train.Saver(keep_checkpoint_every_n_hours=1.0)
            if FLAGS.pre_trained_checkpoint:
                train_utils.restore_fn(FLAGS)

            start_epoch = 0
            training_batches = int(MODELNET10_TRAIN_DATA_SIZE / FLAGS.batch_size)
            if MODELNET10_TRAIN_DATA_SIZE % FLAGS.batch_size > 0:
                training_batches += 1
            val_batches = int(MODELNET10_VALIDATE_DATA_SIZE / FLAGS.batch_size)
            if MODELNET10_VALIDATE_DATA_SIZE % FLAGS.batch_size > 0:
                val_batches += 1

            # The filenames argument to the TFRecordDataset initializer can either
            # be a string, a list of strings, or a tf.Tensor of strings.
            training_tf_filenames = os.path.join(FLAGS.dataset_dir, 'train.record')
            val_tf_filenames = os.path.join(FLAGS.dataset_dir, 'validate.record')
            ##################
            # Training loop.
            ##################
            for n_epoch in range(start_epoch, FLAGS.how_many_training_epochs):
                tf.compat.v1.logging.info('--------------------------')
                tf.compat.v1.logging.info(' Epoch %d' % n_epoch)
                tf.compat.v1.logging.info('--------------------------')

                sess.run(iterator.initializer, feed_dict={tfrecord_names: training_tf_filenames})
                for step in range(training_batches):
                    train_batch_xs, train_batch_ys = sess.run(next_batch)
                    # # Verify image
                    # assert not np.any(np.isnan(train_batch_xs))
                    # n_batch = train_batch_xs.shape[0]
                    # n_view = train_batch_xs.shape[1]
                    # for i in range(n_batch):
                    #     for j in range(n_view):
                    #         img = train_batch_xs[i][j]
                    #         # scipy.misc.toimage(img).show()
                    #         # Or
                    #         img = cv2.cvtColor(img.astype(np.uint8), cv2.COLOR_BGR2RGB)
                    #         cv2.imwrite('/home/ace19/Pictures/' + str(i) +
                    #                     '_' + str(j) + '.png', img)
                    #         # cv2.imshow(str(train_batch_ys[idx]), img)
                    #         cv2.waitKey(100)
                    #         cv2.destroyAllWindows()

                    lr, train_summary, train_accuracy, train_loss, _ = \
                        sess.run([learning_rate, summary_op, accuracy, total_loss, train_op],
                                 feed_dict={X: train_batch_xs,
                                            ground_truth: train_batch_ys,
                                            is_training: True,
                                            dropout_keep_prob: 0.8})

                    # lr, train_summary, train_accuracy, train_loss, grad_vals, _ = \
                    #     sess.run([learning_rate, summary_op, accuracy, total_loss, grad_summ_op, train_op],
                    #     feed_dict={X: train_batch_xs,
                    #                ground_truth: train_batch_ys,
                    #                is_training: True,
                    #                dropout_keep_prob: 0.8})

                    train_writer.add_summary(train_summary, n_epoch)
                    # train_writer.add_summary(grad_vals, n_epoch)
                    tf.compat.v1.logging.info('Epoch #%d, Step #%d, rate %.10f, accuracy %.1f%%, loss %f' %
                                    (n_epoch, step, lr, train_accuracy * 100, train_loss))

                ###################################################
                # Validate the model on the validation set
                ###################################################
                tf.compat.v1.logging.info('--------------------------')
                tf.compat.v1.logging.info(' Start validation ')
                tf.compat.v1.logging.info('--------------------------')

                # Reinitialize iterator with the validation dataset
                sess.run(iterator.initializer, feed_dict={tfrecord_names: val_tf_filenames})

                total_val_accuracy = 0
                validation_count = 0
                total_conf_matrix = None
                for step in range(val_batches):
                    validation_batch_xs, validation_batch_ys = sess.run(next_batch)

                    val_summary, val_accuracy, conf_matrix = \
                        sess.run([summary_op, accuracy, confusion_matrix],
                                 feed_dict={X: validation_batch_xs,
                                            ground_truth: validation_batch_ys,
                                            is_training: False,
                                            dropout_keep_prob: 1.0})

                    validation_writer.add_summary(val_summary, n_epoch)

                    total_val_accuracy += val_accuracy
                    validation_count += 1
                    if total_conf_matrix is None:
                        total_conf_matrix = conf_matrix
                    else:
                        total_conf_matrix += conf_matrix

                total_val_accuracy /= validation_count
                tf.compat.v1.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix))
                tf.compat.v1.logging.info('Validation accuracy = %.1f%% (N=%d)' %
                                (total_val_accuracy * 100, MODELNET10_VALIDATE_DATA_SIZE))

                # Save the model checkpoint periodically.
                if (n_epoch <= FLAGS.how_many_training_epochs-1):
                    checkpoint_path = os.path.join(FLAGS.train_logdir, FLAGS.ckpt_name_to_save)
                    tf.compat.v1.logging.info('Saving to "%s-%d"', checkpoint_path, n_epoch)
                    saver.save(sess, checkpoint_path, global_step=n_epoch)

if __name__ == '__main__':
    if tf.io.gfile.exists(FLAGS.train_logdir):
        tf.io.gfile.rmtree(FLAGS.train_logdir)
    tf.io.gfile.makedirs(FLAGS.train_logdir)

    tf.compat.v1.app.run()


text:
# Copyright (c) OpenMMLab. All rights reserved.
import numpy as np

def bbox_overlaps(bboxes1,
                  bboxes2,
                  mode='iou',
                  eps=1e-6,
                  use_legacy_coordinate=False):
    """Calculate the ious between each bbox of bboxes1 and bboxes2.

    Args:
        bboxes1 (ndarray): Shape (n, 4)
        bboxes2 (ndarray): Shape (k, 4)
        mode (str): IOU (intersection over union) or IOF (intersection
            over foreground)
        use_legacy_coordinate (bool): Whether to use coordinate system in
            mmdet v1.x. which means width, height should be
            calculated as 'x2 - x1 + 1` and 'y2 - y1 + 1' respectively.
            Note when function is used in `VOCDataset`, it should be
            True to align with the official implementation
            `http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCdevkit_18-May-2011.tar`
            Default: False.

    Returns:
        ious (ndarray): Shape (n, k)
    """

    assert mode in ['iou', 'iof']
    if not use_legacy_coordinate:
        extra_length = 0.
    else:
        extra_length = 1.
    bboxes1 = bboxes1.astype(np.float32)
    bboxes2 = bboxes2.astype(np.float32)
    rows = bboxes1.shape[0]
    cols = bboxes2.shape[0]
    ious = np.zeros((rows, cols), dtype=np.float32)
    if rows * cols == 0:
        return ious
    exchange = False
    if bboxes1.shape[0] > bboxes2.shape[0]:
        bboxes1, bboxes2 = bboxes2, bboxes1
        ious = np.zeros((cols, rows), dtype=np.float32)
        exchange = True
    area1 = (bboxes1[:, 2] - bboxes1[:, 0] + extra_length) * (
        bboxes1[:, 3] - bboxes1[:, 1] + extra_length)
    area2 = (bboxes2[:, 2] - bboxes2[:, 0] + extra_length) * (
        bboxes2[:, 3] - bboxes2[:, 1] + extra_length)
    for i in range(bboxes1.shape[0]):
        x_start = np.maximum(bboxes1[i, 0], bboxes2[:, 0])
        y_start = np.maximum(bboxes1[i, 1], bboxes2[:, 1])
        x_end = np.minimum(bboxes1[i, 2], bboxes2[:, 2])
        y_end = np.minimum(bboxes1[i, 3], bboxes2[:, 3])
        overlap = np.maximum(x_end - x_start + extra_length, 0) * np.maximum(
            y_end - y_start + extra_length, 0)
        if mode == 'iou':
            union = area1[i] + area2 - overlap
        else:
            union = area1[i] if not exchange else area2
        union = np.maximum(union, eps)
        ious[i, :] = overlap / union
    if exchange:
        ious = ious.T
    return ious

max_stars_repo_name: Brym-Gyimah/mmdetection


text:
from data_loading import NMRDataLoader, make_pred_file, cd
import keras
import numpy as np
import sklearn.metrics as skm

if __name__ == "__main__":
    type_ = "CNN"
    number = 89
    desktop = True

    if desktop:
        path = "C:/SOFTWARE and giggles/NMR_tuning/t_{}".format(number)
        storage_path = "C:/SOFTWARE and giggles/NMR_tuning/1st_tuning_round/preds{}".format(number)
    else:
        path = "D:/SOFTWARE/numerai_data/t_{}".format(number)
        storage_path = "D:/SOFTWARE/MI Jule tests/preds{}".format(number)

    nmr_loader = NMRDataLoader(path)
    X_train, Y_train, X_val, Y_val, X_tournament, id_tournament = nmr_loader.get_data_small_val(val_era_size=None)

    val_pred_list = []
    # top_performingCNN = [27, 46, 49, 50, 84, 85, 99, 129, 169, 194, 204, 223, 228, 238, 260, 270]
    top_performingCNN = []
    top_performingDNN = [45, 62, 83, 87, 96]
    # top_performingDNN = []

    import os

#    for file in os.listdir(storage_path):
#        if file.endswith(".npy"):
#            top_performingCNN.append(file)

    for i in top_performingCNN:
        model_name = "CNN_models_param_{}".format(i)
        print("Loading model {}".format(model_name))

        with cd(storage_path):
            val_pred_list.append(np.load("{}_predictions.npy".format(model_name)))
            #val_pred_list.append(np.load("{}".format(i)))

    for i in top_performingDNN:
        model_name = "DNN_models_param_{}".format(i)
        print("Loading model {}".format(model_name))

        with cd(storage_path):
            val_pred_list.append(np.load("{}_predictions.npy".format(model_name)))

    combined = None
    for entry in val_pred_list:
        if combined is None:
            combined = entry
        else:
            combined = combined + entry

    ensemble_pred = combined/(len(top_performingCNN)+len(top_performingDNN))

    print("test: {}".format(ensemble_pred[0]))

    make_pred_file(storage_path=storage_path, predictions=ensemble_pred, ids=id_tournament,
                   name="DNN_top{}_pred.csv".format(len(top_performingDNN)))

max_stars_repo_name: Hoeiriis/ConvForTimeSeries-Mirror

max_issues_repo_name: Hoeiriis/ConvForTimeSeries-Mirror

max_forks_repo_name: Hoeiriis/ConvForTimeSeries-Mirror


text:
#!/usr/bin/env python2.7
# -*- mode: python; coding: utf-8; -*-

"""Module for generating lexicon using Rao and Ravichandran's method (2009).

"""

##################################################################
# Imports
from __future__ import unicode_literals, print_function

from blair_goldensohn import seeds2seedpos
from common import FMAX, FMIN, NEGATIVE, POSITIVE, SYNRELS
from rao import POS_IDX, NEG_IDX, NEUT_IDX, POL_IDX, SCORE_IDX

import numpy as np
import sys

##################################################################
# Constants

##################################################################
# Methods
def seeds2synids(a_germanet, a_terms, a_pos):
    """Convert list of lexical terms to synset id's.

    @param a_germanet - GermaNet instance
    @param a_terms - set of terms to check
    @param a_pos - part-of-speech tag of the lexical term or None

    @return list of synset id's

    """
    ret = set()
    for ilex in a_terms:
        for ilexid in a_germanet.lex2lexid[ilex]:
            for isynid in a_germanet.lexid2synids[ilexid]:
                if a_pos is None or a_germanet.synid2pos[isynid] in a_pos:
                    ret.add(isynid)
    return ret

def seedpos_chck(a_germanet, a_term, a_pos):
    """Check whether given term and pos pair is present in Ontology.

    @param a_germanet - GermaNet instance
    @param a_term - lexical term to check
    @param a_pos - part-of-speech tag of the lexical term

    @return \c True if the given term is present in Ontology

    """
    ipos = None
    for lexid in a_germanet.lex2lexid.get(a_term, []):
        for isynid in a_germanet.lexid2synids[lexid]:
            ipos = a_germanet.synid2pos[isynid]
            if ipos == a_pos:
                return True
    return False

def _add_syn(a_scores, a_idx, a_synid, a_germanet, a_term2idx):
    """Update counter of synonyms from a synset.

    @param a_scores - target matrix to update
    @param a_idx - index of the polarity class
    @param a_synid - synset id whose lex counters should be updated
    @param a_germanet - GermaNet instance
    @param a_term2idx - mapping from (term, pos) to matrix index

    @return void

    @note modifies a_scores in place

    """
    ipos = a_germanet.synid2pos[a_synid]
    for ilexid in a_germanet.synid2lexids[a_synid]:
        for ilex in a_germanet.lexid2lex[ilexid]:
            iterm = (ilex, ipos)
            idx = a_term2idx[iterm]
            a_scores[a_idx, idx] += 1

def _add_lex(a_scores, a_idx, a_lexid, a_germanet, a_term2idx):
    """Update counter of synonyms from lexemes.

    @param a_scores - target matrix to update
    @param a_idx - index of the polarity class
    @param a_lexid - id of the lexeme whose counters should be updated
    @param a_germanet - GermaNet instance
    @param a_term2idx - mapping from (term, pos) to matrix index

    @return void

    @note modifies a_scores in place

    """
    ipos = ""
    for isynid in a_germanet.lexid2synid[a_lexid]:
        ipos = a_germanet.synid2pos[isynid]
        for ilex in a_germanet.lexid2lex[a_lexid]:
            iterm = (ilex, ipos)
            idx = a_term2idx[iterm]
            a_scores[a_idx, idx] += 1

def _compute_numerator(a_scores, a_idx, a_seed_synids, a_seed_lexidpos,
                       a_germanet, a_term2idx, a_ext_rel):
    """Compute the number of synsets containing a seed term.

    @param a_scores - target matrix to update
    @param a_idx - index of the polarity class
    @param a_seed_synids - synset id's of the seeds
    @param a_seed_lexidpos - synset id's of the seeds
    @param a_germanet - GermaNet instance
    @param a_term2idx - mapping from (term, pos) to matrix index
    @param a_ext_rel - use an extended set of synonymous relations

    @return \c void

    @note modifies a_scores in place

    """
    idx = 0
    ipos = iterm = None
    # add synonymous relations from synsets
    for isynid in a_seed_synids:
        _add_syn(a_scores, a_idx, isynid, a_germanet, a_term2idx)
        if a_ext_rel:
            for trg_synid, irelname in a_germanet.con_relations[isynid]:
                if irelname in SYNRELS:
                    _add_syn(a_scores, a_idx, trg_synid,
                             a_germanet, a_term2idx)
    # add lexical synonymous relations
    if a_ext_rel:
        src_lexid = set(ilexid
                        for ilex, _ in a_seed_lexidpos
                        for ilexid in a_germanet.lex2lexid)
        for ilexid in src_lexid:
            for itrg, irel in a_germanet.lex_relations[ilexid]:
                if irel in SYNRELS:
                    _add_lex(a_scores, a_idx, itrg,
                             a_germanet, a_term2idx)

def kim_hovy(a_germanet, a_pos, a_neg, a_neut, a_seed_pos,
             a_ext_syn_rels):
    """Extend sentiment lexicons using the min-cut method of Kim/Hovy (2006).

    @param a_germanet - GermaNet instance
    @param a_pos - set of lexemes with positive polarity
    @param a_neg - set of lexemes with negative polarity
    @param a_neut - set of lexemes with neutral polarity
    @param a_seed_pos - part-of-speech class of seed synsets ("none" for no
      restriction)
    @param a_ext_syn_rels - use extended set of synonymous relations

    @return list of polar terms, their polarities, and scores

    """
    # construct a graph from GermaNet
    nodes = set((ilex, ipos)
                for isynid, ipos in a_germanet.synid2pos.iteritems()
                for ilexid in a_germanet.synid2lexids[isynid]
                for ilex in a_germanet.lexid2lex[ilexid]
                )
    # estimate prior polarities of the classes
    Z = float(len(nodes))
    if a_seed_pos == "none":
        a_seed_pos = ["adj", "nomen", "verben"]
    else:
        a_seed_pos = [a_seed_pos]

    pos_seedpos = filter(lambda x: seedpos_chck(a_germanet, *x),
                         seeds2seedpos(a_pos, a_seed_pos))
    N_pos = float(len(pos_seedpos))
    P_pos = N_pos / Z
    assert 0. <= P_pos <= 1., \
        "Invalid value of prior positive probability: {:f}".format(P_pos)
    pos_synids = seeds2synids(a_germanet, a_pos, a_seed_pos)

    neg_seedpos = filter(lambda x: seedpos_chck(a_germanet, *x),
                         seeds2seedpos(a_neg, a_seed_pos))
    N_neg = float(len(neg_seedpos))
    P_neg = N_neg / Z
    assert 0. <= P_neg <= 1., \
        "Invalid value of prior negative probability: {:f}".format(P_neg)
    neg_synids = seeds2synids(a_germanet, a_neg, a_seed_pos)

    neut_seedpos = filter(lambda x: seedpos_chck(a_germanet, *x),
                          seeds2seedpos(a_neut, a_seed_pos))
    N_neut = float(len(neut_seedpos))
    P_neut = N_neut / Z
    assert 0. <= P_neut <= 1., \
        "Invalid value of prior neutral probability: {:f}".format(P_neut)
    neut_synids = seeds2synids(a_germanet, a_neut, a_seed_pos)

    # estimate conditional probabilities of terms given classes
    term2idx = {iterm: i for i, iterm in enumerate(nodes)}
    scores = np.zeros((3, len(term2idx)))
    # compute the number of synsets intersecting with the seeds
    # normalize probability scores
    for iseed_synids, iseed_lexidpos, idx \
        in zip((pos_synids, neg_synids, neut_synids),
               (pos_seedpos, neg_seedpos, neut_seedpos),
               (POS_IDX, NEG_IDX, NEUT_IDX)):
        _compute_numerator(scores, idx, iseed_synids, iseed_lexidpos,
                           a_germanet, term2idx, a_ext_syn_rels)
    # normalize probabilities
    scores[POS_IDX, :] *= P_pos / (float(len(pos_synids)) or 1.)
    scores[NEG_IDX, :] *= P_neg / (float(len(neg_synids)) or 1.)
    scores[NEUT_IDX, :] *= P_neut / (float(len(neut_synids)) or 1.)
    # check
    assert np.max(np.max(scores, axis=1), axis=0) <= 1., \
        "Ivalid maximum probability value."
    assert np.min(np.min(scores, axis=1), axis=0) >= 0., \
        "Ivalid minimum probability value: {:f}.".format(
            np.min(np.min(scores, axis=1), axis=0))
    # determine classes
    idx = 0
    ret = [(w, POSITIVE, FMAX) for w in a_pos] \
        + [(w, NEGATIVE, FMIN) for w in a_neg]
    lex2lidx = {el[0]: i for i, el in enumerate(ret)}
    classes = np.argmax(scores, axis=0)
    for (ilex, ipos), idx in term2idx.iteritems():
        if ilex in a_pos or ilex in a_neg:
            continue
        if classes[idx] == POS_IDX:
            ipol = POSITIVE
            iscore = scores[POS_IDX, idx]
        elif classes[idx] == NEG_IDX:
            ipol = NEGATIVE
            iscore = scores[NEG_IDX, idx]
        else:
            continue
        if iscore == 0.:
            continue
        if ilex in lex2lidx:
            idx = lex2lidx[ilex]
            if abs(iscore) > abs(ret[idx][SCORE_IDX]):
                ret[idx][POL_IDX] = ipol
                ret[idx][SCORE_IDX] = iscore
        else:
            lex2lidx[ilex] = len(ret)
            ret.append([ilex, ipol, iscore])
    return ret

max_stars_repo_name: WladimirSidorenko/SentimentLexicon

max_issues_repo_name: WladimirSidorenko/SentimentLexicon

max_forks_repo_name: WladimirSidorenko/SentimentLexicon


text:
# %%
import pandas as pd
import matplotlib.pyplot as plt
import os,sys
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder
import numpy as np
#%%
TRAIN_TRANS_PATH = "./../raw_dataset/trainset/train_trans.csv"
trans = pd.read_csv(TRAIN_TRANS_PATH)
# 首先，对platform做onehot
encoder = LabelEncoder()
platform = encoder.fit_transform(trans['platform'].values)
platform = np.array([platform]).T
enc = OneHotEncoder()
a=enc.fit_transform(platform)
a=a.toarray()
a=pd.DataFrame(a)
a.columns=['platform1','platform2','platform3','platform4','platform5','platform6']
trans = pd.concat([trans,a],axis=1)
trans = trans.drop(['platform'],axis=1)
trans.head()
# 对tunnel_in 做onehot并把缺失值做成新的一类
p = trans['tunnel_in'].fillna('a')
trans['tunnel_in'] = p
tunnel_in = encoder.fit_transform(trans['tunnel_in'].values)
tunnel_in = np.array([tunnel_in]).T
enc = OneHotEncoder()
a=enc.fit_transform(tunnel_in)
a=a.toarray()
a=pd.DataFrame(a)
a.columns=['tunnel_in1','tunnel_in2','tunnel_in3','tunnel_in4','tunnel_in5','tunnel_in6']
trans = pd.concat([trans,a],axis=1)
trans = trans.drop(['tunnel_in'],axis=1)
trans.head()
# 将ip和ip3两列去掉
trans = trans.drop(['ip'],axis=1)
trans = trans.drop(['ip_3'],axis=1)
trans.head()
# 对tunnel_out进行onehot加上众数填补
trans['tunnel_out'].value_counts()
p = trans['tunnel_out'].fillna('6ee790756007e69a')
trans['tunnel_out'] = p
tunnel_out = encoder.fit_transform(trans['tunnel_out'].values)
tunnel_out = np.array([tunnel_out]).T
enc = OneHotEncoder()
a=enc.fit_transform(tunnel_out)
a=a.toarray()
a=pd.DataFrame(a)
a.columns=['tunnel_out1','tunnel_out2','tunnel_out3','tunnel_out4']
trans = pd.concat([trans,a],axis=1)
trans = trans.drop(['tunnel_out'],axis=1)
trans.head()
# type1做onehot
encoder = LabelEncoder()
type1 = encoder.fit_transform(trans['type1'].values)
type1 = np.array([type1]).T
enc = OneHotEncoder()
a=enc.fit_transform(type1)
a=a.toarray()
a=pd.DataFrame(a)
a.columns=['type1_1','type1_2','type1_3','type1_4','type1_5','type1_6','type1_7','type1_8','type1_9','type1_10',
'type1_11','type1_12','type1_13','type1_14','type1_15','type1_16','type1_17','type1_18','type1_19','type1_20']
trans = pd.concat([trans,a],axis=1)
trans = trans.drop(['type1'],axis=1)
trans.head()

# 对type2 缺失值单独做成一类然后onehot
p = trans['type2'].fillna('a')
trans['type2'] = p
type2 = encoder.fit_transform(trans['type2'].values)
type2 = np.array([type2]).T
enc = OneHotEncoder()
a=enc.fit_transform(type2)
a=a.toarray()
a=pd.DataFrame(a)
a.columns=['type2_1','type2_2','type2_3','type2_4','type2_5','type2_6','type2_7','type2_8','type2_9','type2_10',
'type2_11','type2_12','type2_13']
trans = pd.concat([trans,a],axis=1)
trans = trans.drop(['type2'],axis=1)
trans.head()
trans.to_csv(path_or_buf='./../dataset1/trainset/train_trans.csv')
# %%
TRAIN_TRANS_PATH = "./../raw_dataset/testset/test_a_trans.csv"
trans = pd.read_csv(TRAIN_TRANS_PATH)
trans = trans.drop(['ip'],axis=1)
trans = trans.drop(['ip_3'],axis=1)
p = trans['type2'].fillna('a')
trans['type2'] = p
p = trans['tunnel_out'].fillna('6ee790756007e69a')
trans['tunnel_out'] = p
p = trans['tunnel_in'].fillna('a')
trans['tunnel_in'] = p
trans.head()
trans.to_csv(path_or_buf='./../dataset1/test_a_trans.csv')

# %%


text:
import re
import requests
import warnings
import pandas as pd
import numpy as np
import statsmodels.api as sm
import statsmodels.stats.api as sms
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from statsmodels.compat import lzip
from statsmodels.stats.stattools import durbin_watson
from statsmodels.tsa.stattools import adfuller, coint
from matplotlib import style, cm
from matplotlib.ticker import FuncFormatter, ScalarFormatter
from IPython.display import display, Math, Latex
from tqdm import tqdm
from scipy.optimize import curve_fit
from scipy import stats

def coinmetrics_urls():
    """
    Scapes URLs where data is located.

    :returns: (pd.DataFrame, list) Filename URLs of the csv files where data is located along with a list of tickers for
                                  which a csv file exists with data.
    """
    query = re.findall('[^>]+\.csv', requests.get('https://github.com/coinmetrics/data/tree/master/csv/').text)
    filenames, tickers = [], []
    for filename in query:
        try:
            filenames.append(filename.split('href="')[1])
            tickers.append(filename.split('href="')[1].split('/')[-1].split('.')[0])
        except IndexError:
            continue
    filenames = pd.DataFrame(filenames, columns=['filenames'])
    return filenames, tickers

def download():
    """
    Downloads data updated daily by Coin Metrics from https://github.com/coinmetrics/data/tree/master/csv. CSV files
    containing the data is saved to '../data/coinmetrics/' path.
    """
    filenames, tickers = coinmetrics_urls()
    for ticker in tqdm(tickers):
        filename = filenames[filenames['filenames']=='/coinmetrics/data/blob/master/csv/{}.csv'.format(ticker)].squeeze()
        csv = pd.read_csv('https://github.com' + filename + '?raw=true')
        csv.to_csv('../data/coinmetrics/{}.csv'.format(ticker))

def check_tickers(search_term, filenames, tickers, show=False):
    """
    Helper function to filter out tickers which have no available data based on the search term parameter.

    :param search_term: (str) Search term to look for in all ticker datasets. Type only one search term.
    :param filenames: (pd.DataFrame) DataFrame of filenames where csv files are located.
    :param tickers: (list of str) List of tickers to filter through.
    :param show: (bool) Optional, prints tickers which have data available.
    :returns: (list) List of tickers which have data available.
    """
    filtered_tickers = []
    for ticker in tqdm(tickers):
        filename = filenames[filenames['filenames'] == '/coinmetrics/data/blob/master/csv/{}.csv'.format(ticker)].squeeze()
        df = pd.read_csv('../data/coinmetrics/{}.csv'.format(ticker))
        columns = pd.DataFrame(df.columns, columns=['columns'])
        if columns[(columns['columns'].str.contains(search_term))].empty is not True:
            filtered_tickers.append(ticker)
        else:
            continue
    if show==True:
        print('Tickers with Available Data: {}'.format(filtered_tickers))
    return filtered_tickers

def timeseries(ticker, fields):
    """
    Helper function to filter out tickers which have no available data based on the search term parameter.

    :param ticker: (str) Ticker of cryptocurrency you want to look up.
    :param fields: (str or list of str) Data fields for which to include in timeseries DataFrame.
    :returns: (pd.DateFrame or pd.Series) DataFrame or Series (if only one field) containing requested data.
    """
    df = pd.read_csv('../data/coinmetrics/{}.csv'.format(ticker)).iloc[:, 1:].set_index('time', drop=True)
    df.index = pd.to_datetime(df.index)
    return df[fields].dropna()

def stock_to_flow_data(ticker):
    """
    Compiles required data to compute stock-to-flow model.

    :param ticker: (str) Ticker of cryptocurrency you want to look up.
    :returns: (pd.DateFrame) DataFrame containing stock-to-flow data.
    """
    df = timeseries(ticker, ['CapMrktCurUSD', 'PriceUSD', 'BlkCnt', 'SplyCur'])
    df.insert(3, 'TotalBlks', df.BlkCnt.cumsum().values)
    df['StocktoFlow'] = df['SplyCur'] / ((df['SplyCur'] - df['SplyCur'].shift(365)))
    return df.dropna().round(2)

def objective(x, a, b):
    """
    Power Law Function
    """
    return np.exp(a) * (x ** b)

def stock_to_flow_model(ticker, p0=None, show=False):
    """
    Computes a fitted stock-to-flow model to observed data, computes Spearman Correlation Coefficient, and tests the
    null hypothesis that there is no correlation between the computed stock-to-flow model and observations. Rejecting the null
    hypothesis means accepting the alternative hypothesis that there is correlation between the stock-to-flow model and
    observed values.

    :param ticker: (str) Ticker of cryptocurrency.
    :param p0: (list of floats) Optional, initial guesses for coefficients a and b in the objective function. Defaults
                                to None.
    :param show: (bool) Optional, prints the results from fitting a power law function to the stock-to-flow data.
                        Defaults to False.
    :returns: (pd.DataFrame, np.array) DataFrame containing data necessary to compute stock-to-flow model
                                                  along with a np.array containing the fitted values for coefficients
                                                  a and b in the objective function.
    """
    df = stock_to_flow_data(ticker)
    xdata = df.StocktoFlow.values
    ydata = df.CapMrktCurUSD.values
    params, cov = curve_fit(objective, xdata, ydata, p0)
    drawdown = df.CapMrktCurUSD.fillna(method='ffill')
    drawdown[np.isnan(drawdown)] = -np.Inf
    roll_max = np.maximum.accumulate(drawdown)
    drawdown = drawdown / roll_max - 1.
    df['ModelCapMrktCurUSD'] = (np.exp(params[0]) * (df['StocktoFlow'] ** params[1])).round(4)
    df['Difference%'] = df['ModelCapMrktCurUSD'] / df['CapMrktCurUSD'] - 1
    df['MaxDrawdown%'] = drawdown.round(4)
    df.insert(2, 'BlkCntMonthly', df['TotalBlks'] - df['TotalBlks'].shift(30))
    sf = df.StocktoFlow.values[-1].round(2)
    p0 = df.CapMrktCurUSD[-1].round(2)
    p1 = df.ModelCapMrktCurUSD[-1].round(2)
    r, p = (stats.spearmanr(df['CapMrktCurUSD'], df['ModelCapMrktCurUSD']))
    r2 = r**2
    n = len(xdata)
    k = len(params)
    adj_r2 = 1 - ((1 - r2) * (n - 1) / (n - k - 1))
    if show==True:
        print('Current Stock-to-Flow: {}'.format(sf))
        print('Current Market Value: ${:,.2f}'.format(p0))
        print('Model Prediction: ${:,.2f}'.format(p1))
        print('Potential Return%: {:,.2f}%'.format((p1 / p0 - 1) * 100))
        print('')
        print('Fitted Power Law Model: MarketCapUSD = e^{:.3f} * SF^{:.3f}'.format(*params))
        print('Equivalent Regression Model: ln(MarketCapUSD) = {:.3f} * ln(SF) + {:.3f}'.format(params[1], params[0]))
        print('Spearman R-Squared: {}'.format(r2.round(4)))
        print('Adj. Spearman R-Squared: {}'.format(adj_r2.round(4)))
        print('P-value of Correlation Coefficient: {}'.format(p.round(4)))
        print(' ')
        print('Conclusion: ')
        if p < 0.05:
            print('[1] Correlation detected. Reject null hypothesis that correlation is equal to 0.')
            print('[2] Statistically significant at the 95% confidence level.')
            print('[3] The independent variable explains approximately {}% of the variation in the dependent variable.'.format(
                (r2*100).round(2)))
        else:
            print('[1] No correlation detected. Fail to reject null hypothesis that correlation is equal to 0.')
            print('[2] Statistically insignificant at the 95% confidence level.')
            print('[3] The independent variable explains approximately {}% of the variation in the dependent variable.'.format(
                    (r2 * 100).round(2)))
        print('')
        print('Notes: ')
        print('[1] Assumes model is correctly specified with no violations of the classic normal linear regression model assumptions.')
        print('[2] Conclusion could be the result of spurious correlation. Test for cointegration to confirm. Use with caution.')
    return df, params

def clean_tickers(p0=None):
    """
    Display tickers for which data is available for a stock-to-flow model and for which no Error or RuntimeWarning are
    thrown. Refer to https://docs.coinmetrics.io/asset-metrics/network-usage/blkcnt and
    https://docs.coinmetrics.io/asset-metrics/supply/splycur for more information on the required inputs. The method
    below searches all ticker data specifically for 'blkcnt' and 'splycur' and removes tickers which have no data.
    Finally, the method also tries to fit a power law model to the data and filters out tickers whose data causes the
    parameter optimization algorithm to throw a RuntimeWarning. Tickers which have data but throws a RuntimeWarning may
    contain data which cannot be properly fitted to a power law curve for whatever reasons, one of which is not having
    a large enough sample size.

    :param p0: (list of [float,float]) Optional, initial guesses for coefficients a and b in the objective function.
                                       Defaults to None.
    :returns: (list) List of tickers which have data available and do not produce a RuntimeWarning.
    """
    warnings.filterwarnings('error')
    filenames, tickers = coinmetrics_urls()
    tickers = check_tickers('BlkCnt', filenames, tickers) # Search Term 1
    tickers = check_tickers('SplyCur', filenames, tickers) # Search Term 2
    filtered_tickers = []
    for ticker in tqdm(tickers):
        try:
            df, params = stock_to_flow_model(ticker, p0)
            filtered_tickers.append(ticker)
        except (AttributeError, ValueError, RuntimeWarning):
            continue
    print('Tickers which ran without raising an Error or RuntimeWarning: {}'.format(filtered_tickers))
    print('Number of Tickers: {}'.format(len(filtered_tickers)))
    return filtered_tickers

def regression_analysis(df, show=False, cov_type='HAC'):
    """
    Tests the null hypothesis that the computed stock-to-flow model does not correlate with actual observed values. Rejecting
    the null hypothesis means accepting the alternative hypothesis that the stock-to-flow model does correlate with
    observed data.

    :param df: (pd.DataFrame) DataFrame containing data necessary to compute stock-to-flow model.
    :param show: (bool) Optional, if True, prints the results of the regression analysis and hypothesis test. Defaults
                        to False.
    :param cov_type: (str) Optional, the type of robust sandwich estimator to use. See
                           https://www.statsmodels.org/dev/generated/statsmodels.regression.linear_model.OLSResults.get_robustcov_results.html#statsmodels.regression.linear_model.OLSResults.get_robustcov_results
                           for more information. Defaults to 'HAC'. See
                           https://www.statsmodels.org/devel/generated/statsmodels.stats.sandwich_covariance.cov_hac.html#statsmodels.stats.sandwich_covariance.cov_hac
                           for more information.
    :returns: (obj) Results instance with the requested robust covariance as the default
                   covariance of the parameters. Inferential statistics like p-values and hypothesis tests will be based
                   on this covariance matrix.
    """
    x = df['ModelCapMrktCurUSD']
    y = df['CapMrktCurUSD']
    X = sm.add_constant(x)
    results = sm.OLS(y, X).fit().get_robustcov_results(cov_type, maxlags=1) # 'HAC' uses Newey-West method
    if show==True:
        print(results.summary())
        print('\nConclusion: ')
        if results.f_pvalue < 0.05:

            print('[1] Reject H\N{SUBSCRIPT ZERO} because \N{greek small letter beta}\N{SUBSCRIPT ONE} is statistically different from 0.')
            print('[2] Model may have explanatory value.')
        else:
            print('[1] Fail to reject H\N{SUBSCRIPT ZERO} because \N{greek small letter beta}\N{SUBSCRIPT ONE} is not statistically different from 0.')
            print('[2] Model does not appear to have explanatory value.')
    return results

def model_significance(ticker, results):
    """
    Generates DataFrame containing statistical significance and correlation data for quick reference.

    :param ticker: (str) Ticker of cryptocurrency.
    :param results: (obj) Results instance with the requested robust covariance as the default covariance of the
                          parameters. Inferential statistics like p-values and hypothesis tests will be based on this
                          covariance matrix.
    :returns: (pd.DataFrame) DataFrame containing statistical significance and correlation data.
    """
    return pd.DataFrame(
        index=['f_pvalue', 'const_pvalue', 'beta_pvalue', 'rsquared', 'rsquared_adj'],
        columns=[ticker],
        data=[results.f_pvalue, results.pvalues[0], results.pvalues[1], results.rsquared, results.rsquared_adj]
    ).round(3)

def confidence_interval(df, ticker, results, show=False):
    """
    Generates confidence interval data based on regression analysis.

    :param df: (pd.DataFrame) DataFrame containing data necessary to compute stock-to-flow model.
    :param ticker: (str) Ticker of cryptocurrency.
    :param results: (obj) Results instance with the requested robust covariance as the default covariance of the
                          parameters. Inferential statistics like p-values and hypothesis tests will be
                          based on this covariance matrix.
    :param show: (bool) Optional, if True, prints the results of the regression analysis and hypothesis test. Defaults
                        to false.
    :returns: (pd.Series, pd.Series) Contains tuple of two pd.Series containing the lower confidence level and upper
                                    confidence level.
    """
    get_prediction = results.get_prediction().summary_frame()
    obs_ci_lower, obs_ci_upper = get_prediction.obs_ci_lower, get_prediction.obs_ci_upper
    if show==True:
        print('Ticker: {}'.format(ticker))
        print('Confidence Level: 95%')
        print('Current Market Value: ${:,.2f}'.format(df['CapMrktCurUSD'][-1]))
        print('Lower 95%: ${:,.2f} or {:,.2f}%'.format(obs_ci_lower[-1], (obs_ci_lower[-1] / df['CapMrktCurUSD'][-1] - 1) * 100))
        print('Mean Estimate: ${:,.2f} or {:,.2f}%'.format(results.predict()[-1], (results.predict()[-1] / df['CapMrktCurUSD'][-1] - 1) * 100))
        print('Upper 95%: ${:,.2f} or {:,.2f}%'.format(obs_ci_upper[-1], (obs_ci_upper[-1] / df['CapMrktCurUSD'][-1] - 1) * 100))
    return obs_ci_lower, obs_ci_upper

def markdown_model(params):
    """
    Helper function to display specified regression model in markdown format.

    :param params: (list of [float, float]) Ticker of cryptocurrency.
    :returns: (markdown text) Generates log scale pyplot.
    """
    a, b = params[0].round(3), params[1].round(3)
    print('Power Law Model:')
    display(Math(r'MarketCapUSD = e^{{{}}} * SF^{{{}}}'.format(a, b)))
    print('which is equivalent to the linear function:')
    display(Math(r'ln(MarketCapUSD) = {{{}}} * ln(SF) + {{{}}}'.format(b, a)))
    print('which is a linear function.')

def breuschpagan(results, alpha=0.05):
    """
    Prints results from Breusch-Pagan Lagrange Multiplier test for heteroscedasticity. The null hypothesis of the test
    is that there is no heteroskedasticity.

    :param results: (obj) Results instance with the requested robust covariance as the default covariance of the
                          parameters. Inferential statistics like p-values and hypothesis tests will be
                          based on this covariance matrix.
    :param alpha: (float) Significance level.
    :returns: None
    """
    name = ['Lagrange multiplier statistic', 'p-value', 'f-value', 'f p-value']
    test = sms.het_breuschpagan(results.resid, results.model.exog)
    het_breuschpagan = lzip(name, test)
    print('Breusch-Pagan f p-value = {}'.format(het_breuschpagan[3][1]))
    print('')
    print('Conclusion: ')
    if het_breuschpagan[3][1] < alpha:
        print('[1] Heteroskedasticity detected. Reject null hypothesis of no heteroskedasticity.')
        print('[2] The variance of the error terms may not be the same for all observations.')
        print('[3] OLS standard errors in this regression likely significantly underestimate the true standard errors.')
        print('[4] t-statistics for the significance of individual regression coefficients likely to be inflated and unreliable.')
        print('[5] Estimators of the standard error of regression coefficients likely to be biased and unreliable.')
    else:
        print('[1] Heteroskedasticity was not detected. Fail to reject null hypothesis of no heteroskedasticity.')

def durbinwatson(results, critical_value=1.925):
    """
    Prints results from Durbin-Watson test for serial correlation. The null hypothesis of the test is that there is no
    serial correlation in the residuals.

    :param results: (obj) Results instance with the requested robust covariance as the default covariance of the
                          parameters. Inferential statistics like p-values and hypothesis tests will be
                          based on this covariance matrix.
    :param critical_value: (float) Critical value with which to compare the Durbin-Watson statistic to test for serial
                                   correlation.
    :returns: None
    """
    dw = durbin_watson(results.resid).round(3)
    print('Durbin-Watson = {}'.format(dw))
    print('')
    print('Conclusion: ')
    if dw < critical_value:
        print('[1] Positive serial correlation detected. Reject null hypothesis of no positive serial correlation.')
        print('[2] F-statistic to test overall significance of the regression likely to be inflated.')
        print('[3] OLS standard errors in this regression likely significantly underestimate the true standard errors.')
        print('[4] t-statistics for the significance of individual regression coefficients likely to be inflated and unreliable.')
        print('[5] Estimators of the standard error of regression coefficients likely to be biased and unreliable.')
    else:
        print(
            '[1] No positive serial correlation detected. Fail to reject null hypothesis of no positive serial correlation.')

def shapiro_test(results, alpha=0.05):
    """
    Prints results from Shapiro-Wilk test for normality. The Shapiro-Wilk test tests the null hypothesis that the data
    was drawn from a normal distribution.

    :param results: (obj) Results instance with the requested robust covariance as the default covariance of the
                          parameters. Inferential statistics like p-values and hypothesis tests will be
                          based on this covariance matrix.
    :param alpha: (float) Significance level.
    :returns: None
    """
    shapiro_test = stats.shapiro(results.resid)
    print(shapiro_test)
    print('')
    print('Conclusion:')
    if shapiro_test[1] < alpha:
        print('[1] Non-normality detected. Reject null hypothesis that the residuals are normal.')
        print('[2] Regression model may violate assumption of normality in linear regression.')
    else:
        print('[1] Normality detected. Fail to reject null hypothesis that the residuals are normal.')
        print('[2] Regression model appears to satisfy the assumption of normality in linear regression.')

def adfuller_test(df, alpha=0.05):
    """
    Prints results from Augmented Dickey-Fuller unit root test. The Augmented Dickey-Fuller test can be used to test
    for a unit root in a univariate process in the presence of serial correlation.

    :param df: (pd.DataFrame) DataFrame containing data necessary to compute stock-to-flow model.
    :param alpha: (float) Significance level.
    :returns: None
    """
    def adf_test(timeseries):
        print("Results of Dickey-Fuller Test:")
        dftest = adfuller(timeseries)
        dfoutput = pd.Series(
            dftest[0:4],
            index=[
                "Test Statistic",
                "p-value",
                "#Lags Used",
                "Number of Observations Used",
            ],
        )
        for key, value in dftest[4].items():
            dfoutput["Critical Value (%s)" % key] = value
        print(dfoutput)

    adf_sf = adfuller(df['StocktoFlow'])[1].round(4)
    adf_mktcap = adfuller(df['CapMrktCurUSD'])[1].round(4)
    print('Augmented Dickey-Fuller = {} (StocktoFlow)'.format(adf_sf))
    print('Augmented Dickey-Fuller = {} (CapMrktCurUSD)'.format(adf_mktcap))
    print('')
    print('Conclusion: ')
    if (adf_sf > alpha) & (adf_mktcap > alpha):
        print('[1] Unit root and nonstationarity detected in both time series. Reject null hypothesis of no unit root and stationarity.')
        print('[2] Expected value of the error term may not be 0 which may result in inconsistent regression coefficients and standard erros.')
        print('[3] Variance of the error term may not be constant for all observations indicating presence of heteroskedasticity.')
        print('[4] Error term may be correlated across observations indicating presence of serial correlation.')
        print('[5] Regression model appears to violate nonstationarity assumption of linear regression and may need to be corrected.')
    elif (adf_sf > alpha) | (adf_mktcap > alpha):
        print('[1] Unit root and nonstationarity detected in one of the time series. Reject null hypothesis of no unit root and stationarity.')
        print('[2] Expected value of the error term may not be 0 which may result in inconsistent regression coefficients and standard erros.')
        print('[3] Variance of the error term may not be constant for all observations indicating presence of heteroskedasticity.')
        print('[4] Error term may be correlated across observations indicating presence of serial correlation.')
        print('[5] Regression model appears to violate nonstationarity assumption of linear regression and may need to be corrected.')
    else:
        print('[1] No positive serial correlation detected. Fail to reject null hypothesis of no positive serial correlation. ')
        print('[2] Regression model does not appear to violate nonstationarity assumption of linear regression.')

def cointegration(df, alpha=0.05):
    """
    Prints results from test for no-cointegration of a univariate equation. The null hypothesis is no cointegration.
    This uses the augmented Engle-Granger two-step cointegration test. Constant or trend is included in 1st stage
    regression, i.e. in cointegrating equation.

    :param df: (pd.DataFrame) DataFrame containing data necessary to compute stock-to-flow model.
    :param alpha: (float) Significance level.
    :returns: None
    """
    coint_test = coint(df['StocktoFlow'], df['CapMrktCurUSD'])
    coint_tstat = coint_test[0]
    coint_pvalue = coint_test[1]
    coint_critvalues = coint_test[2]
    print('Cointegration p-value = {}'.format(coint_pvalue.round(3)))
    print('')
    print('Conclusion: ')
    if coint_pvalue > alpha:
        print('[1] No cointegration detected at the 5% level. Fail to reject null hypothesis.')
        print('[2] There may not exist a cointegrated relationship between the dependent and independent variables.')
        print('[3] Regression model is likely to show spurious correlation and be unreliable.')
    else:
        print('[1] Cointegration detected at the 5% level. Reject the null hypothesis.')
        print('[2] There may exist a cointegrated relationship between the dependent and independent variables.')
        print('[3] Regression model is unlikely to show spurious correlation and may be reliable.')

def conf_int_chart(df, ticker, results, figsize=(12,6), save=False, show=True):
    """
    Generates a plot of regression model data and confidence interval.

    :param df: (pd.DataFrame) DataFrame containing data necessary to compute stock-to-flow model
    :param ticker: (str) Ticker of cryptocurrency.
    :param results: (obj) Results instance with the requested robust covariance as the default covariance of the
                          parameters. Inferential statistics like p-values and hypothesis tests will be
                          based on this covariance matrix.
    :param figsize: (float, float) Optional, multiple by which to multiply the maximum weighting constraints at the
                                   ticker level. Defaults to (12,6).
    :param save: (bool) Optional, saves the chart as a png file to charts folder. Defaults to False.
    :param show: (bool) Optional, displays plot. Defaults to True.
    :returns: (plt) Generates log scale pyplot of CapMrktCurUSD over CapMrktCurUSD with 95% confidence interval.
    """
    params = results.params
    ytrue = df['CapMrktCurUSD'].to_numpy()
    ypred = df['ModelCapMrktCurUSD'].to_numpy()
    obs_ci_lower, obs_ci_upper = confidence_interval(df, ticker, results)
    plt.style.use('default')
    fig = plt.gcf()
    fig.set_size_inches(figsize)
    plt.plot(ypred, ytrue, 'bo')
    plt.plot(ypred, results.predict(), 'r-')
    plt.plot(ypred, sorted(obs_ci_lower), 'r--')
    plt.plot(ypred, sorted(obs_ci_upper), 'r--')
    plt.title("CapMrktCurUSD vs. ModelCapMrktCurUSD ({})\n {} to {}".format(
        ticker,
        df.index[0].strftime('%m-%d-%Y'),
        df.index[-1].strftime('%m-%d-%Y')))
    plt.legend([
        'CapMrktCurUSD / ModelCapMrktCurUSD ({})'.format(ticker),
        'Linear Model: {:.4f}x + {:.4f}'.format(params[1], params[0]),
        '95% Confidence Interval'
    ])
    plt.xlabel('ModelCapMrktCurUSD ({})'.format(ticker))
    plt.ylabel('CapMrktCurUSD ({})'.format(ticker))
    plt.xscale('log')
    plt.yscale('log')
    plt.gca().yaxis.set_major_formatter(FuncFormatter(lambda ytrue, _: '{:,.16g}'.format(ytrue)))
    plt.gca().xaxis.set_major_formatter(FuncFormatter(lambda ypred, _: '{:,.16g}'.format(ypred)))
    if save == True: plt.savefig(
        '../charts/conf_int_chart_{}.png'.format(datetime.today().strftime('%m-%d-%Y')), bbox_inches='tight')
    if show == False: plt.close()

def charts(df, ticker, params, chart=1, figsize=(12,6), save=False, show=True):
    """
    Helper function of preformatted charts to show the results of the stock-to-flow model curve fitting.

    :param df: (pd.DataFrame) DataFrame containing data necessary to compute stock-to-flow model
    :param ticker: (str) Ticker of cryptocurrency.
    :param params: (list of [float,float]) Ticker of cryptocurrency.
    :param chart: (int) Select one of 3 pre-formatted charts labeled 1, 2 and 3. Defaults to 1.
    :param figsize: (float, float) Optional, multiple by which to multiply the maximum weighting constraints at the ticker level.
    :param save: (bool) Optional, saves the chart as a png file to charts folder. Defaults to False.
    :param show: (bool) Optional, displays plot. Defaults to True.
    :returns: (pyplot) Generates log scale pyplot.
    """
    dates = np.array(df.index)
    sf = df['StocktoFlow'].to_numpy()
    d = (df['MaxDrawdown%'] * 100).to_numpy()
    ytrue = df['CapMrktCurUSD'].to_numpy()
    ypred = df['ModelCapMrktCurUSD'].to_numpy()
    if chart==1:
        plt.style.use('grayscale')
        fig, ax = plt.subplots(figsize=figsize)
        im = ax.scatter(dates, ytrue, c=d, cmap=cm.jet, lw=1, alpha=1, zorder=5, label=ticker)
        plt.yscale('log', subsy=[1])
        ax.plot(dates, ypred, c='black', label='ModelCapMrktCurUSD: e^{:.3f} * SF^{:.3f}'.format(*params))
        ax.yaxis.set_major_formatter(ScalarFormatter())
        ax.yaxis.set_minor_formatter(ScalarFormatter())
        ax.yaxis.set_major_formatter(FuncFormatter(lambda ytrue, _: '{:,.16g}'.format(ytrue)))
        cbar = fig.colorbar(im, ax=ax)
        cbar.ax.set_ylabel('Maximum Drawdown%')
        plt.xlabel('Year')
        plt.ylabel('CapMrktCurUSD ({})'.format(ticker))
        plt.title("CapMrktCurUSD and ModelCapMrktCurUSD ({})\n {} to {}".format(
            ticker,
            df.index[0].strftime('%m-%d-%Y'),
            df.index[-1].strftime('%m-%d-%Y')))
        plt.legend()
        plt.show()
    elif chart==2:
        plt.style.use('default')
        fig = plt.gcf()
        fig.set_size_inches(figsize)
        plt.yscale('log')
        plt.plot(dates, ytrue, '-b')
        plt.plot(dates, ypred, 'r')
        plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%Y'))
        plt.gca().xaxis.set_major_locator(mdates.YearLocator(1))
        plt.gca().yaxis.set_major_formatter(FuncFormatter(lambda ytrue, _: '{:,.16g}'.format(ytrue)))
        plt.legend(['CapMrktCurUSD ({})'.format(ticker), 'ModelCapMrktCurUSD: e^{:.3f} * SF^{:.3f}'.format(*params)])
        plt.title("CapMrktCurUSD and ModelCapMrktCurUSD ({})\n {} to {}".format(
            ticker,
            df.index[0].strftime('%m-%d-%Y'),
            df.index[-1].strftime('%m-%d-%Y')))
        plt.xlabel('Year')
        plt.ylabel('CapMrktCurUSD ({})'.format(ticker))
    elif chart==3:
        plt.style.use('default')
        fig = plt.gcf()
        fig.set_size_inches(figsize)
        plt.yscale('log')
        plt.plot(sf, ytrue, 'bo', label='data')
        plt.plot(sf, objective(sf, *params), 'r-', label='curve_fit')
        plt.gca().yaxis.set_major_formatter(FuncFormatter(lambda ytrue, _: '{:,.16g}'.format(ytrue)))
        plt.legend(['CapMrktCurUSD ({})'.format(ticker), 'Fitted Power Law Model: e^{:.3f} * SF^{:.3f}'.format(*params)])
        plt.title("CapMrktCurUSD vs. Stock-to-Flow ({})\n {} to {}".format(
            ticker,
            df.index[0].strftime('%m-%d-%Y'),
            df.index[-1].strftime('%m-%d-%Y')))
        plt.xlabel('Stock-to-Flow ({})'.format(ticker))
        plt.ylabel('CapMrktCurUSD ({})'.format(ticker))
    else:
        raise ValueError('Invalid chart number. Type a valid number to the chart parameter.')
    if save == True: plt.savefig(
        '../charts/chart{}_{}.png'.format(chart, datetime.today().strftime('%m-%d-%Y')), bbox_inches='tight')
    if show == False: plt.close()

max_stars_repo_name: vishalbelsare/pyfinlab

max_issues_repo_name: vishalbelsare/pyfinlab

max_forks_repo_name: vishalbelsare/pyfinlab

reason: import numpy,from scipy,import statsmodels,from statsmodels