text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
###########################################
# (c) 2016-2020 Polyvios Pratikakis
# polyvios@ics.forth.gr
###########################################

"""
Statistical analysis of tweets
This file recreates many feature engineering papers
It computes mostly features based on "over all tweets" computations.
"""

import re
import sys
import optparse
import emoji
from datetime import datetime
from collections import Counter, defaultdict
from nltk.tokenize import TweetTokenizer
from nltk.metrics.distance import edit_distance
import calendar
import numpy
import operator
#from urlparse import urlparse
from urllib.parse import urlparse
from progress.bar import Bar
#from itertools import izip
from twkit.utils import *

usage_times_attrs = [
  'seen_total', 'total_inferred', 'seen_greek_total',
  'deleted_tweets',
  'all_intervals',
  'seen_top_tweets', 'top_tweets_pcnt', 'top_intervals',
  'mention_indegree', 'mention_outdegree',
  'mention_inweight', 'mention_outweight',
  'mention_avg_inweight', 'mention_avg_outweight',
  'mention_out_in_ratio', 'mention_pcnt',
  'most_mentioned_users', 'most_mentioned_by',

  'quote_indegree',
  'quote_outdegree',
  'quote_inweight',
  'quote_outweight',
  'quote_avg_inweight',
  'quote_avg_outweight',
  'quote_out_in_ratio',
  'quote_pcnt',
  'most_quoted_users',
  'most_quoted_by',

  'retweet_indegree', 'retweet_outdegree',
  'retweet_inweight', 'retweet_outweight',
  'retweet_avg_inweight', 'retweet_avg_outweight',
  'retweet_out_in_ratio', 'retweet_pcnt',
  'most_retweeted_users', 'most_retweeted_by',
  'rt_intervals',
  'reply_indegree', 'reply_outdegree',
  'reply_inweight', 'reply_outweight',
  'reply_avg_inweight', 'reply_avg_outweight',
  'reply_out_in_ratio', 'replies_pcnt',
  'most_replied_to', 'most_replied_by',
  'reply_intervals',
  'seen_replied_to', 'most_engaging_tweet',
  'plain_tweets',
  'most_used_sources',
  'time_between_any', 'time_between_top', 'time_between_rt', 'time_between_replies',
  'tweets_per_hour_of_day', 'tweets_per_weekday', 'tweets_per_active_day', 'tweets_per_day',
  'last_tweeted_at', 'life_time', 'max_daily_interval',
  'number_of_languages', 'tweets_per_language',
  'last_month',
  'userinfo_timeseries'
  ]
usage_times_buckets_sec = [1, 30, 60, 3600, 3600*24, 3600*24*7, 3600*24*30, 3600*24*365]

def log_event(d, lastd, quanta, intervals):
  if lastd is None: return None
  delta = d - lastd
  i = 0
  sec = delta.total_seconds()
  while(i < 8 and sec > usage_times_buckets_sec[i]): i+=1
  quanta[i] += 1
  intervals.append(sec)
  return sec

def get_all_tweets(db, criteria, batch=None):
  if criteria:
    tweets = db.tweets.find({'created_at' : criteria}).sort('created_at', 1)
  else:
    tweets = db.tweets.find().sort('created_at', 1)
  if batch:
    tweets = tweets.batch_size(batch)
  if verbose():
    return Bar("Loading:", max=tweets.count(), suffix = '%(index)d/%(max)d - %(eta_td)s').iter(tweets)
  return tweets

cached_tweets={}
def get_user_tweets(db, userid, criteria, batch=None, cacheifsmall=True, timesort=1):
  global cached_tweets
  if cacheifsmall:
    if cached_tweets.get('uid', -1) == userid and 'tweets' in cached_tweets:
      # and cached_tweets.get('before', None) == before and cached_tweets.get('after', None) == after:
      return cached_tweets['tweets']
  if criteria:
    tweets = db.tweets.find({'user.id' : userid, 'created_at' : criteria }).sort('created_at', timesort)
  else:
    tweets = db.tweets.find({'user.id' : userid}).sort('created_at', timesort)
  if batch:
    tweets = tweets.batch_size(batch)
  count = tweets.count()
  if cacheifsmall:
    if count < 100000:
      cached_tweets.clear()
      cached_tweets['uid'] = userid
      #cached_tweets['before'] = before
      #cached_tweets['after'] = after
      cached_tweets['tweets'] = list(tweets)
      tweets = cached_tweets['tweets']
  if verbose():
    return Bar("Loading:", max=count, suffix = '%(index)d/%(max)d - %(eta_td)s').iter(tweets)
  return tweets

def get_user_replies(db, userid, criteria):
  if criteria:
    tweets = db.tweets.find({'in_reply_to_user_id' : userid, 'created_at' : criteria})
  else:
    tweets = db.tweets.find({'in_reply_to_user_id' : userid})
  if verbose():
    return Bar("Loading replies:", max=tweets.count(), suffix = '%(index)d/%(max)d - %(eta_td)s').iter(tweets)
  return tweets

def get_user_mentions(db, userid, criteria):
  if criteria:
    tweets = db.tweets.find({'user_mentions' : {'$elemMatch': {'id': userid}}, 'created_at' : criteria})
  else:
    tweets = db.tweets.find({'user_mentions' : {'$elemMatch': {'id': userid}}})
  if verbose():
    return Bar("Loading mentions:", max=tweets.count(), suffix = '%(index)d/%(max)d - %(eta_td)s').iter(tweets)
  return tweets

def get_user_retweets(db, userid, criteria):
  """
  Returns tweets by other users that retweet tweets by the given user.
  """
  if criteria:
    tweets = db.tweets.find({'retweeted_status.user.id' : userid, 'created_at': criteria})
  else:
    tweets = db.tweets.find({'retweeted_status.user.id' : userid})
  if verbose():
    return Bar("Loading retweets:", max=tweets.count(), suffix = '%(index)d/%(max)d - %(eta_td)s').iter(tweets)
  return tweets

def get_retweeters(db, uid, criteria):
  retweeters = Counter()
  for t in get_user_retweets(db, uid, criteria):
    retweeters[t['user']['id']] += 1
  return retweeters

def get_retweeted(db, uid, criteria):
  retweeted = Counter()
  for t in get_user_tweets(db, uid, criteria):
    if 'retweeted_status' not in t: continue
    retweeted[t['retweeted_status']['user']['id']] += 1
  return retweeted

def get_user_quoted_tweets(db, userid, criteria):
  """
  Returns tweets by other users that quote tweets by the given user.
  """
  if criteria:
    tweets = db.tweets.find({'quoted_status.user.id' : userid, 'created_at': criteria})
  else:
    tweets = db.tweets.find({'quoted_status.user.id' : userid})
  if verbose():
    return Bar("Loading quotes:", max=tweets.count(), suffix = '%(index)d/%(max)d - %(eta_td)s').iter(tweets)
  return tweets

def get_quoters(db, uid, criteria):
  """
  Returns users that have quoted the given user.
  """
  quoters = Counter()
  for t in get_user_quoted_tweets(db, uid, criteria):
    quoters[t['user']['id']] += 1
  return quoters

def get_quoted(db, uid, criteria):
  """
  Returns users quoted by the given user.
  """
  quoted = Counter()
  for t in get_user_tweets(db, uid, criteria):
    if 'quoted_status' not in t: continue
    quoted[t['quoted_status']['user']['id']] += 1
  return quoted

def fill_lastmonth_usage(db, u):
  lastday = u['last_tweeted_at'].replace(hour=0, minute=0, second=0, microsecond=0)
  monthstart = lastday - timedelta(days=30)
  lastmonth = {x: Counter() for x in range(0,31)}
  criteria = {}
  criteria['$lte'] = lastday
  criteria['$gte'] = monthstart
  for tweet in get_user_tweets(db, u['id'], criteria):
    d = tweet.get('created_at')
    if d is None: continue
    if d > monthstart:
      sec = int((d - monthstart).total_seconds())
      days = sec // 3600 // 24
      hours = sec // 3600 % 24
      lastmonth[days][hours] += 1
  u['last_month'] = [{'day': monthstart+timedelta(days=i), 'hour': j, 'count': lastmonth[i][j]} for i in lastmonth for j in lastmonth[i]]

def fill_userinfo_timeseries(db, u):
  data = [
    { 'key': 'statuses', 'values': [] },
    { 'key': 'friends', 'values': [] },
    { 'key': 'followers', 'values': [] }
  ]
  users = db.users.find(
    { 'id': u['id'] },
    { 'statuses_count': 1,
      'friends_count': 1,
      'followers_count': 1,
      'timestamp_at':1
    }
  ).sort('_id', 1)
  cnt = users.count()
  #skip = 0
  #if cnt > 1000:
  for x in users:
    t = x['timestamp_at'] if 'timestamp_at' in x else x['_id'].generation_time
    if 'statuses_count' in x: data[0]['values'].append( {'date': t, 'y': x['statuses_count']} )
    if 'friends_count' in x: data[1]['values'].append( {'date': t, 'y': x['friends_count']} )
    if 'followers_count' in x: data[2]['values'].append( {'date': t, 'y': x['followers_count']} )
  u['userinfo_timeseries'] = data

def usage_times_stats(db, u, criteria):
  userid = u['id']

  hcnt = Counter({x: 0 for x in range(0, 24)})
  dcnt = Counter({x: 0 for x in range(0, 7)})

  allquanta = Counter()
  intervals = []
  lasttime = None

  max_daily_interval = Counter()
  tweets_per_day = Counter()

  topquanta = Counter()
  topintervals = []
  lasttoptime = None
  seentop = 0

  rtquanta = Counter()
  rtintervals = []
  lastrttime = None
  rtcnt = Counter()

  replyquanta = Counter()
  replyintervals = []
  lastreplytime = None
  replycnt = Counter()

  mentioncnt = Counter()
  quotecnt = Counter()

  sourcecnt = Counter()
  lastday = datetime.now()

  el_tweets = 0
  languages = Counter()
  plain_tweets = 0
  total = 0
  total_seen = 0
  deleted_tweets = 0

  if verbose(): print(" scan tweets")
  for tweet in get_user_tweets(db, u['id'], criteria):
    total += 1
    if 'created_at' not in tweet: continue
    total_seen += 1
    d = tweet['created_at']
    if tweet.get('deleted', False):
      deleted_tweets += 1
    if 'lang' in tweet:
      if tweet['lang'] == config.lang: el_tweets += 1
      languages[tweet['lang']] += 1
    lastday = d
    sleepseconds = log_event(d, lasttime, allquanta, intervals)
    lasttime = d
    sourcecnt[tweet.get('source', 'empty_source_tag')] += 1
    day = d.replace(hour=0, minute=0, second=0, microsecond=0)
    tweets_per_day[day] += 1
    if sleepseconds:
      max_daily_interval[day] = max(max_daily_interval[day], sleepseconds)

    if 'retweeted_status' in tweet:
      rtid = tweet['retweeted_status']['user']['id']
      if rtid is None: raise "oops!"
      log_event(d, lastrttime, rtquanta, rtintervals)
      lastrttime = d
      rtcnt[rtid] += 1
    else: #orig user content, not RT
      if tweet.get('in_reply_to_user_id', None):
        log_event(d, lastreplytime, replyquanta, replyintervals)
        replycnt[tweet['in_reply_to_user_id']] += 1
        lastreplytime = d
      elif len(tweet.get('user_mentions', [])):
        #mentions that are not replies
        mentioncnt += Counter(x['id'] for x in tweet['user_mentions'] if x is not None)
      else:
        #top level tweets
        log_event(d, lasttoptime, topquanta, topintervals)
        seentop += 1
        lasttoptime = d
      if 'quoted_status_id' in tweet:
        if 'quoted_status' in tweet:
          try:
            quotecnt[tweet['quoted_status']['user']['id']] += 1
          except:
            print("\nerror!")
            print(tweet)
        else:
          print(u"Found quoter tweet with missing quoted status: {} -> {}".format(tweet['id'], tweet['quoted_status_id']))
          qid = int(tweet['quoted_status_id'])
          tt = db.tweets.find_one({'id': qid})
          if tt is None:
            print("Could not find quoted status locally")
          else:
            del tt['_id']
            db.tweets.update_one(tweet, {'$set' : { 'quoted_status_id': qid, 'quoted_status' : tt }})
      if len(tweet.get('urls', [])) == 0 and len(tweet.get('user_mentions',[])) == 0 and len(tweet.get('hashtags', [])) == 0:
        plain_tweets += 1
    hcnt[d.hour] += 1
    dcnt[d.weekday()] += 1

  if verbose(): print(" mentions")
  mentionbycnt = Counter()
  seenmentions = 0
  user_mentions = get_user_mentions(db, u['id'], criteria)
  for tweet in user_mentions:
    mentionbycnt[tweet['user']['id']] += 1
    seenmentions += 1

  if verbose(): print(" retweets")
  rtbycnt = Counter()
  user_retweets = get_user_retweets(db, u['id'], criteria)
  for tweet in user_retweets:
    rtbycnt[tweet['user']['id']] += 1

  if verbose(): print(" replies")
  user_replies = get_user_replies(db, u['id'], criteria)
  repliedcnt = Counter()
  repliedto = Counter()
  for tweet in user_replies:
    repliedcnt[tweet['user']['id']] += 1
    if 'in_reply_to_status_id' not in tweet:
      # if the replied-to tweet is deleted, twitter doesn't populate
      # the field
      #print("oops:",
      #print(tweet)
      continue
    repliedto[tweet['in_reply_to_status_id']] += 1
  most_engaging = repliedto.most_common(1)
  if len(most_engaging):
    most_engaging_tweet = db.tweets.find_one({'id': most_engaging[0]})
  else:
    most_engaging_tweet = None

  if verbose(): print(" quotes")
  user_quotes = get_user_quoted_tweets(db, u['id'], criteria)
  quotedcnt = Counter()
  quotedtw = Counter()
  for tweet in user_quotes:
    quotedcnt[tweet['user']['id']] += 1
    if 'quoted_status_id' not in tweet: continue
    quotedtw[tweet['quoted_status_id']] += 1
  most_quoted = quotedtw.most_common(1)
  if len(most_quoted):
    most_quoted_tweet = db.tweets.find_one({'id': most_quoted[0]})
  else:
    most_quoted_tweet = None

  if verbose(): print(" saving")
  #total = seentop + seenreplies + seenrt + seenmention
  qrange = range(0,len(usage_times_buckets_sec))
  u['seen_total'] = total_seen
  u['total_inferred'] = total
  total = total_seen
  u['seen_greek_total'] = el_tweets
  u['deleted_tweets'] = deleted_tweets
  u['number_of_languages'] = len(languages)
  u['tweets_per_language'] = [{'lang': i[0], 'count': i[1]} for i in languages.most_common(5)]
  u['all_intervals'] = [{'bucket': i, 'count': allquanta[i]} for i in qrange]
  if total == 0: total = 1 #avoid divzero
  u['seen_top_tweets'] = seentop
  u['top_tweets_pcnt'] = 100.0 * seentop / total
  u['top_intervals'] = [{'bucket': i, 'count': topquanta[i]} for i in qrange]

  u['mention_indegree'] = len(mentionbycnt)
  u['mention_outdegree'] = len(mentioncnt)
  u['mention_inweight'] = sum(mentionbycnt.values())
  u['mention_outweight'] = sum(mentioncnt.values())
  u['mention_avg_inweight'] = 1.0 * u['mention_inweight'] / max(1, u['mention_indegree'])
  u['mention_avg_outweight'] = 1.0 * u['mention_outweight'] / max(1, u['mention_outdegree'])
  u['mention_out_in_ratio'] = 1.0 * u['mention_outdegree'] / max(1, u['mention_indegree'])
  u['mention_pcnt'] = 100.0 * u['mention_outweight'] / total
  u['most_mentioned_users'] = [{'id': i[0], 'user': id_to_userstr(db, i[0]), 'count': i[1]} for i in mentioncnt.most_common(500)]
  u['most_mentioned_by'] = [{'id': i[0], 'user': id_to_userstr(db, i[0]), 'count': i[1]} for i in mentionbycnt.most_common(500)]

  u['quote_indegree'] = len(quotedcnt)
  u['quote_outdegree'] = len(quotecnt)
  u['quote_inweight'] = sum(quotedcnt.values())
  u['quote_outweight'] = sum(quotecnt.values())
  u['quote_avg_inweight'] = 1.0 * u['quote_inweight'] / max(1, u['quote_indegree'])
  u['quote_avg_outweight'] = 1.0 * u['quote_outweight'] / max(1, u['quote_outdegree'])
  u['quote_out_in_ratio'] = 1.0 * u['quote_outdegree'] / max(1, u['quote_indegree'])
  u['quote_pcnt'] = 100.0 * u['quote_outweight'] / total
  u['most_quoted_users'] = [{'id': i[0], 'user': id_to_userstr(db, i[0]), 'count': i[1]} for i in quotecnt.most_common(500)]
  u['most_quoted_by'] = [{'id': i[0], 'user': id_to_userstr(db, i[0]), 'count': i[1]} for i in quotedcnt.most_common(500)]

  u['retweet_indegree'] = len(rtbycnt)
  u['retweet_outdegree'] = len(rtcnt)
  u['retweet_inweight'] = sum(rtbycnt.values())
  u['retweet_outweight'] = sum(rtcnt.values())
  u['retweet_avg_inweight'] = 1.0 * u['retweet_inweight'] / max(1, u['retweet_indegree'])
  u['retweet_avg_outweight'] = 1.0 * u['retweet_outweight'] / max(1, u['retweet_outdegree'])
  u['retweet_out_in_ratio'] = 1.0 * u['retweet_outdegree'] / max(1, u['retweet_indegree'])
  u['retweet_pcnt'] = 100.0 * u['retweet_outdegree'] / total
  u['most_retweeted_by'] = [{'id': i[0], 'user': id_to_userstr(db, i[0]), 'count': i[1]} for i in rtbycnt.most_common(500)]
  u['most_retweeted_users'] = [{'id': i[0], 'user': id_to_userstr(db, i[0]), 'count': i[1]} for i in rtcnt.most_common(500)]
  u['rt_intervals'] = [{'bucket': i, 'count': rtquanta[i]} for i in qrange]

  u['reply_indegree'] = len(repliedcnt)
  u['reply_outdegree'] = len(replycnt)
  u['reply_inweight'] = sum(repliedcnt.values())
  u['reply_outweight'] = sum(replycnt.values())
  u['reply_avg_inweight'] = 1.0 * u['reply_inweight'] / max(1, u['reply_indegree'])
  u['reply_avg_outweight'] = 1.0 * u['reply_outweight'] / max(1, u['reply_outdegree'])
  u['reply_out_in_ratio'] = 1.0 * u['reply_outdegree'] / max(1, u['reply_indegree'])
  u['replies_pcnt'] = 100.0 * u['reply_outdegree'] / total
  u['most_replied_to'] = [{'id': i[0], 'user': id_to_userstr(db, i[0]), 'count': i[1]} for i in replycnt.most_common(500)]
  u['most_replied_by'] = [{'id': i[0], 'user': id_to_userstr(db, i[0]), 'count': i[1]} for i in repliedcnt.most_common(500)]
  u['reply_intervals'] = [{'bucket': i, 'count': replyquanta[i]} for i in qrange]
  u['seen_replied_to'] = len(repliedto)
  u['most_engaging_tweet'] = {'id': most_engaging_tweet['id'], 'text': most_engaging_tweet.get('text', '<not crawled>')} if most_engaging_tweet is not None else None

  u['plain_tweets'] = plain_tweets
  u['most_used_sources'] = [{'source': i[0], 'count': i[1]} for i in sourcecnt.most_common(500)]
  u['time_between_any'] = {
    'min': min(intervals) if len(intervals) else None,
    'max': max(intervals) if len(intervals) else None,
    'avg': numpy.mean(intervals) if len(intervals) else None,
    'med': numpy.median(intervals) if len(intervals) else None,
    'std': numpy.std(intervals) if len(intervals) else None
  }
  u['time_between_top'] = {
    'min': min(topintervals) if len(topintervals) else None,
    'max': max(topintervals) if len(topintervals) else None,
    'avg': numpy.mean(topintervals) if len(topintervals) else None,
    'med': numpy.median(topintervals) if len(topintervals) else None,
    'std': numpy.std(topintervals) if len(topintervals) else None
  }
  u['time_between_rt'] = {
    'min': min(rtintervals) if len(rtintervals) else None,
    'max': max(rtintervals) if len(rtintervals) else None,
    'avg': numpy.mean(rtintervals) if len(rtintervals) else None,
    'med': numpy.median(rtintervals) if len(rtintervals) else None,
    'std': numpy.std(rtintervals) if len(rtintervals) else None
  }
  u['time_between_replies'] = {
    'min': min(replyintervals) if len(replyintervals) else None,
    'max': max(replyintervals) if len(replyintervals) else None,
    'avg': numpy.mean(replyintervals) if len(replyintervals) else None,
    'med': numpy.median(replyintervals) if len(replyintervals) else None,
    'std': numpy.std(replyintervals) if len(replyintervals) else None
  }
  maxdinterval = list(max_daily_interval.values())
  u['max_daily_interval'] = {
    'min': min(maxdinterval),
    'minday': min(max_daily_interval, key=max_daily_interval.get),
    'max': max(maxdinterval),
    'maxday': max(max_daily_interval, key=max_daily_interval.get),
    'avg': numpy.mean(maxdinterval),
    'med': numpy.median(maxdinterval),
    'std': numpy.std(maxdinterval)
  } if len(maxdinterval) else {
    'min': None,
    'minday': None,
    'max': None,
    'avg': None,
    'med': None,
    'std': None
  }
  u['last_tweeted_at'] = lastday
  lifetime = int((lastday - u['created_at']).total_seconds())
  seconds = lifetime % 60
  minutes = (lifetime / 60) % 60
  hours = (lifetime / 3600) % 24
  days = lifetime / (3600 * 24) % 365
  years = lifetime / (3600 * 24 * 365)
  u['life_time'] = {
    'years': years,
    'days': days,
    'hours': hours,
    'minutes': minutes,
    'seconds': seconds
  }
  u['tweets_per_hour_of_day'] = [{'hour': i, 'count': hcnt[i]} for i in hcnt]
  u['tweets_per_weekday'] = [{'day': i, 'count': dcnt[i]} for i in dcnt]
  twdinterval = list(tweets_per_day.values())
  u['tweets_per_active_day'] = {
    'min': min(twdinterval) if len(twdinterval) else 0,
    'minday': min(tweets_per_day, key=tweets_per_day.get) if len(twdinterval) else 0,
    'max': max(twdinterval) if len(twdinterval) else 0,
    'maxday': max(tweets_per_day, key=tweets_per_day.get) if len(twdinterval) else 0,
    'avg': numpy.mean(twdinterval) if len(twdinterval) else 0,
    'med': numpy.median(twdinterval) if len(twdinterval) else 0,
    'std': numpy.std(twdinterval) if len(twdinterval) else 0
  }
  startd = u['created_at'].replace(hour=0, minute=0, second=0, microsecond=0)
  endd = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
  while startd <= endd:
    if startd not in tweets_per_day:
      tweets_per_day[startd] = 0  #zero the missing ones
    startd = startd + timedelta(days=1)
  twdinterval = list(tweets_per_day.values()) #now count all zero days
  u['tweets_per_day'] = {
    'min': min(twdinterval) if len(twdinterval) else None,
    'minday': min(tweets_per_day, key=tweets_per_day.get) if len(twdinterval) else None,
    'max': max(twdinterval) if len(twdinterval) else None,
    'maxday': max(tweets_per_day, key=tweets_per_day.get) if len(twdinterval) else None,
    'avg': numpy.mean(twdinterval) if len(twdinterval) else None,
    'med': numpy.median(twdinterval) if len(twdinterval) else None,
    'std': numpy.std(twdinterval) if len(twdinterval) else None
  }
  fill_lastmonth_usage(db, u)
  fill_userinfo_timeseries(db, u)

# end

follower_stats_attrs = [
  'fr_scanned_at', 'seen_fr', 'gr_fr', 'gr_fr_pcnt', 'tr_fr', 'tr_fr_pcnt',
  'fo_scanned_at', 'seen_fo', 'gr_fo', 'gr_fo_pcnt', 'tr_fo', 'tr_fo_pcnt',
  'fr_fo_jaccard', 'fr_and_fo', 'fr_or_fo', 'gr_fr_fo', 'gr_fr_fo_pcnt',
  'greek' ]

def fill_follower_stats(db, u):
  totalfr = 0
  grfrcnt = 0
  trackedfr = 0
  #for fid in db.follow.find({'id': u['id']}).distinct('follows'):
  friends = frozenset(get_friends(db, u['id']))
  followers = frozenset(get_followers(db, u['id']))
  fr_fo_count = len(friends & followers)
  all_count = len(friends | followers)
  fr_fo_jaccard = 1.0 * fr_fo_count / (all_count if all_count > 0 else 1)
  fr_fo_gr = Counter()
  for fid in friends:
    if is_greek(db, fid):
      grfrcnt += 1
      fr_fo_gr[fid] += 1
    x = get_tracked(db, fid)
    if x:
      trackedfr += 1
    totalfr += 1
  totalfo = 0
  grfocnt = 0
  trackedfo = 0
  #for fid in db.follow.find({'follows': u['id']}).distinct('id'):
  for fid in followers:
    if is_greek(db, fid):
      grfocnt += 1
      fr_fo_gr[fid] += 1
    x = get_tracked(db, fid)
    if x:
      trackedfo += 1
    totalfo += 1
  frscan = is_recently_scanned(db, u['id'], 'friends')
  if frscan: frscan = frscan.replace(microsecond=0).isoformat()
  foscan = is_recently_scanned(db, u['id'], 'followers')
  gr_fr_fo = len(fr_fo_gr.keys())
  if foscan: foscan = foscan.replace(microsecond=0).isoformat()
  u['fr_scanned_at'] = frscan
  u['seen_fr'] = totalfr
  u['gr_fr'] = grfrcnt
  if totalfr == 0: totalfr = 1 #smoothing to avoid div by zero
  u['gr_fr_pcnt'] = 100.0*grfrcnt/totalfr
  u['tr_fr'] = trackedfr
  u['tr_fr_pcnt'] = 100*trackedfr/totalfr
  u['fo_scanned_at'] = foscan
  u['seen_fo'] = totalfo
  u['gr_fo'] = grfocnt
  if totalfo == 0: totalfo = 1 #smoothing to avoid div by zero
  u['gr_fo_pcnt'] = 100.0*grfocnt/totalfo
  u['tr_fo'] = trackedfo
  u['tr_fo_pcnt'] = 100.0*trackedfo/totalfo
  u['fr_fo_jaccard'] = fr_fo_jaccard
  u['fr_and_fo'] = fr_fo_count
  u['fr_or_fo'] = all_count
  u['gr_fr_fo'] = gr_fr_fo
  u['gr_fr_fo_pcnt'] = 100.0*gr_fr_fo / (all_count if all_count > 0 else 1)
  u['greek'] = is_greek(db, u['id'])

class itertext(object):
  def __init__(self, cursor):
    self.cursor = cursor
  def __iter__(self):
    return (self)
  def __next__(self):
    n = next(self.cursor)
    while 'text' not in n:
      n = next(self.cursor)
    text = n['text']
    res = re.sub(r'https?://[\S/]+', '', text, flags=re.MULTILINE)
    res = re.sub(r'@[a-zA-Z0-9_]+', '', res, flags=re.MULTILINE)
    res = re.sub(r'#\S+', '', res, flags=re.MULTILINE)
    res = re.sub(r'\b\d+\b', '', res, flags=re.MULTILINE)
    #res = re.sub('ς', 'σ', res)
    #print(text.encode('utf-8'))
    #print(res.encode('utf-8'))
    #print("-----")
    return res

def deaccent(s):
  assert(isinstance(s, str))
  return s \
    .replace(u'ά', u'α') \
    .replace(u'έ', u'ε') \
    .replace(u'ή', u'η') \
    .replace(u'ί', u'ι') \
    .replace(u'ό', u'ο') \
    .replace(u'ύ', u'υ') \
    .replace(u'ώ', u'ω') \
    .replace(u'ς', u'σ')

import os
crawlerdir = os.environ['CRAWLERDIR']

expletives = set()
with open(crawlerdir+"greekdata/expletives", "r") as f:
  for line in f:
    expletives.add(deaccent(line.strip().lower()))
articles = set()
with open(crawlerdir+"greekdata/articles", "r") as f:
  for line in f:
    articles.add(deaccent(line.strip().lower()))
pronouns = set()
with open(crawlerdir+"greekdata/pronouns", "r") as f:
  for line in f:
    pronouns.add(deaccent(line.strip().lower()))
locations = set()
with open(crawlerdir+"greekdata/locations", "r") as f:
  for line in f:
    locations.add(deaccent(line.strip().lower()))
negationwords = set()
with open(crawlerdir+"greekdata/negationwords", "r") as f:
  for line in f:
    negationwords.add(deaccent(line.strip().lower()))
emoticons = set()
with open(crawlerdir+"greekdata/emoticons", "r") as f:
  for line in f:
    emoticons.add(deaccent(line.strip().lower()))
stopwords = set()
with open(crawlerdir+"greekdata/stopwords", "r") as f:
  for line in f:
    w = line.strip()
    stopwords.add(deaccent(line.strip().lower()))
punctuation_chars = u'@![]...;:?«»"\'+-&()\\/,<>`´“”|*%…’'

def is_location(word):
  for l in locations:
    if re.match(l, word, re.I): return True
  return False

def letter_count(word):
  digit = 0
  alpha = 0
  upper = 0
  lower = 0
  greek = 0
  for i in word:
    if i.isdigit(): digit += 1
    if i.isalpha(): alpha += 1
    if i.islower(): lower += 1
    if i.isupper(): upper += 1
    if i.lower() in u'αβγδεζηθικλμνξοπρσςτυφχψωάέήίόύώϊϋΐΰ': greek += 1
  return digit, alpha, upper, lower, greek

def get_phrase_stats(tweetwords):
  allupper = 0
  alllower = 0
  allnumber = 0
  punctuation = 0
  digit = 0
  alpha = 0
  upper = 0
  lower = 0
  total = 0
  greek = 0
  for w in tweetwords:
    if w.isupper() and len(w) > 1: allupper += 1
    if w.islower(): alllower += 1
    if w in punctuation_chars : punctuation += 1
    total += len(w)
    d,a,u,l,g = letter_count(w)
    digit += d
    alpha += a
    upper += u
    lower += l
    greek += g
  return len(tweetwords), allupper, alllower, punctuation, digit, alpha, upper, lower, total, greek

def get_bigrams(wordlist):
  wordlist = [x for x in wordlist if deaccent(x.lower()) not in stopwords and deaccent(x.lower()) not in punctuation_chars and not emoji.get_emoji_regexp().match(x)]
  return zip(wordlist, wordlist[1:])

word_stats_attrs = [
  'total_words', 'min_wptw', 'avg_wptw', 'med_wptw', 'std_wptw',
  'unique_words', 'lex_freq',
  'total_bigrams', 'unique_bigrams', 'bigram_lex_freq',
  'articles', 'pronouns', 'expletives', 'locations', 'emoticons',
  'emoji', 'alltokens',
  'all_caps_words', 'all_caps_words_pcnt',
  'all_caps_tweets', 'all_caps_tweets_pcnt',
  'all_nocaps_words', 'all_nocaps_words_pcnt',
  'punctuation_chars', 'punctuation_pcnt',
  'total_chars',
  'digit_chars', 'digit_pcnt',
  'alpha_chars', 'alpha_pcnt',
  'upper_chars', 'upper_pcnt',
  'lower_chars', 'lower_pcnt',
  'greek_chars', 'greek_pcnt',
  'total_hashtags', 'hashtags_per_tw',
  'uniq_hashtags', 'total_rt_hashtags', 'uniq_rt_hashtags',
  'most_common_words', 'most_common_bigrams',
  'most_common_hashtags', 'most_common_rt_hashtags',
  'most_common_urls', 'most_common_rt_urls',
  'seen_urls', 'urls_per_tw', 'avg_edit_distance'
  ]

def unshort_url(db, url):
  cached = db.shorturl.find_one({'shorturl': url})
  if cached != None and cached['url'] != None:
    if cached['url'] == url:
      db.shorturl.delete_one({'shorturl': url})
    return unshort_url(db, cached['url'])
  return url

def fill_word_stats(db, u, criteria):
  tknzr = TweetTokenizer(strip_handles=True, reduce_len=True)
  own_tweets = []
  rturlcnt = Counter()
  urlcnt = Counter()
  urlpertw = []
  tagcnt = Counter()
  tagpertw = []
  rttags = []
  words = []
  url_to_name = []
  uname = u['screen_name'].lower()
  for t in get_user_tweets(db, u['id'], criteria, batch=1000):
    if 'retweeted_status' in t:
      if 'urls' in t['retweeted_status'] and t['retweeted_status']['urls'] is not None:
        rturlcnt += Counter(urlparse(unshort_url(db, i)).netloc for i in t['retweeted_status'].get('urls',[]))
      if 'hashtags' in t['retweeted_status']:
        rttags.append(t['retweeted_status']['hashtags'])
    else:
      if 'urls' in t and t['urls'] is not None:
        urlcnt += Counter(urlparse(unshort_url(db, i)).netloc for i in t['urls'])
        url_to_name.extend(edit_distance(urlparse(unshort_url(db, i)).netloc, uname) for i in t['urls'])
        urlpertw.append(len(t['urls']))
      else:
        urlpertw.append(0)
      if 'hashtags' in t:
        tagcnt += Counter(t['hashtags'])
        tagpertw.append(len(t['hashtags']))
      else:
        tagpertw.append(0)
      if 'text' in t:
        own_tweets.append({'text': t['text']})

  if verbose(): print(" tokenize")
  words = [tknzr.tokenize(s) for s in itertext(iter(own_tweets))]
  wcounts = [len(s) for s in words]

  #tagfreq = Counter(t for s in tags for t in s)
  #tagpertw = [len(s) for s in tags]
  uniqtags = len(tagcnt)
  totaltags = sum(tagcnt.values())

  rttagfreq = Counter(t for s in rttags for t in s)
  uniqrtags = len(rttagfreq)
  totalrtags = sum(rttagfreq.values())

  if verbose(): print(" wc")
  artcnt  = 0
  proncnt = 0
  explcnt = 0
  loccnt = 0
  emocnt = 0
  emojicnt = 0
  tw = 0
  tuw = 0
  tu2w = 0
  t2w = 0
  ustat = [0]*10
  wc = Counter()
  bifreq = Counter()
  capstweets = 0

  try:
    if verbose(): print("  words")
    capstweets = sum(1 if all(w.isupper() for w in s) else 0 for s in words)
    wc = Counter(w for s in words for w in s)
    bigrams = (get_bigrams(s) for s in words)
    twstat = (get_phrase_stats(s) for s in words)
    ustat = reduce(lambda x, y: tuple(map(operator.add, x, y)), twstat)
    if verbose(): print("  bigrams")
    bc = Counter(b for s in bigrams for b in s)
    if verbose(): print("  dicts")
    tuw = len(wc)
    tw = sum(wc.values())
    tu2w = len(bc)
    t2w = sum(bc.values())
    if verbose(): print("  freqs")
    bifreq = bc
    if verbose(): print("  pos")
    for w,i in wc.items():
      wd = deaccent(w.lower())
      if wd in expletives: explcnt += i
      if wd in articles: artcnt += i
      if wd in pronouns: proncnt += i
      if is_location(wd): loccnt += i
      if wd in emoticons: emocnt += i
      if emoji.get_emoji_regexp().match(wd):
        emojicnt += i
        wc[w] = 0
      if wd in stopwords: wc[w] = 0 #do not count stopwords
      if wd in punctuation_chars: wc[w] = 0 #do not count punctuation
  except:
    pass

  seen_own = len(own_tweets)
  if seen_own == 0: seen_own = 1 #for division
  if verbose(): print(" saving")
  u['total_words'] = tw
  if tw == 0: tw = 1 # avoid divzero
  u['min_wptw'] = min(wcounts) if len(wcounts) else 0
  u['avg_wptw'] = numpy.mean(wcounts) if len(wcounts) else 0
  u['med_wptw'] = numpy.median(wcounts) if len(wcounts) else 0
  u['std_wptw'] = numpy.std(wcounts) if len(wcounts) else 0
  u['unique_words'] = tuw
  u['lex_freq'] = 1.0*tuw/tw
  u['total_bigrams'] = t2w
  if t2w == 0: t2w = 1 # avoid divzero
  u['unique_bigrams'] = tu2w
  u['bigram_lex_freq'] = 1.0*tu2w/t2w
  u['articles'] = artcnt
  u['pronouns'] = proncnt
  u['expletives'] = explcnt
  u['locations'] = loccnt
  u['emoticons'] = emocnt
  u['emoji'] = emojicnt
  u['alltokens'] = ustat[0]
  u['all_caps_words'] = ustat[1]
  u['all_caps_words_pcnt'] = 100.0*ustat[1]/tw
  u['all_caps_tweets'] = capstweets
  u['all_caps_tweets_pcnt'] = 100.0*capstweets/seen_own
  u['all_nocaps_words'] = ustat[2]
  u['all_nocaps_words_pcnt'] = 100.0*ustat[2]/tw
  u['punctuation_chars'] = ustat[3]
  u['total_chars'] = ustat[8]
  u['punctuation_pcnt'] = 100.0*ustat[3]/max(ustat[8], 1)
  u['digit_chars'] = ustat[4]
  u['digit_pcnt'] = 100.0*ustat[4]/max(ustat[8], 1)
  u['alpha_chars'] = ustat[5]
  u['alpha_pcnt'] = 100.0*ustat[5]/max(ustat[8], 1)
  u['upper_chars'] = ustat[6]
  u['upper_pcnt'] = 100.0*ustat[6]/max(ustat[8], 1)
  u['lower_chars'] = ustat[7]
  u['lower_pcnt'] = 100.0*ustat[7]/max(ustat[8], 1)
  u['greek_chars'] = ustat[9]
  u['greek_pcnt'] = 100.0*ustat[9]/max(ustat[8], 1)
  u['total_hashtags'] = totaltags
  u['hashtags_per_tw'] = {
    'min': min(tagpertw) if len(tagpertw) else None,
    'max': max(tagpertw) if len(tagpertw) else None,
    'avg': numpy.mean(tagpertw) if len(tagpertw) else None,
    'med': numpy.median(tagpertw) if len(tagpertw) else None,
    'std': numpy.std(tagpertw) if len(tagpertw) else None
  }
  u['uniq_hashtags'] = uniqtags
  u['total_rt_hashtags'] = totalrtags
  u['uniq_rt_hashtags'] = uniqrtags
  u['most_common_words'] = [{'word': i[0], 'count': i[1]} for i in wc.most_common(500)]
  u['most_common_bigrams'] = [{'bigram': ' '.join(i[0]), 'count': i[1]} for i in bifreq.most_common(500)]
  u['most_common_hashtags'] = [{'hashtag': i[0], 'count': i[1]} for i in tagcnt.most_common(500)]
  u['most_common_rt_hashtags'] = [{'hashtag': i[0], 'count': i[1]} for i in rttagfreq.most_common(500)]
  u['most_common_urls'] = [{'url': i[0], 'count': i[1]} for i in urlcnt.most_common(500)]
  u['most_common_rt_urls'] = [{'url': i[0], 'count': i[1]} for i in rturlcnt.most_common(500)]
  u['seen_urls'] = sum(urlcnt.values())
  u['urls_per_tw'] = {
    'min': min(urlpertw) if len(urlpertw) else None,
    'max': max(urlpertw) if len(urlpertw) else None,
    'avg': numpy.mean(urlpertw) if len(urlpertw) else None,
    'med': numpy.median(urlpertw) if len(urlpertw) else None,
    'std': numpy.std(urlpertw) if len(urlpertw) else None
  }
  u['avg_edit_distance'] = numpy.mean(url_to_name) if len(url_to_name) else None

user_metadata_attrs = [
  'id',
  'screen_name', 'screen_name_len', 'screen_name_upper',
  'screen_name_lower', 'screen_name_digit', 'screen_name_alpha',
  'name', 'name_len', 'name_upper', 'name_lower', 'name_digit',
  'name_alpha', 'name_greek',
  'created_at', 'tweet_count', 'favourites_count',
  'followers_count', 'friends_count', 'fr_fo_ratio',
  'location', 'has_location', 'time_zone', 'lang',
  'protected', 'verified', 'dead', 'suspended',
  'user_url',
  'bio_words', 'bio_upper_words', 'bio_lower_words',
  'bio_punctuation_chars', 'bio_digit_chars', 'bio_alpha_chars',
  'bio_upper_chars', 'bio_lower_chars', 'bio_greek_chars',
  'bio_total_chars'
  ]

def fill_metadata_stats(db, u):
  u2 = lookup_user(db, u['id'])
  digit, alpha, upper, lower, greek = letter_count(u2['screen_name'])
  assert(greek == 0)
  (bio_words, bio_upper_words, bio_lower_words, bio_punctuation_chars,
    bio_digit_chars, bio_alpha_chars, bio_upper_chars, bio_lower_chars,
    bio_total_chars, bio_greek_chars) = get_phrase_stats(u2['description'].split()) if 'description' in u2 else (None,) * 10
  totalfo = u2.get('followers_count', 0)
  totalfr = u2.get('friends_count', 0)
  u['screen_name'] = u2['screen_name']
  u['screen_name_len'] = len(u2['screen_name'])
  u['screen_name_upper'] = upper
  u['screen_name_lower'] = lower
  u['screen_name_digit'] = digit
  u['screen_name_alpha'] = alpha
  u['name'] = u2.get('name', None)
  digit, alpha, upper, lower, greek = letter_count(u2.get('name', u''))
  u['name_len'] = len(u2.get('name',''))
  u['name_upper'] = upper
  u['name_lower'] = lower
  u['name_digit'] = digit
  u['name_alpha'] = alpha
  u['name_greek'] = greek
  u['created_at'] = u2.get('created_at', datetime.now())
  u['tweet_count'] = u2.get('statuses_count', 0)
  u['favourites_count'] = u2.get('favourites_count', 0)
  u['followers_count'] = totalfo
  u['friends_count'] = totalfr
  if totalfo == 0: totalfo = 1 # smoothing to avoid division by zero
  u['fr_fo_ratio'] = 1.0*totalfr/totalfo
  u['location'] = u2.get('location', None)
  u['has_location'] = ('location' in u2)
  u['time_zone'] = u2.get('time_zone', None)
  u['lang'] = u2.get('lang', None)
  u['protected'] = u2.get('protected', False)
  u['verified'] = u2.get('verified', False)
  u['dead'] = is_dead(db, u['id'])
  u['suspended'] = is_suspended(db, u['id'])
  u['user_url'] = u2.get('url', None)
  u['bio_words'] = bio_words
  u['bio_upper_words'] = bio_upper_words
  u['bio_lower_words'] = bio_lower_words
  u['bio_punctuation_chars'] = bio_punctuation_chars
  u['bio_digit_chars'] = bio_digit_chars
  u['bio_alpha_chars'] = bio_alpha_chars
  u['bio_upper_chars'] = bio_upper_chars
  u['bio_lower_chars'] = bio_lower_chars
  u['bio_greek_chars'] = bio_greek_chars
  u['bio_total_chars'] = bio_total_chars
  #u['censored_somewhere'] = u2.get('withheld_in_countries', '') != ''

favoriter_attrs = [
  'favoriters', 'favorited',
  'most_favoriters', 'most_favorited'
]

def get_favoriters(db, uid):
  favoriters = defaultdict(lambda: [])
  if verbose(): sys.stderr.write(u'Fav graph for user {}/{}\n'.format(uid, id_to_userstr(db, uid)))
  for tw in get_user_tweets(db, uid, None, None):
    for faver in db.favorites.find({'tweet_id': tw['id']}):
      favoriters[faver['user_id']].append(tw['id'])
  return favoriters

def get_favorited(db, uid):
  favorited = defaultdict(lambda: [])
  for faved in db.favorites.find({'user_id': uid}):
    twid = faved['tweet_id']
    tw = db.tweets.find_one({'id': twid})
    if tw is None:
      if verbose(): print("missing tweet {}".format(twid))
      continue
    if 'user' in tw:
      twuid = tw['user']['id']
      favorited[twuid].append(twid)
  return favorited

def fill_favoriter_stats(db, u):
  f = Counter({k: len(v) for k,v in get_favoriters(db, u['id']).items()})
  u['favoriters'] = len(f)
  u['most_favoriters'] = [{'id': i[0], 'user': id_to_userstr(db, i[0]), 'count': i[1]} for i in f.most_common(500)]
  fd = Counter({k: len(v) for k,v in get_favorited(db, u['id']).items()})
  u['favorited'] = len(fd)
  u['most_favorited'] = [{'id': i[0], 'user': id_to_userstr(db, i[0]), 'count': i[1]} for i in fd.most_common(500)]

if __name__ == '__main__':
  #print("used as library only")
  parser = optparse.OptionParser()
  parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="List names of tracked users")
  parser.add_option("--id", action="store_true", dest="ids", default=False, help="Input is user ids.")
  parser.add_option("-b", "--before", action="store", dest="before", default=False, help="Before given date.")
  parser.add_option("-a", "--after", action="store", dest="after", default=False, help="After given date.")
  (options, args) = parser.parse_args()

  verbose(options.verbose)

  criteria = {}
  if options.before:
    criteria['$lte'] = dateutil.parser.parse(options.before)
  if options.after:
    criteria['$gte'] = dateutil.parser.parse(options.after)

  if len(args) == 0:
    parser.print_help()
    sys.exit(1)

  db, api = init_state(True, ignore_api=True)
  userlist = [x.lower().replace("@", "") for x in args]
  for user in userlist:
    uid = int(user) if options.ids else None
    uname = None if options.ids else user
    u = get_tracked(db, uid, uname)
    if u == None:
      x = lookup_user(db, uid, uname)
      if x:
        u = { 'id': x['id'], 'screen_name_lower': x['screen_name'].lower() }
      else:
        print("unknown user:", uid, uname)
        continue
  #  fill_word_stats(db, u, True)
  #  fill_follower_stats(db, u, True)
    usage_times_stats(db, u, criteria, True)
    print(u)

max_stars_repo_name: evaperon/twAwler

max_issues_repo_name: evaperon/twAwler

max_forks_repo_name: evaperon/twAwler


text:
#-------------------------------------------------------------------------------
# Name:        opan_utils_vector
# Purpose:     Test objects for opan.utils.vector
#
# Author:      Brian Skinn
#                bskinn@alum.mit.edu
#
# Created:     19 Apr 2016
# Copyright:   (c) Brian Skinn 2016
# License:     The MIT License; see "license.txt" for full license terms
#                   and contributor agreement.
#
#       This file is part of opan (Open Anharmonic), a system for automated
#       computation of anharmonic properties of molecular systems via wrapper
#       calls to computational/quantum chemical software packages.
#
#       http://www.github.com/bskinn/opan
#
#-------------------------------------------------------------------------------

import unittest
from opan.test.utils import inject_tests

class TestOpanUtilsVectorParallelCheck(unittest.TestCase):

    def test_Utils_Vector_ParallelCheck_Good(self):
        from opan.utils.vector import parallel_check as pc
        import numpy as np

        # Parallel vectors
        self.assertTrue(pc(np.array([1, 2, 3]),
                           np.array([1.5, 3, 4.5])))

        # Anti-parallel vectors
        self.assertTrue(pc(np.array([-1, 5.3, 3.1]),
                           np.array([3, -15.9, -9.3])))

        # Non-(anti-)parallel vectors
        self.assertFalse(pc(np.array([4.8, 0.35, -1.822]),
                            np.array([-1.3, 3.77, 19.14])))

    def test_Utils_Vector_ParallelCheck_BadShape(self):
        from opan.utils.vector import parallel_check as pc
        import numpy as np

        self.assertRaises(ValueError, pc,
                          np.array([[1, 2, 3], [3, 2, 1]]),
                          np.array([2, 4, 9]))

    def test_Utils_Vector_ParallelCheck_LenMismatch(self):
        from opan.utils.vector import parallel_check as pc
        import numpy as np

        self.assertRaises(ValueError, pc,
                          np.array(range(3)),
                          np.array(range(4)))

class TestOpanUtilsVectorProjRejAngle(unittest.TestCase):
    import numpy as np
    from opan.const import OpanEnum

    # Enum for the types of data stored for calculations
    class DType(OpanEnum):
        V1 = 'V1'
        V2 = 'V2'
        PROJ = 'PROJ'
        REJ = 'REJ'
        ANG = 'ANG'

    # Enum for the types of vector pairs being tested
    class VecType(OpanEnum): # Types of vectors
        O1 = 'O1'   # Both order-one
        LOL = 'LOL' # Both large (large on large)
        SOS = 'SOS' # Both small (small on small)
        LOS = 'LOS' # Large onto small
        SOL = 'SOL' # Small onto large
        BS = 'BS'   # Badly-scaled

    # Enum for the relationship between the vectors being tested
    class RelType(OpanEnum): # Type of vector relationship
        NS = 'NS'      # Nonspecific
        PAR = 'PAR'    # Nearly parallel
        NORM = 'NORM'  # Nearly normal
        AP = 'AP'      # Nearly anti-parallel

    # Substitution string for naming the vector datasets
    namestr = "{0}_{1}"

    # Dict of dicts of data
    data = {
            # Unremarkable vectors with ~order-one components
            namestr.format(RelType.NS, VecType.O1):
                {DType.V1: np.array([1, 2, 3]),
                 DType.V2: np.array([-1, 3, 8]),
                 DType.PROJ: np.array([-0.391892, 1.175676, 3.135135]),
                 DType.REJ: np.array([1.391892, 0.824324, -0.135135]),
                 DType.ANG: np.float_(25.712002)},

            # Nearly-normal vectors with ~order-one components
            namestr.format(RelType.NORM, VecType.O1):
                {DType.V1: np.array([2, 8, -4, 2.5, -1.4]),
                 DType.V2: np.array([-1, 3, 6.2, 5, 7]),
                 DType.PROJ: np.array([0.000817, -0.002450, -0.005064,
                                       -0.004084, -0.005717]),
                 DType.REJ: np.array([1.999183, 8.002450, -3.994936,
                                      2.504084, -1.394283]),
                 DType.ANG: np.float_(90.053923)},

            # Nearly-parallel vectors with ~order-one components
            namestr.format(RelType.PAR, VecType.O1):
                {DType.V1: np.array([1, 2, 2.9999, 4]),
                 DType.V2: np.array([1.0001, 2, 3, 4]),
                 DType.PROJ: np.array([1.000087, 1.999973, 2.999960, 3.999947]),
                 DType.REJ: np.array([-0.000087, 0.000027,
                                      -0.000060, 0.000053]),
                 DType.ANG: np.float_(0.001267)},

            # Nearly-antiparallel vectors with ~order-one components
            namestr.format(RelType.AP, VecType.O1):
                {DType.V1: np.array([-0.5, 2.3, 1.4, -3.1]),
                 DType.V2: np.array([0.50001, -2.29999, -1.4, 3.1]),
                 DType.PROJ: np.array([-0.500011, 2.299992,
                                        1.400001, -3.100003]),
                 DType.REJ: np.array([0.000011, 0.000008,
                                     -0.000001, 0.000003]),
                 DType.ANG: np.float_(179.999814)},

            # Two large vectors far from parallel/normal
            namestr.format(RelType.NS, VecType.LOL):
                {DType.V1: np.array([376328., 384874.,
                                     992834., 182873.]),
                 DType.V2: np.array([538344., 283747.,
                                     1838447., 929292.]),
                 DType.PROJ: np.array([269185.658799, 141880.699195,
                                       919270.144855, 464669.577884]),
                 DType.REJ: np.array([107142.341201, 242993.300805,
                                      73563.855145, -281796.577884]),
                 DType.ANG: np.float_(20.151554)},

            # Two small vectors far from parallel/normal
            namestr.format(RelType.NS, VecType.SOS):
                {DType.V1: np.array([0.000045, -0.00031,
                                     -0.000915, 0.000002]),
                 DType.V2: np.array([0.0002874, -0.0003987,
                                     0.0000034, 0.000719]),
                 DType.PROJ: np.array([0.000051, -0.000071,
                                       0.000001, 0.000128]),
                 DType.REJ: np.array([-0.000006, -0.000239,
                                      -0.000916, -0.000126]),
                 DType.ANG: np.float_(80.787151)},

            # Large onto small, far from parallel/normal
            namestr.format(RelType.NS, VecType.LOS):
                {DType.V1: np.array([238973., 239884.,
                                     -1092938., -893983.]),
                 DType.V2: np.array([0.0002874, -0.0003987,
                                     0.0000034, 0.000719]),
                 DType.PROJ: np.array([-255163.218951, 353979.037564,
                                       -3018.632375, -638351.963904]),
                 DType.REJ: np.array([494136.218951, -114095.037564,
                                      -1089919.367625, -255631.036096]),
                 DType.ANG: np.float_(122.176632)},

            # Small onto large, far from parallel/normal
            namestr.format(RelType.NS, VecType.SOL):
                {DType.V1: np.array([0.00000374, -0.0000233,
                                     0.0002837, 0.0000026]),
                 DType.V2: np.array([538344., 283747., 1838447., 929292.]),
                 DType.PROJ: np.array([0.000061, 0.000032,
                                       0.000207, 0.000105]),
                 DType.REJ: np.array([-0.000057, -0.000055,
                                       0.000077, -0.000102]),
                 DType.ANG: np.float_(31.859107)},

            # Two badly scaled vectors far from parallel/normal
            namestr.format(RelType.NS, VecType.BS):
                {DType.V1: np.array([0.000015, 6214., -0.000235, 12374.]),
                 DType.V2: np.array([-0.00005, 38184., 0.000045, 21669.]),
                 DType.PROJ: np.array([-0.000013, 10011.853795,
                                       0.000012, 5681.616904]),
                 DType.REJ: np.array([0.000028, -3797.853795,
                                      -0.000247, 6692.383096]),
                 DType.ANG: np.float_(33.760587)},

            # Two large vectors nearly parallel
            namestr.format(RelType.PAR, VecType.LOL):
                {DType.V1: np.array([554387., 38185., -532247., 12374.]),
                 DType.V2: np.array([554389., 38184., -532248., 12375.]),
                 DType.PROJ: np.array([554387.488030, 38183.895862,
                                    -532246.548415, 12374.966250]),
                 DType.REJ: np.array([-0.488030, 1.104138,
                                      -0.451585, -0.966250]),
                 DType.ANG: np.float_(0.000120)},

            # Two small vectors nearly parallel
            namestr.format(RelType.PAR, VecType.SOS):
                {DType.V1: np.array([0.000015, 0.000016, -0.000042,
                                    -0.000034, 0.000065, -0.000033]),
                 DType.V2: np.array([0.000014, 0.000017, -0.000041,
                                    -0.000033, 0.000066, -0.000032]),
                 DType.PROJ: np.array([0.000014, 0.000017, -0.000041,
                                      -0.000033, 0.000066, -0.000032]),
                 DType.REJ: np.array([0.000001, -0.000001, -0.000001,
                                     -0.000001, -0.000001, -0.000001]),
                 DType.ANG: np.float_(1.483536)},

            # Large onto small, nearly parallel
            namestr.format(RelType.PAR, VecType.LOS):
                {DType.V1: np.array([14001., 17002., -41003., -33001.,
                                     66004., -32005., 75008.]),
                 DType.V2: np.array([0.000014, 0.000017, -0.000041,
                                    -0.000033, 0.000066, -0.000032, 0.000075]),
                 DType.PROJ: np.array([14001.205610, 17001.463955,
                                      -41003.530715, -33002.841795,
                                       66005.683590, -32002.755680,
                                       75006.458626]),
                 DType.REJ: np.array([-0.205610, 0.536045, 0.530715,
                                       1.841795, -1.683590, -2.244320,
                                       1.541374]),
                 DType.ANG: np.float_(0.001811)},

            # Small onto large, nearly parallel
            namestr.format(RelType.PAR, VecType.SOL):
                {DType.V1: np.array([1.20E-05, -1.30E-05, 3.80E-05,
                                    -7.90E-05, -8.50E-05, 2.20E-05,
                                     4.50E-05]),
                 DType.V2: np.array([12006., -13009., 38007.,
                                    -79008., -85006., 22001.,
                                     45003.]),
                 DType.PROJ: np.array([0.000012, -0.000013, 0.000038,
                                      -0.000079, -0.000085, 0.000022,
                                       0.000045]),
                 DType.REJ: np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]),
                 DType.ANG: np.float_(0.004356)},

            # Two badly scaled vectors nearly parallel
            namestr.format(RelType.PAR, VecType.BS):
                {DType.V1: np.array([0.00053, -2135., 0.00015, 65548.,
                                    -0.0085, 0.00022, -17815., -0.00035]),
                 DType.V2: np.array([0.00054, -2136., 0.00014, 65549.,
                                    -0.0084, 0.00023, -17816., -0.00036]),
                 DType.PROJ: np.array([0.000540, -2135.960458, 0.000140,
                                       65547.786547, -0.008400, 0.000230,
                                      -17815.670188, -0.000360]),
                 DType.REJ: np.array([-0.000010, 0.960458, 0.000010,
                                       0.213453, -0.000100, -0.000010,
                                       0.670188,0.000010]),
                 DType.ANG: np.float_(0.001004)},

            # Two large vectors nearly normal
            namestr.format(RelType.NORM, VecType.LOL):
                {DType.V1: np.array([654564., 48249., -248896.,
                                    -54789., 24444.]),
                 DType.V2: np.array([-6048., 68589., -34061.,
                                      95549., -108100.]),
                 DType.PROJ: np.array([11.146008, -126.404358, 62.771856,
                                     -176.089606, 199.220153]),
                 DType.REJ: np.array([654552.853992, 48375.404358,
                                     -248958.771856, -54612.910394,
                                       24244.779847]),
                 DType.ANG: np.float_(90.024498)},

            # Two small vectors nearly normal
            namestr.format(RelType.NORM, VecType.SOS):
                {DType.V1: np.array([0.000048, 0.000017, -0.000032,
                                    0.000091, -0.000016]),
                 DType.V2: np.array([0.000015, 0.000007, 0.00001,
                                    0.000001, 0.000035]),
                 DType.PROJ: np.array([0.000000, 0.000000, 0.000000,
                                    0.000000, 0.000001]),
                 DType.REJ: np.array([0.000048, 0.000017, -0.000032,
                                    0.000091, -0.000017]),
                 DType.ANG: np.float_(89.350346)},

            # Large onto small, nearly normal
            namestr.format(RelType.NORM, VecType.LOS):
                {DType.V1: np.array([32185., 27265., -30185.,
                                    108115., -13755.]),
                 DType.V2: np.array([0.000015, 0.000007, 0.00001,
                                    0.000001, 0.000035]),
                 DType.PROJ: np.array([-14.343750, -6.693750, -9.562500,
                                        -0.956250, -33.468750]),
                 DType.REJ: np.array([32199.343750, 27271.693750,
                                    -30175.437500, 108115.956250,
                                    -13721.531250]),
                 DType.ANG: np.float_(90.018157)},

            # Small onto large, nearly normal
            namestr.format(RelType.NORM, VecType.SOL):
                {DType.V1: np.array([0.000028, -0.000035, 0.000017,
                                     0.000098, 0.000072, -0.000055]),
                DType.V2: np.array([16589., 22852., 44185.,
                                    14273., 24599., 65489.]),
                 DType.PROJ: np.array([0., 0., 0., 0., 0., 0.]),
                 DType.REJ: np.array([0.000028, -0.000035, 0.000017,
                                      0.000098, 0.000072, -0.000055]),
                 DType.ANG: np.float_(90.073867)},

            # Two badly scaled vectors nearly normal
            namestr.format(RelType.NORM, VecType.BS):
                {DType.V1: np.array([0.0015, 6214., 2319., 145.]),
                 DType.V2: np.array([8285., -0.0004, 0.0034, 2166.]),
                 DType.PROJ: np.array([35.485053, -0.000002,
                                       0.000015, 9.277082]),
                 DType.REJ: np.array([-35.483553, 6214.000002,
                                      2318.999985, 135.722918]),
                 DType.ANG: np.float_(89.683234)},

            # Two large vectors nearly anti-parallel
            namestr.format(RelType.AP, VecType.LOL):
                {DType.V1: np.array([215484., 665452., -654587.,
                                     541887., 64657., -6546347.,
                                    -687887., -1137889.]),
                 DType.V2: np.array([-215485., -665453., 654589.,
                                     -541882., -64659., 6546378.,
                                      687889., 1137888.]),
                 DType.PROJ: np.array([215484.046715, 665450.056100,
                                      -654586.104161, 541879.602766,
                                       64658.713955, -6546349.039449,
                                      -687885.956845, -1137882.966092]),
                 DType.REJ: np.array([-0.046715, 1.943900, -0.895839,
                                       7.397234, -1.713955, 2.039449,
                                      -1.043155, -6.033908]),
                 DType.ANG: np.float_(179.999914)},

            # Two small vectors nearly anti-parallel
            namestr.format(RelType.AP, VecType.SOS):
                {DType.V1: np.array([0.000041, -0.000038, 0.000091,
                                    -0.000019, 0.000037, -0.000068,
                                    -0.000071, -0.000055]),
                 DType.V2: np.array([-0.000041, 0.000039, -0.00009,
                                      0.000019, -0.000036, 0.000068,
                                      0.00007, 0.000055]),
                 DType.PROJ: np.array([0.000041, -0.000039, 0.000091,
                                      -0.000019, 0.000036, -0.000068,
                                      -0.000070, -0.000055]),
                 DType.REJ: np.array([0.000000, 0.000001, 0.000000,
                                      0.000000, 0.000001, 0.000000,
                                     -0.000001, 0.000000]),
                 DType.ANG: np.float_(179.379006)},

            # Large onto small, nearly anti-parallel
            namestr.format(RelType.AP, VecType.LOS):
                {DType.V1: np.array([41002., -38997., 90004., -18997.,
                                     36001., -68002., -70003., -54988.]),
                 DType.V2: np.array([-0.000041, 0.000039, -0.00009,
                                      0.000019, -0.000036, 0.000068,
                                      0.00007, 0.000055]),
                 DType.PROJ: np.array([40999.983927, -38999.984711,
                                       89999.964717, -18999.992551,
                                       35999.985887, -67999.973342,
                                      -69999.972558, -54999.978438]),
                 DType.REJ: np.array([2.016073, 2.984711, 4.035283,
                                      2.992551, 1.014113, -2.026658,
                                     -3.027442, 11.978438]),
                 DType.ANG: np.float_(179.994978)},

            # Small onto large, nearly anti-parallel
            namestr.format(RelType.AP, VecType.SOL):
                {DType.V1: np.array([0.000038, -0.000044, 0.000057,
                                     0.000098, 0.000089, -0.000072,
                                    -0.000022, 0.000025, -0.000017]),
                 DType.V2: np.array([-38007., 44009., -56995.,
                                     -98004., -88999., 71995.,
                                      22002., -25001., 16996.]),
                 DType.PROJ: np.array([0.000038, -0.000044, 0.000057,
                                       0.000098, 0.000089, -0.000072,
                                      -0.000022, 0.000025, -0.000017]),
                 DType.REJ: np.array([0., 0., 0., 0., 0., 0., 0., 0., 0.]),
                 DType.ANG: np.float_(179.995212)},

            # Two badly scaled vectors nearly anti-parallel
            namestr.format(RelType.AP, VecType.BS):
                {DType.V1: np.array([25778., -35778., 0.000032, -47789.,
                                     -0.000038, 0.000041, 24448., -35779.,
                                     -0.000017]),
                 DType.V2: np.array([-25779., 35772., -0.000031, 47788.,
                                      0.000038, -0.000041, -24444., 35777.,
                                      0.000016]),
                 DType.PROJ: np.array([25780.714146, -35774.378619, 0.000031,
                                      -47791.177610, -0.000038, 0.000041,
                                       24445.625376, -35779.378952, -0.000016]),
                 DType.REJ: np.array([-2.714146, -3.621381, 0.000001,
                                       2.177610, 0.000000, 0.000000,
                                       2.374624, 0.378952, -0.000001]),
                 DType.ANG: np.float_(179.995917)}
            }

    # Template functions
    # Vector projection template
    def template_proj(self, name, data):
        from opan.utils.vector import proj

        v1 = data[self.DType.V1]
        v2 = data[self.DType.V2]
        p = proj(v1, v2)
        for i, t in enumerate(zip(p, data[self.DType.PROJ])):
            self.assertAlmostEqual(*t, delta=1e-6,
                       msg="Test {0}: Index {1}; V1 = {2}; V2 = {3}"
                            .format(name, i, v1, v2))

    # Vector rejection template
    def template_rej(self, name, data):
        from opan.utils.vector import rej

        v1 = data[self.DType.V1]
        v2 = data[self.DType.V2]
        r = rej(v1, v2)
        for i, t in enumerate(zip(r, data[self.DType.REJ])):
            self.assertAlmostEqual(*t, delta=1e-6,
                       msg="Test {0}: Index {1}; V1 = {2}; V2 = {3}"
                            .format(name, i, v1, v2))

    # Vector angle template
    def template_angle(self, name, data):
        from opan.utils.vector import vec_angle

        v1 = data[self.DType.V1]
        v2 = data[self.DType.V2]
        a = vec_angle(v1, v2)
        self.assertAlmostEqual(a, data[self.DType.ANG], delta=1e-6,
                   msg="Test {0}: V1 = {1}; V2 = {2}"
                   .format(name, v1, v2))

    # Populate the local namespace with the auto-generated
    #  test methods
    _locals = locals()
    inject_tests(_locals, data, "test_Vector_Proj_Good_{0}", template_proj)
    inject_tests(_locals, data, "test_Vector_Rej_Good_{0}", template_rej)
    inject_tests(_locals, data, "test_Vector_Angle_Good_{0}", template_angle)

    def setUp(self):
        self.longMessage = True

    def test_Utils_Vector_Proj_BadVec_NotVector(self):
        import numpy as np
        from opan.utils.vector import proj

        self.assertRaises(ValueError, proj,
                          np.array(range(16)).reshape((4, 4)),
                          np.array(range(16)))

    def test_Utils_Vector_Proj_BadVecOnto_NotVector(self):
        import numpy as np
        from opan.utils.vector import proj

        self.assertRaises(ValueError, proj,
                          np.array(range(16)),
                          np.array(range(16)).reshape((4, 4)))

    def test_Utils_Vector_Proj_BadVecsShapeMismatch(self):
        import numpy as np
        from opan.utils.vector import proj

        self.assertRaises(ValueError, proj,
                          np.array(range(5)), np.array(range(6)))

    def test_Utils_Vector_Angle_BadVecShapes(self):
        import numpy as np
        from opan.utils.vector import vec_angle as ang

        self.assertRaises(ValueError, ang,
                            np.array(range(6)).reshape((2,3)),
                            np.array(range(6)))

        self.assertRaises(ValueError, ang,
                            np.array(range(6)),
                            np.array(range(6)).reshape((3,2)))

    def test_Utils_Vector_Angle_VecLenMismatch(self):
        import numpy as np
        from opan.utils.vector import vec_angle as ang

        self.assertRaises(ValueError, ang,
                            np.array(range(4)), np.array(range(12)))

    def test_Utils_Vector_Angle_VecNormTooSmall(self):
        import numpy as np
        from opan.utils.vector import vec_angle as ang

        self.assertRaises(ValueError, ang,
                            1e-12* np.array(range(4)),
                            np.array(range(-3,1)))

        self.assertRaises(ValueError, ang,
                            np.array(range(4)),
                            1e-12 * np.array(range(-3,1)))

class TestOpanUtilsVectorOrthoBasis(unittest.TestCase):

    def setUp(self):
        self.longMessage = True

    def test_Utils_Vector_OrthoBasis_GoodRandomRefVec(self):
        import numpy as np
        from opan.utils.vector import ortho_basis as ob
        from opan.utils.vector import vec_angle as ang

        nv = [8.5, -3.15, -2.1884]

        on1, on2 = ob(nv)

        self.assertAlmostEqual(np.dot(nv, on1), 0.0,
                delta=1e-10,
                msg="Normal vector not normal to first basis vector")

        self.assertAlmostEqual(np.dot(nv, on2), 0.0,
                delta=1e-10,
                msg="Normal vector not normal to second basis vector")

        self.assertAlmostEqual(np.dot(on1, on2), 0.0,
                delta=1e-10,
                msg="Basis vectors not normal")

        self.assertAlmostEqual(np.sum(np.power(on1, 2.)), 1.0,
                delta=1e-10,
                msg="First basis vector not normalized")

        self.assertAlmostEqual(np.sum(np.power(on2, 2.)), 1.0,
                delta=1e-10,
                msg="Second basis vector not normalized")

        self.assertAlmostEqual(ang(nv, np.cross(on1, on2)), 0.0,
                delta=1e-5,
                msg="Incorrect handedness of basis vectors")

    def test_Utils_Vector_OrthoBasis_GoodDefinedRefVec(self):
        import numpy as np
        from opan.utils.vector import ortho_basis as ob
        from opan.utils.vector import vec_angle as ang
        from opan.utils.vector import rej

        nv = [2.5, 31.15, -25.1884]
        rv = [-12.15, 0.0034, 35.18]

        on1, on2 = ob(nv, rv)

        self.assertAlmostEqual(np.dot(nv, on1), 0.0,
                delta=1e-10,
                msg="Normal vector not normal to first basis vector")

        self.assertAlmostEqual(np.dot(nv, on2), 0.0,
                delta=1e-10,
                msg="Normal vector not normal to second basis vector")

        self.assertAlmostEqual(np.dot(on1, on2), 0.0,
                delta=1e-10,
                msg="Basis vectors not normal")

        self.assertAlmostEqual(np.sum(np.power(on1, 2.)), 1.0,
                delta=1e-10,
                msg="First basis vector not normalized")

        self.assertAlmostEqual(np.sum(np.power(on2, 2.)), 1.0,
                delta=1e-10,
                msg="Second basis vector not normalized")

        self.assertAlmostEqual(ang(nv, np.cross(on1, on2)), 0.0,
                delta=1e-5,
                msg="Incorrect handedness of basis vectors")

        self.assertAlmostEqual(ang(rej(rv, nv), on1), 0.0,
                delta=1e-5,
                msg="First basis vector not aligned with reference")

    def test_Utils_Vector_OrthoBasis_TooParallelRefVec(self):
        import numpy as np
        from opan.utils.vector import ortho_basis as ob
        from opan.test.utils import assertErrorAndTypecode
        from opan.error import VectorError as VErr

        nv = np.array([2.81, -3.855, -12.384])

        assertErrorAndTypecode(self, VErr, ob, VErr.NONPRL,
                                nv, nv*1.1111)

    def test_Utils_Vector_OrthoBasis_BadNormArrayShape(self):
        import numpy as np
        from opan.utils.vector import ortho_basis as ob

        with self.assertRaises(ValueError):
            ob(np.array(range(9)).reshape((3,3)))

    def test_Utils_Vector_OrthoBasis_BadNormArrayLen(self):
        import numpy as np
        from opan.utils.vector import ortho_basis as ob

        with self.assertRaises(ValueError):
            ob(np.array(range(6)))

    def test_Utils_Vector_OrthoBasis_BadRefArrayShape(self):
        import numpy as np
        from opan.utils.vector import ortho_basis as ob

        with self.assertRaises(ValueError):
            ob(np.array(range(1,4)), np.array(range(9)).reshape((3,3)))

    def test_Utils_Vector_OrthoBasis_BadRefArrayLen(self):
        import numpy as np
        from opan.utils.vector import ortho_basis as ob

        with self.assertRaises(ValueError):
            ob(np.array(range(1,4)), np.array(range(6)))

class TestOpanUtilsVectorOrthonormCheck(unittest.TestCase):

    def setUp(self):
        self.longMessage = True

    def test_Utils_Vector_ONCheck_Trivial1D(self):
        import numpy as np
        from scipy import linalg as spla
        from opan.utils.vector import orthonorm_check as onchk

        vec = np.array(range(15))
        vec = vec / spla.norm(vec)

        c, nf, of = onchk(vec)

        self.assertTrue(c)
        self.assertIsNone(nf)
        self.assertIsNone(of)

        c, nf, of = onchk(vec, report=True)

        self.assertTrue(c)
        self.assertEqual(len(nf), 0)
        self.assertEqual(len(of), 0)

    def test_Utils_Vector_ONCheck_Trivial2D(self):
        import numpy as np
        from scipy import linalg as spla
        from opan.utils.vector import orthonorm_check as onchk
        from opan.utils.vector import ortho_basis as ob

        vec = np.array([3.18, -2.25, 1.0005])
        on1, on2 = ob(vec)

        m = np.column_stack((vec / spla.norm(vec), on1, on2))

        c, nf, of = onchk(m)

        self.assertTrue(c)
        self.assertIsNone(nf)
        self.assertIsNone(of)

        c, nf, of = onchk(m, report=True)

        self.assertTrue(c)
        self.assertEqual(len(nf), 0)
        self.assertEqual(len(of), 0)

    def test_Utils_Vector_ONCheck_NonNormedVec(self):
        import numpy as np
        from opan.utils.vector import orthonorm_check as onchk
        from opan.utils.vector import ortho_basis as ob

        vec = np.array([-2.112, 2923.3, -0.2323])
        on1, on2 = ob(vec)

        m = np.column_stack((vec, on1, on2))

        c, nf, of = onchk(m)

        self.assertFalse(c)
        self.assertIsNone(nf)
        self.assertIsNone(of)

        c, nf, of = onchk(m, report=True)

        self.assertFalse(c)
        self.assertEqual(len(of), 0)

        self.assertEqual(len(nf), 1)
        self.assertEqual(nf[0], 0)

    def test_Utils_Vector_ONCheck_SkewedVec(self):
        import numpy as np
        from scipy import linalg as spla
        from opan.utils.vector import orthonorm_check as onchk
        from opan.utils.vector import ortho_basis as ob

        vec = np.array([-6.277, 1.345, -23.8734])
        offvec = np.subtract(vec, np.array([-3, 5, -12]))
        offvec = offvec / spla.norm(offvec)

        on1, on2 = ob(vec)

        m = np.column_stack((offvec, on1, on2))

        c, nf, of = onchk(m, report=True)

        self.assertFalse(c)
        self.assertEqual(len(nf), 0)

        self.assertEqual(len(of), 2)
        self.assertEqual(of[0], (0,1))
        self.assertEqual(of[1], (0,2))

def suite():
    s = unittest.TestSuite()
    tl = unittest.TestLoader()
    s.addTests([tl.loadTestsFromTestCase(TestOpanUtilsVectorParallelCheck),
                tl.loadTestsFromTestCase(TestOpanUtilsVectorProjRejAngle),
                tl.loadTestsFromTestCase(TestOpanUtilsVectorOrthoBasis),
                tl.loadTestsFromTestCase(TestOpanUtilsVectorOrthonormCheck)
                ])
    return s

if __name__ == '__main__':  # pragma: no cover
    print("Module not executable.")

ext: py

max_stars_repo_name: bskinn/opan

max_issues_repo_name: bskinn/opan

max_forks_repo_name: bskinn/opan

reason: import numpy,from scipy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 13 11:46:59 2021

@author: cadecastro.com
"""
import numpy as np
import matplotlib.pyplot as plt
#%INGRESO DE PARÁMETROS:
t=0.15#;% Espesor como fracción de la cuerda al 25 % de la misma
c=1740#;%[mm] Cuerda del ala
N=int(1000)#Número de puntos gráfica
#%CURVAS DEL PERFIL:
R1=c*(16*t*t+1)/(32*t)
R2=c*(16*t*t+9)/(32*t)
x=np.linspace(0,c,N)
y_inf=np.zeros(N)
y_sup=np.zeros(N)
for i in range(0,N):
    if x[i]<=0.25*c:
        y_sup[i]=np.sqrt(R1*R1-(0.25*c-x[i])*(0.25*c-x[i]))-R1+t*c
    else:
        y_sup[i]=np.sqrt(R2*R2-(0.25*c-x[i])*(0.25*c-x[i]))-R2+t*c
#Salidas:
print('Cuerda [mm] = ',c)
print('Radio R1 [mm] = ',R1)
print('Radio R2 [mm] = ',R2)
#Gráfica:
plt.plot(x,y_inf,'k-')
plt.plot(x,y_sup,'k-')
plt.grid(True,'both','both')
plt.axis('equal')
plt.title('Gráfica del perfil alar')
plt.xlabel('x [mm]')
plt.ylabel('y [mm]')

reason: import numpy


text:
# -*- coding: utf-8 -*-

import cPickle
import tables
from numpy import *
from RangeBearingMeasurementModel import *
import os
from matplotlib import pyplot

motion_params = {
    'std_encoder':2,
    'std_alpha':0.0873,
    'l':2.83,
    'h':0.76,
    'a':3.78,
    'b':0.50}

sensor_params = {
    'max_range':10,
    'max_bearing':pi,
    'std_range':1.0,
    'std_bearing':0.0349,
    'pd':0.95,
    'clutter_rate':20}
measurement_model = RangeBearingMeasurementModel(sensor_params)

print('loading mat-file...')
file = tables.openFile('groundtruth.mat')
landmarks = file.root.staticMap[:].transpose()
trajectory = file.root.traj[:].transpose()
controls = file.root.controls[:].transpose()
file.close()

n_runs = 50

print('generating simulation inputs...')
for n in xrange(n_runs):
    print(n)
    # make the noisy control inputs
    n_controls = size(controls,1)
    encoder_noise = random.randn(n_controls)*motion_params['std_encoder']
    steering_noise = random.randn(n_controls)*motion_params['std_alpha']
    controls_noisy = controls + vstack((encoder_noise,steering_noise))
    controls_noisy[1,:] = wrap_angle(controls_noisy[1,:])

    # generate noisy measurements
    n_steps = size(trajectory,1)
    measurements = [] ;
    for i in xrange(n_steps):
        z = measurement_model.compute_noisy_measurement(trajectory[:,i],landmarks)
        measurements.append(z)

    # write to text file
    os.mkdir(str(n))
    dir_str = str(n)+os.sep
    controls_file = open(dir_str+'controls_synth.txt','w')
    controls_file.write('% velocity\tsteering angle\n')
    for i in xrange(n_controls):
        control_str = '{u[0]} {u[1]}\n'.format(u=controls_noisy[:,i])
        controls_file.write(control_str)
    controls_file.close()

    measurements_file = open(dir_str+'measurements_synth.txt','w')
    measurements_file.write('% measurements from simulation data. One time step per line, each pair of of numbers is a range/bearing measurement.\n')
    for i in xrange(n_steps):
        Z = measurements[i] ;
        n_z = size(Z,1)
        z_str = ''
        for j in xrange(n_z):
            z_str += '{z[0]} {z[1]} '.format(z=Z[:,j])
        z_str += '\n'
        measurements_file.write(z_str)
    measurements_file.close()

    datafile = open(dir_str+'simdata.pk2','w')
    cPickle.dump( landmarks, datafile, cPickle.HIGHEST_PROTOCOL )
    cPickle.dump( trajectory, datafile, cPickle.HIGHEST_PROTOCOL )
    cPickle.dump( controls, datafile, cPickle.HIGHEST_PROTOCOL )
    cPickle.dump( controls_noisy, datafile, cPickle.HIGHEST_PROTOCOL )
    cPickle.dump( measurements, datafile, cPickle.HIGHEST_PROTOCOL )
    datafile.close()

max_stars_repo_name: cheesinglee/cuda-PHDSLAM

max_issues_repo_name: cheesinglee/cuda-PHDSLAM

max_forks_repo_name: cheesinglee/cuda-PHDSLAM

reason: from numpy


text:
from nltk.tokenize import word_tokenize
from torch.autograd import Variable
import csv
import spacy
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim

# Set hyperparameters
embed_size = 100
batch_size = 20
l_rate = 0.001
num_epochs = 10

# Read in text data
text_file = open('sentiment_dataset.csv', 'r')
reader = csv.reader(text_file)
data = []
full_text = ''
for line in reader:
	data.append(line)
	full_text += line[-1]
text_file.close()

# Create vocabulary
word_list = word_tokenize(full_text.lower())
vocab = np.unique(word_list)
vocab_size = len(vocab)
print(vocab_size)

# Create word to index mapping
w_to_i = {word: ind for ind, word in enumerate(vocab)}

# Load text parser
nlp = spacy.load('en')

class RecurNN(nn.Module):
	def __init__(self):
		super(RecurNN, self).__init__()
		self.embeddings = nn.Embedding(vocab_size, embed_size)
		self.first_linear = nn.Linear(2*embed_size, embed_size)
		self.tanh = nn.Tanh()
		self.final_linear = nn.Linear(embed_size, 1)
		self.sigmoid = nn.Sigmoid()

	def parse(sent):
		doc = nlp(sent)
		root = doc[0]
		for word in doc:
			if word.head == word:
				root = word
				break
		return root

	def compose(a, b):
		ab = torch.cat((a, b), 0)
		out = self.first_linear(ab)
		out = self.tanh(out)
		return out

	def compute_vec(root):
		lookup = torch.LongTensor([w_to_i[root.text]])
		embed_vec = self.embeddings(lookup).view((1, -1))
		if len(root.children) == 0:
			return embed_vec
		vec_list = []
		for child in root.children:
			vec_list.append(compute_vec(child))
			if len(vec_list) == 2:
				comp_vec = compose(vec_list[0], vec_list[1])
				vec_list.clear()
				vec_list.append(comp_vec)
		return compose(vec_list[-1], embed_vec)

	def forward(self, x):
		final_vec = compute_vec(parse(x))
		return self.sigmoid(self.final_linear(final_vec)).view((-1, 1))

model = RecurNN()

for epoch in range(len(num_iterations)):
	for line in data:
		print(line)
		print(word_tokenize(line.lower()).join(' '))


text:
import matplotlib
import matplotlib.pyplot as plt
from floodsystem.datafetcher import fetch_measure_levels
from floodsystem.stationdata import build_station_list
from datetime import datetime, timedelta
from floodsystem.analysis import polyfit
import numpy as np

def plot_water_levels(station, dates, levels):
    plt.plot(dates,levels)
    plt.xlabel('date')
    plt.ylabel('water level (m)')
    plt.xticks(rotation=45)
    plt.title(station.name)
    plt.plot(dates, [station.typical_range[0]] * len(dates), 'g--')
    plt.plot(dates, [station.typical_range[1]] * len(dates), 'g--')
    plt.show()

def plot_water_level_with_fit(station, dates, levels, p, show_typical_range = True):
    """Plot water levels for a station with polyfit.
    Also accepts lists as input, showing up to the first 6 stations."""
    poly, d0 = polyfit(dates, levels, p)
    dates_num = matplotlib.dates.date2num(dates)

    typical_min = station.typical_range[0]
    typical_max = station.typical_range[1]
    typical_min_list = np.full(len(dates), typical_min)
    typical_max_list = np.full(len(dates), typical_max)

    plt.plot(dates, levels, label = "Real data")
    plt.plot(dates, poly(dates_num - dates_num[d0 - 1]), label = "Least-squares polynomial fit")
    plt.plot(dates, typical_min_list, label = "Typical minimum level")
    plt.plot(dates, typical_max_list, label = "Typical maximum level")
    plt.xlabel('date')
    plt.ylabel('water level (m)')
    plt.xticks(rotation=45)
    plt.title(station.name)

    plt.legend()
    plt.tight_layout()
    plt.show()

max_stars_repo_name: angelinaachillea/New-Floods

max_issues_repo_name: angelinaachillea/New-Floods

max_forks_repo_name: angelinaachillea/New-Floods


text:
import numpy as np
import itertools
import pickle
import tictactoe
import pygame
import time
from pygame.locals import *
import sys
from tqdm import tqdm

# https://raw.githubusercontent.com/nyergler/teaching-python-with-pygame/master/ttt-tutorial/tictactoe.py
class State():
    def __init__(self, state):
        self.state = state
        self.beads = self.init_beads()

    def init_beads(self):
        beads = {}
        zero_count = max(self.state.count('0') - 1,2)
        for idx, i in enumerate(self.state):
            if i == '0':
                beads[idx] = zero_count
        return beads

    def get_beads(self):
        rand = np.random.rand(1)
        values = np.array(list(self.beads.values()))
        keys = np.array(list(self.beads.keys()))
        if np.sum(values) == 0:
            values = np.ones(values.shape[0])
        values = values / np.sum(values)
        #print(values)
        prob = 0
        for  idx, i in enumerate(values):
            prob += i
            if rand < prob:
                return keys[idx]

    def set_beads(self, key, reward):
        self.beads[key] += reward

def create_states(all_permutations):
    states = {}
    for string in all_permutations:
        states[string] = State(string)
    return states

def check_win_case(i,j,k, state):
    if(state[i] == state[j] and state[i] == state[k] and state[i] != '0'):
        return True
    return False
def check_win(state):
    if check_win_case(0,4,8, state):
        return True
    elif check_win_case(0,1,2,state):
        return True
    elif check_win_case(0,3,6,state):
        return True
    elif check_win_case(1,4,7,state):
        return True
    elif check_win_case(2,4,6,state):
        return True
    elif check_win_case(2,5,8,state):
        return True
    elif check_win_case(3,4,5,state):
        return True
    elif check_win_case(6,7,8,state):
        return True
    else:
        return False

def check_draw(states):
    if states.count('0') == 0:
        return True
    return False

def give_reward(states, menacing_states, menacing_steps, reward):
    for idx, state in enumerate(menacing_states):
        #print(''.join(list(state)))
        #print(menacing_steps[idx], idx)
        #print(states[''.join(list(state))].beads)
        states[''.join(list(state))].set_beads(menacing_steps[idx], reward)
        #print(states[''.join(list(state))].beads)

def prnt_game(state):
    """
    for idx, i in enumerate(state):
        print(i + ' | ', end='')
        if (idx+1)%3 == 0:
            print('\n- | - | -')
    """
    pass

def quit_prompt():
    print('Wanna Quit? Press Y')
    s = input()
    if s == 'Y':
        return True
    return False

def check_state(states, current_state):
    if ''.join(current_state) in states:
        return states
    states[''.join(current_state)] = State(current_state)
    return states

def game_on(states, path):
    try:
        pickle_in = open(path,"rb")
        states = pickle.load(pickle_in)
        print('Model Loaded')
    except FileNotFoundError:
        pass
    #print(states['000000001'].beads)
    wanna_quit = False
    while(not wanna_quit):
        print("Game Start")
        current_state = list('000000000')
        prnt_game(current_state)
        menacing_steps = []
        menacing_states = []
        pygame.init()
        tictactoe.grid = [ [ None, None, None ], \
                            [ None, None, None ], \
                            [ None, None, None ] ]
        tictactoe.winner = None
        ttt = pygame.display.set_mode ((300, 325))
        pygame.display.set_caption ('Tic-Tac-Toe')
        board = tictactoe.initBoard (ttt)
        new_game = False
        while(not new_game):
            try:
                for event in pygame.event.get():
                    if event.type is QUIT:
                        wanna_quit = True
                        new_game = True
                        break
                    elif event.type is MOUSEBUTTONDOWN:
                        # the user clicked; place an X or O
                        board, row, col = tictactoe.clickBoard(board)
                        # check for a winner
                        tictactoe.gameWon(board)
                        tictactoe.showBoard(ttt, board)
                        if row is None:
                            continue
                        a = row * 3 + col
                        if current_state[a] == '0':
                            current_state[a] = '1'
                            states = check_state(states, current_state)
                        else:
                            print('The place is already  filled! Please fill an unoccupied  place')
                            continue
                        if check_win(current_state):
                            give_reward(states, menacing_states, menacing_steps, -1)
                            prnt_game(current_state)
                            print('User won')
                            # wanna_quit = quit_prompt()
                            time.sleep(1)
                            new_game = True
                            break
                        if check_draw(current_state):
                            give_reward(states, menacing_states, menacing_steps, 1)
                            prnt_game(current_state)
                            print('Game Draw')
                            # wanna_quit = quit_prompt()
                            time.sleep(1)
                            new_game = True
                            break
                        #print('********')
                        #print(''.join(current_state))
                        menacing_states.append(tuple(current_state))
                        current_bead = states[''.join(current_state)].get_beads()
                        menacing_steps.append(current_bead)
                        #print(current_bead)
                        #print('********')
                        row_bead = int(current_bead / 3)
                        col_bead = current_bead % 3
                        tictactoe.drawMove (board, row_bead, col_bead, "O")
                        tictactoe.gameWon(board)
                        tictactoe.showBoard(ttt, board)
                        if current_state[current_bead] == '0':
                            current_state[current_bead] = '2'
                            states = check_state(states, current_state)
                        else:
                            print('The place is already filled! Please fill an unoccupied place')
                            break
                        if check_win(current_state):
                            give_reward(states, menacing_states, menacing_steps, 3)
                            prnt_game(current_state)
                            print('Menace won')
                            # wanna_quit = quit_prompt()
                            time.sleep(1)
                            new_game = True
                            break
                        if check_draw(current_state):
                            give_reward(states, menacing_states, menacing_steps, 1)
                            prnt_game(current_state)
                            print('Game Draw')
                            # wanna_quit = quit_prompt()
                            time.sleep(1)
                            break
                        prnt_game(current_state)
                        tictactoe.showBoard(ttt, board)
                    tictactoe.showBoard(ttt, board)
            except ValueError:
                print('The place is already  filled! Please fill an unoccupied  place')
                continue
def train_game(states, path, iterations):

    for _ in tqdm(range(iterations)):
        #print("Game Start")
        current_state = list('000000000')
        prnt_game(current_state)
        menacing_steps1 = []
        menacing_states1 = []
        menacing_steps2 = []
        menacing_states2 = []
        while(True):
            try:
                #print('PLAYER 1')
                menacing_states1.append(tuple(current_state))
                current_bead = states[''.join(current_state)].get_beads()
                menacing_steps1.append(current_bead)
                current_state[current_bead] = '1'
                states = check_state(states, current_state)
                #print(current_bead)
                #prnt_game(current_state)
                if check_win(current_state):
                    #print('One won')
                    give_reward(states, menacing_states1, menacing_steps1, 3)
                    give_reward(states, menacing_states2, menacing_steps2, -1)
                    prnt_game(current_state)
                    break
                if check_draw(current_state):
                    #print('Game Draw')
                    give_reward(states, menacing_states1, menacing_steps1, 1)
                    give_reward(states, menacing_states2, menacing_steps2, 1)
                    prnt_game(current_state)
                    break
                #print('PLAYER 2')
                #print(''.join(current_state))
                menacing_states2.append(tuple(current_state))
                current_bead = states[''.join(current_state)].get_beads()
                menacing_steps2.append(current_bead)
                current_state[current_bead] = '2'
                states = check_state(states, current_state)
                #print(current_bead)
                #prnt_game(current_state)
                if check_win(current_state):
                    #print('Two won')
                    give_reward(states, menacing_states1, menacing_steps1, -1)
                    give_reward(states, menacing_states2, menacing_steps2, 3)
                    prnt_game(current_state)
                    break
                if check_draw(current_state):
                    #print('Game Draw')
                    give_reward(states, menacing_states1, menacing_steps1, 1)
                    give_reward(states, menacing_states2, menacing_steps2, 1)
                    prnt_game(current_state)
                    break
                #prnt_game(current_state)
            except ValueError:
                print('The place is already  filled! Please fill an unoccupied  place')
                continue
    print('training completed')
    pickle_out = open(path,"wb")
    pickle.dump(states, pickle_out)
    pickle_out.close()

def main():
    argv = sys.argv
    if len(argv) == 3:
        mode = argv[1]
        iterations = int(argv[2] )
    elif len(argv) == 2:
        mode = argv[1]
        iterations = 10000
    else:
        mode =  'test'
    path = 'model.pickle'
    states = {'000000000' : State('000000000')}
    if mode=='test':
        game_on(states, path)
    else:
        print('Training started')
        train_game(states, path, iterations)

if __name__ == "__main__":
    main()
    pass

max_stars_repo_name: Hsankesara/Menace

max_issues_repo_name: Hsankesara/Menace

max_forks_repo_name: Hsankesara/Menace


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright © 2016 Matthew Stone <mstone5@mgh.harvard.edu>
# Distributed under terms of the MIT license.

"""
Modification of Michael Waskom's JointGrid implementation in Seaborn.

Supports multiple JointGrids in single figure
"""

import numpy as np
import pandas as pd
import matplotlib.gridspec as gridspec
import matplotlib.pyplot as plt
import seaborn as sns

class JointGrid(sns.JointGrid):
    """Grid for drawing a bivariate plot with marginal univariate plots."""

    def __init__(self, x, y, data=None, gs=None, ratio=5, space=.2,
                 dropna=True, xlim=None, ylim=None):
        """Set up the grid of subplots.

        Parameters
        ----------
        x, y : strings or vectors
            Data or names of variables in ``data``.
        data : DataFrame, optional
            DataFrame when ``x`` and ``y`` are variable names.
        size : numeric
            Size of each side of the figure in inches (it will be square).
        ratio : numeric
            Ratio of joint axes size to marginal axes height.
        space : numeric, optional
            Space between the joint and marginal axes
        dropna : bool, optional
            If True, remove observations that are missing from `x` and `y`.
        {x, y}lim : two-tuples, optional
            Axis limits to set before plotting.

        See Also
        --------
        jointplot : High-level interface for drawing bivariate plots with
                    several different default plot kinds.

        """

        # Set up the subplot grid
        if gs is None:
            gs = gridspec.GridSpec(ratio + 1, ratio + 1,
                                   hspace=space, wspace=space)

        ax_joint = plt.subplot(gs[1:, :-1])
        ax_marg_x = plt.subplot(gs[0, :-1], sharex=ax_joint)
        ax_marg_y = plt.subplot(gs[1:, -1], sharey=ax_joint)

        self.ax_joint = ax_joint
        self.ax_marg_x = ax_marg_x
        self.ax_marg_y = ax_marg_y

        # Turn off tick visibility for the measure axis on the marginal plots
        plt.setp(ax_marg_x.get_xticklabels(), visible=False)
        plt.setp(ax_marg_y.get_yticklabels(), visible=False)

        # Turn off the ticks on the density axis for the marginal plots
        plt.setp(ax_marg_x.yaxis.get_majorticklines(), visible=False)
        plt.setp(ax_marg_x.yaxis.get_minorticklines(), visible=False)
        plt.setp(ax_marg_y.xaxis.get_majorticklines(), visible=False)
        plt.setp(ax_marg_y.xaxis.get_minorticklines(), visible=False)
        plt.setp(ax_marg_x.get_yticklabels(), visible=False)
        plt.setp(ax_marg_y.get_xticklabels(), visible=False)
        ax_marg_x.yaxis.grid(False)
        ax_marg_y.xaxis.grid(False)

        # Possibly extract the variables from a DataFrame
        if data is not None:
            if x in data:
                x = data[x]
            if y in data:
                y = data[y]

        # Possibly drop NA
        if dropna:
            not_na = pd.notnull(x) & pd.notnull(y)
            x = x[not_na]
            y = y[not_na]

        # Find the names of the variables
        if hasattr(x, "name"):
            xlabel = x.name
            ax_joint.set_xlabel(xlabel)
        if hasattr(y, "name"):
            ylabel = y.name
            ax_joint.set_ylabel(ylabel)

        # Convert the x and y data to arrays for plotting
        self.x = np.asarray(x)
        self.y = np.asarray(y)

        if xlim is not None:
            ax_joint.set_xlim(xlim)
        if ylim is not None:
            ax_joint.set_ylim(ylim)

class JointGrids:
    def __init__(self, data, x, y,
                 col=None, col_order=None,
                 row=None, row_order=None,
                 panel_size=8, ratio=5):
                # row=None, row_order=None,
                # col=None, col_order=None,
                # hue=None, hue_order=None):
        """
        Borrowed heavily from seaborn FacetGrid

        Arguments
        ---------
        panel_size : int, optional
            Height/width of each constituent JointGrid
        ratio : int, optional
            Ratio of joint to marginal axis size
        """

        if row is None:
            row_names = []
        else:
            row_names = sns.utils.categorical_order(data[row], row_order)

        if col is None:
            col_names = []
        else:
            col_names = sns.utils.categorical_order(data[col], col_order)

        #  if col is not None and col_order is None:
            #  col_order = data[col].drop_duplicates().sort_values()
        #  n_cols = len(col_order)
        n_cols = 1 if col is None else len(col_names)
        n_rows = 1 if row is None else len(row_names)

        self.fig = plt.figure(figsize=(n_cols * panel_size,
                                       n_rows * panel_size))

        self.gs = gridspec.GridSpec(n_rows, n_cols)
        self.grids = np.empty((n_rows, n_cols), dtype=object)

        if len(row_names) > 0 and len(col_names) > 0:
            for i, row_val in enumerate(row_names):
                for j, col_val in enumerate(col_names):
                    subdata = data.loc[(data[col] == col_val) &
                                       (data[row] == row_val)]
                    ss = self.gs[i, j]
                    gs = gridspec.GridSpecFromSubplotSpec(ratio + 1, ratio + 1,
                                                          subplot_spec=ss)

                    grid = JointGrid(x, y, data=subdata, gs=gs)
                    self.grids[i, j] = grid

        else:
            if len(row_names) > 0:
                facets = row_names
                facet = row
                grids = self.grids[:, 0]
            else:
                facets = col_names
                facet = col
                grids = self.grids[0]

            for i, val in enumerate(facets):
                subdata = data.loc[data[facet] == val]
                gs = gridspec.GridSpecFromSubplotSpec(ratio + 1, ratio + 1,
                                                      subplot_spec=self.gs[i])
                grid = JointGrid(x, y, data=subdata, gs=gs)
                grids[i] = grid

    def set_xlims(self, xmin, xmax):
        for grid in self.grids.flat:
            grid.ax_joint.set_xlim(xmin, xmax)

    def set_ylims(self, ymin, ymax):
        for grid in self.grids.flat:
            grid.ax_joint.set_ylim(ymin, ymax)

    def set_lims(self, xmin, xmax):
        for grid in self.grids.flat:
            grid.ax_joint.set_xlim(xmin, xmax)
            grid.ax_joint.set_ylim(xmin, xmax)

    def plot_joint(self, func, **kwargs):
        for grid in self.grids.flat:
            grid.plot_joint(func, **kwargs)

    def plot_marginals(self, func, **kwargs):
        for grid in self.grids.flat:
            grid.plot_marginals(func, **kwargs)

max_stars_repo_name: msto/svplot

max_issues_repo_name: msto/svplot

max_forks_repo_name: msto/svplot


text:
# This program generates a dataset of guassian(only along x-axis) with some noise
import matplotlib
import matplotlib.pyplot as plt
plt.switch_backend('Agg')
from matplotlib import cm
import numpy as np
from scipy.stats import norm
import numpy as np
import h5py
from mpl_toolkits.mplot3d import Axes3D
import math
mean1 = 10
sigma1 = 2
nevents =5 # events of one class. Total will be double
mean2 = 10
sigma2 = 4

# Normalized guassian functions. Both give same value
#def gaussian(x, mu, sig):
#    return 1./(math.sqrt(2.*math.pi)*sig)*np.exp(-np.power((x - mu)/sig, 2.)/2)

def gaussian(x,x0,sigma):
  return np.exp(-np.power((x - x0)/sigma, 2.)/2.)

x = np.arange(0, 25, 1)
y = np.arange(0, 25, 1)
z = np.arange(0, 25, 1)

Z = np.zeros(shape=(nevents*2, 25, 25, 25))
T = np.zeros(shape=(nevents*2,1))
index = 0
for i in range(nevents):
    noise = np.random.normal(0, 0.01, 25)
    B = gaussian(x, mean1, sigma1)+ noise
    B = 80 * B
    Z[index,:,12,12]= B
    fig=plt.figure(index)
    fig=plt.plot(x, B)
    plt.title("Training data class1 along x_axis")
    plt.xlabel("Cell location along x_axis")
    plt.ylabel("Value")
    plt.savefig("figs/demo" + str(index) +".png")
    noise = np.random.normal(0, 0.01, 25)
    B = gaussian(x, mean2, sigma2)+ noise
    B = 80 * B
    index= index + 1
    T[index]= 1
    Z[index,:,12,12]= B
    fig=plt.figure(index)
    fig=plt.plot(x, B)
    plt.title("Training data class 2 along x_axis")
    plt.xlabel("Cell location along x_axis")
    plt.ylabel("Value")
    plt.savefig("figs/demo" + str(index) +".png")
    index= index+1
print "mean of 1 = %f" %np.mean(Z[:,:,12,12])
print "sigma of 1= %f" %np.std(Z[:,:,12,12])

print(Z.shape)
print(T.shape)

h5f = h5py.File('data.h5', 'w')
h5f.create_dataset('ECAL', data=Z, dtype='f8')
h5f.create_dataset('TAG', data=T, dtype='i8')
h5f.close()
#fig = plt.figure(nevents1*2)
#ax = fig.add_subplot(111, projection='3d')
#a, b, c, d = Z.nonzero()
#sp = ax.scatter(b, c, d, c=Z[0,b,c,d])
#plt.colorbar(sp)
#plt.savefig("figs/3d2.png")