text:
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Thu Jul  5 12:02:41 2018

@author: isaaclera
"""

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from pandas import Series, date_range
import matplotlib.pyplot as plt
import numpy as np
import json
from scipy import stats
import collections
import networkx as nx

time = 10000
path ="exp1/Results__%i"%time

#T1+T2
# Number of messages from sensors to replicas
# Total de paquetes desde los sensores hasta las replicas (suma de las replicas). (acc. from sensors a replicas)
#Sin identificar elementos.

df = pd.read_csv(path + ".csv")
dfl = pd.read_csv(path + "_link.csv")

print "Total number of perfomed transmissions : %i" %len(df)

#Group by app, src, and id
print "Total n.messages between sources -> replicas by app"
dtmp = df[df["module.src"]=="None"].groupby(['app','TOPO.src'])['id'].apply(list)
print "\tApplication\tMessages "
totalMessages = 0
for app,values in enumerate(dtmp):
    print "\t%i\t%i" %(app,len(values))
    totalMessages+=len(values)

print "Total Message  between sources -> replicas: %i" %totalMessages

print "Total messages between replicas -> cloud: %i" %(len(df)-totalMessages)

#T3
#Latency from replicas to the cloud
#Latency desde las replicas al cloud

#desde df- identificar id que van al cloud
#desde dfl - coger mensajes[ids] and borrar aquellos from M.USER.APP.
dtmp = df[df["module.src"]!="None"].groupby(['app','TOPO.src'])['id'].apply(list)
print "Total n.messages between replicas -> cloud by app:"
print "\tApplication\tMessages "
totalMessages = 0
for app,values in enumerate(dtmp):
    print "\t%i\t%i" %(app,len(values))
    totalMessages+=len(values)
#OJO-Si desglosar latency por app...deberia de estar aqui, lo siguiente:

#TOTAL
idMessagesToCloud = df[df["module.src"]!="None"]["id"].values
#Selecting only transmissions between replicas and cloud
dftmp = dfl[dfl["id"].isin(idMessagesToCloud)]
dftmp = dftmp[dftmp['message'].str.contains("M.CLOUD")]
grm = dftmp.groupby(["id"]).agg({"latency":np.sum})
print "Total avg. latency (replicas->cloud): %f" %grm.mean().latency

#T4
#Latencia desde el sensor - con la replica(modulo) más cercana | más lejana (H)
print "Computing latency sources -> replica (MIN and MAX Path) each app"
PATH = collections.namedtuple('PATH', 'src dst len')
#LOADING NETWORK
data = json.load(open('exp1/networkDefinition.json'))
G = nx.Graph()
for edge in data["link"]:
    G.add_edge(edge["s"], edge["d"])

dtmp = df[df["module.src"]=="None"].groupby(['app','TOPO.src','TOPO.dst','message'])['id'].apply(list)
dmsg = dfl[dfl["message"].str.contains("M.USER.APP")]
for app in dtmp.index.levels[0].values:
    print "app ",app
    minpath = PATH(src=0,dst=0,len=999999)
    maxpath = PATH(src=0,dst=0,len=-1)
    for src in dtmp[app].index.levels[0].values:
#        print "\tsrc ",src
        for dst in dtmp[app][src].index.levels[0].values:
#            print "\t\tdst ",dst
            path = list(nx.shortest_path(G, source=src, target=dst))
#            print "\t\t",path
            if minpath.len>len(path):
                minpath = PATH(src=src,dst=dst,len=len(path))
            if maxpath.len<len(path):
                maxpath = PATH(src=src,dst=dst,len=len(path))
    print "\t",minpath
    print "\t",maxpath

    #MinPATH
    idMin = dtmp[app][minpath.src][minpath.dst].values[0]
    print "\t\t MIN PATH Latency: ", dmsg[dmsg["id"].isin(idMin)].groupby(['id']).agg({"latency":np.sum}).mean().values[0] #Include cloud?

    idMax = dtmp[app][maxpath.src][maxpath.dst].values[0]
    print "\t\t MAX PATH Latency: ", dmsg[dmsg["id"].isin(idMax)].groupby(['id']).agg({"latency":np.sum}).mean().values[0]

reason: import numpy,from scipy,import networkx


text:
data_root = 'C:/Users/parisa/Desktop/csYRPCai/project/train/'

import numpy as np
import pandas as pd
import time
from sklearn import svm, datasets
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import label_binarize
from sklearn.multiclass import OneVsRestClassifier
from sklearn.multiclass import OneVsOneClassifier
from sklearn.preprocessing import MultiLabelBinarizer

train_photos = pd.read_csv(data_root+'train_photo_to_biz_ids.csv')
train_photo_to_biz = pd.read_csv(data_root+'train_photo_to_biz_ids.csv', index_col='photo_id')
train_df = pd.read_csv(data_root+"train_biz_HOGfeatures.csv")

test_photos = pd.read_csv(data_root+'test_photo_to_biz.csv')
test_photo_to_biz = pd.read_csv(data_root+'test_photo_to_biz.csv', index_col='photo_id')
test_df = pd.read_csv(data_root+"test_biz_HOGfeatures.csv")

y_train = train_df['label'].values
X_train = train_df['feature vector'].values
X_test = test_df['feature vector'].values

def convert_label_to_array(str_label):
    str_label = str_label[1:-1]
    str_label = str_label.split(',')
    return [int(x) for x in str_label if len(x)>0]

def convert_feature_to_vector(str_feature):
    str_feature = str_feature[1:-1]
    str_feature = str_feature.split(',')
    return [float(x) for x in str_feature]

y_train = np.array([convert_label_to_array(y) for y in train_df['label']])
X_train = np.array([convert_feature_to_vector(x) for x in train_df['feature vector']])
X_test = np.array([convert_feature_to_vector(x) for x in test_df['feature vector']])

print "X_train: ", X_train.shape
print "X_test: ", X_test.shape
print "y_train: ", y_train.shape

t=time.time()

mlb = MultiLabelBinarizer()
y_train= mlb.fit_transform(y_train)  #Convert list of labels to binary matrix

random_state = np.random.RandomState(0)
classifier = OneVsRestClassifier(svm.SVC(kernel='linear', probability=True))
#classifier = OneVsRestClassifier(svm.SVC(C=1.0, gamma='auto', kernel='rbf', probability=True))

classifier.fit(X_train, y_train)

y_predict = classifier.predict(X_test)

print "Time passed: ", "{0:.1f}".format(time.time()-t), "sec"

print "Samples of predicted labels (in binary matrix):\n", y_predict[0:3]
print "\nSamples of predicted labels:\n", mlb.inverse_transform(y_predict[0:3])

df = pd.DataFrame(columns=['business_id','labels'])

for i in range(len(test_df)):
    biz = test_df.loc[i]['business']
    label = y_predict[i]
    label = str(label)[1:-1].replace(",", " ")
    df.loc[i] = [str(biz), label]

with open(data_root+"submission_HOG.csv",'w') as f:
    df.to_csv(f, index=False)

ext: py

max_stars_repo_name: parisaheidarei/YRPC

max_issues_repo_name: parisaheidarei/YRPC

max_forks_repo_name: parisaheidarei/YRPC

reason: import numpy


text:
# -*- coding: utf-8 -*-
"""BTP_v3

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1IgjR4o7OyXvz1bXPHw_GwwjJsDQ_pI_V
"""

# Commented out IPython magic to ensure Python compatibility.
import numpy as np
import math
import matplotlib.pyplot as plt
import seaborn as sns
import random

# %matplotlib inline
#plt.rcParams["figure.figsize"] = (10,10)

def B(delE):
  # The metropolis funtion. The multiplier 200 is taken by using the constants
  # in ΔE/kT. It will vary is specific systems are used or other values are taken.

  dE = -delE * 200
  if dE > 1:
    dE = 1
  U = math.exp(dE)
  if dE < -5:
    U = 0
  return min(U, 1)

def E( X, Y, W, H):
  # The Energy funtion for the interaction between solute and dislocation. We do
  # not use the values of other variables as they are collectively assumed to a
  # value in the metropolis function.

  pi = np.pi
  ans = 0
  for i in range(-4, 5):
    ans += ( np.sin(2*pi*Y) / ( np.cosh(2*pi*(X - (i*W/H))) - np.cos(2* pi *Y)))

  return ans

def En(sig, mul):
  # Energy function for the dislocation.
  E_prime = sig*mul
  return E_prime

def Bn(delE):
  # Meteropolis function for the dislocation
  deE = -delE
  if deE > 1:
    deE = 1
  U = math.exp(deE)
  if deE < -5:
    U = 0
  return min(U, 1)

def E_tot(sol_loc, x_dis, y_dis, H, W):
  # function for calculating the total energy field of the dislocation-solute
  # interaction

  E_dis = 0
  for [x, y] in sol_loc:
    X = (x - x_dis) / W
    Y = (y - y_dis) / H
    E_dis = E_dis + E(X, Y, W, H)

  return E_dis*200

def randomize(array, loc_set, sol_loc, H, W):
  # function for randimizing the solute atoms when dislocation completes one pass
  # through the simulation cell.

  val = 2*W /5 # val can be set to define the number of dislocation atoms whose
  # position has to be randomised.

  for [x, y] in sol_loc:
    flag = True
    while flag:
      x_new = random.randint(0, W-1)
      y_new = random.randint(0, H-1)
      if x < val:
        flag = False
        break

      if [x_new, y_new] not in sol_loc:
        loc_set.remove((x, y))
        loc_set.add((x_new, y_new))
        sol_loc.remove([x, y])
        sol_loc.append([x_new, y_new])
        array[x][y] = 0
        array[x_new][y_new] = 1
        flag = False

# Code for testing dislocation motion algorithm as described in the paper.
# Runnning ths code gives us the results of the case where no solute atoms are
# present.

W = int (input("Enter Width of the simulation cell: "))
H = int (input("Enter Height of the simulation cell: "))
iter = int (input("Enter number of iterations: "))
mul = float (input("Enter Multiplier for external stress in Meteropolis Function: "))

mix = [] # a list to which we will append the displacement vectors for calculating
# average velocity for different external stresses.

sig = 0.001 # initial external stress.

x = [] # a list to which values of external stresses are appended.

while sig<=0.04:
  array = np.zeros((W, H), int)
  x_dis = int (W/2)
  y_dis = int (H/2)

  array[x_dis][y_dis] = -1

  vel = [] # displacement vector.
  k = 0 # total distance moved.
  i = 0
  for i in range(iter):
    U = random.randint(0, 1)

    if U == 0: #represents negative jump.
      dE = En(sig, mul)
      x_dis_new = (x_dis - 1) % W

    else: # represents postive jump.
      dE = -En(sig, mul)
      x_dis_new = (x_dis + 1) % W

    bd = Bn(dE)
    n = random.random()

    if n<bd:
      array[x_dis][y_dis] = 0
      x_dis = x_dis_new
      array[x_dis][y_dis] = -1
      k = k + (2*U - 1)

    if i%10000 == 0: # calculate the value for every 10000 steps.
      vel.append(k)

  x.append(sig)
  sig += 0.001
  mix.append(vel)

y = []
# code below calculates the average velocity for each external stress in x and
# appends it in y.
for row in mix:
  sum = row[0]
  for i in range(len(row) - 1):
     sum += row[i+1] - row[i]
  sum = sum / len(row)
  y.append(sum/10000)

#plotting the velocity vs stress graph.
plt.plot(x, y)
plt.xlabel("External Stress")
plt.ylabel("Velocity of Dislocation")

#entering parameters.
conc = float (input("Enter percentage concentration of solute: "))
Mr = float (input("Enter mobility ratio: "))
mul = float (input("Enter Multiplier for external stress in Meteropolis Function: ")) #(100 or 50)

sig = float (input("Enter external stress as a function of G: "))

W = int (input("Enter Width of the simulation cell: "))
H = int (input("Enter Height of the simulation cell: "))

#initializing array  and randomly distributing solutes in the matrix

array = np.zeros((W, H), int)

sol_loc = [] # represents an array with the locations of the solute atoms.
loc_set = set() # a set containing the locations of the solute atoms and the dislocation.
sol = (H*W*conc)/100
a = 0
R = sol*Mr # Mobility parameter.

x_dis = int (W/2)
y_dis = int (H/2)

array[x_dis][y_dis] = -1
loc_set.add((x_dis, y_dis))

while (a < sol):
  i = random.randint(0, W-1)
  j = random.randint(0, H-1)

  if array[i][j] == 0:
    array[i][j] = 1
    a += 1
    loc_set.add((i, j))
    sol_loc.append([i, j])

  if (a==sol):
    break

#projecting the array as image
plt.rcParams["figure.figsize"] = (5,10) # represents the dimensions of the image.
# can be varied to suit the dimensions of the simulation cell.

plt.grid(False)
plt.axis('off')
image = plt.imshow(array, interpolation='nearest')
plt.show()

# The main simulation algorithm. It calculates the parameters and then chooses whether
# we move the dislocation or the a solute atom.

iter = int (input("Enter Number of Iterations: "))
vel = [] # velocity vector
disp = 0 # total displacement variable.

for i in range (0, iter):

   # The if statement is the code for downloading the of the simulation cell.
   # Create a folder image on the device and copy the path and replace it with
   # "Image" in the below code. This is the most time consuming step. As such for
   # only observation we can reduce the frequency of image collection.

   # array2 = array[x_dis - k:x_dis + k][y_dis - k:y_dis + k]
   # array2 a subset of main simulation cell can be used in showing the portion of
   # 2k x 2k around the dislocation by replacing array by array2 in the image line.

  if i%100 == 0:
      name = r"Image/" + str (int( i/100)) + ".png"
      plt.rcParams["figure.figsize"] = (5,10)
      plt.grid(False)
      plt.axis('off')
      image = plt.imshow(array, interpolation='nearest')
      plt.savefig(name)

  if i%1000 == 0:
    vel.append(disp)

  n = random.randint(0, int (R))

  if n == 0:
    # dislocation motion condition.

    k = random.randint(0, 1)
    if k == 0:
      dE = En(sig, mul)
      x_dis_new = (x_dis - 1) % W

    else:
      dE = -En(sig, mul)
      x_dis_new = (x_dis + 1) % W

    # The following two lines of code consider the influence of solute atoms on the
    # dislocation. Although very less compared to the influence of external stress,
    # using this should improve the accuracy of the simulation. However it doubles
    # the time required to run the simulation.

    # delE = E_tot(sol_loc, x_dis_new, y_dis, H, W) - E_tot(sol_loc, x_dis, y_dis, H, W)
    # dE = dE + delE

    bd = Bn(dE)
    n = random.random()

    if n < bd:
      if (x_dis_new, y_dis) not in loc_set:
        loc_set.remove((x_dis, y_dis))
        loc_set.add((x_dis_new, y_dis))
        array[x_dis][y_dis] = 0
        array[x_dis_new][y_dis] = -1
      else:
        sol_loc.remove([x_dis_new, y_dis])
        sol_loc.append([x_dis, y_dis])
        array[x_dis][y_dis] = 1
        array[x_dis_new][y_dis] = -1

      x_dis = x_dis_new
      disp = disp + (2*k - 1)

    if x_dis==0:
      randomize(array, loc_set, sol_loc, H, W)

  else:
    # solute motion case
    n = random.randint(0, sol-1) # selecting a random solute.
    x = sol_loc[n][0]
    y = sol_loc[n][1]

    k = random.randint(0, 3)
    for i in range(4):
      k = (k + i) % 4
      xi = x
      yi = y

      # probable jumps
      if (k==0):
        xi = (x - 1) % W
      elif (k==1):
        yi = (y - 1) % H
      elif (k==2):
        xi = (x + 1) % W
      else:
        yi = (y + 1) % H

      if ((xi, yi) not in loc_set): # checking if jump is feasible.

        X = (x - x_dis) / W
        Y = (y - y_dis) / H
        Xi = (xi - x_dis) / W
        Yi = (yi - y_dis) / H

        delE = E(Xi, Yi, W, H) - E(X, Y, W, H)
        n = random.random()
        b = B(delE)

        if n < b:
          m = random.random()
          if delE*200 < 0 or m>0.8:

            # updating simulation cell, set and list.
            array[x][y] = 0
            array[xi][yi] = 1

            loc_set.remove((x, y))
            loc_set.add((xi, yi))
            sol_loc.append([xi, yi])
            sol_loc.remove([x, y])

            break

# if used in colab this code can be used to create a downloadable zip file.
!zip -r /content/Image.zip /content/Image

# plotting graph of displacement vs Montecarlo steps.
X = []
for i in range(1000):
  X.append(i/100)

Y = []
Y.append(vel[0])
for i in range(len(vel) - 1):
  Y.append(vel[i+1] - vel[i])

plt.rcParams["figure.figsize"] = (5,5)
plt.scatter(X, vel)

plt.ylabel("Displacement of the Dislocation")
plt.xlabel("Number of Montecarlo Steps (per 100000)")

#plotting graph of the magnified simulation cell after the process is complete.
array2 = array[x_dis-50:x_dis+50][y_dis-50:y_dis+50]
plt.grid(False)
plt.axis('off')
plt.rcParams['figure.figsize'] = (5, 5)
image = plt.imshow(array2, interpolation='nearest')
plt.show()

#calculating the average velocity of the dislocation.
sum = vel[0]
for i in range(len(vel) - 1):
  sum += vel[i+1] - vel[i]
sum = sum / len(vel)

print(sum/1000)

#plotting image
plt.grid(False)
plt.axis('off')
image = plt.imshow(array, interpolation='nearest')
plt.show()

max_stars_repo_name: AbhayMudgal/BTP

max_issues_repo_name: AbhayMudgal/BTP

max_forks_repo_name: AbhayMudgal/BTP


text:
# --------------------------------------------------------
# Tensorflow Faster R-CNN
# Licensed under The MIT License [see LICENSE for details]
# Written by Zheqi He, Xinlei Chen, based on code from Ross Girshick
# --------------------------------------------------------
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import _init_paths
# from model.train_val import get_training_roidb, train_net
# from model.train_val_rfcn import get_training_roidb, train_net
from model.train_val import get_training_roidb, train_net
from model.config import cfg, cfg_from_file, cfg_from_list, get_output_dir, get_output_tb_dir
from datasets.factory import get_imdb
import datasets.imdb
import argparse
import pprint
import numpy as np
import sys

import tensorflow as tf
from nets.vgg16 import vgg16
from nets.resnet_v1_rfcn_hole import resnetv1
# from nets.resnet_v1_rfcn_hole_local_global import resnetv1
# from nets.resnet_v1_rfcn_hole_local_global_without_bbox import resnetv1
#from nets.resnet_v1_rfcn_hole_local_global_elements_wise import resnetv1

def parse_args():
  """
  Parse input arguments
  """
  parser = argparse.ArgumentParser(description='Train a Fast R-CNN network')
  parser.add_argument('--cfg', dest='cfg_file',
                      help='optional config file',
                      default=None, type=str)
  parser.add_argument('--weight', dest='weight',
                      help='initialize with pretrained model weights',
                      type=str)
  parser.add_argument('--imdb', dest='imdb_name',
                      help='dataset to train on',
                      default='voc_2007_trainval+voc_2012_trainval', type=str)
  parser.add_argument('--imdbval', dest='imdbval_name',
                      help='dataset to validate on',
                      default='voc_2007_test', type=str)
  parser.add_argument('--iters', dest='max_iters',
                      help='number of iterations to train',
                      default=110000, type=int)
  parser.add_argument('--tag', dest='tag',
                      help='tag of the model',
                      default=None, type=str)
  parser.add_argument('--net', dest='net',
                      help='vgg16, res50, res101, res152',
                      default='res101', type=str)
  parser.add_argument('--set', dest='set_cfgs',
                      help='set config keys', default=None,
                      nargs=argparse.REMAINDER)

  if len(sys.argv) == 1:
    parser.print_help()
    sys.exit(1)

  args = parser.parse_args()
  return args

def combined_roidb(imdb_names):
  """
  Combine multiple roidbs
  """

  def get_roidb(imdb_name):
    imdb = get_imdb(imdb_name)
    print('Loaded dataset `{:s}` for training'.format(imdb.name))
    imdb.set_proposal_method(cfg.TRAIN.PROPOSAL_METHOD)
    print('Set proposal method: {:s}'.format(cfg.TRAIN.PROPOSAL_METHOD))
    roidb = get_training_roidb(imdb)
    return roidb

  roidbs = [get_roidb(s) for s in imdb_names.split('+')]
  roidb = roidbs[0]
  if len(roidbs) > 1:
    for r in roidbs[1:]:
      roidb.extend(r)
    tmp = get_imdb(imdb_names.split('+')[1])
    imdb = datasets.imdb.imdb(imdb_names, tmp.classes)
  else:
    imdb = get_imdb(imdb_names)
  return imdb, roidb

if __name__ == '__main__':
  args = parse_args()

  print('Called with args:')
  print(args)

  if args.cfg_file is not None:
    cfg_from_file(args.cfg_file)
  if args.set_cfgs is not None:
    cfg_from_list(args.set_cfgs)

  print('Using config:')
  pprint.pprint(cfg)

  np.random.seed(cfg.RNG_SEED)

  # train set
  imdb, roidb = combined_roidb(args.imdb_name)
  print('{:d} roidb entrie s'.format(len(roidb)))

  # output directory where the models are saved
  output_dir = get_output_dir(imdb, args.tag)
  print('Output will be saved to `{:s}`'.format(output_dir))

  # tensorboard directory where the summaries are saved during training
  tb_dir = get_output_tb_dir(imdb, args.tag)
  print('TensorFlow summaries will be saved to `{:s}`'.format(tb_dir))

  # also add the validation set, but with no flipping images
  orgflip = cfg.TRAIN.USE_FLIPPED
  cfg.TRAIN.USE_FLIPPED = False
  _, valroidb = combined_roidb(args.imdbval_name)
  print('{:d} validation roidb entries'.format(len(valroidb)))
  cfg.TRAIN.USE_FLIPPED = orgflip

  # load network
  if args.net == 'vgg16':
    net = vgg16(batch_size=cfg.TRAIN.IMS_PER_BATCH)
  elif args.net == 'res50':
    net = resnetv1(batch_size=cfg.TRAIN.IMS_PER_BATCH, num_layers=50)
  elif args.net == 'res101':
    net = resnetv1(batch_size=cfg.TRAIN.IMS_PER_BATCH, num_layers=101)
  elif args.net == 'res152':
    net = resnetv1(batch_size=cfg.TRAIN.IMS_PER_BATCH, num_layers=152)
  else:
    raise NotImplementedError

  train_net(net, imdb, roidb, valroidb, output_dir, tb_dir,
            pretrained_model=args.weight,
            max_iters=args.max_iters)


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2021 The TARTRL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""""""

import random

import numpy as np
import torch

from tmarl.configs.config import get_config
from tmarl.runners.base_runner import Runner

def set_seed(seed):
    random.seed(seed)
    np.random.seed(seed)

    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)

class Evaluator(Runner):
    def __init__(self, argv,program_type=None, client=None):
        super().__init__(argv)

        parser = get_config()
        all_args = self.extra_args_func(argv, parser)

        all_args.cuda = not all_args.disable_cuda

        self.algorithm_name = all_args.algorithm_name

        # cuda
        if not all_args.disable_cuda and torch.cuda.is_available():
            device = torch.device("cuda:0")

            if all_args.cuda_deterministic:
                torch.backends.cudnn.benchmark = False
                torch.backends.cudnn.deterministic = True
        else:
            print("choose to use cpu...")
            device = torch.device("cpu")

        # run dir
        run_dir = self.setup_run_dir(all_args)

        # env init
        Env_Class, SubprocVecEnv, DummyVecEnv = self.get_env()
        eval_envs = self.env_init(
            all_args, Env_Class, SubprocVecEnv, DummyVecEnv)
        num_agents = all_args.num_agents

        config = {
            "all_args": all_args,
            "envs": None,
            "eval_envs": eval_envs,
            "num_agents": num_agents,
            "device": device,
            "run_dir": run_dir,
        }
        self.all_args, self.envs, self.eval_envs, self.config \
            = all_args, None, eval_envs, config
        self.driver = self.init_driver()

    def run(self):
        # run experiments
        self.driver.run()
        self.stop()

    def stop(self):
        pass

    def extra_args_func(self, argv, parser):
        raise NotImplementedError

    def get_env(self):
        raise NotImplementedError

    def init_driver(self):
        raise NotImplementedError

    def make_eval_env(self, all_args, Env_Class, SubprocVecEnv, DummyVecEnv):
        def get_env_fn(rank):
            def init_env():
                env = Env_Class(all_args)
                env.seed(all_args.seed * 50000 + rank * 10000)
                return env

            return init_env

        if all_args.n_eval_rollout_threads == 1:
            return DummyVecEnv([get_env_fn(0)])
        else:
            return SubprocVecEnv([get_env_fn(i) for i in range(all_args.n_eval_rollout_threads)])

    def env_init(self, all_args, Env_Class, SubprocVecEnv, DummyVecEnv):
        eval_envs = self.make_eval_env(
            all_args, Env_Class, SubprocVecEnv, DummyVecEnv) if all_args.use_eval else None
        return eval_envs

    def setup_run_dir(self, all_args):
        return None

max_stars_repo_name: TARTRL/TiKick

max_issues_repo_name: TARTRL/TiKick

max_forks_repo_name: TARTRL/TiKick


text:
import DBCV
from sklearn import datasets
from sklearn.cluster import KMeans
from scipy.spatial.distance import euclidean

def generate_data(n_samples=300, noise=0.05):
    noisy_moons = datasets.make_moons(n_samples=n_samples, noise=noise)
    X = noisy_moons[0]
    return X

def generate_labels(X):
    kmeans =  KMeans(n_clusters=2)
    kmeans_labels = kmeans.fit_predict(X)
    return kmeans_labels

if __name__ == '__main__':
    X = generate_data()
    labels = generate_labels(X)
    score = DBCV.DBCV(X, labels, dist_function=euclidean)

reason: from scipy


text:
# *- encoding: utf-8 -*-
"""
Utilities to download NeuroImaging datasets
"""
# Author: Alexandre Abraham, Philippe Gervais
# License: simplified BSD

import os
import urllib
import urllib2
import tarfile
import zipfile
import sys
import shutil
import time
import hashlib
import fnmatch
import warnings
import cPickle as pickle

import numpy as np
from scipy import ndimage
from sklearn.datasets.base import Bunch

import nibabel

def _format_time(t):
    if t > 60:
        return "%4.1fmin" % (t / 60.)
    else:
        return " %5.1fs" % (t)

def _md5_sum_file(path):
    """ Calculates the MD5 sum of a file.
    """
    f = open(path, 'rb')
    m = hashlib.md5()
    while True:
        data = f.read(8192)
        if not data:
            break
        m.update(data)
    return m.hexdigest()

def _read_md5_sum_file(path):
    """ Reads a MD5 checksum file and returns hashes as a dictionary.
    """
    f = open(path, "r")
    hashes = {}
    while True:
        line = f.readline()
        if not line:
            break
        h, name = line.rstrip().split('  ', 1)
        hashes[name] = h
    return hashes

class ResumeURLOpener(urllib.FancyURLopener):
    """Create sub-class in order to overide error 206.  This error means a
       partial file is being sent, which is fine in this case.
       Do nothing with this error.

       Note
       ----
       This was adapted from:
       http://code.activestate.com/recipes/83208-resuming-download-of-a-file/
    """
    def http_error_206(self, url, fp, errcode, errmsg, headers, data=None):
        pass

def _chunk_report_(bytes_so_far, total_size, t0):
    """Show downloading percentage.

    Parameters
    ----------
    bytes_so_far: int
        Number of downloaded bytes

    total_size: int, optional
        Total size of the file. None is valid

    t0: int, optional
        The time in seconds (as returned by time.time()) at which the
        download was started.
    """
    if total_size:
        percent = float(bytes_so_far) / total_size
        percent = round(percent * 100, 2)
        dt = time.time() - t0
        # We use a max to avoid a division by zero
        remaining = (100. - percent) / max(0.01, percent) * dt
        # Trailing whitespace is too erase extra char when message length
        # varies
        sys.stderr.write(
            "Downloaded %d of %d bytes (%0.2f%%, %s remaining)  \r"
            % (bytes_so_far, total_size, percent,
               _format_time(remaining)))
    else:
        sys.stderr.write("Downloaded %d of ? bytes\r" % (bytes_so_far))

def _chunk_read_(response, local_file, chunk_size=8192, report_hook=None,
                 initial_size=0, total_size=None, verbose=0):
    """Download a file chunk by chunk and show advancement

    Parameters
    ----------
    response: urllib.addinfourl
        Response to the download request in order to get file size

    local_file: file
        Hard disk file where data should be written

    chunk_size: int, optional
        Size of downloaded chunks. Default: 8192

    report_hook: bool
        Whether or not to show downloading advancement. Default: None

    initial_size: int, optional
        If resuming, indicate the initial size of the file

    Returns
    -------
    data: string
        The downloaded file.

    """
    if total_size is None:
        total_size = response.info().getheader('Content-Length').strip()
    try:
        total_size = int(total_size) + initial_size
    except Exception, e:
        if verbose > 0:
            print "Warning: total size could not be determined."
            if verbose > 1:
                print "Full stack trace: %s" % e
        total_size = None
    bytes_so_far = initial_size

    t0 = time.time()
    while True:
        chunk = response.read(chunk_size)
        bytes_so_far += len(chunk)

        if not chunk:
            if report_hook:
                sys.stderr.write('\n')
            break

        local_file.write(chunk)
        if report_hook:
            _chunk_report_(bytes_so_far, total_size, t0)

    return

def _get_dataset_dir(dataset_name, data_dir=None, folder=None,
                     create_dir=True):
    """ Create if necessary and returns data directory of given dataset.

    Parameters
    ----------
    dataset_name: string
        The unique name of the dataset.

    data_dir: string, optional
        Path of the data directory. Used to force data storage in a specified
        location. Default: None

    folder: string, optional
        Folder in which the file must be fetched inside the dataset folder.

    create_dir: bool, optional
        If the directory does not exist, determine whether or not it is created

    Returns
    -------
    data_dir: string
        Path of the given dataset directory.

    Notes
    -----
    This function retrieve the datasets directory (or data directory) using
    the following priority :
    1. the keyword argument data_dir
    2. the environment variable ALLENINF_DATA
    3. "alleninf_data" directory into the home directory
    """
    if not data_dir:
        data_dir = os.getenv("ALLENINF_DATA", os.path.expanduser("~/alleninf_data"))
    data_dir = os.path.join(data_dir, dataset_name)
    if folder is not None:
        data_dir = os.path.join(data_dir, folder)
    if not os.path.exists(data_dir) and create_dir:
        os.makedirs(data_dir)
    return data_dir

def _uncompress_file(file_, delete_archive=True):
    """Uncompress files contained in a data_set.

    Parameters
    ----------
    file: string
        path of file to be uncompressed.

    delete_archive: bool, optional
        Wheteher or not to delete archive once it is uncompressed.
        Default: True

    Notes
    -----
    This handles zip, tar, gzip and bzip files only.
    """
    print 'extracting data from %s...' % file_
    data_dir = os.path.dirname(file_)
    # We first try to see if it is a zip file
    try:
        filename, ext = os.path.splitext(file_)
        processed = False
        if ext == '.zip':
            z = zipfile.ZipFile(file_)
            z.extractall(data_dir)
            z.close()
            processed = True
        elif ext == '.gz':
            import gzip
            gz = gzip.open(file_)
            out = open(filename, 'wb')
            shutil.copyfileobj(gz, out, 8192)
            gz.close()
            out.close()
            # If file is .tar.gz, this will be handle in the next case
            if delete_archive:
                os.remove(file_)
            file_ = filename
            filename, ext = os.path.splitext(file_)
            processed = True
        if ext in ['.tar', '.tgz', '.bz2']:
            tar = tarfile.open(file_, "r")
            tar.extractall(path=data_dir)
            tar.close()
            processed = True
        if not processed:
            raise IOError("Uncompress: unknown file extension: %s" % ext)
        if delete_archive:
            os.remove(file_)
        print '   ...done.'
    except Exception as e:
        print 'Error uncompressing file: %s' % e
        raise

def _fetch_file(url, data_dir, resume=True, overwrite=False,
                md5sum=None, verbose=0):
    """Load requested file, downloading it if needed or requested.

    Parameters
    ----------
    url: string
        Contains the url of the file to be downloaded.

    data_dir: string, optional
        Path of the data directory. Used to force data storage in a specified
        location. Default: None

    resume: bool, optional
        If true, try to resume partially downloaded files

    overwrite: bool, optional
        If true and file already exists, delete it.

    md5sum: string, optional
        MD5 sum of the file. Checked if download of the file is required

    verbose: int, optional
        Defines the level of verbosity of the output

    Returns
    -------
    files: string
        Absolute path of downloaded file.

    Notes
    -----
    If, for any reason, the download procedure fails, all downloaded files are
    removed.
    """
    # Determine data path
    if not os.path.exists(data_dir):
        os.makedirs(data_dir)

    # Determine filename using URL
    parse = urllib2.urlparse.urlparse(url)
    file_name = os.path.basename(parse.path)

    temp_file_name = file_name + ".part"
    full_name = os.path.join(data_dir, file_name)
    temp_full_name = os.path.join(data_dir, temp_file_name)
    if os.path.exists(full_name):
        if overwrite:
            os.remove(full_name)
        else:
            return full_name
    if os.path.exists(temp_full_name):
        if overwrite:
            os.remove(temp_full_name)
    t0 = time.time()
    local_file = None
    initial_size = 0
    try:
        # Download data
        print 'Downloading data from %s ...' % url
        if resume and os.path.exists(temp_full_name):
            url_opener = ResumeURLOpener()
            # Download has been interrupted, we try to resume it.
            local_file_size = os.path.getsize(temp_full_name)
            # If the file exists, then only download the remainder
            url_opener.addheader("Range", "bytes=%s-" % (local_file_size))
            try:
                data = url_opener.open(url)
            except urllib2.HTTPError:
                # There is a problem that may be due to resuming. Switch back
                # to complete download method
                return _fetch_file(url, data_dir, resume=False,
                                   overwrite=False)
            local_file = open(temp_full_name, "ab")
            initial_size = local_file_size
        else:
            data = urllib2.urlopen(url)
            local_file = open(temp_full_name, "wb")
        _chunk_read_(data, local_file, report_hook=True,
                     initial_size=initial_size, verbose=verbose)
        # temp file must be closed prior to the move
        if not local_file.closed:
            local_file.close()
        shutil.move(temp_full_name, full_name)
        dt = time.time() - t0
        print '...done. (%i seconds, %i min)' % (dt, dt / 60)
    except urllib2.HTTPError, e:
        print 'Error while fetching file %s.' \
            ' Dataset fetching aborted.' % file_name
        if verbose > 0:
            print "HTTP Error:", e, url
        raise
    except urllib2.URLError, e:
        print 'Error while fetching file %s.' \
            ' Dataset fetching aborted.' % file_name
        if verbose > 0:
            print "URL Error:", e, url
        raise
    finally:
        if local_file is not None:
            if not local_file.closed:
                local_file.close()
    if md5sum is not None:
        if (_md5_sum_file(full_name) != md5sum):
            raise ValueError("File %s checksum verification has failed."
                             " Dataset fetching aborted." % local_file)
    return full_name

def movetree(src, dst):
    """Move an entire tree to another directory. Any existing file is
    overwritten"""
    names = os.listdir(src)

    # Create destination dir if it does not exist
    if not os.path.exists(dst):
        os.makedirs(dst)
    errors = []

    for name in names:
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)
        try:
            if os.path.isdir(srcname) and os.path.isdir(dstname):
                movetree(srcname, dstname)
                os.rmdir(srcname)
            else:
                shutil.move(srcname, dstname)
        except (IOError, os.error) as why:
            errors.append((srcname, dstname, str(why)))
        # catch the Error from the recursive movetree so that we can
        # continue with other files
        except Exception as err:
            errors.extend(err.args[0])
    if errors:
        raise Exception(errors)

def _fetch_files(dataset_name, files, data_dir=None, resume=True, folder=None,
                 mock=False, verbose=0):
    """Load requested dataset, downloading it if needed or requested.

    If needed, _fetch_files download data in a sandbox and check that all files
    are present before moving it to their final destination. This avoids
    corruption of an existing dataset.

    Parameters
    ----------
    dataset_name: string
        Unique dataset name

    files: list of (string, string, dict)
        List of files and their corresponding url. The dictionary contains
        options regarding the files. Options supported are 'uncompress' to
        indicates that the file is an archive, 'md5sum' to check the md5 sum of
        the file and 'move' if renaming the file or moving it to a subfolder is
        needed.

    data_dir: string, optional
        Path of the data directory. Used to force data storage in a specified
        location. Default: None

    resume: bool, optional
        If true, try resuming download if possible

    folder: string, optional
        Folder in which the file must be fetched inside the dataset folder.

    mock: boolean, optional
        If true, create empty files if the file cannot be downloaded. Test use
        only.

    Returns
    -------
    files: list of string
        Absolute paths of downloaded files on disk
    """
    # There are two working directories here:
    # - data_dir is the destination directory of the dataset
    # - temp_dir is a temporary directory dedicated to this fetching call. All
    #   files that must be downloaded will be in this directory. If a corrupted
    #   file is found, or a file is missing, this working directory will be
    #   deleted.
    data_dir = _get_dataset_dir(dataset_name, data_dir=data_dir, folder=folder)
    files_pickle = pickle.dumps(files)
    files_md5 = hashlib.md5(files_pickle).hexdigest()
    temp_dir = os.path.join(data_dir, files_md5)

    # Abortion flag, in case of error
    abort = False

    files_ = []
    for file_, url, opts in files:
        # 3 possibilities:
        # - the file exists in data_dir, nothing to do.
        # - the file does not exists: we download it in temp_dir
        # - the file exists in temp_dir: this can happen if an archive has been
        #   downloaded. There is nothing to do

        # Target file in the data_dir
        target_file = os.path.join(data_dir, file_)
        # Target file in temp dir
        temp_target_file = os.path.join(temp_dir, file_)
        if (not os.path.exists(target_file) and not
                os.path.exists(temp_target_file)):
            if not os.path.exists(temp_dir):
                os.mkdir(temp_dir)
            md5sum = opts.get('md5sum', None)
            dl_file = _fetch_file(url, temp_dir, resume=resume,
                                  verbose=verbose, md5sum=md5sum)
            if 'move' in opts:
                # XXX: here, move is supposed to be a dir, it can be a name
                move = os.path.join(temp_dir, opts['move'])
                move_dir = os.path.dirname(os.path.join(temp_dir, move))
                if not os.path.exists(move_dir):
                    os.makedirs(move_dir)
                shutil.move(os.path.join(temp_dir, dl_file), move)
                dl_file = os.path.join(temp_dir, opts['move'])
            if 'uncompress' in opts:
                try:
                    if not mock or os.path.getsize(dl_file) != 0:
                        _uncompress_file(dl_file)
                    else:
                        os.remove(dl_file)
                except:
                    abort = True
        if (not os.path.exists(target_file) and not
                os.path.exists(temp_target_file)):
            if not mock:
                warnings.warn('An error occured while fetching %s' % file_)
                abort = True
            else:
                if not os.path.exists(os.path.dirname(temp_target_file)):
                    os.makedirs(os.path.dirname(temp_target_file))
                open(temp_target_file, 'w').close()
        if abort:
            if os.path.exists(temp_dir):
                shutil.rmtree(temp_dir)
            raise IOError('Fetching aborted. See error above')
        files_.append(target_file)
    # If needed, move files from temps directory to final directory.
    if os.path.exists(temp_dir):
        #XXX We could only moved the files requested
        #XXX Movetree can go wrong
        movetree(temp_dir, data_dir)
        shutil.rmtree(temp_dir)
    return files_

def _tree(path, pattern=None, dictionary=False):
    """ Return a directory tree under the form of a dictionaries and list

    Parameters:
    -----------
    path: string
        Path browsed

    pattern: string, optional
        Pattern used to filter files (see fnmatch)

    dictionary: boolean, optional
        If True, the function will return a dict instead of a list
    """
    files = []
    dirs = [] if not dictionary else {}
    for file_ in os.listdir(path):
        file_path = os.path.join(path, file_)
        if os.path.isdir(file_path):
            if not dictionary:
                dirs.append((file_, _tree(file_path, pattern)))
            else:
                dirs[file_] = _tree(file_path, pattern)
        else:
            if pattern is None or fnmatch.fnmatch(file_, pattern):
                files.append(file_path)
    files = sorted(files)
    if not dictionary:
        return sorted(dirs) + files
    if len(dirs) == 0:
        return files
    if len(files) > 0:
        dirs['.'] = files
    return dirs

###############################################################################
# Dataset downloading functions

def fetch_microarray_expression(data_dir=None, url=None, resume=True,
        verbose=0):
    """
    """

    if url is None:
        url = "https://www.dropbox.com/s/8dua0ndd72pbstn/"

    files = [('microarray_expression.h5', url + 'microarray_expression.h5?dl=1', {})
             ]

    files_ = _fetch_files('microarray_expression', files, data_dir=data_dir,
            resume=resume)

    return Bunch(microarray_expression=files_[0])

max_stars_repo_name: chrisfilo/alleninf

max_issues_repo_name: chrisgorgo/alleninf

max_forks_repo_name: chrisfilo/alleninf

reason: import numpy,from scipy


text:
#!/usr/bin/env python

"""
This ROS node is responsible for planning a path.

This node takes the field shape as a geometry_msgs/PolygonStamped
and publishes the path as a set of visualization markers.

Once the path has been generated the node can, by configuration or
a service call, start feeding path waypoints as actionlib goals to move base.
"""

import roslib; roslib.load_manifest('automow_planning')
import rospy
import tf
from tf.transformations import quaternion_from_euler as qfe
from actionlib import SimpleActionClient

import numpy as np
from math import radians

from geometry_msgs.msg import PolygonStamped, Point, PoseStamped
from std_msgs.msg import ColorRGBA
from visualization_msgs.msg import Marker, MarkerArray
from nav_msgs.msg import Path, Odometry
from move_base_msgs.msg import MoveBaseAction, MoveBaseGoal
from std_srvs.srv import Empty

import shapely.geometry as geo

class PathPlannerNode(object):
    """
    This is a ROS node that is responsible for planning and executing
    the a path through the field.
    """
    def __init__(self):
        # Setup ROS node
        rospy.init_node('path_planner')

        # ROS params
        self.cut_spacing = rospy.get_param("~cut_spacing", 0.25)

        # Setup publishers and subscribers
        rospy.Subscriber('/field/cut_area', PolygonStamped, self.field_callback)
        self.path_marker_pub = rospy.Publisher('visualization_marker',
                                               MarkerArray,
                                               latch=True)
        rospy.Subscriber('/odom', Odometry, self.odom_callback)

        # Setup initial variables
        self.field_shape = None
        self.field_frame_id = None
        self.path = None
        self.path_status = None
        self.path_markers = None
        self.start_path_following = True
        self.robot_pose = None
        self.goal_state = None
        self.current_destination = None
        self.testing = False
        self.current_distance = None
        self.previous_destination = None
        self.clear_costmaps = rospy.ServiceProxy('/move_base/clear_costmaps', Empty)
        self.just_reset = False

        # Spin until shutdown or we are ready for path following
        rate = rospy.Rate(10.0)
        while not rospy.is_shutdown() and not self.start_path_following:
            rate.sleep()
        # If shutdown, return now
        if rospy.is_shutdown():
            return
        # Run until stopped
        heading = 0
        while not rospy.is_shutdown():
            # Setup path following
            self.setup_path_following(heading)
            # Iterate on path following
            while not rospy.is_shutdown():
                if not self.step_path_following():
                    break
            # Rotate 90 degrees, repeat
            if heading == 0:
                heading = 90
            else:
                heading = 0

    def field_callback(self, msg):
        """
        Handles new field polygons, has to be called
        at least once before planning happens.
        """
        # Convert the PolygonStamped into a shapely polygon
        temp_points = []
        for point in msg.polygon.points:
            temp_points.append( (float(point.x), float(point.y)) )
        self.field_shape = geo.Polygon(temp_points)
        self.field_frame_id = msg.header.frame_id

    def odom_callback(self, msg):
        """
        Watches for the robot's Odometry data, which is used in the path
        planning as the initial robot position.
        """
        self.robot_pose = msg

    def plan_path(self, field_polygon, origin=None, degrees=0):
        """
        This is called after a field polygon has been received.

        This uses the automow_planning.coverage module to plan a
        coverage path using the field geometry.  The path consists of
        a series of waypoints.
        """
        # Get the rotation to align with the longest edge of the polygon
        from automow_planning.maptools import rotation_tf_from_longest_edge, RotationTransform
        rotation = rotation_tf_from_longest_edge(field_polygon)
        rotation = RotationTransform(rotation.w + degrees)
        # Rotate the field polygon
        from automow_planning.maptools import rotate_polygon_to
        transformed_field_polygon = rotate_polygon_to(field_polygon, rotation)
        # Decompose the rotated field into a series of waypoints
        from automow_planning.coverage import decompose
        print origin
        if origin is not None:
            point_mat = np.mat([[origin[0], origin[1], 0]], dtype='float64').transpose()
            origin = rotation.irm * point_mat
            origin = (origin[0,0], origin[1,0])
        transformed_path = decompose(transformed_field_polygon,
                                     origin=(origin[0], origin[1]),
                                     width=self.cut_spacing)
        # Rotate the transformed path back into the source frame
        from automow_planning.maptools import rotate_from
        self.path = rotate_from(np.array(transformed_path), rotation)
        # Calculate headings and extend the waypoints with them
        self.path = self.calculate_headings(self.path)
        # Set the path_status to 'not_visited'
        self.path_status = []
        for waypoint in self.path:
            self.path_status.append('not_visited')
        # Visualize the data
        self.visualize_path(self.path, self.path_status)

    def calculate_headings(self, path):
        """
        Calculates the headings between paths and adds them to the waypoints.
        """
        new_path = []
        for index, waypoint in enumerate(path):
            new_path.append(list(path[index]))
            # If the end, copy the previous heading
            if index == 0:
                new_path[index].append(0)
                continue
            # Calculate the angle between this waypoint and the next
            dx = path[index][0] - path[index-1][0]
            dy = path[index][1] - path[index-1][1]
            from math import atan2, pi
            heading = atan2(dy, dx)
            new_path[index].append(heading)
        return new_path

    def visualize_path(self, path, path_status=None):
        """
        Convenience function, calls both visualize_path{as_path, as_marker}
        """
        # TODO: finish this (path as path viz)
        # self.visualize_path_as_path(path, path_status)
        self.visualize_path_as_marker(path, path_status)

    def visualize_path_as_path(self, path, path_status=None):
        """
        Publishes a nav_msgs/Path msg containing the planned path.

        If path_status is not None then the only waypoints put in the
        nav_msgs/Path msg will be ones that are 'not_visited' or 'visiting'.
        """
        now = rospy.Time.now()
        msg = Path()
        msg.header.stamp = now
        msg.header.frame_id = self.field_frame_id
        for index, waypoint in enumerate(path):
            # Only put 'not_visited', 'visiting', and the most recent 'visited'
            # in the path msg
            if path_status != None: # If not set, ignore
                if path_status[index] == 'visited': # if this one is visited
                    try:
                        # if the next one is visited too
                        if path_status[index+1] == 'visited':
                            # Then continue, because this one doesn't belong
                            # in the path msg
                            continue
                    except KeyError as e: # incase index+1 is too big
                        pass
            # Otherwise if belongs, put it in
            pose_msg = PoseStamped()
            pose_msg.header.stamp = now
            pose_msg.header.frame_id = self.field_frame_id
            pose_msg.pose.position.x = waypoint[0]
            pose_msg.pose.position.y = waypoint[1]
            msg.poses.append(pose_msg)

    def visualize_path_as_marker(self, path, path_status):
        """
        Publishes visualization Markers to represent the planned path.

        Publishes the path as a series of spheres connected by lines.
        The color of the spheres is set by the path_status parameter,
        which is a list of strings of which the possible values are in
        ['not_visited', 'visiting', 'visited'].
        """
        # Get the time
        now = rospy.Time.now()
        # If self.path_markers is None, initialize it
        if self.path_markers == None:
            self.path_markers = MarkerArray()
        # # If there are existing markers, delete them
        # markers_to_delete = MarkerArray()
        # if len(self.path_markers.markers) > 0:
        #     for marker in self.path_markers.markers:
        #         marker.action = Marker.DELETE
        #         markers_to_delete.markers.append(marker)
        #     self.path_marker_pub.publish(markers_to_delete)
        # Clear the path_markers
        self.path_markers = MarkerArray()
        line_strip_points = []
        # Create the waypoint markers
        for index, waypoint in enumerate(path):
            waypoint_marker = Marker()
            waypoint_marker.header.stamp = now
            waypoint_marker.header.frame_id = self.field_frame_id
            waypoint_marker.ns = "waypoints"
            waypoint_marker.id = index
            waypoint_marker.type = Marker.ARROW
            if index == 0:
                waypoint_marker.type = Marker.CUBE
            waypoint_marker.action = Marker.MODIFY
            waypoint_marker.scale.x = 1
            waypoint_marker.scale.y = 1
            waypoint_marker.scale.z = 0.25
            point = Point(waypoint[0], waypoint[1], 0)
            waypoint_marker.pose.position = point
            # Store the point for the line_strip marker
            line_strip_points.append(point)
            # Set the heading of the ARROW
            quat = qfe(0, 0, waypoint[2])
            waypoint_marker.pose.orientation.x = quat[0]
            waypoint_marker.pose.orientation.y = quat[1]
            waypoint_marker.pose.orientation.z = quat[2]
            waypoint_marker.pose.orientation.w = quat[3]
            # Color is based on path_status
            status = path_status[index]
            if status == 'not_visited':
                waypoint_marker.color = ColorRGBA(1,0,0,0.5)
            elif status == 'visiting':
                waypoint_marker.color = ColorRGBA(0,1,0,0.5)
            elif status == 'visited':
                waypoint_marker.color = ColorRGBA(0,0,1,0.5)
            else:
                rospy.err("Invalid path status.")
                waypoint_marker.color = ColorRGBA(1,1,1,0.5)
            # Put this waypoint Marker in the MarkerArray
            self.path_markers.markers.append(waypoint_marker)
        # Create the line_strip Marker which connects the waypoints
        line_strip = Marker()
        line_strip.header.stamp = now
        line_strip.header.frame_id = self.field_frame_id
        line_strip.ns = "lines"
        line_strip.id = 0
        line_strip.type = Marker.LINE_STRIP
        line_strip.action = Marker.ADD
        line_strip.scale.x = 0.1
        line_strip.color = ColorRGBA(0,0,1,0.5)
        line_strip.points = line_strip_points
        self.path_markers.markers.append(line_strip)
        # Publish the marker array
        self.path_marker_pub.publish(self.path_markers)

    def setup_path_following(self, degrees=0):
        """
        Sets up the node for following the planned path.

        Will wait until the initial robot pose is set and until
        the move_base actionlib service is available.
        """
        # Create the actionlib service
        self.move_base_client = SimpleActionClient('move_base', MoveBaseAction)
        connected_to_move_base = False
        dur = rospy.Duration(1.0)
        # If testing prime the robot_pose
        if self.testing:
            self.robot_pose = Odometry()
            self.robot_pose.pose.pose.position.x = 0
            self.robot_pose.pose.pose.position.y = 0
        # Wait for the field shape
        while self.field_shape == None:
            # Check to make sure ROS is ok still
            if rospy.is_shutdown(): return
            # Print message about the waiting
            msg = "Qualification: waiting on the field shape."
            rospy.loginfo(msg)
            rospy.Rate(1.0).sleep()
        # Wait for the robot position
        while self.robot_pose == None:
            # Check to make sure ROS is ok still
            if rospy.is_shutdown(): return
            # Print message about the waiting
            msg = "Qualification: waiting on initial robot pose."
            rospy.loginfo(msg)
            rospy.Rate(1.0).sleep()
        # Now we should plan a path using the robot's initial pose
        origin = (self.robot_pose.pose.pose.position.x,
                  self.robot_pose.pose.pose.position.y)
        self.plan_path(self.field_shape, origin, degrees)
        rospy.loginfo("Path Planner: path planning complete.")
        # Now wait for move_base
        while not connected_to_move_base:
            # Wait for the server for a while
            connected_to_move_base = self.move_base_client.wait_for_server(dur)
            # Check to make sure ROS is ok still
            if rospy.is_shutdown(): return
            # Update the user on the status of this process
            msg = "Path Planner: waiting on move_base."
            rospy.loginfo(msg)
        # Now we are ready to start feeding move_base waypoints
        return

    def get_next_waypoint_index(self):
        """
        Figures out what the index of the next waypoint is.

        Iterates through the path_status's and finds the visiting one,
        or the next not_visited one if not visiting on exists.
        """
        for index, status in enumerate(self.path_status):
            if status == 'visited':
                continue
            if status == 'visiting':
                return index
            if status == 'not_visited':
                return index
        # If I get here then there are no not_visited and we are done.
        return None

    def distance(self, p1, p2):
        """Returns the distance between two points."""
        from math import sqrt
        dx = p2.target_pose.pose.position.x - p1.target_pose.pose.position.x
        dy = p2.target_pose.pose.position.y - p1.target_pose.pose.position.y
        return sqrt(dx**2 + dy**2)

    def step_path_following(self):
        """
        Steps the path following system, checking if new waypoints
        should be sent, if a timeout has occurred, or if path following
        needs to be paused.
        """
        # Visualize the data
        self.visualize_path(self.path, self.path_status)
        # Get the next (or current) waypoint
        current_waypoint_index = self.get_next_waypoint_index()
        # If the index is None, then we are done path planning
        if current_waypoint_index == None:
            rospy.loginfo("Path Planner: Done.")
            return False
        if current_waypoint_index == 0:
            self.path_status[current_waypoint_index] = 'visited'
        # Get the waypoint and status
        current_waypoint = self.path[current_waypoint_index]
        current_waypoint_status = self.path_status[current_waypoint_index]
        # If the status is visited
        if current_waypoint_status == 'visited':
            # This shouldn't happen...
            return True
        # If the status is not_visited then we need to push the goal
        if current_waypoint_status == 'not_visited':
            # Cancel any current goals
            self.move_base_client.cancel_all_goals()
            # Set it to visiting
            self.path_status[current_waypoint_index] = 'visiting'
            # Push the goal to the actionlib server
            destination = MoveBaseGoal()
            destination.target_pose.header.frame_id = self.field_frame_id
            destination.target_pose.header.stamp = rospy.Time.now()
            # Set the target location
            destination.target_pose.pose.position.x = current_waypoint[0]
            destination.target_pose.pose.position.y = current_waypoint[1]
            # Calculate the distance
            if self.previous_destination == None:
                self.current_distance = 5.0
            else:
                self.current_distance = self.distance(self.previous_destination, destination)
            # Set the heading
            quat = qfe(0, 0, current_waypoint[2])
            destination.target_pose.pose.orientation.x = quat[0]
            destination.target_pose.pose.orientation.y = quat[1]
            destination.target_pose.pose.orientation.z = quat[2]
            destination.target_pose.pose.orientation.w = quat[3]
            # Send the desired destination to the actionlib server
            rospy.loginfo("Sending waypoint (%f, %f)@%f" % tuple(current_waypoint))
            self.current_destination = destination
            self.move_base_client.send_goal(destination)
            self.previous_destination = destination
        # If the status is visiting, then we just need to monitor the status
        temp_state = self.move_base_client.get_goal_status_text()
        if current_waypoint_status == 'visiting':
            if temp_state in ['ABORTED', 'SUCCEEDED']:
                self.path_status[current_waypoint_index] = 'visited'
            else:
                duration = rospy.Duration(1.0)
                from math import floor
                count = 0
                while not self.move_base_client.wait_for_result(duration) and count != 6+int(self.current_distance*4):
                    if rospy.is_shutdown(): return False
                    count += 1
                if count == 6+int(self.current_distance*4):
                    rospy.logwarn("Path Planner: move_base goal timeout occurred, clearing costmaps")
                    # Cancel the goals
                    self.move_base_client.cancel_all_goals()
                    # Clear the cost maps
                    self.clear_costmaps()
                    # Wait 1 second
                    rospy.Rate(1.0).sleep()
                    # Then reset the current goal
                    if not self.just_reset:
                        self.just_reset = True
                        self.path_status[current_waypoint_index] = 'not_visited'
                    else:
                        self.just_reset = False
                        self.path_status[current_waypoint_index] = 'visited'
                    return True
                self.path_status[current_waypoint_index] = 'visited'
        return True

if __name__ == '__main__':
    ppn = PathPlannerNode()


text:
import numpy as np
import pandas as pd
import time

np.random.seed(2)

N_STATES = 6
ACTIONS = ['left', 'right']
EPSILON = 0.9 # greedy policy
ALPHA = 0.1 # learning rate
LAMBDA = 0.9 # reward decay
MAX_EPISODES = 13
FRESH_TIME = 0.001

def build_q_table(n_states, actions):
    table = pd.DataFrame(
        np.zeros((n_states, len(actions))),
        columns = actions,
    )
    return table

def choose_action(state, q_table):
    state_actions = q_table.iloc[state, :]
    if (np.random.uniform() > EPSILON) or (state_actions.all() == 0):
        action_name = np.random.choice(ACTIONS)
    else:
        action_name = state_actions.argmax()
    return action_name

def get_env_feedback(S, A):
    if A == 'right':
        if S == N_STATES - 2:
            S_ = 'terminal'
            R = 1
        else:
            S_ = S + 1
            R = 0
    else:
        R = 0
        if S == 0:
            S_ = S
        else:
            S_ = S - 1
    return S_, R

def update_env(S, episode, step_counter):
    # This is how environment be updated
    env_list = ['-']*(N_STATES-1) + ['T']   # '---------T' our environment
    if S == 'terminal':
        interaction = 'Episode %s: total_steps = %s' % (episode+1, step_counter)
        #print('\r{}'.format(interaction), end='')
        time.sleep(2)
        #print('\r                                ', end='')
    else:
        env_list[S] = 'o'
        interaction = ''.join(env_list)
        #print('\r{}'.format(interaction), end='')
        time.sleep(FRESH_TIME)

def rl():
    q_table = build_q_table(N_STATES, ACTIONS)
    for episode in range(MAX_EPISODES):
        step_counter = 0
        S = 0
        is_terminated = False
        update_env(S, episode, step_counter)
        while not is_terminated:
            A = choose_action(S, q_table)
            S_, R = get_env_feedback(S, A)
            q_predict = q_table.ix[S, A]
            if S_ != 'terminal':
                q_target = R + LAMBDA * q_table.iloc[S_, :].max()
            else:
                q_target = R
                is_terminated = True

            q_table.ix[S, A] += ALPHA * (q_target - q_predict)
            S = S_
            update_env(S, episode, step_counter + 1)
            step_counter += 1
    return q_table

if __name__ == "__main__":
    q_table = rl()
    print('\r\nQ-table:\n')
    print(q_table)

max_stars_repo_name: ckbjimmy/active


text:
import argparse
import json
import pymysql
import pandas as pd
import numpy as np
import matplotlib
from matplotlib.backends.backend_pdf import PdfPages
from sshtunnel import SSHTunnelForwarder

import chart_functions

# returns a string with the full query containing the date filtering (year, month, day)
def get_query(table, filter_dict):
    query = "SELECT * FROM %s" % table

    if (len(filter_dict) > 0):
        query = "%s WHERE" % query

        for i, key in enumerate(filter_dict):
            assert (len(filter_dict[key]) == 2), "Number of filter items must be 2 (lower limit and upper limit)"
            assert (filter_dict[key][0] <= filter_dict[key][1]), "Bad %s argumets" % key

            if (key == "year"):
                query = "%s YEAR(date) >= %0d && YEAR(date) <= %0d" % (query, filter_dict[key][0], filter_dict[key][1])
            elif (key == "month"):
                query = "%s MONTH(date) >= %0d && MONTH(date) <= %0d" % (query, filter_dict[key][0], filter_dict[key][1])
            elif (key == "day"):
                query = "%s DAY(date) >= %0d && DAY(date) <= %0d" % (query, filter_dict[key][0], filter_dict[key][1])
            else:
                assert 0, "Unable to handle filter type"

            if (i != len(filter_dict) - 1):
                query = "%s &&" % (query)

    query = "%s;" % query
    return query

# returns a data frame based on the table the data is stored in MySQL and the date filter (to form the query)
def get_data_from_sql(table, filter_dict, conn):
    query = get_query(table, filter_dict)
    print("query = %s" % query)
    return pd.read_sql_query(query, conn, index_col="date")

# used to clean the data (trim values lower or higher than some tresholds) and return the resampled data for mean, min and max
def get_clean_data(df, resample_rule, my_filter = None):
    if my_filter != None:
        assert (len(my_filter) == 2), "Number of filter items must be 2 (lower limit and upper limit)"
        assert (my_filter[0] <= my_filter[1]), "Bad argumets: %d must be lower or equal to %d" % (my_filter[0], my_filter[1])
        df["value"] = df["value"].apply(lambda x: np.NaN if x < my_filter[0] or x > my_filter[1] else x)
    # the sensors might produce spikes and we can end up with the same date (which created problems when we use it as index)
    # remove index duplicates and create equal spaced samples (one per hour)
    df = df[~df.index.duplicated(keep = "last")]
    df = df.resample("H").ffill();
    df["value"] = df["value"].interpolate(method = "linear")

    return df.resample(resample_rule).mean(), df.resample(resample_rule).min(), df.resample(resample_rule).max()

# transform the unqual spaced data (for windows) to equal spaced data (which has been cleaned and resampled)
def get_clean_equal_spaced_data(df, resample_rule):
    # the sensors might produce spikes and we can end up with the same date (which created problems when we use it as index)
    # remove index duplicates and create equal spaced samples (one per hour)
    df = df[~df.index.duplicated(keep="last")]
    df = df.resample("H").ffill()

    # create a sample per day which sums the 1hour equaly spaced samples
    return df.resample(resample_rule).sum()

# the heatindex is a how humidity influences the felt temperature
def compute_heat_index(df_temperature, df_humidity, c):
    # join the outdoor temp data with the humidity to start calculations
    data_heatindex = data_outdoor_mean.join(data_humidity_mean, on = "date", lsuffix = "temp", rsuffix = "humid")
    # the applied formula is for F degees while the measurments are in C degrees
    data_heatindex["valuetemp_f"] = (data_heatindex["valuetemp"] * 9 / 5) + 32
    data_heatindex["value"] = np.where(data_heatindex["valuetemp"] >= 21,
        (c["c1"] + c["c2"] * data_heatindex["valuetemp_f"] + \
        c["c3"] * data_heatindex["valuehumid"] + c["c4"] * data_heatindex["valuetemp_f"] * data_heatindex["valuehumid"] + \
        c["c5"] * (data_heatindex["valuetemp_f"] ** 2) + c["c6"] * (data_heatindex["valuehumid"] ** 2) + \
        c["c7"] * (data_heatindex["valuetemp_f"] ** 2) * data_heatindex["valuehumid"] + \
        c["c8"] * data_heatindex["valuetemp_f"] * (data_heatindex["valuehumid"] ** 2) + \
        c["c9"] * (data_heatindex["valuetemp_f"] ** 2) * (data_heatindex["valuehumid"] ** 2) - 32) * 5/9 , data_heatindex["valuetemp"])
    # heatindex makes sense to be applied when temperatures are high May-Octomber
    data_heatindex = data_heatindex.loc[str(year) + "-05-01" : str(year) + "-10-01"]

    ## apply color labels in order to be able to know the color of the segements when creating the chart
    data_heatindex.loc[(data_heatindex["value"] < 27), "color"] = "green"
    data_heatindex.loc[(data_heatindex["value"] >= 27) & (data_heatindex["value"] < 32), "color"] = "yellow"
    data_heatindex.loc[(data_heatindex["value"] >= 32) & (data_heatindex["value"] < 41), "color"] = "orange"
    data_heatindex.loc[(data_heatindex["value"] >= 41), "color"] = "red"

    return data_heatindex

# read configuration
parser = argparse.ArgumentParser()
parser.add_argument("--config_file", "-f", default="analysis_config.json", help="path to configuration json file (default: analysis_config.json)")
args = parser.parse_args()

with open(args.config_file) as data_file:
    config_options = json.load(data_file)
data_file.close()

matplotlib.style.use(config_options["charts"]["style"])

for year in range(config_options["charts"]["start_report_year"], config_options["charts"]["end_report_year"] + 1):

    pp = PdfPages("sensors_analysis_report_" + str(year) + ".pdf")

    #get raw data from SQL via SSH -> to be removed and replaced with REST call (if there is a REST server)
    ########################################################################
    with SSHTunnelForwarder((config_options["ssh"]["server"], config_options["ssh"]["port"]), ssh_username = config_options["ssh"]["user"],
        ssh_password = config_options["ssh"]["password"], remote_bind_address = (config_options["mysql"]["server"], config_options["mysql"]["port"])) as tunnel:

        conn = pymysql.connect(host = config_options["mysql"]["server"], user = config_options["mysql"]["user"], passwd = config_options["mysql"]["password"],
            db = config_options["mysql"]["database"], port = tunnel.local_bind_port)

        date_filter = {"year": [year, year]}#, "month": [8, 8], "day": [21, 21]}

        data_temperature = get_data_from_sql("temperature", date_filter, conn)
        data_humidity = get_data_from_sql("humidity", date_filter, conn)
        data_pressure = get_data_from_sql("pressure", date_filter, conn)
        data_windows = get_data_from_sql("windows", date_filter, conn)

        conn.close()

    #process and clear SQL data (prepare for charts)
    ########################################################################
    data_outdoor = data_temperature.loc[data_temperature["name"] == "Outdoor Unit"]
    data_indoor = data_temperature.loc[data_temperature["name"] == "Indoor Living"]
    data_window_left = data_windows.loc[data_windows["name"] == "Living LEFT"]
    data_window_right = data_windows.loc[data_windows["name"] == "Living RIGHT"]

    sample_statistics = pd.DataFrame(columns=["Outdoor temp", "Indoor temp", "Humidity", "Pressure", "Window Left", "Window Right"])
    sample_statistics.loc["Collected samples"] = [len(data_outdoor), len(data_indoor), len(data_humidity), len(data_pressure), len(data_window_left), len(data_window_right)]

    data_window_left = get_clean_equal_spaced_data(data_window_left, "M")
    data_window_right = get_clean_equal_spaced_data(data_window_right, "M")

    data_outdoor_mean, data_outdoor_min, data_outdoor_max = get_clean_data(data_outdoor, "D")
    data_outdoor_mean_month, data_outdoor_min_month, data_outdoor_max_month = get_clean_data(data_outdoor, "M")
    data_indoor_mean, data_indoor_min, data_indoor_max = get_clean_data(data_indoor, "D", [config_options["data"]["min_indoor_temp"], config_options["data"]["max_indoor_temp"]])
    data_humidity_mean, data_humidity_min, data_humidity_max = get_clean_data(data_humidity, "D", [config_options["data"]["min_humidity"], config_options["data"]["max_humidity"]])
    data_pressure_mean, data_pressure_min, data_pressure_max = get_clean_data(data_pressure, "D", [config_options["data"]["min_pressure"], config_options["data"]["max_pressure"]])

    c = config_options["charts"]["heat_index_coef"]
    data_heatindex = compute_heat_index(data_outdoor_mean, data_humidity_mean, c)

    sample_statistics.loc["Used samples"] = [len(data_outdoor_mean), len(data_indoor_mean), len(data_humidity_mean), len(data_pressure_mean), len(data_window_left), len(data_window_right)]
    sample_statistics.loc["Min"] = ["%.1f" % data_outdoor_min["value"].min(), "%.1f" % data_indoor_min["value"].min(), data_humidity_min["value"].min(), data_pressure_min["value"].min(), "-", "-"]
    sample_statistics.loc["Max"] = ["%.1f" % data_outdoor_max["value"].max(), "%.1f" % data_indoor_max["value"].max(), data_humidity_max["value"].max(), data_pressure_max["value"].max(), "-", "-"]

    #create charts
    ########################################################################
    xsize = config_options["charts"]["xfig_size"]
    ysize = config_options["charts"]["yfig_size"]
    chart_functions.create_title_page(xsize, ysize, pp, "Sensor data report - " + str(year))
    chart_functions.create_table_samples(xsize, ysize, pp, sample_statistics)
    chart_functions.create_histogram_chart(xsize, ysize, pp, data_outdoor_mean, data_outdoor_min, data_outdoor_max, "Outdoor temperature", "Celsius Degrees")
    chart_functions.create_indoor_outdoor_chart(xsize, ysize, pp, data_outdoor_mean, data_indoor_mean)
    chart_functions.create_temp_min_max_chart(xsize, ysize, pp, "Indoor", data_indoor_mean, data_indoor_max, data_indoor_min)
    chart_functions.create_temp_min_max_chart(xsize, ysize, pp, "Outdoor", data_outdoor_mean, data_outdoor_max, data_outdoor_min)
    chart_functions.create_heat_index_chart(xsize, ysize, pp, data_heatindex)
    chart_functions.create_humidity_pressure_chart(xsize, ysize, pp, data_humidity_mean, data_pressure_mean)
    chart_functions.create_windows_chart(xsize, ysize, pp, data_window_left, data_window_right, data_outdoor_mean_month)

    pp.close()
    print("Finished creating sensors_analysis_report_" + str(year) + ".pdf")