text:
"""
The FastFeatureAdversaries attack
"""
# pylint: disable=missing-docstring
import warnings

import numpy as np
import tensorflow as tf

from cleverhans.attacks.attack import Attack
from cleverhans.compat import reduce_sum
from cleverhans.model import Model
from cleverhans.utils_tf import clip_eta

class FastFeatureAdversaries(Attack):
  """
  This is a fast implementation of "Feature Adversaries", an attack
  against a target internal representation of a model.
  "Feature adversaries" were originally introduced in (Sabour et al. 2016),
  where the optimization was done using LBFGS.
  Paper link: https://arxiv.org/abs/1511.05122

  This implementation is similar to "Basic Iterative Method"
  (Kurakin et al. 2016) but applied to the internal representations.

  :param model: cleverhans.model.Model
  :param sess: optional tf.Session
  :param dtypestr: dtype of the data
  :param kwargs: passed through to super constructor
  """

  def __init__(self, model, sess=None, dtypestr='float32', **kwargs):
    """
    Create a FastFeatureAdversaries instance.
    """
    super(FastFeatureAdversaries, self).__init__(model, sess, dtypestr,
                                                 **kwargs)
    self.feedable_kwargs = ('eps', 'eps_iter', 'clip_min', 'clip_max')
    self.structural_kwargs = ['ord', 'nb_iter', 'layer']

    assert isinstance(self.model, Model)

  def parse_params(self,
                   layer=None,
                   eps=0.3,
                   eps_iter=0.05,
                   nb_iter=10,
                   ord=np.inf,
                   clip_min=None,
                   clip_max=None,
                   **kwargs):
    """
    Take in a dictionary of parameters and applies attack-specific checks
    before saving them as attributes.

    Attack-specific parameters:

    :param layer: (required str) name of the layer to target.
    :param eps: (optional float) maximum distortion of adversarial example
                compared to original input
    :param eps_iter: (optional float) step size for each attack iteration
    :param nb_iter: (optional int) Number of attack iterations.
    :param ord: (optional) Order of the norm (mimics Numpy).
                Possible values: np.inf, 1 or 2.
    :param clip_min: (optional float) Minimum input component value
    :param clip_max: (optional float) Maximum input component value
    """

    # Save attack-specific parameters
    self.layer = layer
    self.eps = eps
    self.eps_iter = eps_iter
    self.nb_iter = nb_iter
    self.ord = ord
    self.clip_min = clip_min
    self.clip_max = clip_max

    # Check if order of the norm is acceptable given current implementation
    if self.ord not in [np.inf, 1, 2]:
      raise ValueError("Norm order must be either np.inf, 1, or 2.")
    if len(kwargs.keys()) > 0:
      warnings.warn("kwargs is unused and will be removed on or after "
                    "2019-04-26.")

    return True

  def attack_single_step(self, x, eta, g_feat):
    """
    TensorFlow implementation of the Fast Feature Gradient. This is a
    single step attack similar to Fast Gradient Method that attacks an
    internal representation.

    :param x: the input placeholder
    :param eta: A tensor the same shape as x that holds the perturbation.
    :param g_feat: model's internal tensor for guide
    :return: a tensor for the adversarial example
    """

    adv_x = x + eta
    a_feat = self.model.fprop(adv_x)[self.layer]

    # feat.shape = (batch, c) or (batch, w, h, c)
    axis = list(range(1, len(a_feat.shape)))

    # Compute loss
    # This is a targeted attack, hence the negative sign
    loss = -reduce_sum(tf.square(a_feat - g_feat), axis)

    # Define gradient of loss wrt input
    grad, = tf.gradients(ys=loss, xs=adv_x)

    # Multiply by constant epsilon
    scaled_signed_grad = self.eps_iter * tf.sign(grad)

    # Add perturbation to original example to obtain adversarial example
    adv_x = adv_x + scaled_signed_grad

    # If clipping is needed,
    # reset all values outside of [clip_min, clip_max]
    if (self.clip_min is not None) and (self.clip_max is not None):
      adv_x = tf.clip_by_value(adv_x, self.clip_min, self.clip_max)

    adv_x = tf.stop_gradient(adv_x)

    eta = adv_x - x
    eta = clip_eta(eta, self.ord, self.eps)

    return eta

  def generate(self, x, g, **kwargs):
    """
    Generate symbolic graph for adversarial examples and return.

    :param x: The model's symbolic inputs.
    :param g: The target value of the symbolic representation
    :param kwargs: See `parse_params`
    """

    # Parse and save attack-specific parameters
    assert self.parse_params(**kwargs)

    g_feat = self.model.fprop(g)[self.layer]

    # Initialize loop variables
    eta = tf.random.uniform(
        tf.shape(input=x), -self.eps, self.eps, dtype=self.tf_dtype)
    eta = clip_eta(eta, self.ord, self.eps)

    def cond(i, _):
      return tf.less(i, self.nb_iter)

    def body(i, e):
      new_eta = self.attack_single_step(x, e, g_feat)
      return i + 1, new_eta

    _, eta = tf.while_loop(cond=cond, body=body, loop_vars=(tf.zeros([]), eta), back_prop=True,
                           maximum_iterations=self.nb_iter)

    # Define adversarial example (and clip if necessary)
    adv_x = x + eta
    if self.clip_min is not None and self.clip_max is not None:
      adv_x = tf.clip_by_value(adv_x, self.clip_min, self.clip_max)

    return adv_x


text:
# coding: utf-8

# # Εργαστηριακή Άσκηση 2. Μη επιβλεπόμενη μάθηση.
# ## Σημασιολογική απεικόνιση δεδομένων με χρήση SOM
#
#  # Στοχεία Ομάδας:
# ### Kουδιγκέλη Χριστιάνα ΑΜ:03113649
# ### Γεωργής Ανδρέας      ΑΜ:03115194
# ### Ομάδα Α43

# ## Δημιουργία dataset
# Στη δεύτερη εφαρμογή θα βασιστούμε στις τοπολογικές ιδιότητες των Self Organizing Maps (SOM) για να φτιάξουμε ενά χάρτη (grid) δύο διαστάσεων όπου θα απεικονίζονται όλες οι ταινίες της συλλογής της ομάδας με τρόπο χωρικά συνεκτικό ως προς το περιεχόμενο και κυρίως το είδος τους.
# Η `build_final_set` αρχικά μετατρέπει την αραιή αναπαράσταση tf-idf της εξόδου της `TfidfVectorizer()` σε πυκνή (η [αραιή αναπαράσταση](https://en.wikipedia.org/wiki/Sparse_matrix) έχει τιμές μόνο για τα μη μηδενικά στοιχεία).
# Στη συνέχεια ενώνει την πυκνή `dense_tf_idf` αναπαράσταση και τις binarized κατηγορίες `catbins` των ταινιών ως επιπλέον στήλες (χαρακτηριστικά). Συνεπώς, κάθε ταινία αναπαρίσταται στο Vector Space Model από τα χαρακτηριστικά του TFIDF και τις κατηγορίες της.
# Τέλος, δέχεται ένα ορισμα για το πόσες ταινίες να επιστρέψει, με default τιμή όλες τις ταινίες (5000).

# In[1]:

get_ipython().system(u'pip install --upgrade pip')
get_ipython().system(u'pip install --upgrade numpy')
get_ipython().system(u'pip install --upgrade pandas')
get_ipython().system(u'pip install --upgrade nltk')
get_ipython().system(u'pip install --upgrade scikit-learn')
get_ipython().system(u'pip install unidecode')

get_ipython().system(u'pip install requests')
from io import StringIO
import requests
import json
import pandas as pd
import numpy as np
from unidecode import unidecode
import nltk
nltk.download('punkt')
from nltk.corpus import stopwords
import string
from nltk.stem import WordNetLemmatizer
from nltk.stem.porter import PorterStemmer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import random
from sklearn.externals import joblib

import matplotlib

# we will plot inside the notebook and not in separate window

from sklearn.cluster import KMeans

# In[2]:

dataset_url = "https://drive.google.com/uc?export=download&id=1PdkVDENX12tQliCk_HtUnAUbfxXvnWuG"
# make direct link for drive docs this way https://www.labnol.org/internet/direct-links-for-google-drive/28356/
df_data_1 = pd.read_csv(dataset_url, sep='\t',  header=None, quoting=3, error_bad_lines=False)
# βάλτε το seed που αντιστοιχεί στην ομάδα σας
team_seed_number = 42 #ομάδα Α43, επιλέξαμε το 42 καθώς η αρίθμηση πινάκων ξεκινάει από το 0

movie_seeds_url = "https://drive.google.com/uc?export=download&id=1NkzL6rqv4DYxGY-XTKkmPqEoJ8fNbMk_"
df_data_2 = pd.read_csv(movie_seeds_url, header=None, error_bad_lines=False)

my_index = df_data_2.iloc[team_seed_number,:].values
titles = df_data_1.iloc[:, [2]].values[my_index - 1] # movie titles (string)
categories = df_data_1.iloc[:, [3]].values[my_index - 1] # movie categories (string)
bins = df_data_1.iloc[:, [4]]
catbins = bins[4].str.split(',', expand=True).values.astype(np.float)[my_index - 1] # movie categories in binary form (1 feature per category)
summaries =  df_data_1.iloc[:, [5]].values[my_index - 1] # movie summaries (string)
corpus = summaries[:,0].tolist() # list form of summaries

vectorizer = TfidfVectorizer(max_df=0.5, min_df=7, stop_words='english')
vectorizer.fit(corpus)
corpus_tf_idf = vectorizer.transform(corpus)

# # SOM with Somoclu
#
# Στο notebook αυτό θα χρησιμοποιήσουμε τη βιβλιοθήκη [Somoclu](https://somoclu.readthedocs.io/en/stable/index.html). Πρόκειται για μια σύγχρονη και βελτιστοποιημένη υλοποίηση των SOM, παραλληλοποιημένη για CPU και GPU. Πρώτα την εγκαθιστούμε:

# In[3]:

get_ipython().system(u'pip install --upgrade somoclu')

# In[4]:

# Αποθήκευση του corpus_tf_idf
joblib.dump(corpus_tf_idf, 'corpus_tf_idf.pkl')

# In[5]:

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import somoclu
get_ipython().magic(u'matplotlib inline')

#θέλουμε δηλαδή ταινίες της ίδιας κατηγορίας να ανήκουν σε γειτονικούς νευρώνες
def build_final_set(doc_limit):
    # convert sparse tf_idf to dense tf_idf representation
    dense_tf_idf = corpus_tf_idf.toarray()[0:doc_limit,:]
    final_set = np.hstack((dense_tf_idf, catbins[0:doc_limit,:]))
    return final_set

# In[6]:

final_set = build_final_set(doc_limit=1000) # αντί για 1000 θα θέσουμε 50000 απλώς αργούσε και θέσαμε 10000
final_set.shape

# ## Εκπαίδευση χάρτη SOM
# Θα δουλέψουμε με τη βιβλιοθήκη SOM ["Somoclu"](http://somoclu.readthedocs.io/en/stable/index.html). Εισάγουμε τις somoclu και matplotlib και λέμε στη matplotlib να τυπώνει εντός του notebook (κι όχι σε pop up window). Την έχουμε κάνει ήδη import στο προηγούμενο βήμα.
# Θα δoυλέψουμε με χάρτη τύπου planar, παραλληλόγραμμου σχήματος νευρώνων με τυχαία αρχικοποίηση (όλα αυτά είναι default). Μπορεούμε να δοκιμάσουμε διάφορα μεγέθη χάρτη ωστόσο όσο ο αριθμός των νευρώνων μεγαλώνει, μεγαλώνει και ο χρόνος εκπαίδευσης. Για το training δεν χρειάζεται να ξεπεράσουμε τα 100 epochs. Σε γενικές γραμμές μπορούμε να βασιστούμε στις default παραμέτρους μέχρι να έχουμε τη δυνατότητα να οπτικοποιήσουμε και να αναλύσουμε ποιοτικά τα αποτελέσματα. Ξεκινάμε με ένα χάρτη 10 x 10, 100 epochs training και ένα υποσύνολο των ταινιών (π.χ. 2000). Χρησιμοποιούμε την magic `time` για να έχουμε μια εικόνα των χρόνων εκπαίδευσης. Ενδεικτικά, με σωστή κωδικοποίηση tf-idf, μικροί χάρτες για λίγα δεδομένα (1000-2000) παίρνουν γύρω στο ένα λεπτό ενώ μεγαλύτεροι χάρτες με όλα τα δεδομένα μπορούν να πάρουν 10-15 λεπτά ή και περισσότερο.

# In[7]:

n_rows, n_columns = 10, 10
som = somoclu.Somoclu(n_columns, n_rows)
get_ipython().magic(u'time som.train(data=final_set, epochs=100)')

# ## Best matching units
#
# Μετά από κάθε εκπαίδευση αποθηκεύουμε σε μια μεταβλητή τα best matching units (bmus) για κάθε ταινία. Τα bmus μας δείχνουν σε ποιο νευρώνα ανήκει η κάθε ταινία. Προσοχή: η σύμβαση των συντεταγμένων των νευρώνων είναι (στήλη, γραμμή) δηλαδή το ανάποδο από την Python. Με χρήση της [np.unique](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.unique.html) (μια πολύ χρήσιμη συνάρτηση στην άσκηση) αποθηκεύουμε τα μοναδικά best matching units και τους δείκτες τους (indices) προς τις ταινίες. Επίσης μπορεί να έχουμε λιγότερα μοναδικά bmus από αριθμό νευρώνων γιατί μπορεί σε κάποιους νευρώνες να μην έχουν ανατεθεί ταινίες. Ως αριθμό νευρώνα θα θεωρήσουμε τον αριθμό γραμμής στον πίνακα μοναδικών bmus.

# In[8]:

surface_state = som.get_surface_state()   # get_surface_state(data=None))  Return the dot product of the codebook and the data.
bmus = som.get_bmus(surface_state)
bmus_unique = np.unique(bmus,axis=0,return_index=True, return_counts=True)

# ## Ομαδοποίηση (clustering)
#
# Τυπικά, η ομαδοποίηση σε ένα χάρτη SOM προκύπτει από το unified distance matrix (U-matrix): για κάθε κόμβο υπολογίζεται η μέση απόστασή του από τους γειτονικούς κόμβους. Εάν χρησιμοποιηθεί μπλε χρώμα στις περιοχές του χάρτη όπου η τιμή αυτή είναι χαμηλή (μικρή απόσταση) και κόκκινο εκεί που η τιμή είναι υψηλή (μεγάλη απόσταση), τότε μπορούμε να πούμε ότι οι μπλε περιοχές αποτελούν clusters και οι κόκκινες αποτελούν σύνορα μεταξύ clusters.
#
# To somoclu δίνει την επιπρόσθετη δυνατότητα να κάνουμε ομαδοποίηση των νευρώνων χρησιμοποιώντας οποιονδήποτε αλγόριθμο ομαδοποίησης του scikit-learn. Στην άσκηση θα χρησιμοποιήσουμε τον k-Means. Για τον αρχικό μας χάρτη δοκιμάζουμε ένα k=20 ή 25.
#

# ## Οπτικοποίηση U-matrix, clustering και μέγεθος clusters
#
# Για την εκτύπωση του U-matrix χρησιμοποιούμε τη `view_umatrix` με ορίσματα `bestmatches=True` και `figsize=(15, 15)` ή `figsize=(20, 20)`. Τα διαφορετικά χρώματα που εμφανίζονται στους κόμβους αντιπροσωπεύουν τα διαφορετικά clusters που προκύπτουν από τον k-Means.Με το όρισμα `colorbar` μπορούμε να εμφανίσουμε τη λεζάντα του U-matrix.Επίσης δεν τυπώνουμε τις ετικέτες (labels) των δειγμάτων,γιατί είναι πολύ μεγάλος ο αριθμός τους.
#
# Για μια δεύτερη πιο ξεκάθαρη οπτικοποίηση του clustering τυπώνουμε απευθείας τη μεταβλητή `clusters`.
#
# Τέλος, χρησιμοποιώντας πάλι την `np.unique` (με διαφορετικό όρισμα) και την `np.argsort` (υπάρχουν και άλλοι τρόποι υλοποίησης) εκτυπώνουμε τις ετικέτες των clusters (αριθμοί από 0 έως k-1) και τον αριθμό των νευρώνων σε κάθε cluster, με φθίνουσα ή αύξουσα σειρά ως προς τον αριθμό των νευρώνων. Ουσιαστικά είναι ένα εργαλείο για να βρίσκούμε εύκολα τα μεγάλα και μικρά clusters.
#

# In[9]:

n_clusters =25

algorithm = KMeans(n_clusters=n_clusters)
som.cluster(algorithm=algorithm)

som.view_umatrix(bestmatches=True,figsize=(20, 20),colorbar=True)

# In[10]:

n_clusters =30

algorithm = KMeans(n_clusters=n_clusters)
som.cluster(algorithm=algorithm)

som.view_umatrix(bestmatches=True,figsize=(20, 20),colorbar=True)

# In[11]:

clusters = som.clusters
clusters_sorted = np.unique(clusters, return_index = True, return_counts = True)
print ('Clusters sorted byincreasing number of neurons:\n',clusters_sorted[0])
print ('Cluster Starting Index:\n',clusters_sorted[1])
print ('Number Of Neurons Per Cluster:\n',clusters_sorted[2])

# In[12]:

print ('Clusters:\n',clusters)

# ## Αποθήκευση του SOM
#
# Επειδή η αρχικοποίηση του SOM γίνεται τυχαία και το clustering είναι και αυτό στοχαστική διαδικασία, οι θέσεις και οι ετικέτες των νευρώνων και των clusters θα είναι διαφορετικές κάθε φορά που τρέχουμε τον χάρτη, ακόμα και με τις ίδιες παραμέτρους. Για να αποθηκεύσουμε ένα συγκεκριμένο som και clustering χρησιμοποιηούμε και πάλι την `joblib`.
#

# In[13]:

# Αποθήκευση του som
joblib.dump(som, 'som.pkl')

# ## Σημασιολογική ερμηνεία των clusters
#
# Προκειμένου να μελετήσουμε τις τοπολογικές ιδιότητες του SOM και το αν έχουν ενσωματώσει σημασιολογική πληροφορία για τις ταινίες διαμέσου της διανυσματικής αναπαράστασης με το tf-idf και των κατηγοριών, χρειαζόμαστε ένα κριτήριο ποιοτικής επισκόπησης των clusters. Θα υλοποιήσουμε το εξής κριτήριο: Λαμβάνουμε όρισμα έναν αριθμό (ετικέτα) cluster. Για το cluster αυτό βρίσκουμε όλους τους νευρώνες που του έχουν ανατεθεί από τον k-Means. Για όλους τους νευρώνες αυτούς βρίσκουμε όλες τις ταινίες που τους έχουν ανατεθεί (για τις οποίες αποτελούν bmus). Για όλες αυτές τις ταινίες τυπώνουμε ταξινομημένη τη συνολική στατιστική όλων των ειδών (κατηγοριών) και τις συχνότητές τους. Αν το cluster διαθέτει καλή συνοχή και εξειδίκευση, θα πρέπει κάποιες κατηγορίες να έχουν σαφώς μεγαλύτερη συχνότητα από τις υπόλοιπες. Θα μπορούμε τότε να αναθέσουμε αυτήν/ές την/τις κατηγορία/ες ως ετικέτες κινηματογραφικού είδους στο cluster.
#
# Μπορούμε να υλοποιήσουμε τη συνάρτηση αυτή όπως θέλουμε. Μια πιθανή διαδικασία θα μπορούσε να είναι η ακόλουθη:
#
# 1. Ορίζουμε συνάρτηση `print_categories_stats` που δέχεται ως είσοδο λίστα με ids ταινιών. Δημιουργούμε μια κενή λίστα συνολικών κατηγοριών. Στη συνέχεια, για κάθε ταινία επεξεργαζόμαστε το string `categories` ως εξής: δημιουργούμε μια λίστα διαχωρίζοντας το string κατάλληλα με την `split` και αφαιρούμε τα whitespaces μεταξύ ετικετών με την `strip`. Προσθέτουμε τη λίστα αυτή στη συνολική λίστα κατηγοριών με την `extend`. Τέλος χρησιμοποιούμε πάλι την `np.unique` για να μετρήσουμε συχνότητα μοναδικών ετικετών κατηγοριών και ταξινομούμε με την `np.argsort`. Τυπώνουμε τις κατηγορίες και τις συχνότητες εμφάνισης ταξινομημένα. Χρήσιμες μπορεί να σας φανούν και οι `np.ravel`, `np.nditer`, `np.array2string` και `zip`.
#
# 2. Ορίζουμε τη βασική μας συνάρτηση `print_cluster_neurons_movies_report` που δέχεται ως όρισμα τον αριθμό ενός cluster. Με τη χρήση της `np.where` μπορούμε να βρούμε τις συντεταγμένες των bmus που αντιστοιχούν στο cluster και με την `column_stack` να φτιάξουμε έναν πίνακα bmus για το cluster. Προσοχή στη σειρά (στήλη - σειρά) στον πίνακα bmus. Για κάθε bmu αυτού του πίνακα ελέγχουμε αν υπάρχει στον πίνακα μοναδικών bmus που έχουμε υπολογίσει στην αρχή συνολικά και αν ναι προσθέτουμε το αντίστοιχο index του νευρώνα σε μια λίστα. Χρήσιμες μπορεί να είναι και οι `np.rollaxis`, `np.append`, `np.asscalar`. Επίσης πιθανώς να πρέπει να υλοποιήσετε ένα κριτήριο ομοιότητας μεταξύ ενός bmu και ενός μοναδικού bmu από τον αρχικό πίνακα bmus.
#
# 3. Υλοποιούμε μια βοηθητική συνάρτηση `neuron_movies_report`. Λαμβάνει ένα σύνολο νευρώνων από την `print_cluster_neurons_movies_report` και μέσω της `indices` φτιάχνει μια λίστα με το σύνολο ταινιών που ανήκουν σε αυτούς τους νευρώνες. Στο τέλος καλεί με αυτή τη λίστα την `print_categories_stats` που τυπώνει τις στατιστικές των κατηγοριών.

# 1. Συνάρτηση print_categories_stats

# In[14]:

def print_categories_stats(movies_to_check):
    total_cats = []

    for movie in movies_to_check:
        temp = categories[movie][0]
        temp = temp.split('"')
        temp = filter(lambda a: a != '', temp)
        temp = filter(lambda a: a != ',  ',temp)
        total_cats.extend(temp)

    total_cats_unique = np.unique(total_cats, return_counts = True)
    indices = np.argsort(total_cats_unique[1])
    indices = reversed(indices)
    tot = sum(total_cats_unique[1])

    print ('-'*25,'Statistics','-'*25)

    print ('{:<50} {:<20}'.format('Category', 'Percentage'))
    print ('-'*62)
    for i in indices:
        print ('{:<50} {:<20}'.format(total_cats_unique[0][i], str(round(total_cats_unique[1][i]*100.0/tot,3))+'%'))

# 2.Συνάρτηση `print_cluster_neurons_movies_report`

# In[15]:

def print_cluster_neurons_movies_report(curr_cluster):
    print ('Cluster =',curr_cluster,'\n')
    new_indices = np.where(clusters == curr_cluster)
    new_indices = np.column_stack((new_indices[1],new_indices[0]))
    bmus_right = []
    for i in new_indices:
        if(i in bmus_unique[0]):
            bmus_right.append(i)
    bmus_right = np.array(bmus_right)
    return bmus_right

# 3.Συνάρτηση `neuron_movies_report`

# In[16]:

def neuron_movies_report(curr_cluster):
    curr_bmus = print_cluster_neurons_movies_report(curr_cluster)
    movies_to_check = []

    for bmu in curr_bmus:
        for id in range(0,len(final_set),1):
            if(np.array_equal(bmu,bmus[id])):
                movies_to_check.append(id)

    print_categories_stats(movies_to_check)

# In[17]:

clusters_to_check = range(0,n_clusters,1)

for curr_cluster in clusters_to_check:
    neuron_movies_report(curr_cluster)
    print ('-'*62,'\n\n')

# ## Ανάλυση τοπολογικών ιδιοτήτων χάρτη SOM
#
#
# Άλλη μία παρατήρηση είναι ότι δεδομένα που έχουν μεγαλύτερη πυκνότητα πιθανότητας στο χώρο εισόδου τείνουν να απεικονίζονται με περισσότερους νευρώνες στο χώρο μειωμένης διαστατικότητας.
#
# Παρατηρούμε ότι μακρινά πρότυπα εισόδου τείνουν να απεικονίζονται απομακρυσμένα στο χάρτη.
#
#

ext: py

reason: import numpy


text:
import numpy as np
from utils import *
def gauss(state): # img is numpy array image
    pass

max_stars_repo_name: Nikitothka/SUAP

max_issues_repo_name: Nikitothka/SUAP

max_forks_repo_name: Nikitothka/SUAP


text:
# ----------------------------------------------------------------------
#
# Brad T. Aagaard, U.S. Geological Survey
# Charles A. Williams, GNS Science
# Matthew G. Knepley, University of Chicago
#
# This code was developed as part of the Computational Infrastructure
# for Geodynamics (http://geodynamics.org).
#
# Copyright (c) 2010-2018 University of California, Davis
#
# See COPYING for license information.
#
# ----------------------------------------------------------------------
#
# @file tests/fullscale/viscoelasticity/nofaults-2d/axialtraction_maxwell_soln.py
#
# @brief Analytical solution to axial traction problem for a Maxwell viscoelastic material.
#
# 2-D axial traction solution for linear Maxwell viscoelastic material.
#
#             Uy=0
#          ----------
#          |        |
# Ux=0     |        |  Tx=T0
#          |        |
#          |        |
#          ----------
#            Uy=0
#
# Dirichlet boundary conditions
# Ux(-4000,y) = 0
# Uy(x,-4000) = 0
# Uy(x,+4000) = 0
#
# Neumann boundary conditions
# Tx(+4000,y) = T0

import numpy

# Physical properties.
p_density = 2500.0
p_vs = 3464.1016
p_vp = 6000.0
p_viscosity = 9.46728e17

p_mu = p_density*p_vs*p_vs
p_lambda = p_density*p_vp*p_vp - 2.0*p_mu
p_youngs = p_mu*(3.0*p_lambda + 2.0*p_mu)/(p_lambda + p_mu)
p_poissons = 0.5*p_lambda/(p_lambda + p_mu)

# Time information.
year = 60.0*60.0*24.0*365.25
dt = 0.025*year
startTime = dt
endTime = 1.0*year
numSteps = 40
timeArray = numpy.linspace(startTime, endTime, num=numSteps, dtype=numpy.float64)

# Uniform stress field (plane strain).
T0 = -1.0e7
sxx = T0*numpy.ones(numSteps, dtype=numpy.float64)
timeFac = numpy.exp(-p_youngs*timeArray/(6.0*p_viscosity*(1.0 - p_poissons)))
poisFac = (2.0*p_poissons - 1.0)/(1.0 - p_poissons)
syy = T0*(1.0 + poisFac*timeFac)
szz = syy
sxy = numpy.zeros(numSteps, dtype=numpy.float64)

# Deviatoric stress.
meanStress = (sxx + syy + szz)/3.0
sDevxx = sxx - meanStress
sDevyy = syy - meanStress
sDevzz = szz - meanStress

# Uniform strain field.
exx = T0*(1.0 - 2.0*p_poissons)*(3.0 + 2.0*poisFac*timeFac)/p_youngs
eyy = numpy.zeros(numSteps, dtype=numpy.float64)
ezz = numpy.zeros(numSteps, dtype=numpy.float64)
exy = numpy.zeros(numSteps, dtype=numpy.float64)

# Get viscous strains from deviatoric stress.
eVisxx = 0.5*sDevxx/p_mu
eVisyy = 0.5*sDevyy/p_mu
eViszz = 0.5*sDevzz/p_mu
eVisxy = 0.5*sxy/p_mu

# ----------------------------------------------------------------------
class AnalyticalSoln(object):
    """Analytical solution to axial extension problem.
    """
    SPACE_DIM = 2
    TENSOR_SIZE = 4

    def __init__(self):
        self.fields = {
            "displacement": self.displacement,
            "density": self.density,
            "shear_modulus": self.shear_modulus,
            "bulk_modulus": self.bulk_modulus,
            "maxwell_time": self.maxwell_time,
            "cauchy_strain": self.strain,
            "cauchy_stress": self.stress,
            "viscous_strain": self.viscous_strain,
            "initial_amplitude": {
                "bc_xneg": self.initial_displacement,
                "bc_yneg": self.initial_displacement,
                "bc_ypos": self.initial_displacement,
                "bc_xpos": self.initial_traction
            }
        }
        self.key = None
        return

    def getField(self, name, pts):
        if self.key is None:
            field = self.fields[name](pts)
        else:
            field = self.fields[name][self.key](pts)
        return field

    def displacement(self, locs):
        """Compute displacement field at locations.
        """
        (npts, dim) = locs.shape
        disp = numpy.zeros((numSteps, npts, self.SPACE_DIM), dtype=numpy.float64)
        disp[:,:, 0] = numpy.dot(exx.reshape(numSteps, 1), (locs[:, 0] + 4000.0).reshape(1, npts))
        return disp

    def initial_displacement(self, locs):
        """Compute initial displacement field at locations.
        """
        (npts, dim) = locs.shape
        disp = numpy.zeros((1, npts, self.SPACE_DIM), dtype=numpy.float64)
        return disp

    def initial_traction(self, locs):
        """Compute initial traction field at locations.
        """
        (npts, dim) = locs.shape
        traction = numpy.zeros((1, npts, self.SPACE_DIM), dtype=numpy.float64)
        traction[:,:, 1] = T0
        return traction

    def density(self, locs):
        """Compute density field at locations.
        """
        (npts, dim) = locs.shape
        density = p_density * numpy.ones((1, npts, 1), dtype=numpy.float64)
        return density

    def shear_modulus(self, locs):
        """Compute shear modulus field at locations.
        """
        (npts, dim) = locs.shape
        shear_modulus = p_mu * numpy.ones((1, npts, 1), dtype=numpy.float64)
        return shear_modulus

    def bulk_modulus(self, locs):
        """Compute bulk modulus field at locations.
        """
        (npts, dim) = locs.shape
        bulk_modulus = (p_lambda + 2.0 / 3.0 * p_mu) * numpy.ones((1, npts, 1), dtype=numpy.float64)
        return bulk_modulus

    def maxwell_time(self, locs):
        """Compute Maxwell time field at locations.
        """
        (npts, dim) = locs.shape
        maxwell_time = p_viscosity * numpy.ones((1, npts, 1), dtype=numpy.float64)/p_mu
        return maxwell_time

    def strain(self, locs):
        """Compute strain field at locations.
        """
        (npts, dim) = locs.shape
        strain = numpy.zeros((numSteps, npts, self.TENSOR_SIZE), dtype=numpy.float64)
        strain[:,:, 0] = exx.reshape(numSteps, 1)
        strain[:,:, 1] = eyy.reshape(numSteps, 1)
        strain[:,:, 2] = ezz.reshape(numSteps, 1)
        strain[:,:, 3] = exy.reshape(numSteps, 1)
        return strain

    def stress(self, locs):
        """Compute stress field at locations.
        """
        (npts, dim) = locs.shape
        stress = numpy.zeros((numSteps, npts, self.TENSOR_SIZE), dtype=numpy.float64)
        stress[:,:, 0] = sxx.reshape(numSteps, 1)
        stress[:,:, 1] = syy.reshape(numSteps, 1)
        stress[:,:, 2] = szz.reshape(numSteps, 1)
        stress[:,:, 3] = sxy.reshape(numSteps, 1)
        return stress

    def viscous_strain(self, locs):
        """Compute viscous strain field at locations.
        """
        (npts, dim) = locs.shape
        viscous_strain = numpy.zeros((numSteps, npts, self.TENSOR_SIZE), dtype=numpy.float64)
        viscous_strain[:,:, 0] = eVisxx.reshape(numSteps, 1)
        viscous_strain[:,:, 1] = eVisyy.reshape(numSteps, 1)
        viscous_strain[:,:, 2] = eViszz.reshape(numSteps, 1)
        viscous_strain[:,:, 3] = eVisxy.reshape(numSteps, 1)
        return viscous_strain

# End of file

max_stars_repo_name: Grant-Block/pylith

max_issues_repo_name: Grant-Block/pylith

max_forks_repo_name: Grant-Block/pylith


text:
%pylab inline
import matplotlib.pyplot as plt
import string
import jieba
import codecs
import os
from numpy import *
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from wordcloud import WordCloud
from sklearn import naive_bayes as bayes
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB, GaussianNB, BernoulliNB
from sklearn.svm import SVC, NuSVC, LinearSVC
ham_dataPath = r""
spma_dataPath = r""
with open(ham_dataPath,encoding='utf-8') as f:
    ham_txt_list = f.readlines()
with open(spma_dataPath,encoding="utf-8") as f:
    spam_txt_list = f.readlines()
# add stop words
stopwords = codecs.open(r'E:/ml_program/stopwords.txt','r','UTF8').read().split('\n')
# filter the stopwords
ham_processed_texts = []
for txt in ham_txt_list:
    words = []
    seg_list = jieba.cut(txt)
    for seg in seg_list:
        if (seg.isalpha()) and seg!='\n' and seg not in stopwords:
            words.append(seg)
    sentence = " ".join(words)
    ham_processed_texts.append(sentence)
# spam
spam_processed_texts = []
for txt in spam_txt_list:
    words = []
    seg_list = jieba.cut(txt)
    for seg in seg_list:
        if (seg.isalpha()) and seg!='\n' and seg not in stopwords:
            words.append(seg)
    sentence = " ".join(words)
    spam_processed_texts.append(sentence)
 # word cloud
def showWordCloud(text):
    wc = WordCloud(
        background_color = "white",
        max_words = 200,
        font_path = "C:\\Users\\dell\\Downloads\\1252935776\\simhei.ttf",
        min_font_size = 15,
        max_font_size = 50,
        width = 400
    )
    wordcloud = wc.generate(text)
    plt.imshow(wordcloud, interpolation="bilinear")
    plt.axis("off")
# print(" ".join(ham_processed_texts))
# showWordCloud(" ".join(ham_processed_texts))
showWordCloud(" ".join(spam_processed_texts))
showWordCloud(" ".join(ham_processed_texts))
def transformTextToSparseMatrix(texts):
    vectorizer = CountVectorizer(binary=False)
    vectorizer.fit(texts) # word list
    vocabulary = vectorizer.vocabulary_
    vector = vectorizer.transform(texts) # transform into a vector
#     print(vector.toarray())
    result = pd.DataFrame(vector.toarray())

    keys = []
    values = []
    for key,value in vectorizer.vocabulary_.items():
        keys.append(key)
        values.append(value)
    df = pd.DataFrame(data={"key":keys, "value": values})
    colnames = df.sort_values("value")["key"].values
    result.columns = colnames
    return result
#     print(vocabulary)
import numpy as np
data = []
data.extend(ham_processed_texts)
data.extend(spam_processed_texts)
textMatrix = transformTextToSparseMatrix(data)
textMatrix.head()
features = pd.DataFrame(textMatrix.apply(sum,axis=0))
extractedfeatures = [features.index[i] for i in range(features.shape[0]) if features.iloc[i,0]>5]
textMatrix = textMatrix[extractedfeatures]
textMatrix = textMatrix[extractedfeatures]
labels = []
labels.extend(ones(5000))
labels.extend(zeros(5001))
# split into a train and a validation data set
train,test,trainlabel,testlabel = train_test_split(textMatrix,labels,test_size=0.1)

# bayes
clf = bayes.BernoulliNB(alpha=1,binarize=True)
model = clf.fit(train,trainlabel)
# SVM
model2 = LinearSVC()
model2.fit(train,trainlabel)

print(model.score(test,testlabel))
print(model2.score(test,testlabel))

0.922077922077922
0.987012987012987
import matplotlib.pyplot as plt
from pylab import *                                 #to support Chinese
mpl.rcParams['font.sans-serif'] = ['SimHei']

names = ['1000', '2000', '4000', '6000', '8000','9000']
x = range(len(names))
y = [0.8609, 0.900, 0.9133, 0.9242, 0.9355,0.9490]
plt.plot(x, y, marker='o', mec='r', mfc='w')
plt.legend()  # add the legend
plt.xticks(x, names, rotation=45)
plt.margins(0)
plt.subplots_adjust(bottom=0.15)
plt.xlabel(u"the number of training sets") #y axis
plt.ylabel("accuracy rate") #y axis
plt.title("贝叶斯中文垃圾邮件分类准确率的变化") #titile
plt.savefig("nb_zh.png")
plt.show()

reason: import numpy,from numpy


text:
# import asyncio
# import json
# import typing
# from datetime import datetime
import discord
import numpy as np
from discord.ext import commands

from pangoBot import PangoPic, Constants

class PangoBot:
    pangoPic_ = PangoPic.PangoPic()
    bot = commands.Bot
    channels = Constants.Channels
    # bank = {}

    def __init__(self, bot):
        self.bot = bot
        for server in self.bot.guilds:
            self.channels = Constants.Channels(server.id, bot)

    async def on_ready(self):
        """starts pangobot"""
        # msg = await self.channels.get_channel(self.channels.GUIDELINES_CHANNEL_ID).fetch_message(self.channels.GUIDELINES_MSG_ID)
        # await msg.add_reaction(Constants.EMOJI_ACCEPT_GUIDELINES)
        print('pangoBot have logged in as {0.user}'.format(self.bot))

    async def pangopic(self, ctx):
        """command for personalised profile picture, input a color (RGB or HEX) output a reply with the profile picture"""
        if ctx.message.channel.id == self.channels.PANGOPIC_CHANNEL_ID:
            try:
                answer = self.pangoPic_.do_profile_picture(ctx.message.content.replace(Constants.PROFILE_PICTURE_COMMAND, "")[1:])
                await ctx.reply(answer[0], file=answer[1])
            except ValueError:
                e = discord.Embed(title="Error on {} command !".format(Constants.PROFILE_PICTURE_COMMAND[1:]),
                                  description=Constants.ERROR_ON_PROFILE_PICTURE,
                                  color=0xF24E4D)
                await ctx.reply(embed=e)
        return

    async def on_command_error(self, ctx, error):
        if ctx.message.channel.id == self.channels.PANGOPIC_CHANNEL_ID and isinstance(error, commands.CommandNotFound):
            e = discord.Embed(title="Error on {} command !".format(Constants.PROFILE_PICTURE_COMMAND[1:]),
                              description=Constants.ERROR_ON_PROFILE_PICTURE,
                              color=0xF24E4D)
            await ctx.reply(embed=e)
            return
        raise error
    #
    # async def suggest(self, ctx):
    #     """command for suggestions"""
    #     msg = ctx.message.content[9:]
    #     if msg != "":
    #         e = discord.Embed(title="Suggestion",
    #                           url=ctx.message.jump_url,
    #                           description=msg,
    #                           color=0xF24E4D)
    #         await self.channels.get_channel(self.channels.SUGGESTION_CHANNEL_ID).send(embed=e)
    #         await ctx.message.add_reaction(Constants.EMOJI_CHECK)
    #     else:
    #         await ctx.reply("To make a suggestion, please do `!suggest [your suggestion here]`")
    #     return
    #
    # async def on_raw_reaction_add(self, payload):
    #     """Add the role VERIFIED_USER_ROLE from people that react to the message in GUIDELINES_MSG_ID with the emoji EMOJI_GUIDELINES"""
    #     if payload.user_id == self.bot.user.id or payload.message_id == self.bot.user.id:  # check if user that reacted is not pangoBot and that the message isn't from pangoBot
    #         return
    #     if payload.channel_id == self.channels.GUIDELINES_CHANNEL_ID and payload.message_id == self.channels.GUIDELINES_MSG_ID and payload.emoji.name == Constants.EMOJI_ACCEPT_GUIDELINES:
    #         member_ = payload.member
    #         role = discord.utils.get(member_.guild.roles, name=Constants.VERIFIED_USER_ROLE)
    #         await member_.add_roles(role)
    #     return
    #
    # async def on_raw_reaction_remove(self, payload):
    #     """Remove the role VERIFIED_USER_ROLE from people that unreact to the message in GUIDELINES_MSG_ID"""
    #     if payload.user_id == self.bot.user.id or payload.message_id == self.bot.user.id:  # check if user that reacted is not pangoBot and that the message isn't from pangoBot
    #         return
    #     if payload.channel_id == self.channels.GUIDELINES_CHANNEL_ID and payload.message_id == self.channels.GUIDELINES_MSG_ID and payload.emoji.name == Constants.EMOJI_ACCEPT_GUIDELINES:
    #         guild = None
    #         for guild in self.bot.guilds:
    #             if guild == payload.guild_id:
    #                 break
    #         if guild is None:
    #             return
    #         member_ = guild.get_member(payload.user_id)
    #         role = discord.utils.get(member_.guild.roles, name=Constants.VERIFIED_USER_ROLE)
    #         await member_.remove_roles(role)
    #     return
    #
    # async def give_all(self, ctx, role):
    #     role = discord.utils.get(ctx.guild.roles, name=role)
    #     for member in ctx.guild.members:
    #         if role not in member.roles:
    #             try:
    #                 await member.add_roles(role)
    #                 print(member.name)
    #             except:
    #                 print(member.name, member.id)
    #     await ctx.reply("Role {} has been added to everyone in the server".format(role))
    #     return
    #
    # async def remove_all(self, ctx, role):
    #     role = discord.utils.get(ctx.guild.roles, name=role)
    #     for member in ctx.guild.members:
    #         if role in member.roles:
    #             try:
    #                 await member.remove_roles(role)
    #                 print(member.name)
    #             except:
    #                 print(member.name, member.id)
    #     await ctx.reply("Role {} has been removed to everyone in the server".format(role))
    #     return
    #
    # async def save_server(self, ctx):
    #     roles = {r.name: i for i, r in enumerate(ctx.guild.roles)}
    #
    #     members = {}
    #     for member in ctx.guild.members:
    #         members[member.id] = [roles[r.name] for r in member.roles]
    #
    #     categories = {channel.id: {"name": channel.name, "type": channel.type[0], "channels": {}} for channel in
    #                   ctx.guild.channels if channel.category_id is None}
    #     for channel in ctx.guild.channels:
    #         if channel.category_id is not None:
    #             categories[channel.category_id]["channels"][channel.name] = channel.type[0]
    #
    #     server = {"server": categories, "member": members, "roles": roles}
    #
    #     with open('{}.json'.format(datetime.now().strftime("%d_%m_%Y %H_%M_%S")), 'w', encoding='utf-8') as f:
    #         json.dump(server, f, ensure_ascii=False, indent=4)
    #
    #     await ctx.reply("Server has been saved")
    #     return
    #
    # async def clear(self, ctx, number):
    #     number = int(number)
    #     await ctx.channel.purge(limit=number)
    #     await ctx.send("deleted {} messages".format(number))
    #
    # async def ban(self, ctx, members: commands.Greedy[discord.Member],
    #               delete_days: typing.Optional[int] = 0, *,
    #               reason: str):
    #     """Mass bans members with an optional delete_days parameter, send a message before banning people to be sure it's not a mistake"""
    #     if ctx.channel.id == self.channels.COMMAND_CHANNEL_ID:  # to change to an id / channel not a name
    #         accept_decline = await ctx.send(
    #             "Do you really want to ban {}".format(" ".join([member.mention for member in members])))
    #         await accept_decline.add_reaction(Constants.EMOJI_CHECK)
    #         await accept_decline.add_reaction(Constants.EMOJI_CROSS)
    #
    #         def check(reaction_, user_):
    #             return user_ == ctx.message.author and (str(reaction_.emoji) in (
    #             Constants.EMOJI_CHECK, Constants.EMOJI_CROSS))
    #
    #         try:
    #             reaction, user = await self.bot.wait_for('reaction_add', timeout=60.0, check=check)
    #         except asyncio.TimeoutError:
    #             await accept_decline.delete()
    #             await ctx.send('Timed out')
    #         else:
    #             await accept_decline.delete()
    #             if reaction.emoji == Constants.EMOJI_CHECK:
    #                 for member in members:
    #                     await member.ban(delete_message_days=delete_days, reason=reason)
    #                 if delete_days is not None:
    #                     await ctx.send(
    #                         "Banned {} for {} days for {}".format(" ".join([member.mention for member in members]),
    #                                                               delete_days, reason))
    #                 else:
    #                     await ctx.send(
    #                         "Banned {} forever, for {}".format(" ".join([member.mention for member in members]),
    #                                                            delete_days, reason))
    #             else:
    #                 await ctx.send("Bans canceled")
    #
    #
    # async def play(self, ctx, number):
    #     try:
    #         number = int(number)
    #     except:
    #         return
    #     if number <= 0:
    #         await ctx.reply("You need to play at least 1 token.")
    #         return
    #     player_id = ctx.message.author.id
    #     if player_id not in self.bank:
    #         self.bank[player_id] = 500
    #     if self.bank[player_id] < number or self.bank[player_id] <= 0:
    #         await ctx.reply("You don't have enough token. (Your balance is : {})".format(self.bank[player_id]))
    #         return
    #     await ctx.send("You played {} token".format(number))
    #     if random.randint(0, 1) == 0:
    #         self.bank[player_id] += number
    #         await ctx.send("You won {} token !".format(number * 2))
    #     else:
    #         self.bank[player_id] -= number
    #         await ctx.send("You lost {} token :(".format(number))
    #     await ctx.send("Your current balance is {} token.".format(self.bank[player_id]))

max_stars_repo_name: LouisMeMyself/PangoBot

max_issues_repo_name: LouisMeMyself/PangoBot

max_forks_repo_name: LouisMeMyself/PangoBot


text:
from __future__ import absolute_import
from __future__ import division
from builtins import str
from past.utils import old_div
import anuga
import numpy
from . import inlet

import warnings

class Inlet_operator(anuga.Operator):
    """Inlet Operator - add water to an inlet.
    Sets up the geometry of problem

    Inherit from this class (and overwrite
    discharge_routine method for specific subclasses)

    Input: domain, Two points
    """

    def __init__(self,
                 domain,
                 region,
                 Q = 0.0,
                 velocity = None,
                 zero_velocity = False,
                 default = 0.0,
                 description = None,
                 label = None,
                 logging = False,
                 verbose = False):

        anuga.Operator.__init__(self, domain, description, label, logging, verbose)

        self.inlet = inlet.Inlet(self.domain, region, verbose= verbose)

        # should set this up to be a function of time and or space)
        self.Q = Q

        if velocity is not None:
            assert len(velocity)==2

        self.velocity = velocity

        self.zero_velocity = zero_velocity

        self.applied_Q = 0.0

        self.set_default(default)

        self.activate_logging()

    def __call__(self):

        timestep = self.domain.get_timestep()

        t = self.domain.get_time()

        # Need to run global command on all processors
        current_volume = self.inlet.get_total_water_volume()
        total_area = self.inlet.get_area()

        assert current_volume >= 0.0

        Q1 = self.update_Q(t)
        Q2 = self.update_Q(t + timestep)

        #print Q1,Q2
        Q = 0.5*(Q1+Q2)
        volume = Q*timestep

        #print volume

        #print Q, volume

        # store last discharge
        self.applied_Q = Q

        # Distribute positive volume so as to obtain flat surface otherwise
        # just pull water off to have a uniform depth.
        if volume >= 0.0 :
            self.inlet.set_stages_evenly(volume)
            self.domain.fractional_step_volume_integral+=volume
            if self.velocity is not None:
                depths = self.inlet.get_depths()
                self.inlet.set_xmoms(self.inlet.get_xmoms()+depths*self.velocity[0])
                self.inlet.set_ymoms(self.inlet.get_ymoms()+depths*self.velocity[1])
            if self.zero_velocity:
                self.inlet.set_xmoms(0.0)
                self.inlet.set_ymoms(0.0)
        elif current_volume + volume >= 0.0 :
            depth = (current_volume + volume)/total_area
            self.inlet.set_depths(depth)
            self.domain.fractional_step_volume_integral+=volume
            if self.zero_velocity:
                self.inlet.set_xmoms(0.0)
                self.inlet.set_ymoms(0.0)
        else: #extracting too much water!
            self.inlet.set_depths(0.0)

            self.domain.fractional_step_volume_integral-=current_volume
            self.applied_Q = - current_volume/timestep
            if self.zero_velocity:
                self.inlet.set_xmoms(0.0)
                self.inlet.set_ymoms(0.0)

            #msg =  'Requesting too much water to be removed from an inlet! \n'
            #msg += 'current_water_volume = %5.2e Increment volume = %5.2e' % (current_volume, volume)

    def update_Q(self, t):
        """Allowing local modifications of Q
        """
        from anuga.fit_interpolate.interpolate import Modeltime_too_early, Modeltime_too_late

        if callable(self.Q):
            try:
                Q = self.Q(t)
            except Modeltime_too_early as e:
                Q = self.get_default(t,err_msg=e)
            except Modeltime_too_late as e:
                Q = self.get_default(t,err_msg=e)
        else:
            Q = self.Q

        return Q

    def statistics(self):

        message  = '=====================================\n'
        message += 'Inlet Operator: %s\n' % self.label
        message += '=====================================\n'

        message += 'Description\n'
        message += '%s' % self.description
        message += '\n'

        inlet = self.inlet

        message += '-------------------------------------\n'
        message +=  'Inlet\n'
        message += '-------------------------------------\n'

        message += 'inlet triangle indices and centres\n'
        message += '%s' % inlet.triangle_indices
        message += '\n'

        message += '%s' % self.domain.get_centroid_coordinates()[inlet.triangle_indices]
        message += '\n'

        message += 'region\n'
        message += '%s' % inlet
        message += '\n'

        message += '=====================================\n'

        return message

    def timestepping_statistics(self):

        message = '---------------------------\n'
        message += 'Inlet report for %s:\n' % self.label
        message += '--------------------------\n'
        message += 'Q [m^3/s]: %.2f\n' % self.applied_Q

        return message

    def set_default(self, default=None):

        """ Either leave default as None or change it into a function"""

        if default is not None:
            # If it is a constant, make it a function
            if not callable(default):
                tmp = default
                default = lambda t: tmp

            # Check that default_rate is a function of one argument
            try:
                default(0.0)
            except:
                msg = "could not call default"
                raise Exception(msg)

        self.default = default
        self.default_invoked = False

    def get_default(self,t, err_msg=' '):
        """ Call get_default only if exception
        Modeltime_too_late(msg) has been raised
        """

        # Pass control to default rate function
        value = self.default(t)

        if self.default_invoked is False:
            # Issue warning the first time
            msg = ('%s\n'
                   'Instead I will use the default rate: %s\n'
                   'Note: Further warnings will be suppressed'
                   % (str(err_msg), str(self.default(t))))

            warnings.warn(msg)

            # FIXME (Ole): Replace this crude flag with
            # Python's ability to print warnings only once.
            # See http://docs.python.org/lib/warning-filter.html
            self.default_invoked = True

        return value

    def set_Q(self, Q):

        self.Q = Q

    def get_Q(self):

        return self.Q

    def get_inlet(self):

        return self.inlet


text:
import re

import numpy as np

class Excluder:
    """
    In the Vehicle dataset evaluation, we need to exclude the same PID in
    the same camera (CID).
    """
    def __init__(self, gallery_fids):
        # Setup a regexp for extracing the PID and camera (CID) form a FID.
        self.regexp = re.compile('(\S+)/(\d+)_.*')

        # Parse the gallery_set
        self.gallery_pids, self.gallery_cids = self._parse(gallery_fids)

    def __call__(self, query_fids):
        # Extract both the PIDs and CIDs from the query filenames:
        query_pids, query_cids = self._parse(query_fids)

        # Ignore same pid image within the same camera
        cid_matches = self.gallery_cids[None] == query_cids[:,None]
        pid_matches = self.gallery_pids[None] == query_pids[:,None]
        mask = np.logical_and(cid_matches, pid_matches)

        return mask

    def _parse(self, fids):
        """ Return the PIDs and CIDs extracted from the FIDs. """
        pids = []
        cids = []
        for fid in fids:
            cid, pid = self.regexp.match(fid).groups()
            pids.append(pid)
            cids.append(cid)
        return np.asarray(pids), np.asarray(cids)

max_stars_repo_name: bmvc-reid/re-id

max_issues_repo_name: bmvc-reid/re-id

max_forks_repo_name: bmvc-reid/re-id


text:
# -*- coding: utf-8 -*-
from __future__ import print_function, unicode_literals, absolute_import

from . import *
from gtabview.models import as_model

def test_model_id():
    model = as_model([1, 2, 3])
    assert(as_model(model) is model)

def test_model_vect():
    model = as_model([1, 2, 3])
    assert(model.header_shape == (0, 0))
    assert(model.shape == (3, 1))
    assert(materialize(model) == [[1], [2], [3]])

def test_model_vect_str():
    model = as_model(['aaa', 'bbb', 'ccc'])
    assert(model.header_shape == (0, 0))
    assert(model.shape == (3, 1))
    assert(materialize(model) == [['aaa'], ['bbb'], ['ccc']])

def test_model_set():
    model = as_model({'a', 'b', 'c'})
    assert(model.header_shape == (0, 0))
    assert(model.shape == (3, 1))
    assert(sorted(materialize(model)) == [['a'], ['b'], ['c']])

def test_model_iterable():
    model = as_model(sorted(['c', 'b', 'a']))
    assert(model.header_shape == (0, 0))
    assert(model.shape == (3, 1))
    assert(materialize(model) == [['a'], ['b'], ['c']])

def test_model_dict():
    model = as_model({'a': [1, 2, 3], 'b': [1, 2, 3]})
    assert(model.header_shape == (1, 0))
    assert(model.shape == (3, 2))
    assert(materialize(model) == [[1, 1], [2, 2], [3, 3]])
    assert(sorted(materialize_header(model, 0)) == [['a', 'b']])
    assert(materialize_header(model, 1) == [])

def test_model_ordered_dict():
    from collections import OrderedDict
    model = as_model(OrderedDict([('b', [1, 2, 3]), ('a', [1, 2, 3])]))
    assert(model.header_shape == (1, 0))
    assert(model.shape == (3, 2))
    assert(materialize(model) == [[1, 1], [2, 2], [3, 3]])
    assert(materialize_header(model, 0) == [['b', 'a']])
    assert(materialize_header(model, 1) == [])

def test_model_ordered_dict_sort():
    from collections import OrderedDict
    model = as_model(OrderedDict([('b', [1, 2, 3]), ('a', [1, 2, 3])]), sort=True)
    assert(model.header_shape == (1, 0))
    assert(model.shape == (3, 2))
    assert(materialize(model) == [[1, 1], [2, 2], [3, 3]])
    assert(materialize_header(model, 0) == [['a', 'b']])
    assert(materialize_header(model, 1) == [])

def test_model_dict_ragged():
    model = as_model({'a': [1, 2], 'b': [1, 2, 3]})
    assert(model.header_shape == (1, 0))
    assert(model.shape == (3, 2))
    assert(materialize(model) == [[1, 1], [2, 2], [None, 3]])
    assert(materialize_header(model, 0) == [['a', 'b']])
    assert(materialize_header(model, 1) == [])

def test_model_list():
    model = as_model([[1, 2, 3], [1, 2, 3]])
    assert(model.header_shape == (0, 0))
    assert(model.shape == (2, 3))
    assert(materialize(model) == [[1, 2, 3], [1, 2, 3]])

def test_model_list_hdr_idx():
    model = as_model([[None, 'a', 'b', 'c'], ['x', 1, 2, 3], ['y', 1, 2, 3]], hdr_rows=1, idx_cols=1)
    assert(model.header_shape == (1, 1))
    assert(model.shape == (2, 3))
    assert(materialize(model) == [[1, 2, 3], [1, 2, 3]])
    assert(materialize_header(model, 0) == [['a', 'b', 'c']])
    assert(materialize_header(model, 1) == [['x', 'y']])

def test_model_list_ragged():
    model = as_model([[1, 2], [1, 2, 3]])
    assert(model.header_shape == (0, 0))
    assert(model.shape == (2, 3))
    assert(materialize(model) == [[1, 2, None], [1, 2, 3]])

@require('pandas')
def test_model_frame():
    import pandas as pd
    model = as_model(pd.DataFrame([[1, 2, 3], [1, 2, 3]],
                                  columns=['a', 'b', 'c'],
                                  index=['x', 'y']))
    assert(model.header_shape == (1, 1))
    assert(model.shape == (2, 3))
    assert(materialize(model) == [[1, 2, 3], [1, 2, 3]])
    assert(materialize_header(model, 0) == [['a', 'b', 'c']])
    assert(materialize_header(model, 1) == [['x', 'y']])

@require('pandas')
def test_model_frame_multiindex():
    import pandas as pd
    col_index = pd.MultiIndex.from_tuples(
        list(zip(['A', 'B', 'C'], ['a', 'b', 'c'])),
        names=['CL0', 'CL1'])
    row_index = pd.MultiIndex.from_tuples(
        list(zip(['X', 'Y'], ['x', 'y'])),
        names=['RL0', 'RL1'])
    model = as_model(pd.DataFrame([[1, 2, 3], [1, 2, 3]],
                                  columns=col_index,
                                  index=row_index))
    assert(model.header_shape == (2, 2))
    assert(model.shape == (2, 3))
    assert(materialize(model) == [[1, 2, 3], [1, 2, 3]])
    assert(materialize_header(model, 0) == [['A', 'B', 'C'], ['a', 'b', 'c']])
    assert(materialize_header(model, 1) == [['X', 'Y'], ['x', 'y']])
    assert(materialize_names(model, 0) == ['CL0', 'CL1'])
    assert(materialize_names(model, 1) == ['RL0', 'RL1'])

@require('pandas')
def test_model_transpose():
    import pandas as pd
    col_index = pd.MultiIndex.from_tuples(
        list(zip(['A', 'B', 'C'], ['a', 'b', 'c'])),
        names=['CL0', 'CL1'])
    row_index = pd.MultiIndex.from_tuples(
        list(zip(['X', 'Y'], ['x', 'y'])),
        names=['RL0', 'RL1'])
    model = as_model(pd.DataFrame([[1, 2, 3], [1, 2, 3]],
                                  columns=col_index,
                                  index=row_index),
                     transpose=True)
    assert(model.header_shape == (2, 2))
    assert(model.shape == (3, 2))
    assert(materialize(model) == [[1, 1], [2, 2], [3, 3]])
    assert(materialize_header(model, 0) == [['X', 'Y'], ['x', 'y']])
    assert(materialize_header(model, 1) == [['A', 'B', 'C'], ['a', 'b', 'c']])
    assert(materialize_names(model, 0) == ['RL0', 'RL1'])
    assert(materialize_names(model, 1) == ['CL0', 'CL1'])

@require('pandas')
def test_model_series():
    import pandas as pd
    model = as_model(pd.Series([1, 2, 3], name='serie', index=['x', 'y', 'z']))
    assert(model.header_shape == (1, 1))
    assert(model.shape == (3, 1))
    assert(materialize(model) == [[1], [2], [3]])
    assert(materialize_header(model, 0) == [['serie']])
    assert(materialize_header(model, 1) == [['x', 'y', 'z']])

@require('numpy')
def test_model_array():
    import numpy as np
    model = as_model(np.array([[1, 2, 3], [1, 2, 3]]))
    assert(model.header_shape == (0, 0))
    assert(model.shape == (2, 3))
    assert(materialize(model) == [[1, 2, 3], [1, 2, 3]])

@require('numpy')
def test_model_matrix():
    import numpy as np
    import warnings
    with warnings.catch_warnings():
        warnings.simplefilter('ignore', PendingDeprecationWarning)
        model = as_model(np.matrix([[1, 2, 3], [1, 2, 3]]))
    assert(model.header_shape == (0, 0))
    assert(model.shape == (2, 3))
    assert(materialize(model) == [[1, 2, 3], [1, 2, 3]])

max_stars_repo_name: TabViewer/gtabview

max_issues_repo_name: wavexx/gtabview

max_forks_repo_name: wavexx/gtabview


text:
import numpy as np
import tensorflow as tf

from garage.misc import logger, special
from garage.sampler import parallel_sampler
from garage.sampler import singleton_pool
from garage.sampler.base import BaseSampler
from garage.sampler.utils import truncate_paths
from garage.tf.misc import tensor_utils

def worker_init_tf(g):
    g.sess = tf.Session()
    g.sess.__enter__()

def worker_init_tf_vars(g):
    g.sess.run(tf.global_variables_initializer())

class BatchSampler(BaseSampler):
    def start_worker(self):
        if singleton_pool.n_parallel > 1:
            singleton_pool.run_each(worker_init_tf)
        parallel_sampler.populate_task(self.algo.env, self.algo.policy)
        if singleton_pool.n_parallel > 1:
            singleton_pool.run_each(worker_init_tf_vars)

    def shutdown_worker(self):
        parallel_sampler.terminate_task(scope=self.algo.scope)

    def obtain_samples(self, itr):
        cur_policy_params = self.algo.policy.get_param_values()
        cur_env_params = self.algo.env.get_param_values()
        paths = parallel_sampler.sample_paths(
            policy_params=cur_policy_params,
            env_params=cur_env_params,
            max_samples=self.algo.batch_size,
            max_path_length=self.algo.max_path_length,
            scope=self.algo.scope,
        )
        if self.algo.whole_paths:
            return paths
        else:
            paths_truncated = truncate_paths(paths, self.algo.batch_size)
            return paths_truncated

    def process_samples(self, itr, paths):
        baselines = []
        returns = []

        max_path_length = self.algo.max_path_length

        if hasattr(self.algo.baseline, "predict_n"):
            all_path_baselines = self.algo.baseline.predict_n(paths)
        else:
            all_path_baselines = [
                self.algo.baseline.predict(path) for path in paths
            ]

        for idx, path in enumerate(paths):
            path_baselines = np.append(all_path_baselines[idx], 0)
            deltas = path["rewards"] + \
                     self.algo.discount * path_baselines[1:] - \
                     path_baselines[:-1]
            path["advantages"] = special.discount_cumsum(
                deltas, self.algo.discount * self.algo.gae_lambda)
            path["deltas"] = deltas

        for idx, path in enumerate(paths):
            # baselines
            path['baselines'] = all_path_baselines[idx]
            baselines.append(path['baselines'])

            # returns
            path["returns"] = special.discount_cumsum(path["rewards"],
                                                      self.algo.discount)
            returns.append(path["returns"])

        # make all paths the same length
        obs = [path["observations"] for path in paths]
        obs = tensor_utils.pad_tensor_n(obs, max_path_length)

        actions = [path["actions"] for path in paths]
        actions = tensor_utils.pad_tensor_n(actions, max_path_length)

        rewards = [path["rewards"] for path in paths]
        rewards = tensor_utils.pad_tensor_n(rewards, max_path_length)

        returns = [path["returns"] for path in paths]
        returns = tensor_utils.pad_tensor_n(returns, max_path_length)

        advantages = [path["advantages"] for path in paths]
        advantages = tensor_utils.pad_tensor_n(advantages, max_path_length)

        baselines = tensor_utils.pad_tensor_n(baselines, max_path_length)

        agent_infos = [path["agent_infos"] for path in paths]
        agent_infos = tensor_utils.stack_tensor_dict_list([
            tensor_utils.pad_tensor_dict(p, max_path_length)
            for p in agent_infos
        ])

        env_infos = [path["env_infos"] for path in paths]
        env_infos = tensor_utils.stack_tensor_dict_list([
            tensor_utils.pad_tensor_dict(p, max_path_length) for p in env_infos
        ])

        valids = [np.ones_like(path["returns"]) for path in paths]
        valids = tensor_utils.pad_tensor_n(valids, max_path_length)

        average_discounted_return = (np.mean(
            [path["returns"][0] for path in paths]))

        undiscounted_returns = [sum(path["rewards"]) for path in paths]

        ent = np.sum(
            self.algo.policy.distribution.entropy(agent_infos) *
            valids) / np.sum(valids)

        samples_data = dict(
            observations=obs,
            actions=actions,
            rewards=rewards,
            advantages=advantages,
            baselines=baselines,
            returns=returns,
            valids=valids,
            agent_infos=agent_infos,
            env_infos=env_infos,
            paths=paths,
            average_return=np.mean(undiscounted_returns),
        )

        logger.record_tabular('Iteration', itr)
        logger.record_tabular('AverageDiscountedReturn',
                              average_discounted_return)
        logger.record_tabular('AverageReturn', np.mean(undiscounted_returns))
        logger.record_tabular('NumTrajs', len(paths))
        logger.record_tabular('Entropy', ent)
        logger.record_tabular('Perplexity', np.exp(ent))
        logger.record_tabular('StdReturn', np.std(undiscounted_returns))
        logger.record_tabular('MaxReturn', np.max(undiscounted_returns))
        logger.record_tabular('MinReturn', np.min(undiscounted_returns))

        return samples_data


text:
from copy import deepcopy
from scipy.special import binom
import numpy as np
import time
from numpy.polynomial.polynomial import Polynomial

class Function:
    def __iadd__(self, _f):
        f1 = deepcopy(self)
        return FunctionSum(f1, _f)

    def __imul__(self, _f):
        f1 = deepcopy(self)
        return FunctionScalarMul(f1, _f)

    def compose(self, _f):
        f1 = deepcopy(self)
        return FunctionComposition(f1, _f)

    def __add__(self, _f):
        f1 = deepcopy(self)
        return FunctionSum(f1, _f)

    def deriv_once(self):
        return self.deriv()

    def deriv(self, _deg=1):
        raise NotImplementedError()

class ConstFunction(Function):
    def __init__(self, _val, _exec_time, _dim):
        self.dim_ = _dim
        self.val_ = _val
        self.T_ = _exec_time

    def __call__(self, _t):
        _t = np.atleast_1d(_t)
        return np.ones((self.dim_, len(_t))) * self.val_

    def deriv(self, _deg=1):
        if _deg == 0:
            return self
        return ZeroFunction(self.T_, self.dim_)

class ZeroFunction(ConstFunction):
    def __init__(self, _exec_time, _dim):
        ConstFunction.__init__(self, 0.0, _exec_time, _dim)

    def __call__(self, _t):
        _t = np.atleast_1d(_t)
        return np.zeros((self.dim_, len(_t)))

class OneFunction(ConstFunction):
    def __init__(self, _dim):
        ConstFunction.__init__(self, 1.0, _dim)

class FunctionSum(Function):
    def __init__(self, _f1, *args):
        self.dim_ = _f1.dim_
        self.T_ = _f1.T_

        self.function_array_ = []

        for fun in (_f1, ) + args:
            assert fun.dim_ == self.dim_
            assert np.abs(fun.T_ - self.T_) < 1.0e-4
            if isinstance(fun, FunctionSum):
                self.function_array_ += fun.function_array_
            else:
                self.function_array_.append(deepcopy(fun))

    def __call__(self, _t):
        result = self.function_array_[0](_t)
        for fun in self.function_array_[1:]:
            result += fun(_t)
        return result

    def deriv(self, _deg=1):
        if _deg == 0:
            return self
        fun_ders = [fun.deriv(_deg) for fun in self.function_array_]
        return FunctionSum(*fun_ders)

class FunctionScalarMul(Function):
    def __init__(self, _f1, _scalarfun):
        assert _scalarfun.dim_ == 1
        self.dim_ = _f1.dim_
        self.f1_ = deepcopy(_f1)
        self.scalarfun_ = deepcopy(_scalarfun)

        self.T_ = _f1.T_
        assert np.abs(_f1.T_ - _scalarfun.T_) < 1.0e-4

    def __call__(self, _t):
        _t = np.atleast_1d(_t)
        f1 = self.f1_(_t)
        scalar = self.scalarfun_(_t)
        return np.einsum('i,ij ->ij', scalar.ravel(), f1)

    def deriv(self, _deg=1):
        if _deg == 0:
            return self
        result = self
        for _ in range(_deg):
            result = result.deriv_once()

        return result

    def deriv_once(self):
        fun1 = self.f1_
        sfun = self.scalarfun_

        term_1 = FunctionScalarMul(fun1.deriv(), sfun)
        term_2 = FunctionScalarMul(fun1, sfun.deriv())

        return FunctionSum(term_1, term_2)

class FunctionComposition(Function):
    def __init__(self, _f1, _sfun):
        self.f1_ = _f1
        assert _sfun.dim_ == 1
        self.scalarfun_ = _sfun
        self.T_ = _sfun.T_

        assert _sfun(_sfun.T_) <= _f1.T_ + 1.0e-7,\
            'Domain of the parametrization' +\
            ' is larger than the domain of the funciton'
        self.dim_ = _f1.dim_

    def __call__(self, _t):
        return self.f1_(self.scalarfun_(_t).ravel())

    def deriv(self, _deg=1):
        if _deg == 0:
            return self
        result = self
        for _ in range(_deg):
            result = result.deriv_once()

        return result

    def deriv_once(self):
        f1 = FunctionComposition(self.f1_.deriv(), self.scalarfun_)
        f2 = self.scalarfun_.deriv()
        return FunctionScalarMul(f1, f2)

class FunctionConcat(Function):
    def __init__(self, _f1, _f2):
        self.T_ = _f2.T_
        self.dim_ = _f1.dim_
        self.func_array_ = [_f1, _f2]
        self.f1_ = deepcopy(_f1)
        self.f2_ = deepcopy(_f2)
        assert _f1.dim_ == _f2.dim_

    def __call__(self, _time):
        _time = np.atleast_1d(_time)
        time_prev = _time[_time < self.f1_.T_]
        time_post = _time[_time >= self.f1_.T_]
        result = np.vstack([
            self.f1_(time_prev).reshape(-1, self.dim_),
            self.f2_(time_post).reshape(-1, self.dim_)
        ])
        return result

    def deriv(self, _ord=1):
        return FunctionConcat(self.f1_.deriv(_ord), self.f2_.deriv(_ord))

class ScalarPolWrapper(Function):
    def __init__(self, _pol):
        self.T_ = _pol.domain[-1]
        self.dim_ = 1
        self.pol_ = _pol

    def __call__(self, _time):
        _time = np.atleast_1d(_time)
        return self.pol_(_time).reshape(-1, 1)

    def deriv(self, _ord=1):
        return ScalarPolWrapper(self.pol_.deriv(_ord))

class Identity(ScalarPolWrapper):
    def __init__(self, _exec_time):
        pol = Polynomial(
            [0, 1], domain=[0, _exec_time], window=[0, _exec_time])
        ScalarPolWrapper.__init__(self, pol)

max_stars_repo_name: rafaelrojasmiliani/gsplines

max_issues_repo_name: rafaelrojasmiliani/gsplines

max_forks_repo_name: rafaelrojasmiliani/gsplines

reason: import numpy,from numpy,from scipy


text:
import os
import json
from six import iteritems

import h5py
import numpy as np
from tqdm import tqdm

import torch
import torch.nn.functional as F
from torch.utils.data import Dataset

class VisDialDataset(Dataset):

    @staticmethod
    def add_cmdline_args(parser):
        parser.add_argument_group('Dataloader specific arguments')
        parser.add_argument('-input_img', default='data/data_img.h5',
                                help='HDF5 file with image features')
        parser.add_argument('-input_ques', default='data/visdial_data.h5',
                                help='HDF5 file with preprocessed questions')
        parser.add_argument('-input_json', default='data/visdial_params.json',
                                help='JSON file with image paths and vocab')
        parser.add_argument('-img_norm', default=1, choices=[1, 0],
                                help='normalize the image feature. 1=yes, 0=no')
        return parser

    def __init__(self, args, subsets):
        """Initialize the dataset with splits given by 'subsets', where
        subsets is taken from ['train', 'val', 'test']
        """
        super().__init__()
        self.args = args
        self.subsets = tuple(subsets)

        print("Dataloader loading json file: {}".format(args.input_json))
        with open(args.input_json, 'r') as info_file:
            info = json.load(info_file)
            # possible keys: {'ind2word', 'word2ind', 'unique_img_(split)'}
            for key, value in iteritems(info):
                setattr(self, key, value)

        # add <START> and <END> to vocabulary
        word_count = len(self.word2ind)
        self.word2ind['<START>'] = word_count + 1
        self.word2ind['<END>'] = word_count + 2
        self.start_token = self.word2ind['<START>']
        self.end_token = self.word2ind['<END>']

        # padding + <START> + <END> token
        self.vocab_size = word_count + 3
        print("Vocab size with <START>, <END>: {}".format(self.vocab_size))

        # construct reverse of word2ind after adding tokens
        self.ind2word = {
            int(ind): word
            for word, ind in iteritems(self.word2ind)
        }

        print("Dataloader loading h5 file: {}".format(args.input_ques))
        ques_file = h5py.File(args.input_ques, 'r')

        print("Dataloader loading h5 file: {}".format(args.input_img))
        img_file = h5py.File(args.input_img, 'r')

        # load all data mats from ques_file into this
        self.data = {}

        # map from load to save labels
        io_map = {
            'ques_{}': '{}_ques',
            'ques_length_{}': '{}_ques_len',
            'ans_{}': '{}_ans',
            'ans_length_{}': '{}_ans_len',
            'img_pos_{}': '{}_img_pos',
            'cap_{}': '{}_cap',
            'cap_length_{}': '{}_cap_len',
            'opt_{}': '{}_opt',
            'opt_length_{}': '{}_opt_len',
            'opt_list_{}': '{}_opt_list',
            'num_rounds_{}': '{}_num_rounds',
            'ans_index_{}': '{}_ans_ind'
        }

        # processing every split in subsets
        for dtype in subsets:  # dtype is in ['train', 'val', 'test']
            print("\nProcessing split [{}]...".format(dtype))
            # read the question, answer, option related information
            for load_label, save_label in iteritems(io_map):
                if load_label.format(dtype) not in ques_file:
                    continue
                self.data[save_label.format(dtype)] = torch.from_numpy(
                    np.array(ques_file[load_label.format(dtype)], dtype='int64'))

            print("Reading image features...")
            img_feats = torch.from_numpy(np.array(img_file['images_' + dtype]))

            if args.img_norm:
                print("Normalizing image features...")
                img_feats = F.normalize(img_feats, dim=1, p=2)

            # save image features
            self.data[dtype + '_img_fv'] = img_feats
            img_fnames = getattr(self, 'unique_img_' + dtype)
            self.data[dtype + '_img_fnames'] = img_fnames

            # record some stats, will be transferred to encoder/decoder later
            # assume similar stats across multiple data subsets
            # maximum number of questions per image, ideally 10
            self.max_ques_count = self.data[dtype + '_ques'].size(1)
            # maximum length of question
            self.max_ques_len = self.data[dtype + '_ques'].size(2)
            # maximum length of answer
            self.max_ans_len = self.data[dtype + '_ans'].size(2)

        # reduce amount of data for preprocessing in fast mode
        if args.overfit:
            self.data[dtype + '_img_fv'] = self.data[dtype + '_img_fv'][:5]
            self.data[dtype + '_img_fnames'] = self.data[dtype + '_img_fnames'][:5]

        self.num_data_points = {}
        for dtype in subsets:
            self.num_data_points[dtype] = len(self.data[dtype + '_img_fv'])
            print("[{0}] no. of threads: {1}".format(dtype, self.num_data_points[dtype]))
        print("\tMax no. of rounds: {}".format(self.max_ques_count))
        print("\tMax ques len: {}".format(self.max_ques_len))
        print("\tMax ans len: {}".format(self.max_ans_len))

        # prepare history
        for dtype in subsets:
            self._process_history(dtype)

            # 1 indexed to 0 indexed
            self.data[dtype + '_opt'] -= 1
            if dtype + '_ans_ind' in self.data:
                self.data[dtype + '_ans_ind'] -= 1

        # default pytorch loader dtype is set to train
        if 'train' in subsets:
            self._split = 'train'
        else:
            self._split = subsets[0]

    @property
    def split(self):
        return self._split

    @split.setter
    def split(self, split):
        assert split in self.subsets  # ['train', 'val', 'test']
        self._split = split

    # ------------------------------------------------------------------------
    # methods to override - __len__ and __getitem__ methods
    # ------------------------------------------------------------------------

    def __len__(self):
        return self.num_data_points[self._split]

    def __getitem__(self, idx):
        dtype = self._split
        item = {'index': idx}
        item['num_rounds'] = self.data[dtype + '_num_rounds'][idx]

        # get image features
        item['img_feat'] = self.data[dtype + '_img_fv'][idx]
        item['img_fnames'] = self.data[dtype + '_img_fnames'][idx]

        # get question tokens
        item['ques'] = self.data[dtype + '_ques'][idx]
        item['ques_len'] = self.data[dtype + '_ques_len'][idx]

        # get history tokens
        item['hist_len'] = self.data[dtype + '_hist_len'][idx]
        item['hist'] = self.data[dtype + '_hist'][idx]

        # get options tokens
        opt_inds = self.data[dtype + '_opt'][idx]
        opt_size = list(opt_inds.size())
        new_size = torch.Size(opt_size + [-1])
        ind_vector = opt_inds.view(-1)

        option_in = self.data[dtype + '_opt_list'].index_select(0, ind_vector)
        option_in = option_in.view(new_size)

        opt_len = self.data[dtype + '_opt_len'].index_select(0, ind_vector)
        opt_len = opt_len.view(opt_size)

        item['opt'] = option_in
        item['opt_len'] = opt_len
        if dtype != 'test':
            ans_ind = self.data[dtype + '_ans_ind'][idx]
            item['ans_ind'] = ans_ind.view(-1)

        # convert zero length sequences to one length
        # this is for handling empty rounds of v1.0 test, they will be dropped anyway
        if dtype == 'test':
            item['ques_len'][item['ques_len'] == 0] += 1
            item['opt_len'][item['opt_len'] == 0] += 1
            item['hist_len'][item['hist_len'] == 0] += 1
        return item

    #-------------------------------------------------------------------------
    # collate function utilized by dataloader for batching
    #-------------------------------------------------------------------------

    def collate_fn(self, batch):
        dtype = self._split
        merged_batch = {key: [d[key] for d in batch] for key in batch[0]}
        out = {}
        for key in merged_batch:
            if key in {'index', 'num_rounds', 'img_fnames'}:
                out[key] = merged_batch[key]
            elif key in {'cap_len'}:
                out[key] = torch.Tensor(merged_batch[key]).long()
            else:
                out[key] = torch.stack(merged_batch[key], 0)

        # Dynamic shaping of padded batch
        out['hist'] = out['hist'][:, :, :torch.max(out['hist_len'])].contiguous()
        out['ques'] = out['ques'][:, :, :torch.max(out['ques_len'])].contiguous()
        out['opt'] = out['opt'][:, :, :, :torch.max(out['opt_len'])].contiguous()

        batch_keys = ['img_fnames', 'num_rounds', 'img_feat', 'hist',
                      'hist_len', 'ques', 'ques_len', 'opt', 'opt_len']
        if dtype != 'test':
            batch_keys.append('ans_ind')
        return {key: out[key] for key in batch_keys}

    #-------------------------------------------------------------------------
    # preprocessing functions
    #-------------------------------------------------------------------------

    def _process_history(self, dtype):
        """Process caption as well as history. Optionally, concatenate history
        for lf-encoder."""
        captions = self.data[dtype + '_cap']
        questions = self.data[dtype + '_ques']
        ques_len = self.data[dtype + '_ques_len']
        cap_len = self.data[dtype + '_cap_len']
        max_ques_len = questions.size(2)

        answers = self.data[dtype + '_ans']
        ans_len = self.data[dtype + '_ans_len']
        num_convs, num_rounds, max_ans_len = answers.size()

        if self.args.concat_history:
            self.max_hist_len = min(num_rounds * (max_ques_len + max_ans_len), 300)
            history = torch.zeros(num_convs, num_rounds, self.max_hist_len).long()
        else:
            history = torch.zeros(num_convs, num_rounds, max_ques_len + max_ans_len).long()
        hist_len = torch.zeros(num_convs, num_rounds).long()

        # go over each question and append it with answer
        for th_id in range(num_convs):
            clen = cap_len[th_id]
            hlen = min(clen, max_ques_len + max_ans_len)
            for round_id in range(num_rounds):
                if round_id == 0:
                    # first round has caption as history
                    history[th_id][round_id][:max_ques_len + max_ans_len] \
                        = captions[th_id][:max_ques_len + max_ans_len]
                else:
                    qlen = ques_len[th_id][round_id - 1]
                    alen = ans_len[th_id][round_id - 1]
                    # if concat_history, string together all previous question-answer pairs
                    if self.args.concat_history:
                        history[th_id][round_id][:hlen] = history[th_id][round_id - 1][:hlen]
                        history[th_id][round_id][hlen] = self.word2ind['<END>']
                        if qlen > 0:
                            history[th_id][round_id][hlen + 1:hlen + qlen + 1] \
                                = questions[th_id][round_id - 1][:qlen]
                        if alen > 0:
                            # print(round_id, history[th_id][round_id][:10], answers[th_id][round_id][:10])
                            history[th_id][round_id][hlen + qlen + 1:hlen + qlen + alen + 1] \
                                = answers[th_id][round_id - 1][:alen]
                        hlen = hlen + qlen + alen + 1
                    # else, history is just previous round question-answer pair
                    else:
                        if qlen > 0:
                            history[th_id][round_id][:qlen] = questions[th_id][round_id - 1][:qlen]
                        if alen > 0:
                            history[th_id][round_id][qlen:qlen + alen] \
                                = answers[th_id][round_id - 1][:alen]
                        hlen = alen + qlen
                # save the history length
                hist_len[th_id][round_id] = hlen

        self.data[dtype + '_hist'] = history
        self.data[dtype + '_hist_len'] = hist_len


text:
import numpy as np

class UniformPrior():
    def __init__(self, a, b):
        assert a < b
        self.a = a # lower bound
        self.b = b # upper bound

    def log_pdf(self, x):
        """Return the log-probability density
        """
        if (x < self.a) or (x > self.b):
            return -np.inf
        return -np.log(self.b-self.a)

    def draw(self):
        """Return a random sample
        """
        return np.random.uniform(self.a, self.b)

    def bounds(self):
        return self.a, self.b

max_stars_repo_name: pmplewa/GC-OrbitFit

max_issues_repo_name: pmplewa/GC-OrbitFit

max_forks_repo_name: pmplewa/GC-OrbitFit


text:
'''
Created 2017-09-05 10:16:55-04:00
@author: jzacsh@gmail.com
'''

import numpy as np
from mpl_toolkits.mplot3d import Axes3D # needed for param plot='3d' below
from scipy.optimize import minimize
from matplotlib import pyplot

def costsViaSquare(inputs, outputs, weight, bias):
    return np.square(weight*inputs + bias - outputs)

def costsViaAbsVal(inputs, outputs, weight, bias):
    return np.absolute(weight*inputs + bias - outputs)

class TrainingSet:
    def __init__(self, inputs, labels):
        self.inputs = np.array(inputs)
        self.labels = np.array(labels)

    def costhandler(self, weightAndBias):
        """Handler for numpy's minimize() function"""
        return self.costof(weightAndBias[0], weightAndBias[1])

    def costof(self, weight, bias):
        """calculates cost using default methodology"""
        return costsViaSquare(self.inputs, self.labels, weight, bias).sum()

    def randGuessMimizes(self, debugMode=False):
        """returns "optimal" weight, bias, success (ie: whether vals are trustworthy)"""
        if debugMode:
            print("randomly guessing & minimizing.... Knowns are\n\tx: %s\n\ty: %s" %(self.inputs, self.labels))

        for i in range(0, 5):
            ithGuess = np.random.randn(2) # two guesses: one for weight, one for bias
            res = minimize(self.costhandler, ithGuess, method='Nelder-Mead')
            if debugMode:
                print("\tminimized: %s\t[init guess #%d: %s]" %(res.x, i, ithGuess))

        # whatever the last mimizer returned
        return [ res.x[0], res.x[1], res.success ]

    def buildRandomTrainer(setsize=2):
        inputs = np.random.randn(setsize) * 10 # entry-wise multiply by 10
        # see: https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.shape.html
        #print("x scalars, of shape: %s\n%s" % (inputs.shape, inputs))

        # labels subset of {1, -1}
        labels = 2*np.random.randint(size=setsize, low=0, high=2)-1
        #print("y scalars, of shape: %s\n%s" % (labels.shape, labels))
        return TrainingSet(inputs, labels)

def generateWeightBiasSpace(weight, bias):
    sampleFrom = -5
    sampleTo = 5
    sampleRate = 0.5

    print("\tgenerating weights & biases\n\t\t%.2f <- {weight=%0.3f, bias=%0.3f} -> %.2f @%.3f steps\n"%(
        sampleFrom, weight, bias, sampleTo, sampleRate))
    return np.meshgrid(
            np.arange(weight+sampleFrom,weight+sampleTo,sampleRate),
            np.arange(bias+sampleFrom,bias+sampleTo,sampleRate))

def main():
    set = TrainingSet.buildRandomTrainer()
    optimalWeight, optimalBias, minimOK = set.randGuessMimizes()

    print("""rand set's cost was %0.05f
    for minimization with: (optimal) weight=%0.04f, (optimal) bias=%0.04f
    [minimize success: %s]""" % (
        set.costof(optimalWeight, optimalBias),
        optimalWeight, optimalBias,
        minimOK
    ))

    #TODO fix this line + scatterplot graphing section; totally broken
    # plot coordinates of (input,costs) and graph the minimized result
#   ax = pyplot.figure().add_subplot(111, projection='3d')
#
#   # graph a line of our minimized (optimal) weight + bias:
#   #TODO: graph a line on pyplot; subplot??
#   #mx+b form is: m=optimalWeight, b=optimalBias
#
#   #TODO: graph plot of dots, whose x,y coords are:
#   #x = set.inputs
#   #y = set.labels
#
#   #TODO: place above two plots on single x-y plane:
    costs = costsViaSquare(set.inputs, set.labels, optimalWeight, optimalBias)
    pyplot.scatter(set.inputs, costs)
    pyplot.plot(set.inputs, optimalWeight*set.inputs+optimalBias, '-')

    # grid of sampling points
    weights, biases = generateWeightBiasSpace(optimalWeight, optimalBias)

    # calc cost of field of weight/biases surrounding our supposed optimal
    costs = np.array([
        set.costof(w,b) for w,b in zip(np.ravel(weights),np.ravel(biases))
    ]).reshape(weights.shape)

    print("The costs (shape=%s) after sampling:\n%s\n\n"
          %(costs.shape, costs))

    # 2d-graphing machinery
    ax = pyplot.figure().add_subplot(111, projection='3d')
    ax.plot_surface(weights, biases, costs)
    ax.set_xlabel('weights')
    ax.set_ylabel('biases')
    ax.set_zlabel('costs')
    pyplot.show()

if __name__ == '__main__':
    main()

reason: import numpy,from scipy


text:
import pytest
from math import isclose
from numpy import log10
from pandas import Series

from cellengine.utils.scale_utils import apply_scale

@pytest.fixture(scope="module")
def scale():
    return {"minimum": 5, "maximum": 10, "type": "LogScale"}

def test_should_apply_scale(scale):
    # fmt: off
    input = Series([
        -20, 0, 1e-40, 0.01, 0.2, 0.5, 0.9999, 1, 1.00001,
        2, 5, 10, 100, 250, 500, 1000, 5000, 10000, 50000,
        5e5, 5e6, 5e7, 5e8, 5e9, 5e10, 5e11, 5e12, 5e13, 5e14,
        5e15, 5e16, 5e17
    ])
    # fmt: on
    output = Series([], dtype="float64")
    output = input.map(lambda a: apply_scale(scale, a, False))
    # fmt: off
    expected = Series([
        0, 0, 0, 0, 0, 0, 0, 0, 0.00000434292310445319, 0.30102999566398114,
        0.6989700043360186, 1, 2, 2.397940008672037, 2.6989700043360183, 3,
        3.6989700043360187, 4, 4.698970004336018, 5.698970004336018,
        6.698970004336018, 7.698970004336018, 8.698970004336018,
        9.698970004336018, 10.698970004336018, 11.698970004336018,
        12.698970004336018, 13.698970004336018, 14.698970004336018,
        15.698970004336018, 16.698970004336018, 17.698970004336018,
    ])
    # fmt: on
    assert [isclose(a, b, rel_tol=0.00001) for a, b in zip(output, expected)]

def test_should_apply_clamped(scale):
    # fmt: off
    input = Series([
        -20, 0, 0.01, 0.2, 0.5, 1, 2, 5, 10,
        100, 250, 500, 1000, 5000, 10000, 50000
    ])
    # fmt: on
    output = Series([], dtype="float64")
    MINV = 0.6989700043360186
    MAXV = 1
    output = input.map(lambda a: apply_scale(scale, a, True))
    # fmt: off
    expected = Series([
        MINV, MINV, MINV, MINV, MINV, MINV, MINV,
        0.6989700043360186, 1, MAXV, MAXV, MAXV,
        MAXV, MAXV, MAXV, MAXV,
    ])
    # fmt: on
    assert [isclose(a, b, rel_tol=0.00001) for a, b in zip(output, expected)]

def test_should_handle_0_length_arrays(scale):
    input = Series([], dtype="float64")
    output = Series([], dtype="float64")
    output = Series([], dtype="float64")
    output = input.map(lambda a: apply_scale(scale, a, True))
    assert type(output) is Series
    assert output.size == 0

def test_correctly_applies_scale_of_length_n(scale):
    for n in range(1, 32):
        input = Series([1] * n)
        output = Series([], dtype="float64")
        output = input.map(lambda a: apply_scale(scale, a, True))
        for i in range(0, n):
            assert isclose(output[i], log10(scale["minimum"]), rel_tol=0.00001)

max_stars_repo_name: primitybio/cellengine-python-toolk

max_issues_repo_name: primitybio/cellengine-python-toolk

max_forks_repo_name: primitybio/cellengine-python-toolkit

reason: from numpy


text:
from .Environment import Environment
from .Engine import Engine
from .Spacecraft import Spacecraft
from .Orbit import Orbit

import util as t
import numpy as np

if __name__ == '__main__':

	#initialise environment object
	env = environment.Environment("Earth")

	# initialise spacecraft
	eng1 = engine.Engine(100)
	sc1 = spacecraft.Spacecraft(eng1)
	sc1.set_state0([50e6,0.1,0,0,0,0], 'koe')

	eng2 = engine.Engine(200)
	sc2 = spacecraft.Spacecraft(eng2)
	sc2.set_state0([40e6,0.3,np.radians(25),0,0,0], 'koe')

	# define simulation parameters
	tspan = 3*24*60*60 # 1 day in seconds
	dt = 600 # 10 minutes

	# calcualte orbits
	orb1= orbit.Orbit(sc1,env,tspan,dt)
	orb2 = orbit.Orbit(sc2,env,tspan,dt)

	# plot orbits
	t.plot_n_orbits([orb1,orb2],env)

max_stars_repo_name: alexander-minchin/orbitsim

max_issues_repo_name: alexander-minchin/orbitsim

max_forks_repo_name: alexander-minchin/orbitsim


text:
# chapter 5 code as per published

# ## braitenberg
# sl_braitenberg
# sim('sl_braitenberg')

## simple automata

from scipy.io import loadmat
from roboticstoolbox import Bug2, DXform

vars = loadmat("/Users/corkep/code/robotics-toolbox-python/data/house.mat", squeeze_me=True, struct_as_record=False)
house = vars['house']
place = vars['place']
# bug = Bug2(house)
# p = bug.query(place.br3, place.kitchen, animate=True)

vars = loadmat("/Users/corkep/code/robotics-toolbox-python/data/map1.mat", squeeze_me=True, struct_as_record=False)
map = vars['map']
bug = Bug2(map)
# # bug.plot()
# p = bug.query([20, 10], [50, 35], animate=True)
# print(p)
# p = bug.query(place.br3, place.kitchen)
# about(p)
# p = bug.query([], place.kitchen)

# bug = Bug2(house), inflate=7)
# p = bug.query(place.br3, place.kitchen, animate=False)

# bug.plot(p, 'inflated')

# ## making a map (sidebar)
# map = zeros(100, 100)
# map(40:50,20:80) = 1
# map = makemap(100)

## map based planning

dx = DXform(house)
dx.plan(place.kitchen)

dx.plot()

p = dx.query(place.br3) #, animate=True)
print(p)

dx.plot(path=p, block=True)

# p = dx.query(place.br3)

# dx.plot(p)

# numrows(p)

# dx.plan(goal, 'animate')

# dx.plot3d(p)

# # inflation
# dx = DXform(house, 'inflate', 5)
# dx.plan(place.kitchen)
# p = dx.query(place.br3)
# dx.plot(p)

# ## Navigation class (sidebar)
# nav = MyNavClass(world)
# nav.plan()
# nav.plan(goal)

# p = nav.query(start, goal)
# p = nav.query(start)
# nav.plot()
# nav.plot(p)

# ## D* planner
# ds = Dstar(house)
# c = ds.costmap()
# ds.plan(place.kitchen)
# ds.niter
# ds.query(place.br3)

# ds.modify_cost( [300,325 115,125], 5 )

# ds.plan()
# ds.niter
# ds.query(place.br3)

# ## Roadmap methods

# free = 1 - house
# free(1,:) = 0 free(100,:) = 0
# free(:,1) = 0 free(:,100) = 0
# skeleton = ithin(free)

# ## 5.2 PRM
# prm = PRM(house)
# randinit
# prm.plan('npoints', 150)
# prm
# prm.plot()

# p = prm.path(place.br3, place.kitchen)
# about p

# ## random number sidebar

# rand
# rand
# rand
# randinit
# rand
# rand

# ## 5.3 Lattice planner

# lp = Lattice()
# lp.plan('iterations', 2)
# lp.plot()

# lp.plan('iterations', 8)
# lp.plot()

# lp.query( [1 2 pi/2], [2 -2 0] )

# lp.plot()

# p = lp.query( [1 2 pi/2], [2 -2 0] )
# about p

# lp.plan('cost', [1 10 10])
# lp.query(start, goal)
# lp.plot()

# load road
# lp = Lattice(road, 'grid', 5, 'root', [50 50 0])
# lp.plan()
# lp.query([30 45 0], [50 20 0])

# ## RRT planner
# car = Bicycle('steermax', 0.5)
# rrt = RRT(car, 'npoints', 1000)

# rrt.plan()
# rrt.plot()

# rrt = RRT(car, road, 'root', [50 22 0], 'npoints', 1000, 'simtime', 4)

# p = rrt.query([40 45 0], [50 22 0])

# about p

# rrt.plot(p)

# plot_vehicle(p, 'box', 'size', [20 30], 'fi ll', 'r', 'alpha', 0.1)

max_stars_repo_name: LabRobPL/robotics-toolbox-python

max_issues_repo_name: LabRobPL/robotics-toolbox-python

max_forks_repo_name: LabRobPL/robotics-toolbox-python

reason: from scipy


text:
import datetime
import os

import numpy as np
from sqlalchemy import Column, DateTime, Integer, MetaData, String
from sqlalchemy.ext.declarative import DeclarativeMeta, declarative_base

from repalette.constants import RGB_IMAGES_DIR

rgb_meta = MetaData()

RGBBase: DeclarativeMeta = declarative_base()  # use DeclarativeMeta to calm down mypy

class RGBImage(RGBBase):
    __tablename__ = "rgb_images"
    id = Column("id", Integer, primary_key=True)
    url = Column("url", String, unique=True)
    name = Column("name", String, default="")
    height = Column("height", Integer, default=0)
    width = Column("width", Integer, default=0)
    created_at = Column(
        "created_at",
        DateTime,
        default=datetime.datetime.utcnow,
    )

    def __init__(
        self,
        np_palette,
        url="",
        name="",
        height=0,
        width=0,
    ):
        self.url = url
        self.name = name
        self.height = height
        self.width = width

        self.set_palette(np_palette)

    def set_palette(self, np_palette):
        """
        Sets colors
        :param np_palette: numpy rgb palette with shape of [1, 6, 3]
        """
        for i, color in enumerate(np_palette.reshape(6, 3)):
            for value, c in zip(color, "rgb"):
                setattr(
                    self,
                    f"color_{i}_{c}",
                    int(value),
                )

    @property
    def palette(self):
        """
        Returns numpy rgb palette with shape of [1, 6, 3]
        """
        palette = []
        for i in range(6):
            color = []
            for c in "rgb":
                color.append(getattr(self, f"color_{i}_{c}"))
            palette.append(color)
        return np.array(palette).reshape(1, 6, 3).astype(np.uint8)

    @property
    def path(self):
        path = os.path.join(RGB_IMAGES_DIR, self.name)
        return path

# add 18 columns for 6 RGB colors - don't want to do it manually :)
for i in range(6):
    for c in "rgb":
        setattr(
            RGBImage,
            f"color_{i}_{c}",
            Column(
                f"color_{i}_{c}",
                Integer,
                default=0,
            ),
        )

max_stars_repo_name: danielgafni/repalette

max_issues_repo_name: danielgafni/repalette

max_forks_repo_name: danielgafni/repalette