text:
# # def hello(input):
# #     output = "Hello " + input + "!!"
# #     print (output)

# # if __name__ == '__main__':
# #     raw_input()
# #     hello("Shang")

# import matplotlib.pyplot as plt
# import numpy as np

# x = np.arange(0,360)
# y = np.sin(x * np.pi / 180.0)
# z = np.sin(x * np.pi / 90.0)

# plt.plot(x,y)
# plt.plot(x,z)
# plt.xlim(-30, 390)
# plt.ylim(-1.5, 1.5)
# plt.xlabel("x-axis")
# plt.ylabel("y-axis")
# plt.title("The Title")

# # plots = plt.plot(x, y)
# # plt.legend(plots, ('Apple', 'Facebook', 'Google'), loc='best', framealpha=0.5, prop={'size': 'large', 'family': 'monospace'})
# plt.grid(True)
# plt.show()

# import math

# while True:
#     output = float(input("Please enter your output: "))

#     if (output - (-12.5)) <= 0:
#         qos_level = 0
#     elif (output - (-12.5)) > 25:
#         qos_level = 11
#     else:
#         qos_level = math.ceil((output - (-12.5))/2.5)

#     print("QoS Level = ", qos_level + 1)

# while True:
#     output = float(input("Please enter your output: "))
#     matrix = [-12.5, -10, -7.5, -5, -2.5, 0, 2.5, 5, 7.5, 10, 12.5, 15]
#     compare = [(output - i) ** 2 for i in matrix]
#     qos_level = compare.index(min(compare))
#     print("QoS Level = ", qos_level + 1)

# import os

# max_x, max_y = 15, 25

# response = float(input("Please enter your response: "))
# allocated_BW = float(input("Please enter your allocated_BW: "))

# if (response >= 0): #給太多，所以這次的搜尋範圍就不超過上次的allocated_BW
#     print("Too Much !")
#     p, q = 0, 0
#     while p <= max_x:
#         while q <= max_y:
#             #if (response >= 7.5 and (p + q < allocated_BW)) or (response < 7.5 and (p + q >= allocated_BW)):
#             print("p =", p, ", q =", q)
#             if (q < allocated_BW and p + q < allocated_BW):
#                 q += 0.25
#             else:
#                 break
#         q = 0
#         if (p < allocated_BW):
#             p += 0.25
#         os.system("pause")

# else: #給太少，所以這次的搜尋範圍就從上次的allocated_BW開始找起
#     print("Too Less !")
#     if (max_y < allocated_BW): #q_max < allocated_BW
#         p = allocated_BW - max_y
#         q = max_y
#         q_pointer = max_y
#     else: #q_max > allocated_BW
#         p = 0
#         q = allocated_BW
#         q_pointer = allocated_BW

#     while p <= max_x:
#         while q <= max_y:
#             #if (response >= 7.5 and (p + q < allocated_BW)) or (response < 7.5 and (p + q >= allocated_BW)):
#             print("p =", p, ", q =", q)
#             q += 0.25
#         q = q_pointer
#         p += 0.25
#         os.system("pause")

import matplotlib.pyplot as plt
import numpy as np
import math

sigma = 1.0
x_array = np.arange(-100, 100, 0.25)
y_array = (np.exp(-(x_array ** 2)/(2 * (sigma ** 2)))) * (1 / (sigma * math.sqrt(2 * math.pi)))
# y_array = (np.exp(-(x_array ** 2)/(sigma ** 2)))

plt.plot(x_array, y_array)
plt.xlim(-10, 10)
plt.ylim(0, 1)
plt.xlabel("x")
plt.ylabel("f(x)")
plt.title("Gaussian distribution")
plt.grid(True)
plt.show()


text:
import pandas as pd
import numpy as np
from datetime import datetime, timedelta
import os
import glob

def coerce_tz(dt):
    try:
        dt = dt.tz_convert('UTC')
    except TypeError:
        dt = dt.tz_localize('UTC')
    return dt

def read_csv(path):
    return pd.read_csv(path, index_col='time', parse_dates=['time'])

def read_data(path, since):
    """
    Read meteo data from csv files.
    :param path: Path to the csv files.
    :param since (datetime): Oldest data needed.
    :return: Dataframe.
    """
    pattern = os.path.join(path, 'meteo_????-??-??.csv')
    files = sorted(list(glob.glob(pattern)))
    since = coerce_tz(pd.to_datetime(since))

    if files:
        df = read_csv(files.pop())
        while df.index.min() > since and files:
            df2 = read_csv(files.pop())
            df = pd.concat([df, df2])
        return df.sort_index()
    return None

def resample(df, freq='10min'):
    groups = df.groupby(pd.Grouper(freq=freq))
    return groups.apply(reduce)

def reduce(df, minmax=False):
    if df.empty:
        return pd.Series(index=df.columns)

    df_min = df.min()
    df_max = df.max()
    df_mean = df.mean()
    df_idxmin = df.idxmin()
    df_idxmax = df.idxmax()

    if minmax:
        index2 = list(df_mean.index).append([
            'air_temperature_min',
            'rel_humidity_min',
            'air_pressure_min',
            'air_temperature_max',
            'rel_humidity_max',
            'air_pressure_max',
        ])
    else:
        index2 = df_mean.index

    df2 = pd.Series(index=index2)

    df2['air_temperature'] = df_mean['air_temperature']
    df2['rel_humidity'] = df_mean['rel_humidity']
    df2['air_pressure'] = df_mean['air_pressure']
    df2['wind_speed_avg'] = df_mean['wind_speed_avg']
    df2['wind_speed_min'] = df_min['wind_speed_min']
    df2['wind_speed_max'] = df_max['wind_speed_max']
    df2['rain_accumulation'] = df_max['rain_accumulation']
    df2['rain_duration'] = df_max['rain_duration']
    df2['rain_intensity'] = df_mean['rain_intensity']
    df2['rain_peak_intensity'] = df_max['rain_peak_intensity']
    df2['heating_temperature'] = df_mean['heating_temperature']

    if minmax:
        df2['air_temperature_min'] = df_min['air_temperature']
        df2['rel_humidity_min'] = df_min['rel_humidity']
        df2['air_pressure_min'] = df_min['air_pressure']
        df2['rain_intensity_min'] = df_min['rain_intensity']
        df2['air_temperature_max'] = df_max['air_temperature']
        df2['rel_humidity_max'] = df_max['rel_humidity']
        df2['air_pressure_max'] = df_max['air_pressure']
        df2['rain_intensity_max'] = df_max['rain_intensity']

    # wind direction is special
    dir_avg = df['wind_dir_avg'].apply(np.deg2rad)
    dir_avg2 = np.arctan2(np.sum(np.sin(dir_avg)), np.sum(np.cos(dir_avg)))
    df2['wind_dir_avg'] = np.rad2deg(dir_avg2)
    df2['wind_dir_min'] = df['wind_dir_min'].loc[df_idxmax['wind_speed_min']]
    df2['wind_dir_max'] = df['wind_dir_max'].loc[df_idxmin['wind_speed_max']]

    return df2

def main():
    p = Path('data')
    stamp = datetime.utcnow()-timedelta(days=3)
    print(stamp)
    df = read_data(p, stamp)
    print('min: {}, max: {}, num: {}'.format(df.index.min(), df.index.max(), len(df)))

    print(reduce(df, minmax=True))
    print()
    print(reduce(df, minmax=True).index)

if __name__ == '__main__':
    main()

ext: py

max_stars_repo_name: tcpluess/meteo-logger-server

max_issues_repo_name: tcpluess/meteo-logger-server

max_forks_repo_name: tcpluess/meteo-logger-server

reason: import numpy


text:
#!/usr/bin/env python
u"""
plot_locations.py
by Yara Mohajerani (12/2018)

Make a plot of the locations of the study sites
"""
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from pyproj import Proj,transform
from matplotlib.font_manager import FontProperties
from mpl_toolkits.basemap import Basemap

def main():
    #-- directory setup
    #- current directory
    current_dir = os.path.dirname(os.path.realpath(__file__))
    headDirectory = os.path.join(current_dir,'..','FrontLearning_data')

    glaciersFolder=os.path.join(headDirectory,'Glaciers')

    glaciers = ['Helheim','Jakobshavn','Kangerlussuaq','Sverdrup']
    h_align = ['left','left','left','left']
    v_space = [2e5,-6e4,2e5,-5e4]

    #-- output projection
    outputCRS = 4326  #(WGS84) #3413
    outProj = Proj(init='epsg:'+str(outputCRS))

    #-- initialize figure
    fig, ax1 = plt.subplots(1,1,figsize=(6,6))

    ll = {'lat': [58,82],'lon': [302,20]}
    cent = {'lat': 72.225676431028518,
        'lon': 316.18118891332665}

    #-- ellipsoid parameters for Basemap transformation
    a_axis = 6378137.0#-- [m] semimajor axis of the ellipsoid
    flat = 1.0/298.257223563#-- flattening of the ellipsoid
    rad_e = a_axis*(1.0 -flat)**(1.0/3.0)

    m = Basemap(ax=ax1, projection='stere',lon_0=cent['lon'], lat_0=cent['lat'], \
                lat_ts=70.0, llcrnrlat=ll['lat'][0], urcrnrlat=ll['lat'][1],\
                llcrnrlon=ll['lon'][0], urcrnrlon=ll['lon'][1],\
                rsphere=rad_e, resolution='h', area_thresh=10)

    #-- draw coastlines
    m.drawcoastlines()

    #-- set up font properties
    font = FontProperties()
    font.set_weight('bold')

    #-- go through each glacier and list the list of months
    for g,a,v in zip(glaciers,h_align,v_space):
        if g == 'Helheim':
            infile = os.path.join(glaciersFolder,g,'%s Image Data_newCorners.csv'%g)
        else:
            infile = os.path.join(glaciersFolder,g,'%s Image Data.csv'%g)
        #-- get corners
        c = pd.read_csv(infile)
        x1,x2,x3,x4 = c['ulX'][0],c['urX'][0],c['lrX'][0],c['llX'][0]
        y1,y2,y3,y4 = c['ulY'][0],c['urY'][0],c['lrY'][0],c['llY'][0]
        #-- get input projection
        inputCRS = c['Projection'][0]
        inProj = Proj(init='epsg:'+str(inputCRS))

        x_mean,y_mean = np.mean([x1,x2,x3,x4]),np.mean([y1,y2,y3,y4])
        x,y = transform(inProj,outProj,x_mean,y_mean)
        x_mean,y_mean = m(x,y)

        ax1.scatter(x_mean,y_mean,s=200,c='dodgerblue',zorder=10)

        #-- set up text location
        if a == 'left':
            x_str = x_mean + 4e4
        else:
            x_str = x_mean - 4e4
        y_str = y_mean - v

        #-- Add glacier name
        ax1.text(x_str, y_str, g, fontproperties=font, verticalalignment='bottom',\
            horizontalalignment=a,color='indigo', fontsize=15)

    plt.axis('off')
    m.drawmapboundary(fill_color='aqua')
    m.fillcontinents(color='peachpuff',lake_color='aqua')

    m.drawparallels(range(0, 90, 10),labels=[1,0,0,0])
    m.drawmeridians(range(0, 360, 20),labels=[0,0,0,1])

    m.drawmapscale(305, 60, cent['lon'], cent['lat'], 400, barstyle='fancy')

    plt.savefig(os.path.join(headDirectory,'Figure_S1-2_map.pdf'),format='pdf')
    plt.close(fig)

if __name__ == '__main__':
    main()

max_stars_repo_name: yaramohajerani/FrontLearning

max_issues_repo_name: yaramohajerani/FrontLearning

max_forks_repo_name: yaramohajerani/FrontLearning


text:
import numpy as np

from sklearn.tree import DecisionTreeClassifier

from sklearn.metrics import roc_auc_score
from sklearn.metrics import recall_score
from sklearn.metrics import precision_score

from skopt import BayesSearchCV
from skopt.space import Integer, Real, Categorical

from lokki.model import ModelChoice

class DecisionTree(ModelChoice):

    def __init__(self):
        pass

    def fit(self, X, y):
        self.model = self.get_model(X, y)

    def predict(self, X):
        return self.model.predict(X)

    def evaluate(self, parameters, X_train, X_test, y_train, y_test):

        model = self.get_model(X_train, y_train)

        score = None
        pred = model.predict(X_test)

        if parameters['metric'] == 'auc':
            score = roc_auc_score(np.array(y_test).astype(bool), pred.astype(bool))

        elif parameters['metric'] == 'precision':
            score = precision_score(np.array(y_test).astype(bool), pred.astype(bool))

        elif parameters['metric'] == 'recall':
            score = recall_score(np.array(y_test).astype(bool), pred.astype(bool))

        return score

    def get_model(self, X, y):
        search_space = {'criterion' : Categorical(['gini', 'entropy']),
                        'splitter'  : Categorical(['best', 'random']),
                        'max_features'  : Categorical(['auto', 'sqrt', 'log2', None]),
                        'min_samples_split' : Integer(2, 100),
                        'max_depth' : Integer(2, 1000)}
        model = BayesSearchCV(DecisionTreeClassifier(random_state = 0), search_space, random_state = 0, n_iter = 1, cv = 3, n_jobs = -1)
        model.fit(X, y)

        return model

    def get_name(self):
        return 'Decision_Tree'

max_stars_repo_name: bzhanglab/Lokki

max_issues_repo_name: bzhanglab/Lokki

max_forks_repo_name: bzhanglab/Lokki


text:
# Copyright (c) 2018 NVIDIA Corporation
from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals
from six.moves import range

import os
import six
import librosa
import numpy as np
import tensorflow as tf
import pandas as pd

from six import string_types

from open_seq2seq.data.data_layer import DataLayer
from open_seq2seq.data.utils import load_pre_existing_vocabulary
from .speech_utils import get_speech_features_from_file, get_mel,\
                          inverse_mel, normalize, denormalize

class Text2SpeechDataLayer(DataLayer):
  """Text-to-speech data layer class
  """

  @staticmethod
  def get_required_params():
    return dict(
        DataLayer.get_required_params(), **{
            'num_audio_features': int,
            'output_type': ['magnitude', 'mel', 'magnitude_disk', 'mel_disk'],
            'vocab_file': str,
            'dataset_location': str,
            'feature_normalize': bool,
        }
    )

  @staticmethod
  def get_optional_params():
    return dict(
        DataLayer.get_optional_params(), **{
            'dataset_files': list,
            'pad_to': int,
            'mag_power': int,
            'pad_EOS': bool,
            'feature_normalize_mean': float,
            'feature_normalize_std': float
        }
    )

  def __init__(self, params, model, num_workers=None, worker_id=None):
    """Text-to-speech data layer constructor.

    See parent class for arguments description.

    Config parameters:

    * **num_audio_features** (int) --- number of audio features to extract.
    * **output_type** (str) --- could be either "magnitude", or "mel".
    * **vocab_file** (str) --- path to vocabulary file.
    * **dataset_files** (list) --- list with paths to all dataset .csv files.
      File is assumed to be separated by "|".
    * **dataset_location** (string) --- string with path to directory where wavs
      are stored.
    * **feature_normalize** (bool) --- whether to normlize the data with a
      preset mean and std
    * **feature_normalize_mean** (bool) --- used for feature normalize.
      Defaults to 0.
    * **feature_normalize_std** (bool) --- used for feature normalize.
      Defaults to 1.
    * **mag_power** (int) --- the power to which the magnitude spectrogram is
      scaled to:
      1 for energy spectrogram
      2 for power spectrogram
      Defaults to 2.
    * **pad_EOS** (bool) --- whether to apply EOS tokens to both the text and
      the speech signal. Will pad at least 1 token regardless of pad_to value.
      Defaults to True.
    * **pad_to** (int) --- we pad such that the resulting datapoint is a
      multiple of pad_to.
      Defaults to 8.

    """
    super(Text2SpeechDataLayer, self).__init__(
        params,
        model,
        num_workers,
        worker_id
    )
    # Character level vocab
    self.params['char2idx'] = load_pre_existing_vocabulary(
        self.params['vocab_file'],
        read_chars=True,
    )
    self.params['idx2char'] = {i: w for w, i in self.params['char2idx'].items()}
    # add one for implied blank token
    self.params['src_vocab_size'] = len(self.params['char2idx']) + 1

    # This assumes that the LJSpeech dataset is used
    if "mel" in self.params["output_type"]:
      self._mel = True
      self._mel_basis = librosa.filters.mel(
          sr=22050, n_fft=1024, n_mels=self.params['num_audio_features']
      )
    else:
      self._mel = False

    # The rest of the code is not needed for interactive infer
    if self.params["interactive"]:
      return

    if "dataset_files" not in self.params:
      raise ValueError("dataset_files parameter has to be specified")

    names = ['wav_filename', 'transcript', 'transcript_normalized']

    if "disk" in self.params["output_type"]:
      self._load_from_disk = True
    else:
      self._load_from_disk = False

    # Load csv files
    self._files = None
    for csvs in params['dataset_files']:
      files = pd.read_csv(
          csvs,
          encoding='utf-8',
          sep='\x7c',
          header=None,
          names=names,
          quoting=3
      )
      if self._files is None:
        self._files = files
      else:
        self._files = self._files.append(files)

    if self.params['mode'] != 'infer':
      cols = ['wav_filename', 'transcript_normalized']
    else:
      cols = 'transcript_normalized'

    all_files = self._files.loc[:, cols].values
    self._files = self.split_data(all_files)

    self._size = self.get_size_in_samples()
    self._dataset = None
    self._iterator = None
    self._input_tensors = None

  def split_data(self, data):
    if self.params['mode'] != 'train' and self._num_workers is not None:
      size = len(data)
      start = size // self._num_workers * self._worker_id
      if self._worker_id == self._num_workers - 1:
        end = size
      else:
        end = size // self._num_workers * (self._worker_id + 1)
      return data[start:end]
    else:
      return data

  @property
  def iterator(self):
    return self._iterator

  def build_graph(self):
    """Builds data reading graph."""
    self._dataset = tf.data.Dataset.from_tensor_slices(self._files)
    if self.params['shuffle']:
      self._dataset = self._dataset.shuffle(self._size)
    self._dataset = self._dataset.repeat()

    if self.params['mode'] != 'infer':
      self._dataset = self._dataset.map(
          lambda line: tf.py_func(
              self._parse_audio_transcript_element,
              [line],
              [tf.int32, tf.int32, self.params['dtype'], self.params['dtype'],\
               tf.int32],
              stateful=False,
          ),
          num_parallel_calls=8,
      )
      self._dataset = self._dataset.padded_batch(
          self.params['batch_size'],
          padded_shapes=(
              [None], 1, [None, self.params['num_audio_features']], [None], 1
          ),
          padding_values=(
              0, 0, tf.cast(0., dtype=self.params['dtype']),
              tf.cast(1., dtype=self.params['dtype']), 0
          )
      )
    else:
      self._dataset = self._dataset.map(
          lambda line: tf.py_func(
              self._parse_transcript_element,
              [line],
              [tf.int32, tf.int32],
              stateful=False,
          ),
          num_parallel_calls=8,
      )
      self._dataset = self._dataset.padded_batch(
          self.params['batch_size'], padded_shapes=([None], 1)
      )

    self._iterator = self._dataset.prefetch(tf.contrib.data.AUTOTUNE)\
                                  .make_initializable_iterator()

    if self.params['mode'] != 'infer':
      text, text_length, spec, stop_token_target, spec_length = self._iterator\
                                                                    .get_next()
      # need to explicitly set batch size dimension
      # (it is employed in the model)
      spec.set_shape(
          [self.params['batch_size'], None, self.params['num_audio_features']]
      )
      stop_token_target.set_shape([self.params['batch_size'], None])
      spec_length = tf.reshape(spec_length, [self.params['batch_size']])
    else:
      text, text_length = self._iterator.get_next()
    text.set_shape([self.params['batch_size'], None])
    text_length = tf.reshape(text_length, [self.params['batch_size']])

    self._input_tensors = {}
    self._input_tensors["source_tensors"] = [text, text_length]
    if self.params['mode'] != 'infer':
      self._input_tensors['target_tensors'] = [
          spec, stop_token_target, spec_length
      ]

  def _parse_audio_transcript_element(self, element):
    """Parses tf.data element from TextLineDataset into audio and text.

    Args:
      element: tf.data element from TextLineDataset.

    Returns:
      tuple: text_input text as `np.array` of ids, text_input length,
      target audio features as `np.array`, stop token targets as `np.array`,
      length of target sequence,
      .
    """
    audio_filename, transcript = element
    if not six.PY2:
      transcript = str(transcript, 'utf-8')
    transcript = transcript.lower()
    text_input = np.array(
        [self.params['char2idx'][c] for c in unicode(transcript, "utf-8")]
    )
    pad_to = self.params.get('pad_to', 8)
    if self.params.get("pad_EOS", True):
      # num_pad = pad_to - (len(text_input) % pad_to)
      num_pad = pad_to - ((len(text_input) + 1) % pad_to) + 1
      text_input = np.pad(
          text_input, ((0, num_pad)),
          "constant",
          constant_values=self.params['char2idx']["~"]
      )
    if self._load_from_disk:
      file_path = os.path.join(
          self.params['dataset_location'], audio_filename + ".npy"
      )
      spectrogram = np.load(file_path)
      mag_power = self.params.get('mag_power', 2)
      if self._mel:
        spectrogram = get_mel(
            spectrogram,
            power=mag_power,
            feature_normalize=self.params["feature_normalize"],
            mean=self.params.get("feature_normalize_mean", 0.),
            std=self.params.get("feature_normalize_std", 1.),
            mel_basis=self._mel_basis,
            n_mels=self.params['num_audio_features']
        )
      else:
        if mag_power != 1:
          spectrogram = spectrogram * mag_power
          spectrogram = np.clip(spectrogram, a_min=np.log(1e-5), a_max=None)
        # Else it is a magnitude spec, and we need to normalize
        if self.params["feature_normalize"]:
          spectrogram = normalize(
              spectrogram,
              mean=self.params.get("feature_normalize_mean", 0.),
              std=self.params.get("feature_normalize_std", 0.)
          )
    else:
      file_path = os.path.join(
          self.params['dataset_location'], audio_filename + ".wav"
      )
      spectrogram = get_speech_features_from_file(
          file_path,
          self.params['num_audio_features'],
          features_type=self.params['output_type'],
          mag_power=self.params.get('mag_power', 2),
          feature_normalize=self.params["feature_normalize"],
          mean=self.params.get("feature_normalize_mean", 0.),
          std=self.params.get("feature_normalize_std", 1.)
      )
    stop_token_target = np.zeros(
        [len(spectrogram)], dtype=self.params['dtype'].as_numpy_dtype()
    )
    if self.params.get("pad_EOS", True):
      # num_pad = pad_to - (len(spectrogram) % pad_to)
      num_pad = pad_to - ((len(spectrogram) + 1) % pad_to) + 1
      spectrogram = np.pad(
          spectrogram, ((0, num_pad), (0, 0)), "constant", constant_values=0
      )
      stop_token_target = np.pad(
          stop_token_target, ((0, num_pad)), "constant", constant_values=1
      )
    else:
      stop_token_target[-1] = 1.

    assert len(text_input) % pad_to == 0
    assert len(spectrogram) % pad_to == 0
    return np.int32(text_input), \
           np.int32([len(text_input)]), \
           spectrogram.astype(self.params['dtype'].as_numpy_dtype()), \
           stop_token_target.astype(self.params['dtype'].as_numpy_dtype()), \
           np.int32([len(spectrogram)])

  def _parse_transcript_element(self, transcript):
    """Parses text from file and returns array of text features.

    Args:
      transcript: the string to parse.

    Returns:
      tuple: target text as `np.array` of ids, target text length.
    """
    if not six.PY2:
      transcript = str(transcript, 'utf-8')
    transcript = transcript.lower()
    text_input = np.array(
        [self.params['char2idx'][c] for c in unicode(transcript, "utf-8")]
    )
    pad_to = self.params.get('pad_to', 8)
    if self.params.get("pad_EOS", True):
      # num_pad = pad_to - (len(text_input) % pad_to)
      num_pad = pad_to - ((len(text_input) + 1) % pad_to) + 1
      text_input = np.pad(
          text_input, ((0, num_pad)),
          "constant",
          constant_values=self.params['char2idx']["~"]
      )

    return np.int32(text_input), \
           np.int32([len(text_input)])

  def create_interactive_placeholders(self):
    self._text = tf.placeholder(
        dtype=tf.int32,
        shape=[self.params["batch_size"], None]
    )
    self._text_length = tf.placeholder(
        dtype=tf.int32,
        shape=[self.params["batch_size"]]
    )

    self._input_tensors = {}
    self._input_tensors["source_tensors"] = [self._text, self._text_length]

  def create_feed_dict(self, model_in):
    """ Creates the feed dict for interactive infer

    Args:
      model_in (str): The string to be spoken.

    Returns:
      feed_dict (dict): Dictionary with values for the placeholders.
    """
    if not isinstance(model_in, string_types):
      raise ValueError(
          "Text2Speech's interactive inference mode only supports string.",
          "Got {}". format(type(model_in))
      )
    text, text_length = self._parse_transcript_element(model_in)

    text = np.reshape(text, [self.params["batch_size"], -1])
    text_length = np.reshape(text_length, [self.params["batch_size"]])

    feed_dict = {
        self._text: text,
        self._text_length: text_length,
    }
    return feed_dict

  @property
  def input_tensors(self):
    return self._input_tensors

  def get_size_in_samples(self):
    """Returns the number of audio files."""
    return len(self._files)

  def get_magnitude_spec(self, spectrogram):
    """Returns an energy magnitude spectrogram. The processing depends on the
    data leyer params.

    Args:
      spectrogram: output spec from model

    Returns:
      mag_spec: mag spec
    """
    spectrogram = spectrogram.astype(float)
    if self._mel:
      return inverse_mel(
          spectrogram,
          n_mels=self.params['num_audio_features'],
          power=self.params.get('mag_power', 2),
          feature_normalize=self.params["feature_normalize"],
          mean=self.params.get("feature_normalize_mean", 0.),
          std=self.params.get("feature_normalize_std", 1.),
          mel_basis=self._mel_basis
      )
    else:
      if self.params["feature_normalize"]:
        spectrogram = self._denormalize(spectrogram)
      spectrogram = spectrogram * 1.0 / self.params.get('mag_power', 2)
      mag_spec = np.exp(spectrogram)
      return mag_spec

  def _denormalize(self, spectrogram):
    return denormalize(
        spectrogram,
        mean=self.params.get("feature_normalize_mean", 0.),
        std=self.params.get("feature_normalize_std", 1.)
    )


text:
import numpy as np
import time
import mmap
import cosFuncs as cf
import WWio
from scipy.spatial import cKDTree
from utils import CalculateMerit,MergeHalo

def StartTrack(opt,snap,trackIndx,trackMergeDesc,trackDispFlag,allpid,allpartpos,allpartvel,partOffsets,GadHeaderInfo,snapdata,treedata,TrackData,pidOffset,WWstat):

	#Find the physical boxsize
	boxsize = GadHeaderInfo["BoxSize"]*GadHeaderInfo["Scalefactor"]/GadHeaderInfo["h"]

	keepPIDS = np.zeros(len(allpid),dtype = bool)
	newPartOffset=pidOffset
	newPartOffsets=[]

	for i,Indx in enumerate(trackIndx):
		#Get the particle offset
		partOffset=partOffsets[i]

		#Extract all the required properties
		halopos=np.asarray([snapdata["Xc"][Indx],snapdata["Yc"][Indx],snapdata["Zc"][Indx]]) * snapdata["UnitInfo"]["Length_unit_to_kpc"]/1000 #Mpc
		halovel=np.asarray([snapdata["VXc"][Indx],snapdata["VYc"][Indx],snapdata["VZc"][Indx]]) * snapdata["UnitInfo"]["Velocity_unit_to_kms"]
		conc=snapdata["cNFW"][Indx]
		Mass_200crit=snapdata["Mass_200crit"][Indx]*snapdata["UnitInfo"]["Mass_unit_to_solarmass"]
		R_200crit=snapdata["R_200crit"][Indx] * snapdata["UnitInfo"]["Length_unit_to_kpc"]/1000 #Mpc
		npart =snapdata["npart"][Indx]

		#Find the id and index of the host
		if(snapdata["hostHaloID"][Indx]>-1):

			Hostindx=int(snapdata["hostHaloID"][Indx]%opt.Temporal_haloidval-1)
			tmpHostHead=treedata["Descen"][Hostindx]

			#Lets check this host halo is in thr next snapshot
			hostHeadSnap = int(tmpHostHead/opt.Temporal_haloidval)
			if(hostHeadSnap==snap+1):
				hostHead=tmpHostHead
			else:
				hostHead=-1

		else:
			hostHead=-1

		partpos = allpartpos[partOffset:partOffset+npart]
		partvel = allpartvel[partOffset:partOffset+npart]

		#Correct for the periodicity
		for j in range(3):
			if(np.ptp(partpos[:,j])>0.5*boxsize):

				#Find which boundary the halo lies
				boundary = -boxsize if(halopos[j]<0.5*boxsize) else boxsize

				# prevpos = np.mean(partpos[:,0][TrackData["boundSel"][i]])
				sel = abs(partpos[:,j]-halopos[j])>0.5*boxsize

				partpos[:,j][sel] = partpos[:,j][sel] + boundary

		#Get the radial posistions and velocities relative to the halo's
		r=np.sqrt((partpos[:,0]-halopos[0])**2 + (partpos[:,1]-halopos[1])**2 + (partpos[:,2]-halopos[2])**2)
		vr=np.sqrt((partvel[:,0]-halovel[0])**2 + (partvel[:,1]-halovel[1])**2 + (partvel[:,2]-halovel[2])**2)

		#Calculate the circular velocities for the particles in the halo
		Vesc_calc=cf.cosNFWvesc(r, Mass_200crit, conc,z = GadHeaderInfo["Redshift"],f=1,Munit=1,Dist="Ang")

		#Iterate on posistion until less than 100 particles or tracking less than 10% of the halos paricles
		j=0

		mbpsel = (vr<Vesc_calc*0.8/(2**j)) & ((r/R_200crit)<0.8/(2**j))
		while(True):
			mbpnpart=np.sum(mbpsel)
			if((mbpnpart<100) | ((mbpnpart/npart)<0.1)):
				if((j>1) & (mbpnpart<10)):
					# print("Iterated to core with less than 10 mpb",np.sum(mbpsel),"returning to the previous selection of",np.sum(tmpMBPsel),"from a",npart,"size halo")
					mbpsel = tmpMBPsel
				break

			tmpMBPsel = mbpsel
			mbpsel=(vr/Vesc_calc<0.8/(2**j) ) & ((r/R_200crit)<0.8/(2**j))
			j+=1

		#Check to see if the halo can still be tracked
		if(mbpnpart<=5):
			WWstat["NoStart"]+=1
			# print(i,"There are less than 10 highly bound particles to track posistion terminating the tracking")
			continue
		WWstat["Start"]+=1

		#If it can be tracked then add it to the tracked data to be tracked
		TrackData["progenitor"].append(snapdata["ID"][Indx])

		#Lets see if the halo is tracked until dispersed or filling in of a gap
		TrackData["prevpos"].append(halopos)
		TrackData["TrackDisp"].append(trackDispFlag[i])
		TrackData["endDesc"].append(trackMergeDesc[i])
		TrackData["mbpSel"].append(mbpsel)
		TrackData["boundSel"].append(np.ones(npart,dtype=bool))
		TrackData["Conc"].append(conc)
		TrackData["host"].append(hostHead)
		TrackData["CheckMerged"].append({})
		TrackData["TrackedNsnaps"].append(0)
		TrackData["idel"].append(0)
		TrackData["Rvir"].append(R_200crit)
		TrackData["Mvir"].append(Mass_200crit)

		#Update the keepPIDS array so the PID is kept
		keepPIDS[partOffset:partOffset+npart]=True

		#Update the offsets
		newPartOffsets.append(np.uint64(newPartOffset))

		#Add to the offset for the next halo
		newPartOffset+=npart

	startpids = allpid[keepPIDS]

	return newPartOffsets,startpids

def ContinueTrack(opt,snap,TrackData,allpid,allpartpos,allpartvel,partOffsets,snapdata,treedata,progenBool,filenumhalos,pfiles,upfiles,grpfiles,GadHeaderInfo,appendHaloData,appendTreeData,prevappendTreeData,prevupdateTreeData,prevNhalo,WWstat,treeOpt):

	#Set the size of the box
	boxsize=snapdata["SimulationInfo"]["Period"]

	#Find rho crit and use that to find R200crit and M200crit
	rhocrit = cf.cosgrowRhoCrit(z = GadHeaderInfo["Redshift"])
	rhomean = cf.cosgrowRhoMean(z = GadHeaderInfo["Redshift"])

	keepPIDS = np.zeros(len(allpid),dtype = bool)
	newPartOffset=0
	newPartOffsets=[]

	imerittype=6

	#Create a spatial kdtree to find the nearest centrals neighbours to the halo
	pos=np.array([snapdata["Xc"],snapdata["Yc"],snapdata["Zc"]]) * snapdata["UnitInfo"]["Length_unit_to_kpc"]/1000 #Mpc
	pos_tree=cKDTree(pos.T,boxsize=boxsize)

	for i in range(len(TrackData["progenitor"])):

		partOffset = partOffsets[i]
		totnpart = np.uint64(TrackData["boundSel"][i].size)

		partpos = allpartpos[partOffset:partOffset+totnpart]
		partvel = allpartvel[partOffset:partOffset+totnpart]
		partIDs = allpid[partOffset:partOffset+totnpart]

		#Correct for the periodicity
		for j in range(3):
			if(np.ptp(partpos[:,j][TrackData["boundSel"][i]])>0.5*boxsize):

				#Find which boundary the core lies
				boundary = -boxsize if(TrackData["prevpos"][i][j]<0.5*boxsize) else boxsize

				# prevpos = np.mean(partpos[:,0][TrackData["boundSel"][i]])
				sel = (abs(partpos[:,j]-TrackData["prevpos"][i][j])>0.5*boxsize) & (TrackData["boundSel"][i])

				partpos[:,j][sel] = partpos[:,j][sel] + boundary

		#Find the average position and velocities of the most bound particles
		mbppos=partpos[TrackData["mbpSel"][i]]
		meanpos=np.mean(mbppos,axis= 0)
		mbpvel=partvel[TrackData["mbpSel"][i]]
		meanvel=np.mean(mbpvel,axis=0)

		#Update the prevoius posistion of the halo
		TrackData["prevpos"][i] = meanpos

		#Store the previous bound selection
		prevboundSel = TrackData["boundSel"][i].copy()

		#Get the radial posistions and velocities relative to the halo's bound particles
		r=np.sqrt(np.sum((partpos-meanpos)**2,axis=1))
		vr=np.sqrt(np.sum((partvel-meanvel)**2,axis=1))

		#Make any zeros be very small number so don't get errors when dividing by it
		r[r==0] = 1e-5

		#Find the R_200crit and Mass_200crit from rhocrit
		R_200crit=cf.cosFindR200(r[TrackData["boundSel"][i]][r[TrackData["boundSel"][i]]<15*np.median(r[TrackData["boundSel"][i]])],rhocrit,200,GadHeaderInfo["partMass"]*1e10)
		Mass_200crit=cf.coshaloRvirToMvir(R_200crit,rhocrit,200,Munit=1)

		#Calculate the escape velocity from the potential
		Vesc_calc=cf.cosNFWvesc(r, Mass_200crit, c=TrackData["Conc"][i], z = GadHeaderInfo["Redshift"],f=1,Munit=1,Dist="Ang")

		#Find the particles which are bound to the halo
		boundSel = (vr<Vesc_calc) & (r<R_200crit)
		npart=np.sum(boundSel)

		# Check if the halo has gone below the particle limit or is diffuse so its mass has gone above 2 times the initial mass from VELOCIraptor
		if((npart<=20) | (2*TrackData["Mvir"][i]<Mass_200crit)):

			if(TrackData["TrackedNsnaps"][i]>0):
				#Get the index of the WW halo in the previous snapshot
				progen=TrackData["progenitor"][i]
				progenIndx = int(TrackData["progenitor"][i]%opt.Temporal_haloidval-1)

				progenIndx = progenIndx - prevNhalo

				#Find the halo it has merged with if it has been lost using the particles that were previously bound to the halo
				if(TrackData["TrackDisp"][i]):
					MergeHalo(opt,treeOpt,meanpos,partIDs[TrackData["boundSel"][i]],progenIndx,snapdata,TrackData["host"][i],filenumhalos,pfiles,upfiles,grpfiles,prevappendTreeData,pos_tree,WWstat)

				#If not to be tracked until dispersed then connect it up with its endDesc
				else:
					prevappendTreeData["Descendants"][progenIndx] = TrackData["endDesc"][i]
					prevappendTreeData["NumDesc"][progenIndx] = 1

					#Since it is not possible to easily extract the endDesc particles (which could be many snapshots away)
					#the merit is just set to 1.0 for these halos
					prevappendTreeData["Merits"][progenIndx] = 1.0
					WWstat["ConnectPartLimit"]+=1
			else:
				WWstat["PartLimitStart"]+=1

			#Mark the halo to be deleted
			TrackData["idel"][i] = 1
			continue

		# Update the boundSel in the track data
		TrackData["boundSel"][i]=boundSel

		#Iterate on posistion until less than 100 particles or tracking less than 10% of the halos paricles
		j=0
		mbpsel = (vr/Vesc_calc<0.8/(2**j) ) & ((r/R_200crit)<0.8/(2**j))
		while(True):
			if((np.sum(mbpsel)<100) | ((np.sum(mbpsel)/npart)<0.1)):

				#Check if we have iterated to a core with less than 10 bound particles then return to the previous selection
				if((j>1) & (np.sum(mbpsel)<10)):
					# print("Iterated to core with less than 10 mpb",np.sum(mbpsel),"returning to the previous selection of",np.sum(tmpMBPsel),"from a",npart,"size halo")
					mbpsel = tmpMBPsel
				break

			tmpMBPsel = mbpsel
			mbpsel=(vr/Vesc_calc<0.8/(2**j) ) & ((r/R_200crit)<0.8/(2**j))
			j+=1

		# Update the mbpSel for the track data
		TrackData["mbpSel"][i] = mbpsel

		nmbpsel = np.sum(mbpsel)
		boundIDs = np.zeros(npart,dtype=np.uint64)

		#Order the IDs first the mbp radially sorted and then the less bound particles also radially sorted
		boundIDs[:nmbpsel] = partIDs[TrackData["mbpSel"][i]][np.argsort(r[TrackData["mbpSel"][i]])]
		boundIDs[nmbpsel:] = partIDs[(~TrackData["mbpSel"][i]) & (TrackData["boundSel"][i])][np.argsort(r[(~TrackData["mbpSel"][i]) & (TrackData["boundSel"][i])])]
		unboundIDs = partIDs[(~TrackData["mbpSel"][i]) & (~TrackData["boundSel"][i])]

		#Only try to match to a VELOCIraptor halo if this halo is to be tracked until it is dispersed
		if(TrackData["TrackDisp"][i]):

			#Find all the halos within 0.6 R200crit
			indx_list=np.sort(pos_tree.query_ball_point(meanpos,R_200crit))

			#Lets check if it has any matches
			if(indx_list.size):

				#Store all the WhereWolf merit, Matched merits and MatchedID's for
				meritList = []
				MatchedIDList = []
				start=time.time()

				matched=False

				#Open up the first VELOCIraptor file
				fileno=0
				offset = 0

				#If doing core matching finc out how many particles to match in the core of the WW halo
				if((treeOpt["Core_fraction"]<1.0) & (treeOpt["Core_fraction"]>0.0)):
					WWCoreNpart = np.max([np.rint(treeOpt["Core_fraction"]*npart),treeOpt["Core_min_number_of_particles"]])
					WWCoreNpart = np.min([npart,WWCoreNpart]).astype(int)

				#Loop over all the matches
				for indx in indx_list:

					# if(halodata["hostHaloID"][indx]>-1):
					# 	continue

					MatchedID=snapdata["ID"][indx]
					if("Descen" in treedata.keys()):
						MatchedDesc = treedata["Descen"][indx]
						#Lets also extract the merit for the matched halo and lets see if this has a better match
					else:
						break

					#Add the halo to the check merged dict if it does not exist to see how long it has been phased mixed
					if(MatchedID not in TrackData["CheckMerged"][i].keys()):
						TrackData["CheckMerged"][i][MatchedID]=0

					#Loop over the VELOCIraptor files to find which one it is in
					while((indx+1)>(offset + filenumhalos[fileno])):
						offset+=filenumhalos[fileno]
						fileno+=1

					#Get the matched halo particles and properties
					matched_partIDs = WWio.GetHaloParticles(grpfiles[fileno],pfiles[fileno],upfiles[fileno],int(indx-offset))

					######## First match all particles to all particles  ########
					merit = CalculateMerit(treeOpt,boundIDs,matched_partIDs)

					#Only add it if the halo has no progenitor
					if(progenBool[indx]==False):
						meritList.append(merit)
						MatchedIDList.append(MatchedID)

					Matchedx=snapdata["Xc"][indx]*snapdata["UnitInfo"]["Length_unit_to_kpc"]/1000
					Matchedy=snapdata["Yc"][indx]*snapdata["UnitInfo"]["Length_unit_to_kpc"]/1000
					Matchedz=snapdata["Zc"][indx]*snapdata["UnitInfo"]["Length_unit_to_kpc"]/1000
					MatchedR_200crit=snapdata["R_200crit"][indx]*snapdata["UnitInfo"]["Length_unit_to_kpc"]/1000

					#Find the ratio of how far away the halo is to see if it is within 0.1Rvir
					ratioradius2 = ((meanpos[0]-Matchedx)**2 + (meanpos[1]-Matchedy)**2 + (meanpos[2]-Matchedz)**2)/(MatchedR_200crit*MatchedR_200crit)

					if((ratioradius2<0.01) & (snapdata["Mass_200crit"][indx]*snapdata["UnitInfo"]["Mass_unit_to_solarmass"]>0.5*Mass_200crit)):
						#If it is within 0.1 Rvir add one to the counter
						TrackData["CheckMerged"][i][MatchedID]+=1

						#If it is within 0.1 Rvir more than 3 times stop the tracking
						if(TrackData["CheckMerged"][i][MatchedID]==opt.NumSnapsWithinCoreMerge):

							progen=TrackData["progenitor"][i]

							progenIndx = int(TrackData["progenitor"][i]%opt.Temporal_haloidval-1)
							progenIndx = progenIndx - prevNhalo
							prevappendTreeData["Descendants"][progenIndx]= MatchedID
							prevappendTreeData["NumDesc"][progenIndx] = 1
							prevappendTreeData["Ranks"][progenIndx] = 1

							#Loop over the VELOCIraptor files to find which one it is in
							while((indx+1)>(offset + filenumhalos[fileno])):
								offset+=filenumhalos[fileno]
								fileno+=1
							#Calculate the merit between the halos
							prevappendTreeData["Merits"][progenIndx] = merit

							WWstat["Mixed"]+=1

							matched = True

							break

						#Now found a match need to update the ID in the CheckMerged dictonary to point to the descendant but only if not at the final snapshot
						if("Descen" in treedata.keys()):
							TrackData["CheckMerged"][i][MatchedDesc] = TrackData["CheckMerged"][i].pop(MatchedID)

					else:
						#If it goes outside of 0.1Rvir then delete the entry
						TrackData["CheckMerged"][i].pop(MatchedID)

				#The connection to the VELOCIraptor halo is made if the halo has no connection and the WhereWolf halo's merit is above the merit limit

				#First lets find which ones have no connection and are above the merit limit
				meritList=np.asarray(meritList,dtype=np.float32)
				Sel = meritList>treeOpt["Merit_limit_for_next_step"]

				#Find if there are any halos which satisfy the above conditions
				if(np.sum(Sel)):

					#Use a masked array to find the argmax
					maxindx = np.argmax(meritList)

					progen=TrackData["progenitor"][i]

					#Only update if this halo has been tracked for at least 1 snapshot
					if(TrackData["TrackedNsnaps"][i]>0):
						progenIndx = int(TrackData["progenitor"][i]%opt.Temporal_haloidval-1)
						progenIndx = progenIndx - prevNhalo
						prevappendTreeData["Descendants"][progenIndx]= MatchedIDList[maxindx]
						prevappendTreeData["NumDesc"][progenIndx] = 1
						prevappendTreeData["Merits"][progenIndx] = meritList[maxindx]
						WWstat["Matched"]+=1

					matched = True

				# #Remove the indexes where the MatchedMeritList==0 so where halos don't have any connections
				# meritList = meritList[MatchedMeritList>0]
				# MatchedIDList = MatchedIDList[MatchedMeritList>0]
				# MatchedMeritList = MatchedMeritList[MatchedMeritList>0]

				# #Since the halo has a connection(s) lets compare them and see if WhereWolf's connection is 50% better and if the existing merit is poor <0.1
				# Sel = (((meritList-MatchedMeritList)/MatchedMeritList)>0.5) & (MatchedMeritList<0.1)

				# #Find if there are any halos which satisfy the above conditions
				# if(np.sum(Sel)):

				# 	#Use a masked array to find the argmax
				# 	maxindx = meritList.argmax()

				# 	progen=TrackData["progenitor"][i]

				# 	print(meritList[maxindx],MatchedMeritList[maxindx],npart,MatchedIDList[maxindx])

				# 	#Only update if this halo has been tracked for at least 1 snapshot
				# 	if(TrackData["TrackedNsnaps"][i]>0):
				# 		progenIndx = int(TrackData["progenitor"][i]%opt.Temporal_haloidval-1)
				# 		progenIndx = progenIndx - prevNhalo
				# 		prevappendTreeData["Descendants"][progenIndx]= MatchedIDList[maxindx]
				# 		prevappendTreeData["NumDesc"][progenIndx] = 1
				# 		prevappendTreeData["Merits"][progenIndx] = meritList[maxindx]
				# 		WWstat["Switched"]+=1

				# 	matched = True

				if(matched):
					TrackData["idel"][i] = 1
					continue

		#From the caustic line can see if particles are bound to the halo
		ID=(snap+opt.Snapshot_offset)*opt.Temporal_haloidval +(len(snapdata["ID"]) + len(appendHaloData["ID"])) +1

		# Calculate properties of the halo
		R_200mean=cf.cosFindR200(r[TrackData["boundSel"][i]][r[TrackData["boundSel"][i]]<15*np.median(r[TrackData["boundSel"][i]])],rhomean,200,GadHeaderInfo["partMass"]*1e10)
		Mass_200mean=cf.coshaloRvirToMvir(R_200mean,rhomean,200,Munit=1)
		Mass_tot = npart * GadHeaderInfo["partMass"]*1e10 #Units of Msun
		sigV = np.mean(np.sqrt((vr-np.mean(vr))**2))
		Vmax = vr[vr.argmax()]
		Rmax = r[r.argmax()]

		# Now lets append the data to the lists
		appendHaloData["ID"].append(ID)
		appendHaloData["Mass_200crit"].append(Mass_200crit)
		appendHaloData["Mass_200mean"].append(Mass_200mean)
		appendHaloData["Mass_tot"].append(Mass_tot)
		appendHaloData["R_200crit"].append(R_200crit)
		appendHaloData["R_200mean"].append(R_200mean)
		appendHaloData["npart"].append(npart)
		appendHaloData["Xc"].append(meanpos[0])
		appendHaloData["Yc"].append(meanpos[1])
		appendHaloData["Zc"].append(meanpos[2])
		appendHaloData["VXc"].append(meanvel[0])
		appendHaloData["VYc"].append(meanvel[1])
		appendHaloData["VZc"].append(meanvel[2])
		appendHaloData["sigV"].append(sigV)
		appendHaloData["Vmax"].append(Vmax)
		appendHaloData["Rmax"].append(Rmax)
		appendHaloData["Group_Size"].append(npart)
		appendHaloData["Particle_IDs"].extend(boundIDs.tolist())
		appendHaloData["Offset"].append(appendHaloData["Offset"][-1]+npart)
		appendHaloData["Offset_unbound"].append(0)
		appendHaloData["Num_of_groups"]+=1

		#Lets store the previous host of this halo
		prevhost = TrackData["host"][i]

		#Lets see if the halo has a host if not then we can try and set one
		if((TrackData["host"][i]>0) & (int(TrackData["host"][i]/opt.Temporal_haloidval)==snap+opt.Snapshot_offset)):
				appendHaloData["Parent_halo_ID"].append(int(TrackData["host"][i]%opt.Temporal_haloidval+1))
				appendHaloData["hostHaloID"].append(TrackData["host"][i])

				#Only need to do this if not at the final snapshot
				if("Descen" in treedata.keys()):
					HostIndx=int(TrackData["host"][i]%opt.Temporal_haloidval -1)
					hostHeadID=treedata["Descen"][HostIndx]

					#Lets check this host halo is in thr next snapshot
					hostHeadSnap = int(hostHeadID/opt.Temporal_haloidval)

					#Update the host in the trackdata
					if((TrackData["host"][i]!=hostHeadID) & (hostHeadSnap==snap+1)):
						TrackData["host"][i]=hostHeadID
					else:
						TrackData["host"][i]=-1
		else:
			#Check if the halo has a new host by first searching the 100 closest halos
			dists,indexes=pos_tree.query(meanpos,100)

			# Loop over the closet matches
			for dist,indx in zip(dists,indexes):
				hostR200 = snapdata["R_200crit"][indx]*snapdata["UnitInfo"]["Length_unit_to_kpc"]/1000 #Mpc
				hostM200 = snapdata["Mass_200crit"][indx]*snapdata["UnitInfo"]["Mass_unit_to_solarmass"]

				#Check if the WhereWolf halo is within the viral radius of the halo
				ratioradius = dist/hostR200
				if(ratioradius<1.0):

					#If it is within the viral radius check if it is gravitationally bound
					v2=np.sum(meanvel**2)
					G = 4.30211349e-9 #Mpc  Msun^-1 (km/s)^2
					boundCal=0.5 * Mass_200crit * v2 - (G * hostM200 * Mass_200crit)/dist
					if((boundCal<0.0) & (Mass_200crit<hostM200)):
						#If gravitationall bound then set is as the host and subtract the mass from it of the WW halo
						appendHaloData["hostHaloID"].append(snapdata["ID"][indx])
						appendHaloData["Parent_halo_ID"].append(indx+1)

						#Lets uppdate the host to point to this halos head if its in the next snapshot
						if("Descen" in treedata.keys()):
							HostIndx=int(snapdata["ID"][indx]%opt.Temporal_haloidval -1)
							hostHeadID=treedata["Descen"][HostIndx]

							#Lets check this host halo is in thr next snapshot
							hostHeadSnap = int(hostHeadID/opt.Temporal_haloidval)
							if(hostHeadSnap==snap+1):
								TrackData["host"][i]=hostHeadID
							else:
								TrackData["host"][i] = -1

						break

			#If there was nothing found set it to -1
			else:
				appendHaloData["hostHaloID"].append(-1)
				appendHaloData["Parent_halo_ID"].append(-1)

		#Set the descendant of this halos progenitor to point to this halo
		progen=TrackData["progenitor"][i]

		#Add in a entry in the tree for this halo
		appendTreeData["ID"].append(ID)
		appendTreeData["Descendants"].append(np.int64(0))# Will be updated in the next snapshot
		appendTreeData["Ranks"].append(0)
		appendTreeData["NumDesc"].append(0)
		appendTreeData["Merits"].append(0)

		# #For debugging
		# appendTreeData["endDesc"].append(TrackData["endDesc"][i])

		if(TrackData["TrackedNsnaps"][i]==0):
			prevupdateTreeData["ID"].append(progen)
			prevupdateTreeData["Descendants"].append(ID)

			#Calculate its merit with the particles in the original VELOCIraptor halo
			#to the ones which are currently bound to the halo
			merit = CalculateMerit(treeOpt,partIDs[TrackData["boundSel"][i]],partIDs)
			prevupdateTreeData["Merits"].append(merit)

		else:

			progenIndx = int(progen%opt.Temporal_haloidval-1)

			progenIndx = progenIndx - prevNhalo
			prevappendTreeData["Descendants"][progenIndx]= ID
			prevappendTreeData["NumDesc"][progenIndx] = 1

			#Now calculate the merit between the particles that were previously bound to the halo
			#to the ones that were previously bound
			merit = CalculateMerit(treeOpt,partIDs[TrackData["boundSel"][i]],partIDs[prevboundSel])
			prevappendTreeData["Merits"][progenIndx] = merit

		#Set this halo as the halo in the next snapshot progenitor
		TrackData["progenitor"][i] = ID

		#Iterate how many snaps it has been tracked for
		TrackData["TrackedNsnaps"][i]+=1

		#Lets see if the halo has reached the last snapshot it should be tracked for
		if((int(TrackData["endDesc"][i]/opt.Temporal_haloidval)==snap+opt.Snapshot_offset+1) & (TrackData["TrackDisp"][i]==False) & ("Descen" in treedata.keys())):

			appendTreeData["Descendants"][-1] = TrackData["endDesc"][i]
			appendTreeData["NumDesc"][-1] = 1

			#Since it is not possible to easily extract the endDesc particles (which could be many snapshots away)
			#the merit is just set to 1.0 for these halos
			appendTreeData["Merits"][-1] = 1.0
			WWstat["Connect"]+=1

			TrackData["idel"][i] = 1

			continue

		# Check if the halo still has a bound center of at least 5 particles
		if((np.sum(TrackData["mbpSel"][i])<=5) & ("Descen" in treedata.keys())):

			progenIndx = -1

			#Find the halo it has merged with if it has been lost
			if(TrackData["TrackDisp"][i]):
				 MergeHalo(opt,treeOpt,meanpos,partIDs[TrackData["boundSel"][i]],-1,snapdata,prevhost,filenumhalos,pfiles,upfiles,grpfiles,appendTreeData,pos_tree,WWstat)
				 WWstat["MergedMBP"]+=1

			#If not to be tracked until dispersed then connect it up with its endDesc
			else:
				appendTreeData["Descendants"][-1] = TrackData["endDesc"][i]
				appendTreeData["NumDesc"][-1] = 1

				#Since it is not possible to easily extract the endDesc particles (which could be many snapshots away)
				#the merit is just set to 1.0 for these halos
				appendTreeData["Merits"][-1] = 1.0
				WWstat["ConnectMBP"]+=1

			#Mark the halo to be deleted
			TrackData["idel"][i] = 1
			continue

		WWstat["contNSnap"]+=1

		#Update the keepPIDS array so the PID is kept
		keepPIDS[partOffset:partOffset+totnpart]=True

		#Lets also alter the order of the particles in the allpid array
		npart = np.uint64(npart)
		allpid[partOffset:partOffset+npart] = boundIDs
		allpid[partOffset+npart:partOffset+totnpart] = unboundIDs

		#Update the offsets
		newPartOffsets.append(np.uint64(newPartOffset))

		#Add to the offset for the next halo
		newPartOffset+=totnpart

	nextpids = allpid[keepPIDS].copy() #Need to be copy as will be used by startTrack()

	#Update the TrackData by removing all the halos to be removed
	for i in range(len(TrackData["progenitor"])-1,-1,-1):
		if(TrackData["idel"][i]):
			for field in TrackData.keys():
				del TrackData[field][i]

	del pos_tree

	return newPartOffsets,nextpids

max_stars_repo_name: rhyspoulton/WhereWolf

max_issues_repo_name: rhyspoulton/WhereWolf

max_forks_repo_name: rhyspoulton/WhereWolf

reason: import numpy,from scipy


text:
__author__ = 'dmitrii'
import numpy as np
from tools.format import featureFormat, targetFeatureSplit
import csv
from tools.technique import pickIndexesStrategy, randomPicker
import numpy
import os
import random
import re
import geopy
import geopy.distance
from tools.split_list import SplitList
from sys import stdout

class Indices:
    dict = []
    features_list = ["EN", "TR", "FR"]
    features = []
    target = []
    indices = []

    def __init__(self, file_name, data_type, data_amount, additional_configs, max_number_of_configs):
        param_list = {'app': "app4", 'flac': "cr_audio1.flac", 'wav': "cr_audio1.wav",
                                          'enw8': "enwik8", 'enw9': "enwik9", 'game': "game1",
                                          '01': "01",'02': "02",'03': "03",'04': "04",'05': "05",'06': "06",'07': "07",
                                          '08': "08",'09': "09",'10': "10",'11': "11",'12': "12",'13': "13",'14': "14",
                                          '16': "16",'17': "17",'18': "18",'19': "19",'20': "20",'21': "21",'22': "22", 'sort':"sort", 'encrypt':"encrypt", 'decrypt':"decrypt"}
        del self.dict[:]
        del self.indices[:]
        del self.target[:]
        del self.features[:]
        #brise/fedorov
        a_c = np.array(additional_configs)
        a_c.shape = (int(len(additional_configs)/2),2)

        print(data_type)
        self.d_a = data_amount
        with open(file_name, 'r') as csv_file:
            reader = csv.DictReader(csv_file)
            for row in reader:
                if row["Name"] == param_list[data_type]:
                    self.dict.append(row)

        data = featureFormat(self.dict, self.features_list)
        target, features = targetFeatureSplit( data )
        print(len(features))
        print("3")
        print(data_amount)

        #fedorov
        '''
        os.system("Rscript fedorov.R " + str(data_amount))
        r_indices_dict = {}
        freqs = list(set([features[i][1] for i in xrange(len(features))]))
        threads = list(set([int(features[i][0]) for i in xrange(len(features))]))
        num = 1
        for t in threads:
            for f in freqs:
                r_indices_dict[tuple([t,f])] = num
                num += 1
        #print r_indices_dict
        with open("somefile") as file:
            content = file.read()
        r_indices = re.split(" ", content)
        picked_features = []
        for k in r_indices_dict.keys():
            #print r_indices_dict[k]
            if r_indices.__contains__(str(r_indices_dict[k])):
                picked_features.append(numpy.asarray(k))
        '''
        #sobol
        #sort search space
        temp_filename = "somefile_" + str(random.randint(1,1000000)) + ".csv"

        os.system("Rscript sobol.R " + str(max_number_of_configs) + " " + temp_filename)
        all_features = []
        with open(temp_filename, 'r') as csv_file:
            reader = csv.DictReader(csv_file)
            for row in reader:
                all_features.append(tuple(list([float(row["TR"]), float(row["FR"])])))
        all_features.sort(key= lambda  x: x[0])
        split_list = SplitList(all_features, max_number_of_configs/16)
        all_features = split_list.list
        #pick features
        os.system("Rscript sobol.R " + str(data_amount) + " " + temp_filename)
        picked_features = []
        with open(temp_filename, 'r') as csv_file:
            reader = csv.DictReader(csv_file)
            for row in reader:
                picked_features.append(tuple(list([float(row["TR"]),float(row["FR"])])))
        picked_features.pop()
        picked_features.append(tuple(all_features[6-1][15-1]))

        # picked_coords = []
        # for pf in picked_features:
            # for i, row in enumerate(all_features):
            #     try:
            #         picked_coords.append(tuple([i+1, row.index(pf)+1]))
            #     except:
            #         continue
        #    picked_coords.append(tuple(list([int(pf[0])+1,int(pf[1])+1])))

        coordinates = []
        tr_list = [i for i in range(1,7,1)]
        fr_list = [i for i in range(1,17,1)]
        for tr in tr_list:
            for fr in fr_list:
                coordinates.append([tr,fr])
        points = [geopy.Point(p[0],p[1]) for p in coordinates]
        picked_coords = []
        for pf in picked_features:
            pf = geopy.Point(pf[0],pf[1])
            all_distances = [(p,geopy.distance.distance(p,pf).km) for p in points]
            i = 0
            while picked_coords.__contains__(sorted(all_distances, key=lambda x: (x[1]))[i][0]):
                i+=1
            picked_coords.append(sorted(all_distances, key=lambda d: (d[1]))[i][0])
        #print picked_features
        # exit()
        picked_features = []
        freqs = [1200., 1300., 1400., 1600., 1700., 1800., 1900., 2000., 2200., 2300., 2400., 2500., 2700., 2800.,
                 2900., 2901.]
        threads = [1., 2., 4., 8., 16., 32.]
        # threads = [1.,2.,3.,4.,5.,6.,7.,8.,9.,10.,11.,12.,13.,14.,15.,16.,17.,18.,19.,20.,21.,22.,23.,24.,25.,26.,27.,28.,29.,30.,31.,32.]

        for pc in picked_coords:
            picked_features.append([threads[int(pc[0])-1],freqs[int(pc[1]) - 1]])
        print picked_features

        #brise
        # picked features should be a set
        #picked_features = pickIndexesStrategy(features=features, data_amount=data_amount)

        #random
        #picked_features = randomPicker(features=features, data_amount=data_amount, additional_configs=additional_configs)
        #print picked_features

        #print "4"
        #print len(picked_features)
        for i in xrange(len(features)):
            self.features.append(features[i])
            self.target.append(target[i])
            for pf in picked_features:
                if np.setdiff1d(features[i], pf).size == 0:
                    self.indices.append(i)
                    break
            # brise/fedorov
            for ac in a_c:
                # print ac
                # print features[i]
                if np.setdiff1d(features[i], ac).size == 0:
                    self.indices.append(i)
                    break
        print(len(self.indices))
        #print self.indices

        #print "5"

max_stars_repo_name: dpukhkaiev/BRISE

max_issues_repo_name: dpukhkaiev/BRISE

max_forks_repo_name: dpukhkaiev/BRISE


text:
# -*- coding: utf-8 -*-

import numpy as np

class TeamDraftMultileave(object):

  def __init__(self, n_results=10):
    self._name = 'Team-Draft Multileave'
    self._k = n_results
    self.uses_inverted_rankings = False
    self.needs_inverted = False
    self.needs_descending = True
    self.needs_oracle = False
    self.vector_aggregation = False

  def clean(self):
    del self.teams

  def next_index_to_add(self, inter_result, inter_n, ranking, index):
    while index < ranking.shape[0] and np.any(ranking[index] == inter_result[:inter_n]):
      index += 1
    return index

  def make_multileaving(self, descending_rankings):

    rankings = descending_rankings

    n_rankings = rankings.shape[0]
    k = min(self._k, rankings.shape[1])
    teams = np.zeros(k, dtype=np.int32)
    multileaved = np.zeros(k, dtype=np.int32)

    multi_i = 0
    while multi_i < k and np.all(rankings[1:, multi_i] == rankings[0, multi_i]):
      multileaved[multi_i] = rankings[0][multi_i]
      teams[multi_i] = -1
      multi_i += 1

    indices  = np.zeros(n_rankings, dtype=np.int32) + multi_i
    assignment = np.arange(n_rankings)
    assign_i = n_rankings
    while multi_i < k:
      if assign_i == n_rankings:
        np.random.shuffle(assignment)
        assign_i = 0

      rank_i = assignment[assign_i]
      indices[rank_i] = self.next_index_to_add(multileaved, multi_i,
                           rankings[rank_i,:],
                           indices[rank_i])
      multileaved[multi_i] = rankings[rank_i, indices[rank_i]]
      teams[multi_i] = rank_i
      indices[rank_i] += 1
      multi_i += 1
      assign_i += 1

    self.teams = teams
    self.n_rankers = n_rankings
    return multileaved

  def infer_preferences(self, clicked_docs):
    clicked_docs = clicked_docs.astype(bool)
    assigned_clicks = np.sum(np.arange(self.n_rankers)[:,None] == self.teams[clicked_docs][None,:],axis=1)
    return np.sign(assigned_clicks[:,None] - assigned_clicks[None,:])

  def winning_rankers(self, clicked_docs):
    ranker_range = np.arange(self.n_rankers)
    match_matrix = ranker_range[:,None] == self.teams[clicked_docs][None,:]
    ranker_clicks = np.sum(match_matrix.astype(np.int32), axis=1)
    # print self.teams, clicked_docs.astype(int),
    # print ranker_range[ranker_clicks[0] < ranker_clicks]
    return ranker_range[ranker_clicks[0] < ranker_clicks]

max_stars_repo_name: yilingjia/FairExp

max_issues_repo_name: yilingjia/FairExp

max_forks_repo_name: yilingjia/FairExp


text:
from __future__ import absolute_import, division, print_function

import argparse
import os
import numpy as np
import timeit

import tensorflow as tf
import byteps.tensorflow as bps
from tensorflow.keras import applications

# Benchmark settings
parser = argparse.ArgumentParser(description='TensorFlow Synthetic Benchmark',
                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--fp16-allreduce', action='store_true', default=False,
                    help='use fp16 compression during allreduce')

parser.add_argument('--model', type=str, default='ResNet50',
                    help='model to benchmark')
parser.add_argument('--batch-size', type=int, default=32,
                    help='input batch size')

parser.add_argument('--num-warmup-batches', type=int, default=10,
                    help='number of warm-up batches that don\'t count towards benchmark')
parser.add_argument('--num-batches-per-iter', type=int, default=10,
                    help='number of batches per benchmark iteration')
parser.add_argument('--num-iters', type=int, default=10,
                    help='number of benchmark iterations')

parser.add_argument('--no-cuda', action='store_true', default=False,
                    help='disables CUDA training')

args = parser.parse_args()
args.cuda = not args.no_cuda

bps.init()

# pin GPU to be used to process local rank (one GPU per process)
if args.cuda:
    gpus = tf.config.experimental.list_physical_devices('GPU')
    for gpu in gpus:
        tf.config.experimental.set_memory_growth(gpu, True)
    if gpus:
        tf.config.experimental.set_visible_devices(gpus[bps.local_rank()], 'GPU')
else:
    os.environ["CUDA_VISIBLE_DEVICES"] = "-1"

# Set up standard model.
model = getattr(applications, args.model)(weights=None)
opt = tf.optimizers.SGD(0.01)

data = tf.random.uniform([args.batch_size, 224, 224, 3])
target = tf.random.uniform([args.batch_size, 1], minval=0, maxval=999, dtype=tf.float64)

@tf.function
def benchmark_step(first_batch):
    # BytePS: (optional) compression algorithm.
    compression = bps.Compression.fp16 if args.fp16_allreduce else bps.Compression.none

    # BytePS: use DistributedGradientTape
    with tf.GradientTape() as tape:
        probs = model(data, training=True)
        loss = tf.losses.sparse_categorical_crossentropy(target, probs)

    # BytePS: add bps Distributed GradientTape.
    tape = bps.DistributedGradientTape(tape, compression=compression)

    gradients = tape.gradient(loss, model.trainable_variables)
    opt.apply_gradients(zip(gradients, model.trainable_variables))

    # Broadcast initial variable states from rank 0 to all other processes.
    # This is necessary to ensure consistent initialization of all workers when
    # training is started with random weights or restored from a checkpoint.
    #
    # Note: broadcast should be done after the first gradient step to ensure optimizer
    # initialization.
    if first_batch:
        bps.broadcast_variables(model.variables, root_rank=0)
        bps.broadcast_variables(opt.variables(), root_rank=0)

def log(s, nl=True):
    if bps.rank() != 0:
        return
    print(s, end='\n' if nl else '')

log('Model: %s' % args.model)
log('Batch size: %d' % args.batch_size)
device = 'GPU' if args.cuda else 'CPU'
log('Number of %ss: %d' % (device, bps.size()))

with tf.device(device):
    # Warm-up
    log('Running warmup...')
    benchmark_step(first_batch=True)
    timeit.timeit(lambda: benchmark_step(first_batch=False),
                  number=args.num_warmup_batches)

    # Benchmark
    log('Running benchmark...')
    img_secs = []
    for x in range(args.num_iters):
        time = timeit.timeit(lambda: benchmark_step(first_batch=False),
                             number=args.num_batches_per_iter)
        img_sec = args.batch_size * args.num_batches_per_iter / time
        log('Iter #%d: %.1f img/sec per %s' % (x, img_sec, device))
        img_secs.append(img_sec)

    # Results
    img_sec_mean = np.mean(img_secs)
    img_sec_conf = 1.96 * np.std(img_secs)
    log('Img/sec per %s: %.1f +-%.1f' % (device, img_sec_mean, img_sec_conf))
    log('Total img/sec on %d %s(s): %.1f +-%.1f' %
        (bps.size(), device, bps.size() * img_sec_mean, bps.size() * img_sec_conf))

max_stars_repo_name: ChenYuHo/byteps

max_issues_repo_name: ChenYuHo/byteps

max_forks_repo_name: ChenYuHo/byteps


text:
# Copyright (c) 2012, GPy authors (see AUTHORS.txt).
# Licensed under the BSD 3-clause license (see LICENSE.txt)

import sys
import numpy as np
from ...core.parameterization.parameterized import Parameterized
from .kernel_slice_operations import KernCallsViaSlicerMeta
from ...util.caching import Cache_this
from GPy.core.parameterization.observable_array import ObsAr
from functools import reduce
import six

@six.add_metaclass(KernCallsViaSlicerMeta)
class Kern(Parameterized):
    #===========================================================================
    # This adds input slice support. The rather ugly code for slicing can be
    # found in kernel_slice_operations
    # __meataclass__ is ignored in Python 3 - needs to be put in the function definiton
    #__metaclass__ = KernCallsViaSlicerMeta
    #Here, we use the Python module six to support Py3 and Py2 simultaneously
    #===========================================================================
    _support_GPU=False
    def __init__(self, input_dim, active_dims, name, useGPU=False, *a, **kw):
        """
        The base class for a kernel: a positive definite function
        which forms of a covariance function (kernel).

        input_dim:

            is the number of dimensions to work on. Make sure to give the
            tight dimensionality of inputs.
            You most likely want this to be the integer telling the number of
            input dimensions of the kernel.
            If this is not an integer (!) we will work on the whole input matrix X,
            and not check whether dimensions match or not (!).

        active_dims:

            is the active_dimensions of inputs X we will work on.
            All kernels will get sliced Xes as inputs, if active_dims is not None
            Only positive integers are allowed in active_dims!
            if active_dims is None, slicing is switched off and all X will be passed through as given.

        :param int input_dim: the number of input dimensions to the function
        :param array-like|None active_dims: list of indices on which dimensions this kernel works on, or none if no slicing

        Do not instantiate.
        """
        super(Kern, self).__init__(name=name, *a, **kw)
        self.input_dim = int(input_dim)

        if active_dims is None:
            active_dims = np.arange(input_dim)

        self.active_dims = np.atleast_1d(active_dims).astype(int)

        assert self.active_dims.size == self.input_dim, "input_dim={} does not match len(active_dim)={}, active_dims={}".format(self.input_dim, self.active_dims.size, self.active_dims)

        self._sliced_X = 0
        self.useGPU = self._support_GPU and useGPU

        from .psi_comp import PSICOMP_GH
        self.psicomp = PSICOMP_GH()

    @Cache_this(limit=20)
    def _slice_X(self, X):
        return X[:, self.active_dims]

    def K(self, X, X2):
        """
        Compute the kernel function.

        .. math::
            K_{ij} = k(X_i, X_j)

        :param X: the first set of inputs to the kernel
        :param X2: (optional) the second set of arguments to the kernel. If X2
                   is None, this is passed throgh to the 'part' object, which
                   handLes this as X2 == X.
        """
        raise NotImplementedError
    def Kdiag(self, X):
        """
        The diagonal of the kernel matrix K

        .. math::
            Kdiag_{i} = k(X_i, X_i)
        """
        raise NotImplementedError
    def psi0(self, Z, variational_posterior):
        """
        .. math::
            \psi_0 = \sum_{i=0}^{n}E_{q(X)}[k(X_i, X_i)]
        """
        return self.psicomp.psicomputations(self, Z, variational_posterior)[0]
    def psi1(self, Z, variational_posterior):
        """
        .. math::
            \psi_1^{n,m} = E_{q(X)}[k(X_n, Z_m)]
        """
        return self.psicomp.psicomputations(self, Z, variational_posterior)[1]
    def psi2(self, Z, variational_posterior):
        """
        .. math::
            \psi_2^{m,m'} = \sum_{i=0}^{n}E_{q(X)}[ k(Z_m, X_i) k(X_i, Z_{m'})]
        """
        return self.psicomp.psicomputations(self, Z, variational_posterior, return_psi2_n=False)[2]
    def psi2n(self, Z, variational_posterior):
        """
        .. math::
            \psi_2^{n,m,m'} = E_{q(X)}[ k(Z_m, X_n) k(X_n, Z_{m'})]

        Thus, we do not sum out n, compared to psi2
        """
        return self.psicomp.psicomputations(self, Z, variational_posterior, return_psi2_n=True)[2]
    def gradients_X(self, dL_dK, X, X2):
        """
        .. math::

            \\frac{\partial L}{\partial X} = \\frac{\partial L}{\partial K}\\frac{\partial K}{\partial X}
        """
        raise NotImplementedError
    def gradients_X_X2(self, dL_dK, X, X2):
        return self.gradients_X(dL_dK, X, X2), self.gradients_X(dL_dK.T, X2, X)
    def gradients_XX(self, dL_dK, X, X2):
        """
        .. math::

            \\frac{\partial^2 L}{\partial X\partial X_2} = \\frac{\partial L}{\partial K}\\frac{\partial^2 K}{\partial X\partial X_2}
        """
        raise(NotImplementedError, "This is the second derivative of K wrt X and X2, and not implemented for this kernel")
    def gradients_XX_diag(self, dL_dKdiag, X):
        """
        The diagonal of the second derivative w.r.t. X and X2
        """
        raise(NotImplementedError, "This is the diagonal of the second derivative of K wrt X and X2, and not implemented for this kernel")
    def gradients_X_diag(self, dL_dKdiag, X):
        """
        The diagonal of the derivative w.r.t. X
        """
        raise NotImplementedError

    def update_gradients_diag(self, dL_dKdiag, X):
        """ update the gradients of all parameters when using only the diagonal elements of the covariance matrix"""
        raise NotImplementedError

    def update_gradients_full(self, dL_dK, X, X2):
        """Set the gradients of all parameters when doing full (N) inference."""
        raise NotImplementedError

    def update_gradients_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
        """
        Set the gradients of all parameters when doing inference with
        uncertain inputs, using expectations of the kernel.

        The essential maths is

        .. math::

            \\frac{\partial L}{\partial \\theta_i} & = \\frac{\partial L}{\partial \psi_0}\\frac{\partial \psi_0}{\partial \\theta_i}\\
                & \quad + \\frac{\partial L}{\partial \psi_1}\\frac{\partial \psi_1}{\partial \\theta_i}\\
                & \quad + \\frac{\partial L}{\partial \psi_2}\\frac{\partial \psi_2}{\partial \\theta_i}

        Thus, we push the different derivatives through the gradients of the psi
        statistics. Be sure to set the gradients for all kernel
        parameters here.
        """
        dtheta = self.psicomp.psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior)[0]
        self.gradient[:] = dtheta

    def gradients_Z_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior,
                                psi0=None, psi1=None, psi2=None):
        """
        Returns the derivative of the objective wrt Z, using the chain rule
        through the expectation variables.
        """
        return self.psicomp.psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior)[1]

    def gradients_qX_expectations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior):
        """
        Compute the gradients wrt the parameters of the variational
        distruibution q(X), chain-ruling via the expectations of the kernel
        """
        return self.psicomp.psiDerivativecomputations(self, dL_dpsi0, dL_dpsi1, dL_dpsi2, Z, variational_posterior)[2:]

    def plot(self, x=None, fignum=None, ax=None, title=None, plot_limits=None, resolution=None, **mpl_kwargs):
        """
        plot this kernel.
        :param x: the value to use for the other kernel argument (kernels are a function of two variables!)
        :param fignum: figure number of the plot
        :param ax: matplotlib axis to plot on
        :param title: the matplotlib title
        :param plot_limits: the range over which to plot the kernel
        :resolution: the resolution of the lines used in plotting
        :mpl_kwargs avalid keyword arguments to pass through to matplotlib (e.g. lw=7)
        """
        assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
        from ...plotting.matplot_dep import kernel_plots
        kernel_plots.plot(self, x, fignum, ax, title, plot_limits, resolution, **mpl_kwargs)

    def plot_ARD(self, *args, **kw):
        """
        See :class:`~GPy.plotting.matplot_dep.kernel_plots`
        """
        import sys
        assert "matplotlib" in sys.modules, "matplotlib package has not been imported."
        from ...plotting.matplot_dep import kernel_plots
        return kernel_plots.plot_ARD(self,*args,**kw)

    def input_sensitivity(self, summarize=True):
        """
        Returns the sensitivity for each dimension of this kernel.
        """
        return np.zeros(self.input_dim)

    def __add__(self, other):
        """ Overloading of the '+' operator. for more control, see self.add """
        return self.add(other)

    def __iadd__(self, other):
        return self.add(other)

    def add(self, other, name='sum'):
        """
        Add another kernel to this one.

        :param other: the other kernel to be added
        :type other: GPy.kern

        """
        assert isinstance(other, Kern), "only kernels can be added to kernels..."
        from .add import Add
        return Add([self, other], name=name)

    def __mul__(self, other):
        """ Here we overload the '*' operator. See self.prod for more information"""
        return self.prod(other)

    def __imul__(self, other):
        """ Here we overload the '*' operator. See self.prod for more information"""
        return self.prod(other)

    def __pow__(self, other):
        """
        Shortcut for tensor `prod`.
        """
        assert np.all(self.active_dims == range(self.input_dim)), "Can only use kernels, which have their input_dims defined from 0"
        assert np.all(other.active_dims == range(other.input_dim)), "Can only use kernels, which have their input_dims defined from 0"
        other.active_dims += self.input_dim
        return self.prod(other)

    def prod(self, other, name='mul'):
        """
        Multiply two kernels (either on the same space, or on the tensor
        product of the input space).

        :param other: the other kernel to be added
        :type other: GPy.kern

        """
        assert isinstance(other, Kern), "only kernels can be multiplied to kernels..."
        from .prod import Prod
        #kernels = []
        #if isinstance(self, Prod): kernels.extend(self.parameters)
        #else: kernels.append(self)
        #if isinstance(other, Prod): kernels.extend(other.parameters)
        #else: kernels.append(other)
        return Prod([self, other], name)

    def _check_input_dim(self, X):
        assert X.shape[1] == self.input_dim, "{} did not specify active_dims and X has wrong shape: X_dim={}, whereas input_dim={}".format(self.name, X.shape[1], self.input_dim)

    def _check_active_dims(self, X):
        assert X.shape[1] >= len(self.active_dims), "At least {} dimensional X needed, X.shape={!s}".format(len(self.active_dims), X.shape)

class CombinationKernel(Kern):
    """
    Abstract super class for combination kernels.
    A combination kernel combines (a list of) kernels and works on those.
    Examples are the HierarchicalKernel or Add and Prod kernels.
    """
    def __init__(self, kernels, name, extra_dims=[]):
        """
        Abstract super class for combination kernels.
        A combination kernel combines (a list of) kernels and works on those.
        Examples are the HierarchicalKernel or Add and Prod kernels.

        :param list kernels: List of kernels to combine (can be only one element)
        :param str name: name of the combination kernel
        :param array-like extra_dims: if needed extra dimensions for the combination kernel to work on
        """
        assert all([isinstance(k, Kern) for k in kernels])
        extra_dims = np.array(extra_dims, dtype=int)
        input_dim, active_dims = self.get_input_dim_active_dims(kernels, extra_dims)
        # initialize the kernel with the full input_dim
        super(CombinationKernel, self).__init__(input_dim, active_dims, name)
        self.extra_dims = extra_dims
        self.link_parameters(*kernels)

    @property
    def parts(self):
        return self.parameters

    def get_input_dim_active_dims(self, kernels, extra_dims = None):
        #active_dims = reduce(np.union1d, (np.r_[x.active_dims] for x in kernels), np.array([], dtype=int))
        #active_dims = np.array(np.concatenate((active_dims, extra_dims if extra_dims is not None else [])), dtype=int)
        input_dim = reduce(max, (k.active_dims.max() for k in kernels)) + 1

        if extra_dims is not None:
            input_dim += extra_dims.size

        active_dims = np.arange(input_dim)
        return input_dim, active_dims

    def input_sensitivity(self, summarize=True):
        """
        If summize is true, we want to get the summerized view of the sensitivities,
        otherwise put everything into an array with shape (#kernels, input_dim)
        in the order of appearance of the kernels in the parameterized object.
        """
        raise NotImplementedError("Choose the kernel you want to get the sensitivity for. You need to override the default behaviour for getting the input sensitivity to be able to get the input sensitivity. For sum kernel it is the sum of all sensitivities, TODO: product kernel? Other kernels?, also TODO: shall we return all the sensitivities here in the combination kernel? So we can combine them however we want? This could lead to just plot all the sensitivities here...")

    def _check_active_dims(self, X):
        return

    def _check_input_dim(self, X):
        # As combination kernels cannot always know, what their inner kernels have as input dims, the check will be done inside them, respectively
        return

max_stars_repo_name: beckdaniel/GPy

max_issues_repo_name: beckdaniel/GPy

max_forks_repo_name: beckdaniel/GPy


text:
import numpy as np
import cv2

# cap = cv2.VideoCapture('/Users/ariel/funana/mouse_position_tracking/resouces/mouse_cut.avi')
cap = cv2.VideoCapture(0)

fgbg = cv2.createBackgroundSubtractorMOG2()
# kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
# fgbg = cv2.createBackgroundSubtractorKNN()
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
# fgbg = cv2.bgsegcreateBackgroundSubtractorGMG()

while(1):
    ret, frame = cap.read()

    fgmask = fgbg.apply(frame)
    fgmask = cv2.morphologyEx(fgmask, cv2.MORPH_OPEN, kernel)

    cv2.imshow('frame',fgmask)
    k = cv2.waitKey(30) & 0xff
    if k == 27:
        break

cap.release()
cv2.destroyAllWindows()


text:
"""
Routes for querying supervised fastText model
"""
from sanic.blueprints import Blueprint

from numpy import ndarray

from dp4py_logging.time import timeit
from dp4py_sanic.api.response.json_response import json

from dp_fasttext.ml.unsupervised import UnsupervisedModel

from dp_fasttext.app.fasttext_server import FasttextServer
from dp_fasttext.api.request.fasttext_request import FasttextRequest

unsupervised_blueprint = Blueprint('unsupervised', url_prefix='/unsupervised')

@unsupervised_blueprint.route('/similar/vector', methods=['POST'])
@timeit
async def get_similar_words(request: FasttextRequest):
    """
    Returns labels similar to the input vector
    :param request:
    :return:
    """
    app: FasttextServer = request.app
    model: UnsupervisedModel = app.get_unsupervised_model()

    vector: ndarray = request.get_query_vector()
    num_labels: int = request.get_num_labels()

    # Get similar labels to the input vector
    words = model.similar_by_vector(vector, top_n=num_labels, return_similarity=False)

    # Return
    body = {
        "words": words
    }
    return json(request, body, 200)

max_stars_repo_name: ONSdigital/dp-fastText

max_issues_repo_name: ONSdigital/dp-fastText

max_forks_repo_name: ONSdigital/dp-fastText

reason: from numpy


text:
import cv2
import numpy as np
import random
import time
import math
from multiprocessing import Pool
import os

from npgol import iterate
# Z = np.random.randint(0,4,(1080,1920))

totaltimestart = time.time()
width = 1920
height = 1080
framesToProduce = 350
finalDirectory = 'cvart/'
videoDirectory = 'video/'

def check_or_create_directory(directory):
  pathNeeded = os.path.isdir(os.getcwd() + '/' + directory)
  if pathNeeded == False:
    os.mkdir(os.getcwd() + '/' + directory)

def write_frame(frame):
  start = time.time()
  img = np.random.randint(0,2,(1080,1920))

  print 'printing frame:', frame

  past_image = finalDirectory + str('%04d') % (frame - 1) + '.png'
  current_image = finalDirectory + str('%04d') % frame + '.png'
  video_image = videoDirectory + str('%04d') % frame + '.png'

  if os.path.exists(past_image):
    print 'comparing ', past_image, ' to, ', current_image
    img = iterate_gol(past_image, img, video_image)

  print_image(img, frame, start)

def iterate_gol(past_image, current_image, video_image):
  past_image = cv2.imread(past_image)
  video_image = cv2.imread(video_image)

  for yPos in range(0, height):
      for xPos in range(0, width):
        if any(val > 1 for val in past_image[yPos, xPos]):
          past_image[yPos, xPos] = 1

  newImg = iterate(past_image)
  blankImg = np.zeros(past_image.shape, dtype=int)
  video_image

  for yPos in range(0, height):
    for xPos in range(0, width):
      if all(val == 1 for val in past_image[yPos, xPos]):
        blankImg[yPos, xPos] = video_image[yPos, xPos]

  return blankImg

def print_image(img, frame, start):
  file_name = finalDirectory + str('%04d') % frame + '.png'
  cv2.imwrite(file_name, img)
  print 'It took', time.time()-start, 'seconds.'
  print 'To make', str(file_name)

if __name__ == '__main__':
    check_or_create_directory(finalDirectory) #check if directory exists, if not create
    # pool = Pool(6)  # Create a multiprocessing Pool
    # pool.map(write_frame, xrange(framesToProduce))  # process write_frame iterable with pool
    for frame in range (1, framesToProduce):
      write_frame(frame)

    os.chdir(os.getcwd() + '/' + finalDirectory)
    cmdBuild = 'ffmpeg -f image2 -r 30 -i %04d.png -c:v libx264 -pix_fmt yuv420p out.mp4'
    os.system(cmdBuild)

print 'It took', time.time()-totaltimestart, 'seconds.'

max_stars_repo_name: xasopheno/gol

max_issues_repo_name: xasopheno/gol

max_forks_repo_name: xasopheno/gol


text:
# -*- coding: utf-8 -*-
import numpy as np
import scipy.signal

def signal_filter(
    signal,
    sampling_rate=1000,
    lowcut=None,
    highcut=None,
    method="butterworth",
    order=2,
    window_size="default",
    powerline=50,
):
    """Filter a signal using 'butterworth', 'fir' or 'savgol' filters.

    Apply a lowpass (if 'highcut' frequency is provided), highpass (if 'lowcut' frequency is provided)
    or bandpass (if both are provided) filter to the signal.

    Parameters
    ----------
    signal : list or array or Series
        The signal (i.e., a time series) in the form of a vector of values.
        or "bandstop".
    sampling_rate : int
        The sampling frequency of the signal (in Hz, i.e., samples/second).
    lowcut : float
        Lower cutoff frequency in Hz. The default is None.
    highcut : float
        Upper cutoff frequency in Hz. The default is None.
    method : str
        Can be one of 'butterworth', 'fir', 'bessel' or 'savgol'. Note that for Butterworth, the function
        uses the SOS method from `scipy.signal.sosfiltfilt`, recommended for general purpose filtering.
        One can also specify "butterworth_ba' for a more traditional and legacy method (often implemented
        in other software).
    order : int
        Only used if method is 'butterworth' or 'savgol'. Order of the filter (default is 2).
    window_size : int
        Only used if method is 'savgol'. The length of the filter window (i.e. the number of coefficients).
        Must be an odd integer. If 'default', will be set to the sampling rate divided by 10
        (101 if the sampling rate is 1000 Hz).
    powerline : int
        Only used if method is 'powerline'. The powerline frequency (normally 50 Hz or 60 Hz).

    See Also
    --------
    signal_detrend, signal_psd

    Returns
    -------
    array
        Vector containing the filtered signal.

    Examples
    --------
    >>> import numpy as np
    >>> import pandas as pd
    >>> import neurokit2 as nk
    >>>
    >>> signal = nk.signal_simulate(duration=10, frequency=0.5) # Low freq
    >>> signal += nk.signal_simulate(duration=10, frequency=5) # High freq
    >>>
    >>> # Lowpass
    >>> fig1 = pd.DataFrame({"Raw": signal,
    ...                      "Butter_2": nk.signal_filter(signal, highcut=3, method='butterworth', order=2),
    ...                      "Butter_2_BA": nk.signal_filter(signal, highcut=3, method='butterworth_ba', order=2),
    ...                      "Butter_5": nk.signal_filter(signal, highcut=3, method='butterworth', order=5),
    ...                      "Butter_5_BA": nk.signal_filter(signal, highcut=3, method='butterworth_ba', order=5),
    ...                      "Bessel_2": nk.signal_filter(signal, highcut=3, method='bessel', order=2),
    ...                      "Bessel_5": nk.signal_filter(signal, highcut=3, method='bessel', order=5),
    ...                      "FIR": nk.signal_filter(signal, highcut=3, method='fir')}).plot(subplots=True)
    >>> fig1 #doctest: +SKIP

    >>> # Highpass
    >>> fig2 = pd.DataFrame({"Raw": signal,
    ...                      "Butter_2": nk.signal_filter(signal, lowcut=2, method='butterworth', order=2),
    ...                      "Butter_2_ba": nk.signal_filter(signal, lowcut=2, method='butterworth_ba', order=2),
    ...                      "Butter_5": nk.signal_filter(signal, lowcut=2, method='butterworth', order=5),
    ...                      "Butter_5_BA": nk.signal_filter(signal, lowcut=2, method='butterworth_ba', order=5),
    ...                      "Bessel_2": nk.signal_filter(signal, lowcut=2, method='bessel', order=2),
    ...                      "Bessel_5": nk.signal_filter(signal, lowcut=2, method='bessel', order=5),
    ...                      "FIR": nk.signal_filter(signal, lowcut=2, method='fir')}).plot(subplots=True)
    >>> fig2 #doctest: +SKIP

    >>> # Bandpass in real-life scenarios
    >>> original = nk.rsp_simulate(duration=30, method="breathmetrics", noise=0)
    >>> signal = nk.signal_distort(original, noise_frequency=[0.1, 2, 10, 100], noise_amplitude=1,
    ...                            powerline_amplitude=1)
    >>>
    >>> # Bandpass between 10 and 30 breaths per minute (respiratory rate range)
    >>> fig3 = pd.DataFrame({"Raw": signal,
    ...                      "Butter_2": nk.signal_filter(signal, lowcut=10/60, highcut=30/60,
    ...                                                   method='butterworth', order=2),
    ...                      "Butter_2_BA": nk.signal_filter(signal, lowcut=10/60, highcut=30/60,
    ...                                                      method='butterworth_ba', order=2),
    ...                      "Butter_5": nk.signal_filter(signal, lowcut=10/60, highcut=30/60,
    ...                                                   method='butterworth', order=5),
    ...                      "Butter_5_BA": nk.signal_filter(signal, lowcut=10/60, highcut=30/60,
    ...                                                      method='butterworth_ba', order=5),
    ...                      "Bessel_2": nk.signal_filter(signal, lowcut=10/60, highcut=30/60,
    ...                                                   method='bessel', order=2),
    ...                      "Bessel_5": nk.signal_filter(signal, lowcut=10/60, highcut=30/60,
    ...                                                   method='bessel', order=5),
    ...                      "FIR": nk.signal_filter(signal, lowcut=10/60, highcut=30/60,
    ...                                              method='fir'),
    ...                      "Savgol": nk.signal_filter(signal, method='savgol')}).plot(subplots=True)
    >>> fig3 #doctest: +SKIP

    """
    method = method.lower()

    # Sanity checks
    if method != "powerline":
        if lowcut is None and highcut is None:
            return signal

    if method in ["sg", "savgol", "savitzky-golay"]:
        filtered = _signal_filter_savgol(signal, sampling_rate, order, window_size=window_size)
    else:
        if method in ["butter", "butterworth"]:
            filtered = _signal_filter_butterworth(signal, sampling_rate, lowcut, highcut, order)
        elif method in ["butter_ba", "butterworth_ba"]:
            filtered = _signal_filter_butterworth_ba(signal, sampling_rate, lowcut, highcut, order)
        elif method in ["bessel"]:
            filtered = _signal_filter_bessel(signal, sampling_rate, lowcut, highcut, order)
        elif method in ["fir"]:
            filtered = _signal_filter_fir(signal, sampling_rate, lowcut, highcut, window_size=window_size)
        elif method in ["powerline"]:
            filtered = _signal_filter_powerline(signal, sampling_rate, powerline)
        else:
            raise ValueError(
                "NeuroKit error: signal_filter(): 'method' should be "
                "one of 'butterworth', 'butterworth_ba', 'bessel',"
                " 'savgol' or 'fir'."
            )
    return filtered

# =============================================================================
# Savitzky-Golay (savgol)
# =============================================================================

def _signal_filter_savgol(signal, sampling_rate=1000, order=2, window_size="default"):
    """Filter a signal using the Savitzky-Golay method.

    Default window size is chosen based on `Sadeghi, M., & Behnia, F. (2018). Optimum window length of
    Savitzky-Golay filters with arbitrary order. arXiv preprint arXiv:1808.10489.
    <https://arxiv.org/ftp/arxiv/papers/1808/1808.10489.pdf>`_.

    """
    window_size = _signal_filter_windowsize(window_size=window_size, sampling_rate=sampling_rate)

    filtered = scipy.signal.savgol_filter(signal, window_length=window_size, polyorder=order)
    return filtered

# =============================================================================
# FIR
# =============================================================================
def _signal_filter_fir(signal, sampling_rate=1000, lowcut=None, highcut=None, window_size="default"):
    """Filter a signal using a FIR filter.
    """
    try:
        import mne
    except ImportError:
        raise ImportError(
            "NeuroKit error: signal_filter(): the 'mne' module is required for this method to run. ",
            "Please install it first (`pip install mne`).",
        )

    if isinstance(window_size, str):
        window_size = "auto"

    filtered = mne.filter.filter_data(
        signal,
        sfreq=sampling_rate,
        l_freq=lowcut,
        h_freq=highcut,
        method="fir",
        fir_window="hamming",
        filter_length=window_size,
        l_trans_bandwidth="auto",
        h_trans_bandwidth="auto",
        phase="zero-double",
        fir_design="firwin",
        pad="reflect_limited",
        verbose=False,
    )
    return filtered

# =============================================================================
# Butterworth
# =============================================================================

def _signal_filter_butterworth(signal, sampling_rate=1000, lowcut=None, highcut=None, order=5):
    """Filter a signal using IIR Butterworth SOS method.
    """
    freqs, filter_type = _signal_filter_sanitize(lowcut=lowcut, highcut=highcut, sampling_rate=sampling_rate)

    sos = scipy.signal.butter(order, freqs, btype=filter_type, output="sos", fs=sampling_rate)
    filtered = scipy.signal.sosfiltfilt(sos, signal)
    return filtered

def _signal_filter_butterworth_ba(signal, sampling_rate=1000, lowcut=None, highcut=None, order=5):
    """Filter a signal using IIR Butterworth B/A method.
    """
    # Get coefficients
    freqs, filter_type = _signal_filter_sanitize(lowcut=lowcut, highcut=highcut, sampling_rate=sampling_rate)

    b, a = scipy.signal.butter(order, freqs, btype=filter_type, output="ba", fs=sampling_rate)
    try:
        filtered = scipy.signal.filtfilt(b, a, signal, method="gust")
    except ValueError:
        filtered = scipy.signal.filtfilt(b, a, signal, method="pad")

    return filtered

# =============================================================================
# Bessel
# =============================================================================

def _signal_filter_bessel(signal, sampling_rate=1000, lowcut=None, highcut=None, order=5):
    freqs, filter_type = _signal_filter_sanitize(lowcut=lowcut, highcut=highcut, sampling_rate=sampling_rate)

    sos = scipy.signal.bessel(order, freqs, btype=filter_type, output="sos", fs=sampling_rate)
    filtered = scipy.signal.sosfiltfilt(sos, signal)
    return filtered

# =============================================================================
# Powerline
# =============================================================================

def _signal_filter_powerline(signal, sampling_rate, powerline=50):
    """Filter out 50 Hz powerline noise by smoothing the signal with a moving average kernel with the width of one period
    of 50Hz.
    """

    if sampling_rate >= 100:
        b = np.ones(int(sampling_rate / powerline))
    else:
        b = np.ones(2)
    a = [len(b)]
    y = scipy.signal.filtfilt(b, a, signal, method="pad")
    return y

# =============================================================================
# Utility
# =============================================================================
def _signal_filter_sanitize(lowcut=None, highcut=None, sampling_rate=1000, normalize=False):

    # Sanity checks
    if isinstance(highcut, int):
        if sampling_rate <= 2 * highcut:
            print(
                "NeuroKit warning: the sampling rate is too low. Sampling rate"
                " must exceed the Nyquist rate to avoid aliasing problem. "
                "In this analysis, the sampling rate has to be higher than",
                2 * highcut,
                "Hz.",
            )

    # Replace 0 by none
    if lowcut is not None and lowcut == 0:
        lowcut = None
    if highcut is not None and highcut == 0:
        highcut = None

    # Format
    if lowcut is not None and highcut is not None:
        if lowcut > highcut:
            filter_type = "bandstop"
        else:
            filter_type = "bandpass"
        freqs = [lowcut, highcut]
    elif lowcut is not None:
        freqs = [lowcut]
        filter_type = "highpass"
    elif highcut is not None:
        freqs = [highcut]
        filter_type = "lowpass"

    # Normalize frequency to Nyquist Frequency (Fs/2).
    # However, no need to normalize if `fs` argument is provided to the scipy filter
    if normalize is True:
        freqs = np.array(freqs) / (sampling_rate / 2)

    return freqs, filter_type

def _signal_filter_windowsize(window_size="default", sampling_rate=1000):
    if isinstance(window_size, str):
        window_size = int(np.round(sampling_rate / 3))
        if (window_size % 2) == 0:
            window_size + 1  # pylint: disable=W0104
    return window_size

max_stars_repo_name: kassyray/NeuroKit

max_issues_repo_name: kassyray/NeuroKit

max_forks_repo_name: kassyray/NeuroKit

reason: import numpy,import scipy