text:
import numpy as np
import math
from sklearn.metrics import pairwise_distances
from scipy.optimize import curve_fit
import pandas as pd
import os
import matplotlib.pyplot as plt

def angular_distance(degree_1, degree_2):
    # Calculate angular distance in arcminutes
    if not (degree_1 == degree_2).all():
        degree_1_rad = degree_1 / 360 * 2 * math.pi
        degree_2_rad = degree_2 / 360 * 2 * math.pi
        angular_distance_rad = math.acos(math.sin(degree_1[1]) * math.sin(degree_2[1]) + math.cos(
            degree_1[1]) * math.cos(degree_2[1]) * math.cos(degree_1[0] - degree_2[0]))
        angular_distance_arcmin = angular_distance_rad * \
            360 / (2 * math.pi) * 60
    else:
        angular_distance_arcmin = 0
    return angular_distance_arcmin

def decay_law(x, a, b):
    return a * np.exp(-b * x)

def fit_decay(x_vals, y_vals):
    # ,method='trf',loss='arctan')
    popt, pcov = curve_fit(decay_law, x_vals, y_vals, p0=[y_vals.max(), 0.01])
    errors = np.sqrt(np.diag(pcov))
    return popt, errors

def label_spatial_rfi(df_cands, config, plot_arguments=[False, '']):
    # Determine if the clusters show RFI like behaviour spatially

    # Count size oif cluster and order them, could be later be replaced
    # when DataFrame containing the data about the cluster is created
    unique, counts = np.unique(df_cands['cluster_id'], return_counts=True)
    count_sort_ind = np.argsort(-counts)
    unique = unique[count_sort_ind]
    counts = counts[count_sort_ind]

    if plot_arguments[0]:
        try:
            os.mkdir(plot_arguments[1])
        except FileExistsError:
            pass

    # Create list that contains the new cluster DataFrame
    rows = []

    # Cycle thorugh all clusters, beginning with the largest
    for i in unique[:]:
        # Create DataFrame with only the cluster candidates
        df_truncated = df_cands[df_cands['cluster_id'] == i]

        # First read out some basic values
        best_candidate = df_truncated.iloc[0]
        new_row = {}
        new_row['cluster_id'] = i
        new_row['cluster_size'] = len(df_truncated)
        new_row['cluster_beams'] = df_truncated['file_index'].nunique()
        new_row['max_snr'] = df_truncated['snr'].max()
        new_row['min_snr'] = df_truncated['snr'].min()
        new_row['best_candidate_index'] = best_candidate.name
        new_row['best_candidate_file'] = best_candidate['file']
        new_row['best_period'] = best_candidate['period']
        new_row['best_acc'] = best_candidate['acc']
        new_row['best_dm'] = best_candidate['dm']
        new_row['max_dm'] = df_truncated['dm'].max()
        new_row['min_dm'] = df_truncated['dm'].min()
        new_row['period_range'] = df_truncated['period'].max() - \
            df_truncated['period'].min()

        total_nassoc = len(df_truncated) + df_truncated['nassoc'].sum()

        new_row['total_nassoc'] = total_nassoc

        # Create DataFrame with only the strongest candidate per beam
        unique_beams = df_truncated['file_index'].unique()
        df_beams = pd.DataFrame([])
        for beam in unique_beams:
            # When the same beam is in multiple files this does not quite correctly
            row = df_truncated[df_truncated['file_index'] == beam].iloc[0]
            df_beams = df_beams.append(row, ignore_index=True)

        # Fit an exponential decay to the maximum snr in each beam where a candidate is seen
        coords = df_beams[['src_rajd', 'src_dejd']].values

        if len(df_beams) > 1:

            # Currently calculates all distances, not fast, but gives largest value
            distances = pairwise_distances(coords, metric=angular_distance)

            try:
                max_distance = distances[distances > 0].max()
                min_distance = distances[distances > 0].min()
            except:
                max_distance = np.nan
                min_distance = np.nan
            if len(df_beams) > config['min_size_cluster_for_fit']:
                max_pos = df_beams['snr'].values.argmax()
                distances_from_max = distances[max_pos, :]
                snr_vals = df_beams['snr'].values
                try:
                    fit_parameters, fit_errors = fit_decay(
                        distances_from_max, snr_vals)
                except:
                    fit_parameters, fit_errors = [
                        np.nan, np.nan], [np.nan, np.nan]
                fitted = True
            else:
                fit_parameters, fit_errors = [np.nan, np.nan], [np.nan, np.nan]
                fitted = False
        else:
            max_distance = np.nan
            min_distance = np.nan
            fitted = False

        new_row['max_distance'] = max_distance
        new_row['min_distance'] = min_distance
        new_row['fit_decay'] = fit_parameters[1]
        new_row['fit_decay_error'] = fit_errors[1]
        new_row['fit_amplitude'] = fit_parameters[0]
        new_row['fit_amplitude_error'] = fit_errors[0]

        rows.append(new_row)

        if plot_arguments[0] and fitted:
            # Plot diagnostic plots
            fig, (ax1, ax2, ax3, ax4) = plt.subplots(1, 4, figsize=(25, 4))

            ax1.scatter(df_beams['src_rajd'],
                        df_beams['src_dejd'], s=df_beams['snr'] * 1.5)
            ax1.set_xlim(df_cands['src_rajd'].min() - 0.01,
                         df_cands['src_rajd'].max() + 0.01)
            ax1.set_ylim(df_cands['src_dejd'].min() - 0.01,
                         df_cands['src_dejd'].max() + 0.01)
            ax1.set(xlabel='src_rajd', ylabel='src_dejd')
            ax1.set_title(f"{new_row['cluster_size']} candidates in {new_row['cluster_beams']} beams")

            if len(snr_vals) > 3:
                new_xvals = np.linspace(0, distances_from_max.max(), 20)
                new_yvals = decay_law(new_xvals, *fit_parameters)
                ax2.plot(new_xvals, new_yvals)
            ax2.scatter(distances_from_max, snr_vals)
            ax2.set_title(f"a*exp(-b*x); b={fit_parameters[1]}")
            ax2.set(xlabel='Distance (´)', ylabel='snr')

            ax3.scatter(df_cands['period'], df_cands['acc'],
                        s=df_cands['snr'] * 2, c='r')
            ax3.scatter(
                df_truncated['period'], df_truncated['acc'], s=df_truncated['snr'] * 2)
            border = (df_truncated['period'].max() -
                      df_truncated['period'].min()) * 0.1
            if border == 0:
                border = new_row['best_period'] * 0.02
            ax3.set_xlim(df_truncated['period'].min(
            ) - border, df_truncated['period'].max() + border)
            ax3.set(xlabel='period', ylabel='acc')
            ax3.set_title(f"P0:  {new_row['best_period']}; Range: {new_row['period_range']}")

            ax4.scatter(df_truncated['period'],
                        df_truncated['dm'], s=df_truncated['snr'] * 2)
            ax4.set_xlim(df_truncated['period'].min(
            ) - border, df_truncated['period'].max() + border)
            ax4.set(xlabel='period', ylabel='DM')
            ax4.set_title(f"DM: {new_row['min_dm']} - {new_row['max_dm']}")

            plt.savefig(f"{plot_arguments[1]}cluster_{i}_Size_{new_row['cluster_size']}_P0_{new_row['best_period']}.png",
                        bbox_inches='tight')
    df_clusters = pd.DataFrame(rows)

    return df_clusters


text:
from argparse import ArgumentParser

import numpy as np
import tensorflow as tf

from ..mechanics.pong import Pong, S
from ..mechanics.policies import RandomPolicy, Follow, NNPolicy
from ..utils.tf_machinery import NeuralNetwork
from ..utils.replay_db import ReplayDB
from ..utils.greenlet_learner import GreenletLearner
from ..utils import common

class DeepValueIteration:
    """Learn using a function-approximation version of the Value Iteration
    algorithm."""

    def __init__(self, args):
        """Create a DeepValueIteration instance using args."""

        self.args = args
        self.sim = Pong()
        self.nn = NeuralNetwork(Pong.STATE_DIM, Pong.NUM_ACTIONS, [50, 50, 50])
        self.db = ReplayDB(Pong.STATE_DIM, args.db_size)
        self.save_path, self.load_path = common.read_save_load_args(args)

        self.actions = tf.placeholder(tf.int32, (None,), "actions")
        self.q_values = self.nn.take(self.actions)
        self.q_estimation = tf.placeholder(tf.float32, (None,),
            name="q_estimation")
        self.loss = tf.reduce_mean((self.q_estimation - self.q_values) ** 2)

        self.optimizer = tf.train.AdamOptimizer(args.learning_rate)
        self.train_op = self.optimizer.minimize(self.loss)

        self.n_iter = self.nn.load(self.load_path)

        if args.rival_policy == "random":
            self.l_pol = RandomPolicy("l")
        elif args.rival_policy == "follow":
            self.l_pol = Follow("l")
        elif args.rival_policy == "self":
            self.l_pol = NNPolicy("l")
            self.l_pol.nn = self.nn

        if args.discrete_samples or args.discrete_world:
            n_bins = [args.n_cells, args.n_cells, 3, args.n_cells,
                args.n_cells, args.n_cells, args.n_cells, args.n_cells]
            self.d = common.Discretization(Pong.RANGES, n_bins)

    def add_data_points(self):
        """Use the simulator to create new data points."""

        n = self.args.states_per_iteration
        states = np.zeros((n, Pong.STATE_DIM), np.float32)
        states2 = np.zeros((n, Pong.STATE_DIM), np.float32)
        actions = np.zeros(n, np.int32)
        rewards = np.zeros(n, np.float32)
        done = np.zeros(n, np.bool)

        for i in range(n):
            self.sim.random_state()
            states[i] = self.sim.get_state()
            if self.args.discrete_world:
                states[i] = self.d.discretize(states[i])
                self.sim.set_state(states[i])

            l_a = self.l_pol.get_action(states[i])
            r_a = np.random.choice(Pong.NUM_ACTIONS)
            self.sim.step(l_a, r_a)
            states2[i] = self.sim.get_state()
            actions[i] = r_a
            rewards[i] = self.sim.reward("r", not self.args.partial)
            done[i] = self.sim.done

            if self.args.discrete_samples:
                states[i] = self.d.discretize(states[i])
                states2[i] = self.d.discretize(states2[i])

        if self.args.erase_left:
            states[:, [S.L_Y, S.L_VY]] = 0.
            states2[:, [S.L_Y, S.L_VY]] = 0.

        self.db.store(states, states2, actions, rewards, done)

    def q_iteration(self):
        """Perform a single q_iteration."""

        sample = self.db.sample()
        v = self.nn.predict_max(sample.s2)
        q_estimation = sample.r + (~sample.done * self.args.gamma * v)

        feed_dict = {
            self.nn.input: sample.s1,
            self.actions: sample.a,
            self.q_estimation: q_estimation
        }

        self.nn.train_in_batches(self.train_op, feed_dict,
            self.args.batches_per_iteration, self.args.batch_size)

        return self.nn.session.run(self.loss, feed_dict)

    def learn(self):
        """Create data and perform Q-iterations in a loop."""

        for self.n_iter in range(self.n_iter, self.args.n_iters):
            self.add_data_points()
            loss = self.q_iteration()

            if (self.n_iter + 1) % self.args.save_frequency == 0:
                self.nn.save(self.save_path, self.n_iter + 1)
                print("{:04d} : loss={:05.3f}".format(self.n_iter + 1, loss))

#===============================================================================
# Main
#===============================================================================

def main(argv):
    parser = ArgumentParser()

    common.add_save_load_args(parser, "/tmp/Pong/DeepValueIteration/")

    parser.add_argument("--learning_rate", "-lr", default=0.001, type=float)
    parser.add_argument("--partial", action='store_true', default=False)
    parser.add_argument("--gamma", "-g", default=0.99, type=float)

    parser.add_argument("--batches_per_iteration", "-bpi", default=100,
        type=int)
    parser.add_argument("--batch_size", "-bs", default=1000, type=int)
    parser.add_argument("--states_per_iteration", "-spi", default=10000,
        type=int)
    parser.add_argument("--n_iters", "-ni", default=1000, type=int)
    parser.add_argument("--db_size", type=int, default=100000)
    parser.add_argument("--save_frequency", "-sf", default=10, type=int)

    parser.add_argument("--discrete_samples", "-ds", action='store_true',
        default=False)
    parser.add_argument("--discrete_world", "-dw", action='store_true',
        default=False)
    parser.add_argument("--n_cells", "-nc", default=200, type=int)
    parser.add_argument("--erase_left", "-el", action='store_true',
        default=False)

    parser.add_argument("--rival_policy", "-rp",
        choices=["follow", "random", "self"], default="follow")

    args = parser.parse_args(argv)

    DVI = DeepValueIteration(args)
    DVI.learn()

ext: py

max_stars_repo_name: jonathanfiat/ApproxiPong

max_issues_repo_name: jonathanfiat/ApproxiPong

max_forks_repo_name: jonathanfiat/ApproxiPong

reason: import numpy


text:
import pandas as pd
from ._exceptions import ValidationError
from ._utils import _fetch
import matplotlib.dates as mdates
import matplotlib.cbook as cbook
from datetime import datetime
import numpy as np

def _normalize_data(chart):
    """Converts the data to a Pandas dataframe

    Parameters
    ----------
    chart : altair.Chart
    The vega-lite specification in json format

    Returns
    -------
    None

    Raises
    ------
    ValidationError
    Raised when the specification does not contain any data attribute

    NotImplementedError
    Raised when the data specification has an unsupported data source
    """

    spec = chart.to_dict()

    if spec['data'].get('url'):
        df = pd.DataFrame(_fetch(spec['data']['url']))
    elif spec['data'].get('values'):
        return
    else:
        raise NotImplementedError('Given data specification is unsupported at the moment.')

    chart.data = df

def _convert_to_mpl_date(data):
    """Converts datetime, datetime64, strings, and Altair DateTime objects to Matplotlib dates.

    Parameters
    ----------
    data : datetime.datetime, numpy.datetime64, str, Altair DateTime, or sequences of any of these
        The data to be converted to a Matplotlib date

    Returns
    -------
    new_data : list
        A list containing the converted date(s)
    """

    if cbook.iterable(data) and not isinstance(data, str) and not isinstance(data, dict):
        if len(data) == 0:
            return []
        else:
            return np.asarray([_convert_to_mpl_date(i) for i in data])
    else:
        if isinstance(data, str):  # string format for dates
            data = mdates.datestr2num(data)
        elif isinstance(data, np.datetime64):  # sequence of datetimes, datetime64s
            data = mdates.date2num(data)
        elif isinstance(data, dict):  # Altair DateTime
            data = mdates.date2num(_altair_DateTime_to_datetime(data))
        else:
            raise TypeError
        return data

def _altair_DateTime_to_datetime(dt):
    """Convert dictionary representation of an Altair DateTime to datetime object

    Parameters
    ----------
    dt : dict
        The dictionary representation of the Altair DateTime object to be converted.

    Returns
    -------
    A datetime object
    """
    MONTHS = {'Jan': 1, 'January': 1, 'Feb': 2, 'February': 2, 'Mar': 3, 'March': 3, 'Apr': 4, 'April': 4,
              'May': 5, 'Jun': 6, 'June': 6, 'Jul': 7, 'July': 7, 'Aug': 8, 'August': 8, 'Sep': 9, 'Sept': 9,
              'September': 9, 'Oct': 10, 'October': 10, 'Nov': 11, 'November': 11, 'Dec': 12, 'December': 12}

    alt_to_datetime_kw_mapping = {'date': 'day', 'hours': 'hour', 'milliseconds': 'microsecond', 'minutes': 'minute',
                       'month': 'month', 'seconds': 'second', 'year': 'year'}

    datetime_kwargs = {'year': 0, 'month': 1, 'day': 1, 'hour': 0, 'minute': 0, 'second': 0, 'microsecond': 0}

    if 'day' in dt or 'quarter' in dt:
        raise NotImplementedError
    if 'year' not in dt:
        raise KeyError('A year must be provided.')
    if 'month' not in dt:
        dt['month'] = 1  # Default to January
    else:
        if isinstance(dt['month'], str):  # convert from str to number form for months
            dt['month'] = MONTHS[dt['month']]
    if 'date' not in dt:
        dt['date'] = 1  # Default to the first of the month
    if 'milliseconds' in dt:
        dt['milliseconds'] = dt['milliseconds']*1000  # convert to microseconds
    if 'utc' in dt:
        raise NotImplementedError("mpl-altair currently doesn't support timezones.")

    for k, v in dt.items():
        datetime_kwargs[alt_to_datetime_kw_mapping[k]] = v

    return datetime(**datetime_kwargs)

max_stars_repo_name: samuelsinayoko/mpl-altair

max_issues_repo_name: samuelsinayoko/mpl-altair


text:
from gym.envs.atari import AtariEnv
import numpy as np
from utils import *
from dqn import DQN

model=DQN()
images,result=model.build()
init = tf.global_variables_initializer()
env = AtariEnv(game='pong', obs_type='image', frameskip=4)
action_space=6
episodes=1
done=False
reward=0
states = RecentStateBuffer(4, preprocess=preprocess_image)
with tf.Session() as sess:
    sess.run(init)
    for i in range(episodes):
        states.reset()
        state = env.reset()
        states.add(state)
        last_states=states.get()
        action=0
        while len(last_states)<4:
            action = np.random.randint(action_space)
            state, reward, done, _ = env.step(action)
            states.add(state)
            new_states=states.get()
            #DQN.add_transition(last_states,action,reward,new_states,done)
            last_states=new_states
        state_images=preprocess_images(last_states)
        print(state_images)
        action_values=sess.run(result,feed_dict={images:state_images})
        targets=
        loss=l2_loss()

max_stars_repo_name: abefetterman/rl-testing

max_issues_repo_name: abefetterman/rl-testing

max_forks_repo_name: abefetterman/rl-testing


text:
#!/usr/bin/python

import numpy as np
import json
import time
import sys

import matplotlib.pyplot as plt

from sklearn import svm
from sklearn.model_selection import KFold
from sklearn.model_selection import StratifiedKFold

from sklearn.metrics import precision_recall_curve
from sklearn.metrics import average_precision_score
from sklearn.preprocessing import MinMaxScaler

sys.path.append('../../utility')
import yamanishi_data_util as yam

sys.path.append('../cluster/kmedoid')
import kmedoid

from generateNegativeData import genNegativeData
from selfblm import SELFBLM

def main():
    if len(sys.argv)!=5:
        print ("python blmniisvm_experiment.py [DataSetCode] [evalMode]"
                " [dataPath] [outPath]")
        return
    classParam = dict(name='blmnii',proba=True)

    dataset = sys.argv[1]
    evalMode = sys.argv[2]
    dataPath = sys.argv[3]
    outPath = sys.argv[4]

    print "Building Data"
    connMat,comList,proList = yam.loadComProConnMat(dataset,dataPath+"/Adjacency")
    kernel = yam.loadKernel(dataset,dataPath)

    comListIdx = [i for i,_ in enumerate(comList)]
    proListIdx = [i for i,_ in enumerate(proList)]

    nComp = len(comList)
    nProtein = len(proList)

    comSimMat = np.zeros((nComp,nComp), dtype=float)
    proSimMat = np.zeros((nProtein,nProtein), dtype=float)
    for row,i in enumerate(comList):
        for col,j in enumerate(comList):
            comSimMat[row][col] = (kernel[(i,j)]+kernel[(j,i)])/2

    for row,i in enumerate(proList):
        for col,j in enumerate(proList):
            proSimMat[row][col] = (kernel[(i,j)]+kernel[(j,i)])/2

    comSimMat = regularizationKernel(comSimMat)
    proSimMat = regularizationKernel(proSimMat)

    print "Clustering"
    comDisMat = kmedoid.simToDis(comSimMat)
    proDisMat = kmedoid.simToDis(proSimMat)

    _,proClust = kmedoid.kMedoids(len(proList)/2, proDisMat)
    _,comClust = kmedoid.kMedoids(len(comList)/2, comDisMat)

    print "Generate Negative Data"
    connMat = genNegativeData(connMat,proClust,comClust)
    # PLACEHOLDER

    # Split Data
    pairData = []
    connList = []
    print "Split Dataset..."
    if evalMode == "loocv":
        nFold = len(comListIdx)
        kSplit = KFold(n_splits=nFold,shuffle=True)
        comSplit = kSplit.split(comListIdx)

        nFold = len(proListIdx)
        kSplit = KFold(n_splits=nFold,shuffle=True)
        proSplit = kSplit.split(proListIdx)

    elif evalMode == "kfold":
        nFold = 10
        kSplit = KFold(n_splits=nFold, shuffle=True)
        comSplit = kSplit.split(comListIdx)
        proSplit = kSplit.split(proListIdx)

    else:
        assert(False)

    predictedData = np.zeros((len(comList),len(proList)),dtype=float)
    splitPred = []
    proTestList = []
    proTrainList = []
    comTestList = []
    comTrainList = []

    for trainIndex, testIndex in proSplit:
        proTestList.append([i for i in testIndex])
        proTrainList.append([i for i in trainIndex])
    for trainIndex, testIndex in comSplit:
        comTestList.append([i for i in testIndex])
        comTrainList.append([i for i in trainIndex])

    predRes = []
    testData = []

    print "Predicting..."
    for ii,i in enumerate(comTestList):
        for jj,j in enumerate(proTestList):
            sys.stdout.write("\r%03d of %03d||%03d of %03d" %
                                (jj+1, len(proTestList), ii+1,len(comTestList),))
            sys.stdout.flush()

            predictor = SELFBLM(classParam, connMat, comSimMat, proSimMat,
                            [comTrainList[ii],proTrainList[jj]],[i,j])
            for comp in i:
                for prot in j:
                    predRes.append(predictor.predict([(comp,prot)]))
                    if connMat[comp][prot] == 1:
                        testData.append(1)
                    else:
                        testData.append(-1)

    # run core selfBLM
    # Evaluate prediction
    print "\nCalculate Performance"
    key = 'PredictionUsingSelfBLM'
    precision, recall, _ = precision_recall_curve(testData, predRes)
    prAUC = average_precision_score(testData, predRes, average='micro')

    print "Visualiation"
    lineType = 'k-.'

    perf = {'precision': precision, 'recall': recall, 'prAUC': prAUC,
                 'lineType': lineType}
    perf2 = {'prAUC': prAUC, 'nTest': nComp*nProtein}

    with open(outPath+'perf_selfblm_'+evalMode+'_'+dataset+'_perf.json', 'w') as fp:
        json.dump(perf2, fp, indent=2, sort_keys=True)

    plt.clf()
    plt.figure()
    plt.plot(perf['recall'], perf['precision'], perf['lineType'], label= key+' (area = %0.2f)' % perf['prAUC'], lw=2)
    plt.ylim([-0.05, 1.05])
    plt.xlim([-0.05, 1.05])
    plt.xlabel('Recall')
    plt.ylabel('Precision')
    plt.title('Precision-Recall Curve')
    plt.legend(loc="lower left")
    plt.savefig(outPath+'/pr_curve_'+ dataset +'_'+evalMode+'_selfblm.png', bbox_inches='tight')

# http://stackoverflow.com/questions/29644180/gram-matrix-kernel-in-svms-not-positive-semi-definite?rq=1
def regularizationKernel(mat):
    eps = 0.1
    m,n = mat.shape
    assert(m==n)
    while isPSDKernel(mat) == False:
        for i in range(m):
            mat[i][i] = mat[i][i] + eps

    return mat

def isPSDKernel(mat,eps = 1e-8):
  E,V = np.linalg.eigh(mat)
  return np.all(E > -eps) and np.all(np.isreal(E))

if __name__ == '__main__':
    start_time = time.time()
    main()
    print "Program is running for :"+str(time.time()-start_time)

max_stars_repo_name: tttor/csipb-jamu-prj

max_issues_repo_name: tttor/csipb-jamu-prj

max_forks_repo_name: tttor/csipb-jamu-prj


text:
import numpy as np
from collections import defaultdict
from scipy.special import erf
from sklearn.neighbors import KernelDensity

# Version: June/2020
# This file implements the environment construction for bandits algorithm.
# Functions:
#   setup_env:  Setup environment for simulations.
#   generate_samples: Generate samples for each env (creating fixed sample list for debug)
#   est_L_test: Test of convergence of estimation of L.

# Simulated environments:
# mixture absolute Gaussian, Exponential

# ---------------------------------------------------------------------------------

def setup_env(envs_setting, ss_list = ['quantile_0.5'], random_set = None):
    """Setup environment for simulations.

    Parameter:
    --------------------------------------------------
    envs_setting: list of env dict
        keys: instance of environment class
            (AbsGau, Exp, AbsGau_Outlier, Exp_Outlier)
        values: list of parameters
        e.g.environments = [
                {AbsGau: [0.5, 1.0, 1.5]},
                {Exp:    [2.0, 1.0, 1.5]},
                {AbsGau: [0.5], Exp: [1.0, 1.5]}
               ]
    ss_list: list of string
        list of summary statistics names, with the name and parameter

    Return:
    --------------------------------------------------
    results_env: dict of list
        keys: name of environment
            e.g. AbsGau_Outlier_[0.5, 1, 1.5]
        values: list of env instances
    tau-quantiles: dict of list for tau-quantiles
        keys: name of environment
            e.g. AbsGau_Outlier_[0.5, 1, 1.5]
        values: list of medians
    means: dict of list for means
    mvs: dict of list for mean-variance
    samples: dict of list for samples
    """

    rewards_env = defaultdict(list)
    samples = defaultdict(list)
    true_ss_dict = {}
    num_samples = 50000

    for envs_dict in envs_setting:
        name = ''
        print(envs_dict)
        for env, para_list in envs_dict.items():
            env_name = str(env).split('.')[-1][:-2]
            name += env_name + '_' + str(para_list)
        true_ss_dict[name] = defaultdict(list)

        for env, para_list in envs_dict.items():
            for para in para_list:
                if random_set == None:
                    if type(para) is list:
                        current_env = env(*para)
                    else:
                        current_env = env(para)
                else:
                    current_env = env(para, random_set)
                rewards_env[name].append(current_env)
                sample = current_env.sample(num_samples)
                samples[name].append(sample)

                # for ss in ss_list:
                #     ss_name = ss.split('_')[0]
                #     ss_para = float(ss.split('_')[-1])
                #     true_ss_dict[name][ss] = []
                for ss in ss_list:
                    ss_name = ss.split('_')[0]
                    if len(ss.split('_'))> 1:
                        ss_para = float(ss.split('_')[-1])
                    if ss_name == 'quantile':
                        true_ss_dict[name][ss].append(np.quantile(sample, ss_para))
                    elif ss_name == 'mean':
                        true_ss_dict[name][ss_name].append(np.mean(sample))
                    else:
                        assert True, 'Unknown summary statistics!'

    return rewards_env, true_ss_dict, samples

def generate_samples(envs_dict, tau =0.5, num_samples = 5000):
    """Generate samples for each env.

    Parameter:
    --------------------------------------------------
    envs_dict: env dict
        keys: instance of environment class
            (AbsGau, Exp, AbsGau_Outlier, Exp_Outlier)
        values: list of parameters
        e.g. {AbsGau: [[0.5, 1.0, 1.5]], Exp: [[1/4]]},

    tau: float (0,1)
        level of quantile
    num_samples: int
        number of samples for each env.

    Return:
    --------------------------------------------------
    samples: dict of list for samples
        key: arm/env idx
        value: list of sampes (size: num_sample) for arm idx
    """

    samples = defaultdict(list)

    arm_idx = 0

    for env, para_list in envs_dict.items():
        for para in para_list:
            if type(para) is list:
                current_env = env(*para)
            else:
                current_env = env(para)

            sample = current_env.sample(num_samples)
            samples[arm_idx] = sample
            arm_idx += 1

    return samples

def est_L_test(envs_setting, num_exper = 100, sample_size = 500, est_method = 'naive', bandwidth = 1):
    """Test of convergence of estimation of L.

    Parameter:
    --------------------------------------------------
    envs_setting: list of env dict
        keys: instance of environment class
            (AbsGau, Exp, AbsGau_Outlier, Exp_Outlier)
        values: list of parameters
        e.g.environments = [
                {AbsGau: [0.5, 1.0, 1.5]},
                {Exp:    [2.0, 1.0, 1.5]},
                {AbsGau: [0.5], Exp: [1.0, 1.5]}
               ]
    sample_size: int
        number of samples
    est_method: str
        can be one of the choice of 'kde', 'naive'
        'kde': kernel density estimation
        'naive': count the samples insides [0,dt]
    bandwidth: float

    Return:
    --------------------------------------------------
    est_Ls: dict
        keys: name of environment
            e.g. AbsGau_Outlier_[0.5, 1, 1.5]
        values: list of est Ls for each round
    """

    est_Ls = defaultdict(list)

    for envs_dict in envs_setting:
        #name = ''
        #for env, para_list in envs_dict.items():
            #env_name = str(env).split('.')[-1][:-2]
            #name += env_name + '_' + str(para_list)

        arm_idx = 0
        for env, para_list in envs_dict.items():
            for para in para_list:
                if type(para) is list:
                    current_env = env(*para)
                else:
                    current_env = env(para)

                #est_Ls[arm_idx] = []
                for exper in range(num_exper):
                    current_samples = []
                    current_est_Ls = []
                    for t in range(sample_size):
                        current_samples.append(current_env.sample())
                        current_est_Ls.append(est_L(current_samples, est_method, bandwidth))
                    est_Ls[arm_idx].append(current_est_Ls)
                arm_idx +=1
    return est_Ls

def est_L(sample_list, est_method, bandwidth = 0.5):
    """Estimate L from a list of samples.

    Parameter
    ------------------------------------------
    sample_list: list
        a list of samples for arm i at time t
    est_method: str
        can be one of the choice of 'kde', 'naive'
        'kde': kernel density estimation
        'naive': count the samples insides [0,dt]
    """
    if est_method == 'kde':
        kde = KernelDensity(kernel='tophat', bandwidth=bandwidth).fit(np.asarray(sample_list)[:, np.newaxis])
        log_den_0 = kde.score_samples(np.asarray([0])[:, np.newaxis])
        estL = np.exp(log_den_0)[0]
    elif est_method == 'naive':
        sorted_data = np.asarray(sorted(sample_list))
        estL = len(sorted_data[sorted_data <= bandwidth])/len(sorted_data)
        #if len(sample_list) ==1 or estL  == 0:
            # TODO: init value
        #    L = 0.01
    else:
        print('Unkown estimation method.')
    return estL

#------------------------------------------------------------------------------------

class Base_env():
    def __init__(self, para):
        self.para = para

    def sample(self, size = None):
        pass

class Mixture_AbsGau():
    """Env for Absolute Gaussian Distribution with outliers.
    f(t) = p AbsGaux|mu1, sigma1) + (1-p) AbsGau(x|mu2, sigma2) with mu1 < mu2
    To have a IHR distribution, p needs to be small.
    When p = 1, then it's AbsGau with mu1 and sigma1.
    """
    def __init__(self, mu1, sigma1, mu2, sigma2, p):
        self.mu1 = mu1
        self.mu2 = mu2
        self.sigma1 = sigma1
        self.sigma2 = sigma2
        self.p = p

    def pdf(self, x, mu, sigma):
        return 1.0/np.sqrt(2 * np.pi * sigma ** 2) * (np.exp(- 1.0/(2 * sigma**2) * (x - mu)** 2) + np.exp(- 1.0/(2 * sigma**2) * (x + mu)** 2 ))

    def cdf(self, x, mu, sigma):
        return 1.0/2 * (erf((x-mu)/ np.sqrt(2 * sigma ** 2)) + erf((x+ mu)/ np.sqrt(2 * sigma ** 2)))

    def mix_pdf(self, x):
        return (self.p * self.pdf(x, self.mu1, self.sigma1) + (1-self.p)*self.pdf(x, self.mu2,self.sigma2))

    def mix_cdf(self, x):
        return (self.p * self.cdf(x, self.mu1, self.sigma1) + (1-self.p)*self.cdf(x, self.mu2,self.sigma2))

    def hazard_rate(self, x):
        return self.mix_pdf(x)/(1 - self.mix_cdf(x))

    def sample(self, size = None):
        if size == None:
            if np.random.uniform() <= self.p:
                return np.abs(np.random.normal(self.mu1, self.sigma1))
            else:
                return np.abs(np.random.normal(self.mu2, self.sigma2))
        else:
            samples = []
            for i in range(size):
                if np.random.uniform() <= self.p:
                    s = np.abs(np.random.normal(self.mu1, self.sigma1))
                else:
                    s = np.abs(np.random.normal(self.mu2, self.sigma2))
                samples.append(s)
            return np.asarray(samples)

class Exp(Base_env):
    """Env for Exponential Distribution.
    """
    def __init__(self,para):
        super().__init__(para)

    def pdf(self, x):
        return self.para * np.exp(- self.para * x)

    def cdf(self,x):
        return 1 - np.exp(- self.para * x)

    def hazard_rate(self, x):
        return self.pdf(x)/(1 - self.cdf(x))

    def sample(self, size = None):
        return np.random.exponential(1.0/self.para, size)

reason: import numpy,from scipy


text:
# Titanic Dataset Analysis
# Mihir Garg

#Importing Libraries
import pandas as pd
from pandas import Series, DataFrame

#Importing Dataset
titanic=pd.read_csv('train.csv')
titanic.head()

#Info of Dataset
titanic.info()

#Statistics of Dataset
titanic.describe()

# Q1. Who were the passengers on the titanic?

#Importing Libraries for Visualization
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

#Count of male and female in dataset
sns.factorplot('Sex',data=titanic,kind='count',color='red')

#count of male and female by classes
sns.factorplot('Sex',data=titanic,hue='Pclass',kind='count')

#count of male and female by classes
sns.factorplot('Pclass',data=titanic,hue='Sex',kind='count')

# What about split between males,females,and children? How can we go about this?

def split(passenger):
    age,sex=passenger
    if age < 16:
        return 'child'
    else:
        return sex

titanic['person']=titanic[['Age','Sex']].apply(split,axis=1)
titanic[0:10]

#count of male, female and child by classes
sns.factorplot('Pclass',hue='person',data=titanic,kind='count')

#Distribution of ages (one more method)
titanic['Age'].hist(bins=70,color='blue')

plt.hist(titanic['Age'].dropna(),bins=70)

#mean of ages
titanic['Age'].mean()

#count of male, female and child
titanic['person'].value_counts()

#KDE Plot of age with respect to gender
fig=sns.FacetGrid(titanic,hue='Sex',aspect=4)
fig.map(sns.kdeplot,'Age',shade=True)
oldest = titanic['Age'].max()
fig.set(xlim=(0,oldest))
fig.add_legend()

#KDE Plot of age with respect to person
fig=sns.FacetGrid(titanic,hue='person',aspect=4)
fig.map(sns.kdeplot,'Age',shade=True)
oldest = titanic['Age'].max()
fig.set(xlim=(0,oldest))
fig.add_legend()

#KDE Plot of age with respect to Pclass
fig=sns.FacetGrid(titanic,hue='Pclass',aspect=4)
fig.map(sns.kdeplot,'Age',shade=True)
oldest = titanic['Age'].max()
fig.set(xlim=(0,oldest))
fig.add_legend()

# Q2. What deck were the passengers on and how does that relate to their class?

titanic.head()

deck=titanic['Cabin'].dropna()

deck.head()

# We only need the first letter of the deck to classify its level (e.g. A,B,C,D,E,F,G)

levels = []

for level in deck:
    levels.append(level[0])

cabin = DataFrame(levels)
cabin.columns = ['Cabin']
sns.factorplot('Cabin',data=cabin,palette='winter_d',kind='count')

#We have a 'T' deck value there which doesn't make sense, we can drop it out

cabin=cabin[cabin.Cabin!='T']

sns.factorplot('Cabin',data=cabin,palette='rainbow',kind='count')

# Q3.) Where did the passengers come from?

titanic.head()

#plot of passengers embarked from different places
sns.factorplot('Embarked',data=titanic,kind='count',color='blue')

#count of embarkment by classes
sns.factorplot('Embarked',hue='Pclass',data=titanic,kind='count',x_order=['C','Q','S'],palette='ocean')

# Q4. Who was alone and who was with family?

titanic.head()

titanic['Alone']=titanic.SibSp+titanic.Parch

titanic.head()

titanic['Alone']

titanic['Alone'].loc[titanic['Alone']>0]='With Family'
titanic['Alone'].loc[titanic['Alone']==0]='Alone'

titanic.head()

# count of passengers with family or alone
sns.factorplot('Alone',data=titanic,kind='count',palette='hot')

#Q5.What factors helped someone survive the sinking?

titanic['Survivor']=titanic.Survived.map({0:'No',1:'Yes'})
titanic.head()

#count of passengers who survived or not
sns.factorplot('Survivor',data=titanic,kind='count')

#count of passengers who survived or not by classes
sns.factorplot('Pclass','Survived',data=titanic)

#count of passengers who survived or not by person
sns.factorplot('Pclass','Survived',hue='person',data=titanic)

#count of passengers who survived or not with respect to age
sns.lmplot('Age','Survived',data=titanic)

#count of passengers who survived or not with respect to age by classes
sns.lmplot('Age','Survived',hue='Pclass',data=titanic,palette='winter')

generations=[10,20,40,60,80]
sns.lmplot('Age','Survived',hue='Pclass',data=titanic,palette='summer',x_bins=generations)

#count of passengers who survived or not with respect to age by gender
sns.lmplot('Age','Survived',hue='Sex',data=titanic,palette='winter',x_bins=generations)

# Q6. Did the deck have an effect on the passengers survival rate?

def decklevel(cabin):
    deck=cabin
    for level in deck:
        levels=level[0]
        return levels
titanic['decklevels'] = titanic[['Cabin']].dropna().apply(decklevel,axis=1)

titanic.head()

# plot of survivors by decklevels
sns.factorplot('decklevels','Survived',data=titanic)

titanic=titanic[titanic.Cabin!='T']

sns.factorplot('decklevels','Survived',data=titanic)

# plot of survivors by decklevels with respect to person
sns.factorplot('decklevels','Survived',hue='person',data=titanic)

# plot of survivors by decklevels with respect to pclass
sns.factorplot('decklevels','Survived',hue='Pclass',data=titanic)

# plot of survivors by decklevels with respect to sex
sns.factorplot('decklevels','Survived',hue='Sex',data=titanic)

# Q7. Did having a family member increase the odds of surviving the crash?

#plot of surviors with family or not
sns.factorplot('Alone','Survived',data=titanic)

#plot of surviors with family or not by sex
sns.factorplot('Alone','Survived',hue='Sex',data=titanic)

#plot of surviors with family or not by pclass
sns.factorplot('Alone','Survived',hue='Pclass',data=titanic)

#plot of surviors with family or not by decklevels
sns.factorplot('Alone','Survived',hue='decklevels',data=titanic)

max_stars_repo_name: gargmihir/Titanic-Dataset-Analysis

max_issues_repo_name: gargmihir/Titanic-Dataset-Analysis

max_forks_repo_name: gargmihir/Titanic-Dataset-Analysis


text:
#!/usr/bin/env python3
##########################################################
## Jose F. Sanchez									  ##
## Copyright (C) 2019 Lauro Sumoy Lab, IGTP, Spain	  ##
##########################################################
'''
Calls FASTQC analysis and parses results generated.
'''
## useful imports
import time
import io
import os
import re
import sys
from sys import argv
from io import open
import pandas as pd

import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
from pandas.plotting import table
from matplotlib.backends.backend_pdf import PdfPages
import numpy as np

## import my modules
import HCGB
import HCGB.functions.system_call_functions as HCGB_sys
from BacterialTyper.config import set_config

############
def	help_options():
	print ("\nUSAGE:\npython %s folder file1 [file2] name fastqc threads\n"  %os.path.realpath(__file__))

############
def call_fastqc(path, file1, file2, sample, fastqc_bin, threads):
	## call system for fastqc sample given
	#name = functions.create_subfolder(sample, path)
	logFile = path + '/' + sample + '.log'

	if os.path.isfile(logFile):
		return ('OK')

	if not file2:
		cmd_fastqc = '%s --extract -t %s -o %s %s > %s 2> %s' %(fastqc_bin, threads, path, file1, logFile, logFile)
	else:
        ##print ("+ Calling fastqc for samples...")
	       cmd_fastqc = '%s --extract -t %s -o %s %s %s > %s 2> %s' %(fastqc_bin, threads, path, file1, file2, logFile, logFile)

    ## send command
	return (HCGB_sys.system_call( cmd_fastqc ))

############
def run_module_fastqc(path, files, sample, threads):
	## Arguments provided via ARGVs
	fastqc_bin = set_config.get_exe('fastqc')

	if (len(files) == 1):
		codeReturn = call_fastqc(path, files[0], "", sample, fastqc_bin, threads)
	elif (len(files) == 2):
		codeReturn = call_fastqc(path, files[0], files[1], sample, fastqc_bin, threads)
	else:
		print("ERROR: Some error ocurred while executing fastqc. Check samples provided")

	if codeReturn == 'FAIL':
		exit()
	path_to_sample = path + '/' + sample
	return path_to_sample

############
def main():
	## this code runs when call as a single script

  	## control if options provided or help
	if len(sys.argv) > 1:
		print ("")
	else:
		help_options()
		exit()

	path = os.path.abspath(argv[1])
	file1 = os.path.abspath(argv[2])
	file2 = os.path.abspath(argv[3])
	sample = argv[4]
	fastqc_bin = argv[5]
	threads = argv[5]

	## check if paired end
	if not (file2):
		print ('+ No implementation yet for single-end. Sorry.')
		exit()

	##
	path_to_sample = call_fastqc(path, file1, file2, sample, fastqc_bin, threads)

############
'''******************************************'''
if __name__== "__main__":
	main()

max_stars_repo_name: HCGB-IGTP/BacterialTyper

max_issues_repo_name: HCGB-IGTP/BacterialTyper

max_forks_repo_name: HCGB-IGTP/BacterialTyper


text:
# %%
import os
import pandas as pd
import numpy as np
import datetime

# %% CARGA DE DATOS

# MODELO
base = pd.read_csv(r'D:\Basededatos\esquema.csv')

# %%
path = r'D:\Basededatos\Origen\Chile'
os.chdir(path)
files = os.listdir(path)
files

# %%
files_xls = [f for f in files if f[-4:] == 'xlsx']
files_xls

# %%
columnas = ['PPU', 'TIPO VEHICULO', 'MARCA', 'MODELO', 'AÑO FABRICACION']
chile = pd.DataFrame(columns=columnas)

# %%
for f in files_xls:
    data = pd.read_excel(f, engine='openpyxl')
    chile = pd.concat([chile , data], ignore_index=True, join='outer')

# %%
chile.duplicated().sum()

# %%
chile.drop_duplicates(inplace=True)

# %%
chile.info()

# %% DATOS LIMPIOS
chile.to_csv(r'D:\Basededatos\Limpioparaentregar\chile.csv', index=False)

# %%
chile = pd.read_csv(r'D:\Basededatos\Limpioparaentregar\chile.csv')

# %% EQUIVALENCIA
chile["MERCADO"] = "CHILE"
chile["CANTIDAD"] = 1
chile.rename(columns={
                     'TIPO VEHICULO': 'SEGMENTO.1',
                     'AÑO FABRICACION': 'AÑO',
                     'PPU': 'PATENTE',
                     'MODELO': 'MODELO/VERSION'},
                     inplace=True)

# %% FORMATO PARA UNIR
columnasutiles = [
                  "MERCADO",
                  "SEGMENTO.1",
                  "MARCA",
                  "MODELO/VERSION",
                  "AÑO",
                  "PATENTE",
                  "CANTIDAD"]
chile = chile[columnasutiles]

# %%
chile["AÑO"].replace('nan', None, inplace=True)
chile["AÑO"].replace('NaN', None, inplace=True)

# %%
chile["AÑO"] = chile["AÑO"].astype("float").map('{:.0f}'.format)
chile["AÑO"] = chile["AÑO"].astype(int)
chile["AÑO"] = chile["AÑO"].astype(str)

# %% Dividir MODELO/VERSION
chile["CILINDRADA"] = chile["MODELO/VERSION"].str.extract(r'(\d\.\d)', expand=False).str.strip()
new = chile["MODELO/VERSION"].str.split(" ", n = 1, expand = True)
chile["MODELO"] = new[0]

# %%
chile.info()

# %%
chile.to_csv(r'D:\Basededatos\Limpioparaunir\chile.csv', index=False)

# %%
dict(chile["MODELO"].value_counts())
# %%

max_stars_repo_name: Demonliquid/cars-python-cleaning

max_issues_repo_name: Demonliquid/cars-python-cleaning

max_forks_repo_name: Demonliquid/cars-python-cleaning


text:
import os
import json
import numpy as np
import re
import torch
import torch_rl
import gym
from torchvision import transforms
from PIL import Image

import utils

def get_obss_preprocessor(env_id, obs_space, model_dir):
    # Check if it is a MiniGrid environment
    if re.match("MiniGrid-.*", env_id):
        obs_space = {"image": obs_space.spaces['image'].shape, "text": 500} #100

        vocab = Vocabulary(model_dir, obs_space["text"])
        def preprocess_obss(obss, device=None):
            return torch_rl.DictList({
                "image": preprocess_matrix([obs["image"] for obs in obss], device=device),
                "text": preprocess_texts([obs["mission"] for obs in obss], vocab, device=device)
            })
        preprocess_obss.vocab = vocab

    # Part for Hyrule env
    elif re.match("Hyrule-.*", env_id):
        obs_space = {"image": obs_space.shape}

        def preprocess_obss(obss, device=None):
            return torch_rl.DictList({
                "image": preprocess_natural_images([obs["image"] for obs in obss], device=device),
                "goal": preprocess_visible_text([obs["mission"] for obs in obss], device=device),
                "rel_gps": preprocess_matrix([obs["rel_gps"] for obs in obss], device=device),
                "visible_text": preprocess_visible_text([obs["visible_text"] for obs in obss], device=device)
            })

    # Check if the obs_space is of type Box([X, Y, 3])
    elif isinstance(obs_space, gym.spaces.Box) and len(obs_space.shape) == 3 and obs_space.shape[2] == 3:
        obs_space = {"image": obs_space.shape}

        def preprocess_obss(obss, device=None):
            return torch_rl.DictList({
                "image": preprocess_matrix(obss, device=device)
            })
    else:
        raise ValueError("Unknown observation space: " + str(obs_space))

    return obs_space, preprocess_obss

def preprocess_matrix(images, device=None):
    # Bug of Pytorch: very slow if not first converted to numpy array
    images = np.array(images)
    return torch.tensor(images, device=device, dtype=torch.float)

def preprocess_visible_text(visible_text_dict, device=None):
    house_numbers = []
    street_names = []
    for idx in range(len(visible_text_dict)):
        house_numbers.append(visible_text_dict[idx]["house_numbers"])
        street_names.append(visible_text_dict[idx]["street_names"])
    house_numbers = np.array(house_numbers)
    street_names = np.array(street_names)
    return {"house_numbers": torch.tensor(house_numbers, device=device, dtype=torch.float),
            "street_names": torch.tensor(street_names, device=device, dtype=torch.float)}

def preprocess_natural_images(images, device=None):
    images = np.array(images, dtype=np.uint8)
    transform = transforms.Compose([transforms.ToTensor(),
                                    transforms.Normalize(mean=[0.437, 0.452, 0.479], std=[0.2495, 0.2556, 0.2783])])
    tensor_list = []
    for idx, image in enumerate(images):
        tensor_list.append(transform(image))
    return torch.stack(tensor_list).to(device)

def preprocess_texts(texts, vocab, device=None):
    var_indexed_texts = []
    max_text_len = 0

    for text in texts:
        tokens = re.findall("[+-]*[a-z0-9]+", text.lower())
        var_indexed_text = np.array([vocab[token] for token in tokens])
        var_indexed_texts.append(var_indexed_text)
        max_text_len = max(len(var_indexed_text), max_text_len)

    indexed_texts = np.zeros((len(texts), max_text_len))

    for i, indexed_text in enumerate(var_indexed_texts):
        indexed_texts[i, :len(indexed_text)] = indexed_text

    return torch.tensor(indexed_texts, device=device, dtype=torch.long)

class Vocabulary:
    """A mapping from tokens to ids with a capacity of `max_size` words.
    It can be saved in a `vocab.json` file."""

    def __init__(self, model_dir, max_size):
        self.path = utils.get_vocab_path(model_dir)
        self.max_size = max_size
        self.vocab = {}
        if os.path.exists(self.path):
            self.vocab = json.load(open(self.path))

    def __getitem__(self, token):
        if not token in self.vocab.keys():
            if len(self.vocab) >= self.max_size:
                raise ValueError("Maximum vocabulary capacity reached")
            self.vocab[token] = len(self.vocab) + 1
        return self.vocab[token]

    def save(self):
        utils.create_folders_if_necessary(self.path)
        json.dump(self.vocab, open(self.path, "w"))

max_stars_repo_name: mandanasmi/torch-rl

max_issues_repo_name: mandanasmi/torch-rl

max_forks_repo_name: mandanasmi/torch-rl


text:
"""
Created on Feb 28, 2017

@author: Siyuan Huang

Data Processing for Affordance

"""

import numpy as np
import pickle
import config
import os
import subactivity_lstm
from keras.preprocessing import sequence

def load_affordance_data(path):
    relative_path = 'flipped/all/activity_corpus.p'
    if os.path.exists(path + relative_path):
        activity_corpus = pickle.load(open(path + relative_path, 'rb'))
    affordance = list()
    skeleton = list()
    object_pos = list()
    object_category = list()
    for activity, tpgs in activity_corpus.items():
        for tpg in tpgs:
            for spg in tpg.terminals:
                for obj_poses, obj_affordance, obj_object in zip(spg.obj_positions, spg.affordance, spg.objects):
                    empty = 0
                    for obj_pos in obj_poses:
                        if obj_pos.shape[0] == 0:
                            empty = 1
                    if empty == 1:
                        continue
                    else:
                        affordance.append(obj_affordance)
                        skeleton.append(spg.skeletons)
                        object_pos.append(obj_poses)
                        object_category.append(obj_object)
    return feature_selection_completion(object_category, object_pos, affordance, skeleton)

def feature_selection(obj_category, obj_pos, obj_affordance, skeleton):
    maxlen = 200
    affordance_label = ['reachable', 'movable', 'pourable', 'pourto', 'containable', 'drinkable', 'openable', 'placeable', 'closeable', 'cleanable', 'cleaner', 'stationary']
    object_label = ['medcinebox', 'cup', 'bowl', 'box', 'milk', 'book', 'microwave', 'plate', 'remote', 'cloth']
    num_sample = len(obj_category)
    object_feature = np.zeros((num_sample, 10))
    affordance_feature = np.zeros((num_sample, 12))
    distance_feature = np.zeros((num_sample, maxlen, 11))
    orientation_feature = np.zeros((num_sample, maxlen, 1))
    for i in range(num_sample):
        object_feature[i, object_label.index(obj_category[i])] = 1
        affordance_feature[i, affordance_label.index(obj_affordance[i])] = 1
        num_frame = len(skeleton[i])
        distance_feature_sub = np.zeros((num_frame, 11))
        orientation_feature_sub = np.zeros((num_frame, 1))
        frame = 0
        for skeleton_frame, obj_pos_frame in zip(skeleton[i], obj_pos[i]):
            skeleton_frame_temp = subactivity_lstm.skeleton_prune(skeleton_frame)
            obj_pos_mid = np.mean(obj_pos_frame, axis=0)
            distance_feature_sub[frame, :] = np.array([np.linalg.norm(obj_pos_mid - skeleton_frame_temp[j*3:(j+1)*3]) for j in range(11)])
            n_1 = skeleton_frame_temp[6:9] - skeleton_frame_temp[9:12]
            n_2 = skeleton_frame_temp[6:9] - skeleton_frame_temp[15:18]
            n_normal = np.cross(n_1, n_2)
            n_center = obj_pos_mid - np.mean([skeleton_frame_temp[6:9], skeleton_frame_temp[9:12], skeleton_frame_temp[15:18]], axis=0)
            orientation_feature_sub[frame, :] = np.arccos(np.clip(np.dot(n_normal, n_center) / np.linalg.norm(n_normal) / np.linalg.norm(n_center), -1, 1))
            frame += 1
        distance_feature[i, :, :] = np.transpose(sequence.pad_sequences(np.transpose(distance_feature_sub), dtype=float, maxlen=maxlen))
        orientation_feature[i, :, :] = np.transpose(sequence.pad_sequences(np.transpose(orientation_feature_sub), dtype=float, maxlen=maxlen))
    return object_feature, affordance_feature, distance_feature, orientation_feature

def feature_selection_completion(obj_category, obj_pos, obj_affordance, skeleton):
    maxlen = 200
    affordance_label = ['reachable', 'movable', 'pourable', 'pourto', 'containable', 'drinkable', 'openable', 'placeable', 'closeable', 'cleanable', 'cleaner', 'stationary']
    object_label = ['medcinebox', 'cup', 'bowl', 'box', 'milk', 'book', 'microwave', 'plate', 'remote', 'cloth']
    num_sample = len(obj_category)
    object_feature = np.zeros((num_sample, 10))
    affordance_feature = np.zeros((num_sample, 12))
    normal_feature = np.zeros((num_sample, maxlen, 33))
    orientation_feature = np.zeros((num_sample, maxlen, 1))
    for i in range(num_sample):
        object_feature[i, object_label.index(obj_category[i])] = 1
        affordance_feature[i, affordance_label.index(obj_affordance[i])] = 1
        num_frame = len(skeleton[i])
        normal_feature_sub = np.zeros((num_frame, 33))
        orientation_feature_sub = np.zeros((num_frame, 1))
        frame = 0
        for skeleton_frame, obj_pos_frame in zip(skeleton[i], obj_pos[i]):
            skeleton_frame_temp = subactivity_lstm.skeleton_prune(skeleton_frame)
            obj_pos_mid = np.mean(obj_pos_frame, axis=0)
            normal_feature_sub[frame, :] = np.array([obj_pos_mid - skeleton_frame_temp[j*3:(j+1)*3] for j in range(11)]).flatten()
            n_1 = skeleton_frame_temp[6:9] - skeleton_frame_temp[9:12]
            n_2 = skeleton_frame_temp[6:9] - skeleton_frame_temp[15:18]
            n_normal = np.cross(n_1, n_2)
            n_center = obj_pos_mid - np.mean([skeleton_frame_temp[6:9], skeleton_frame_temp[9:12], skeleton_frame_temp[15:18]], axis=0)
            orientation_feature_sub[frame, :] = np.arccos(np.clip(np.dot(n_normal, n_center) / np.linalg.norm(n_normal) / np.linalg.norm(n_center), -1, 1))
            frame += 1
        normal_feature[i, :, :] = np.transpose(sequence.pad_sequences(np.transpose(normal_feature_sub), dtype=float, maxlen=maxlen))
        orientation_feature[i, :, :] = np.transpose(sequence.pad_sequences(np.transpose(orientation_feature_sub), dtype=float, maxlen=maxlen))
    return object_feature, affordance_feature, normal_feature, orientation_feature

def main():
    paths = config.Paths()
    paths.path_huang()
    load_affordance_data(paths.metadata_root)

if __name__ == '__main__':
    main()


text:
import torch
import argparse
import pandas as pd
from tqdm import tqdm
import networkx as nx
from os import listdir
from train import GAT_wrapper
from os.path import isfile, join
from sklearn.metrics import f1_score
from sklearn.metrics import accuracy_score
from event_graph_utils import process_event_dataset_from_networkx, regularization

if __name__ == "__main__":

    # Training settings
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--no-cuda", action="store_true", default=False, help="Disables CUDA training."
    )
    parser.add_argument(
        "--fastmode",
        action="store_true",
        default=False,
        help="Validate during training pass.",
    )
    parser.add_argument(
        "--sparse",
        action="store_true",
        default=False,
        help="GAT with sparse version or not.",
    )
    parser.add_argument("--seed", type=int, default=72, help="Random seed.")
    parser.add_argument(
        "--epochs", type=int, default=10000, help="Number of epochs to train."
    )
    parser.add_argument(
        "--lr", type=float, default=0.005, help="Initial learning rate."
    )
    parser.add_argument(
        "--weight_decay",
        type=float,
        default=5e-4,
        help="Weight decay (L2 loss on parameters).",
    )
    parser.add_argument("--hidden", type=int, default=8, help="Number of hidden units.")
    parser.add_argument(
        "--nb_heads", type=int, default=8, help="Number of head attentions."
    )
    parser.add_argument(
        "--dropout",
        type=float,
        default=0.6,
        help="Dropout rate (1 - keep probability).",
    )
    parser.add_argument(
        "--alpha", type=float, default=0.2, help="Alpha for the leaky_relu."
    )
    parser.add_argument("--patience", type=int, default=100, help="Patience")

    # Parsing our args...
    args = parser.parse_args()
    args.cuda = not args.no_cuda and torch.cuda.is_available()

    path_datasets = "datasets_runs/"
    network_files = [
        f for f in listdir(path_datasets) if isfile(join(path_datasets, f))
    ]

    experimental_results = []

    # Runs the full pipeline 10 times for all the datasets in dataset_runs/
    for i in range(1, 11):
        for network_file in tqdm(network_files):

            print("Networkfile", network_file)

            G = nx.read_gpickle(path_datasets + network_file)

            regularization(G, 512, embedding_feature="features")

            (
                adj,
                features,
                labels,
                idx_train,
                idx_test,
                df_labels,
            ) = process_event_dataset_from_networkx(G)

            print(adj.shape, features.shape, len(idx_train), len(idx_test))

            gat = GAT_wrapper(args)
            gat.train_pipeline(adj, features, labels, idx_train, idx_train, idx_test)

            loss, acc, output = gat.compute_test()

            y_pred = output.cpu().numpy() if args.cuda else output.numpy()
            y_true = []

            for event_id in idx_test:
                for node in G.nodes():
                    if ":event" in node:
                        if G.nodes[node]["id"] == event_id:
                            y_true.append(
                                df_labels[
                                    df_labels.event_id == event_id
                                ].label_code.values[0]
                            )

            f1_macro = f1_score(y_true, y_pred, average="macro")
            acc = accuracy_score(y_true, y_pred)

            print("--->", network_file, "f1_macro", f1_macro, "acc", acc)
            experimental_results.append(
                (network_file, "f1_macro", f1_macro, "acc", acc, y_true, y_pred)
            )
            del gat
            del adj
            del features
            del G

        df_results = pd.DataFrame(experimental_results)
        df_results.to_csv(
            f"./gat_results_{i}_news_cluster_5w1h_graph_hin.csv", index=False
        )

max_stars_repo_name: joaopedromattos/GNEE

max_issues_repo_name: joaopedromattos/GNEE

max_forks_repo_name: joaopedromattos/GNEE

reason: import networkx


text:
import cv2
import numpy as np
import os

# create_watermark ===============================================================
# return:
#	(<Boolean> True/False), depending on the transformation process
def create_watermark(nude):

	# Add alpha channel if missing
	# if nude.shape[2] < 4:
	# 	nude = np.dstack([nude, np.ones((512, 512), dtype="uint8") * 255])

	# watermark = cv2.imread("fake.png", cv2.IMREAD_UNCHANGED)

	# f1 = np.asarray([0, 0, 0, 250])   # red color filter
	# f2 = np.asarray([255, 255, 255, 255])
	# mask = cv2.bitwise_not(cv2.inRange(watermark, f1, f2))
	# mask_inv = cv2.bitwise_not(mask)

	# res1 = cv2.bitwise_and(nude, nude, mask = mask)
	# # res2 = cv2.bitwise_and(nude, nude, mask = mask)
	# # res2 = cv2.bitwise_and(watermark, watermark, mask = mask_inv)
	# res = res1

	# alpha = 0.6
	# return cv2.addWeighted(res, alpha, nude, 1 - alpha, 0)
	return nude


text:
import os
import pandas as pd
import numpy as np

def grepper(name,method):
    cmd = 'grep -A1 ' + "'Alpha  occ. eigenvalues' " + str(name) + '/' + str(method) + '/mexc.out' + '>' + str(name)+ '/' + str(method)+ '/' + 'orbs.out'
    #print(cmd)
    os.system(cmd)

    return

def LUMO(name,method):
    filename = open(str(name)+ '/' + str(method)+ '/' + 'orbs.out','r')
    data = filename.readlines()
    lll = data[-1].split(' ')
   #  print(lll)
    LUMO = float(lll[7])*27.2114
    #print((name,'LUMO',method,LUMO))
  #  for i in filename:
  #      print(i)

    return [name,'LUMO',method,str(LUMO)]

def HOMO(name,method):
    filename = open(str(name)+ '/' + str(method)+ '/' + 'orbs.out','r')
    data = filename.readlines()
  #  print(data[-2])
    lll = data[-2].split(' ')
    #print(lll)
    HOMO = float(lll[-1])*27.2114
    #print((name,'HOMO',method,HOMO))

    return [name,'HOMO',method,str(HOMO)]

def pandass(HOMO,LUMO):
    hom = np.array(HOMO)
    lum = np.array(LUMO)

    #print(HOMO)
    df =  pd.DataFrame(hom,columns=['Name','Type','Method','Energy (eV)'])
    print(df)
    df_2 = pd.DataFrame(lum,columns=['Name','Type','Method','Energy (eV)'])
    print(df_2)

    return

def Main():
    path_to_files = '/Users/tsantaloci/Desktop/python_projects/austin/Dyes/Benchmark/results'
    os.chdir(path_to_files)
    #name = ['NL2','AP25','JW1','R4']
    method = ['mexc','mexc_dichloromethane','pbe1pbe','pbe1pbe_dichloromethane']
    name =  ['DQ5', 'S-DAHTDTT', 'NKX-2883', 'S3', 'HKK-BTZ4', 'TPA-T-TTAR-A', 'NL7', 'NL8', 'AP3', 'NL11', 'C271',  'IQ4', 'WS-55', 'SGT-130', 'FNE52', 'IQ21', 'SGT-136', 'D-DAHTDTT', 'R6', 'TPA-TTAR-A', 'T-DAHTDTT', 'TH304', 'NL4', 'C258', 'TTAR-9', 'SGT-121', 'TTAR-15', 'NL2', 'SGT-129', 'FNE32', 'BTD-1', 'Y123', 'C272', 'FNE34', 'IQ6', 'TP1', 'TTAR-B8', 'R4', 'TPA-T-TTAR-T-A','ZL003','WS-6','NL4','NL6','JW1','AP25']
    method = ['mexc','mexc_dichloromethane','pbe1pbe','pbe1pbe_dichloromethane']
   # method = ['mexc','mexc_dichloromethane',]
    HOMO_1 = []
    LUMO_1 = []
    for bench in name:
        for meth in method:
            grepper(bench,meth)
            HOMO_1.append(HOMO(bench,meth))
            LUMO_1.append(LUMO(bench,meth))
    pandass(HOMO_1,LUMO_1)

    return
Main()


text:
"""
Entry module and class module for training a GroundedTranslation model.
"""

from __future__ import print_function

import argparse
import logging
from math import ceil
import sys

from Callbacks import CompilationOfCallbacks
from data_generator import VisualWordDataGenerator
import models

import keras.callbacks

# Set up logger
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
logger = logging.getLogger(__name__)

class GroundedTranslation(object):

    def __init__(self, args, datagen=None):
        '''
        Initialise the model and set Theano debugging model if
        self.args.debug is true. Prepare the data generator if necessary.
        '''

        self.args = args
        self.data_generator = datagen
        self.use_sourcelang = args.source_vectors is not None
        self.use_image = not args.no_image
        self.log_run_arguments()
        self.data_generator=datagen
        self.prepare_datagenerator()

        if self.args.debug:
            theano.config.optimizer = 'fast_compile'
            theano.config.exception_verbosity = 'high'

    def train_model(self):
        '''
        Initialise the data generator to process the data in a memory-friendly
        manner. Then build the Keras model, given the user-specified arguments
        (or the initial defaults). Train the model for self.args.max_epochs
        and return the training and validation losses.

        The losses object contains a history variable. The history variable is
        a dictionary with a list of training and validation losses:

        losses.history.['loss']
        losses.history.['val_loss']
        '''

        if not self.use_sourcelang:
            hsn_size = 0
        else:
            hsn_size = self.data_generator.hsn_size  # ick

        if self.args.mrnn:
            m = models.MRNN(self.args.embed_size, self.args.hidden_size,
                            self.V, self.args.dropin,
                            self.args.optimiser, self.args.l2reg,
                            hsn_size=hsn_size,
                            weights=self.args.init_from_checkpoint,
                            gru=self.args.gru,
                            clipnorm=self.args.clipnorm,
                            t=self.data_generator.max_seq_len,
                            lr=self.args.lr)
        else:
            m = models.NIC(self.args.embed_size, self.args.hidden_size,
                           self.V, self.args.dropin,
                           self.args.optimiser, self.args.l2reg,
                           hsn_size=hsn_size,
                           weights=self.args.init_from_checkpoint,
                           gru=self.args.gru,
                           clipnorm=self.args.clipnorm,
                           t=self.data_generator.max_seq_len,
                           lr=self.args.lr)

        model = m.buildKerasModel(use_sourcelang=self.use_sourcelang,
                                  use_image=self.use_image)

        callbacks = CompilationOfCallbacks(self.data_generator.word2index,
                                           self.data_generator.index2word,
                                           self.args,
                                           self.args.dataset,
                                           self.data_generator,
                                           use_sourcelang=self.use_sourcelang,
                                           use_image=self.use_image)

        train_generator = self.data_generator.random_generator('train')
        train_size = self.data_generator.split_sizes['train']
        val_generator = self.data_generator.fixed_generator('val')
        val_size = self.data_generator.split_sizes['val']

        losses = model.fit_generator(generator=train_generator,
                                     samples_per_epoch=train_size,
                                     nb_epoch= self.args.max_epochs,
                                     verbose=1,
                                     callbacks=[callbacks],
                                     nb_worker=1,
                                     validation_data=val_generator,
                                     nb_val_samples=val_size)

        return losses

    def prepare_datagenerator(self):
        '''
        Initialise the data generator and its datastructures, unless a valid
        data generator was already passed into the
        GroundedTranslation.__init() function.
        '''

        # Initialise the data generator if it has not yet been initialised
        if self.data_generator == None:
            self.data_generator = VisualWordDataGenerator(self.args,
                                                          self.args.dataset)

            # Extract the working vocabulary from the training dataset
            if self.args.existing_vocab != "":
                self.data_generator.set_vocabulary(self.args.existing_vocab)
            else:
                self.data_generator.extract_vocabulary()
        self.V = self.data_generator.get_vocab_size()

    def log_run_arguments(self):
        '''
        Save the command-line arguments, along with the method defaults,
        used to parameterise this run.
        '''
        logger.info("Run arguments:")
        for arg, value in self.args.__dict__.iteritems():
            logger.info("%s: %s" % (arg, str(value)))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Train an neural image description model")

    # General options
    parser.add_argument("--run_string", default="", type=str,
                        help="Optional string to help you identify the run")
    parser.add_argument("--debug", action="store_true",
                        help="Print debug messages to stdout?")
    parser.add_argument("--init_from_checkpoint", help="Initialise the model\
                        parameters from a pre-defined checkpoint? Useful to\
                        continue training a model.", default=None, type=str)
    parser.add_argument("--fixed_seed", action="store_true",
                        help="Start with a fixed random seed? Useful for\
                        reproding experiments. (default = False)")
    parser.add_argument("--num_sents", default=5, type=int,
                        help="Number of descriptions/image for training")
    parser.add_argument("--meteor_lang", type=str, required=True,
                        help="Language of the input dataset. Required for\
			correct Meteor evaluation. See\
                        http://www.cs.cmu.edu/~alavie/METEOR/README.html#languages\
                        for options.")

    # Define the types of input data the model will receive
    parser.add_argument("--dataset", default="", type=str, help="Path to the\
                        HDF5 dataset to use for training / val input\
                        (defaults to flickr8k)")
    parser.add_argument("--supertrain_datasets", nargs="+", help="Paths to the\
                        datasets to use as additional training input (defaults\
                        to None)")
    parser.add_argument("--unk", type=int,
                        help="unknown character cut-off. Default=3", default=3)
    parser.add_argument("--maximum_length", type=int, default=50,
                        help="Maximum length of sequences permissible\
			in the training data (Default = 50)")
    parser.add_argument("--existing_vocab", type=str, default="",
                        help="Use an existing vocabulary model to define the\
                        vocabulary and UNKing in this dataset?\
                        (default = "", which means we will derive the\
                        vocabulary from the training dataset")
    parser.add_argument("--no_image", action="store_true",
                        help="Do not use image data.")
    parser.add_argument("--source_vectors", default=None, type=str,
                        help="Path to final hidden representations of\
                        encoder/source language VisualWordLSTM model.\
                        (default: None.) Expects a final_hidden_representation\
                        vector for each image in the dataset")
    parser.add_argument("--source_enc", type=str, default=None,
                        help="Which type of source encoder features? Expects\
                        either 'mt_enc' or 'vis_enc'. Required.")
    parser.add_argument("--source_type", type=str, default=None,
                        help="Source features over gold or predicted tokens?\
                        Expects 'gold' or 'predicted'. Required")
    parser.add_argument("--source_merge", type=str, default="sum",
                        help="How to merge source features. Only applies if \
                        there are multiple feature vectors. Expects 'sum', \
                        'avg', or 'concat'.")

    # Model hyperparameters
    parser.add_argument("--batch_size", default=100, type=int)
    parser.add_argument("--embed_size", default=256, type=int)
    parser.add_argument("--hidden_size", default=256, type=int)
    parser.add_argument("--dropin", default=0.5, type=float,
                        help="Prob. of dropping embedding units. Default=0.5")
    parser.add_argument("--gru", action="store_true", help="Use GRU instead\
                        of LSTM recurrent state? (default = False)")
    parser.add_argument("--mrnn", action="store_true",
                        help="Use a Mao-style multimodal recurrent neural\
                        network?")
    parser.add_argument("--peeking_source", action="store_true",
                        help="Input the source features at every timestep?\
                        Default=False.")

    # Optimisation details
    parser.add_argument("--optimiser", default="adam", type=str,
                        help="Optimiser: rmsprop, momentum, adagrad, etc.")
    parser.add_argument("--lr", default=0.001, type=float)
    parser.add_argument("--beta1", default=None, type=float)
    parser.add_argument("--beta2", default=None, type=float)
    parser.add_argument("--epsilon", default=None, type=float)
    parser.add_argument("--stopping_loss", default="bleu", type=str,
                        help="minimise cross-entropy or maximise BLEU?")
    parser.add_argument("--l2reg", default=1e-8, type=float,
                        help="L2 cost penalty. Default=1e-8")
    parser.add_argument("--clipnorm", default=-1, type=float,
                        help="Clip gradients? (default = -1, which means\
                        don't clip the gradients.")
    parser.add_argument("--max_epochs", default=50, type=int,
                        help="Maxmimum number of training epochs. Used with\
                        --predefined_epochs")
    parser.add_argument("--patience", type=int, default=10, help="Training\
                        will be terminated if validation BLEU score does not\
                        increase for this number of epochs")
    parser.add_argument("--no_early_stopping", action="store_true")

    # Language generation details
    parser.add_argument("--generation_timesteps", default=30, type=int,
                        help="Maximum number of words to generate for unseen\
                        data (default=10).")

    # Legacy options
    parser.add_argument("--generate_from_N_words", type=int, default=0,
                        help="Use N words as starting point when generating\
                        strings. Useful mostly for mt-only model (in other\
                        cases, image provides enough useful starting\
                        context.)")
    parser.add_argument("--predefined_epochs", action="store_true",
                        help="Do you want to stop training after a specified\
                        number of epochs, regardless of early-stopping\
                        criteria? Use in conjunction with --max_epochs.")

    # Neccesary but unused in this module
    parser.add_argument("--h5_writeable", action="store_true",
                        help="Open the H5 file for write-access? Useful for\
                        serialising hidden states to disk. (default = False)")
    parser.add_argument("--use_predicted_tokens", action="store_true",
                        help="Generate final hidden state\
                        activations over oracle inputs or from predicted\
                        inputs? Default = False ( == Oracle)")

    arguments = parser.parse_args()

    if arguments.source_vectors is not None:
        if arguments.source_type is None or arguments.source_enc is None:
            parser.error("--source_type and --source_enc are required when\
                        using --source_vectors")

    if arguments.fixed_seed:
        import numpy as np
        np.random.seed(1234)

    import theano
    model = GroundedTranslation(arguments)
    model.train_model()

max_stars_repo_name: terpiljenya/GroundedTranslation

max_issues_repo_name: terpiljenya/GroundedTranslation

max_forks_repo_name: terpiljenya/GroundedTranslation

reason: import numpy,import theano