text:
# Acoustic wave equations with Devito
# Forward/adjoint nonlinear and Born modeling
# Authors: Mathias Louboutin, Philipp Witte
# Date: November 2017
#

# Import modules
from __future__ import print_function
import numpy as np
import gc, os, psutil
from numpy.random import randint
from sympy import solve, cos, sin, expand, symbols
from sympy import Function as fint
from devito.logger import set_log_level
from devito import Eq, Function, TimeFunction, Dimension, Operator, clear_cache, ConditionalDimension, DefaultDimension, Inc
from devito import first_derivative, left, right
from PySource import PointSource, Receiver
from PyModel import Model
from checkpoint import DevitoCheckpoint, CheckpointOperator
from pyrevolve import Revolver
from utils import freesurface

def J_adjoint(model, src_coords, wavelet, rec_coords, recin, space_order=12,
                t_sub_factor=20, h_sub_factor=2, checkpointing=False, free_surface=False,
                n_checkpoints=None, maxmem=None, dt=None, isic=False):
    if checkpointing:
        F = forward_modeling(model, src_coords, wavelet, rec_coords, save=True, space_order=space_order,
                             t_sub_factor=t_sub_factor, h_sub_factor=h_sub_factor, dt=dt, op_return=True,
                             free_surface=free_surface)
        grad = adjoint_born(model, rec_coords, recin, op_forward=F, space_order=space_order,
                            is_residual=True, isic=isic, n_checkpoints=n_checkpoints, maxmem=maxmem,
                            free_surface=free_surface)
    else:
        u0 = forward_modeling(model, src_coords, wavelet, None, save=True, space_order=space_order,
                              t_sub_factor=t_sub_factor, h_sub_factor=h_sub_factor, dt=dt, free_surface=free_surface)
        grad = adjoint_born(model, rec_coords, recin, u=u0, space_order=space_order, isic=isic, dt=dt, free_surface=free_surface)

    return grad

def acoustic_laplacian(v, rho):
    if rho is None:
        Lap = v.laplace
        rho = 1
    else:
        if isinstance(rho, Function):
            Lap = sum([first_derivative(first_derivative(v, fd_order=int(v.space_order/2), side=left, dim=d) / rho,
                       fd_order=int(v.space_order/2), dim=d, side=right) for d in v.space_dimensions])
        else:
            Lap = 1 / rho * v.laplace
    return Lap, rho

def forward_modeling(model, src_coords, wavelet, rec_coords, save=False, space_order=8,
                     free_surface=False, op_return=False, u_return=False, dt=None, tsub_factor=1):
    clear_cache()
    # If wavelet is file, read it
    if isinstance(wavelet, str):
        wavelet = np.load(wavelet)

    # Parameters
    nt = wavelet.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, rho, damp = model.m, model.rho, model.damp

    # Create the forward wavefield
    if save is False and rec_coords is not None:
        u = TimeFunction(name='u', grid=model.grid, time_order=2, space_order=space_order)
        eqsave = []
    elif save is True and tsub_factor > 1:
        u = TimeFunction(name='u', grid=model.grid, time_order=2, space_order=space_order)
        time_subsampled = ConditionalDimension(name='t_sub', parent=u.grid.time_dim, factor=tsub_factor)
        nsave = (nt-1)//tsub_factor + 2
        usave = TimeFunction(name='us', grid=model.grid, time_order=2, space_order=space_order, time_dim=time_subsampled, save=nsave)
        eqsave = [Eq(usave.forward, u.forward)]
    else:
        u = TimeFunction(name='u', grid=model.grid, time_order=2, space_order=space_order, save=nt)
        eqsave = []

    # Set up PDE and rearrange
    ulaplace, rho = acoustic_laplacian(u, rho)
    # H = symbols('H')
    # eqn = m / rho * u.dt2 - H + damp * u.dt
    stencil = damp * ( 2.0 * u - damp * u.backward + dt**2 * rho / m * ulaplace)

    # Input source is wavefield
    if isinstance(wavelet, TimeFunction):
        wf_src = TimeFunction(name='wf_src', grid=model.grid, time_order=2, space_order=space_order, save=nt)
        wf_src._data = wavelet._data
        eqn -= wf_src

    # Rearrange expression
    # stencil = solve(eqn, u.forward)
    expression = [Eq(u.forward, stencil)]

    # Data is sampled at receiver locations
    if rec_coords is not None:
        rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
        rec_term = rec.interpolate(expr=u)
        expression += rec_term

    # Create operator and run
    if save:
        expression += eqsave

    # Free surface
    kwargs = dict()
    if free_surface is True:
        expression += freesurface(u, space_order//2, model.nbpml)

    # Source symbol with input wavelet
    if src_coords is not None:
        src = PointSource(name='src', grid=model.grid, ntime=nt, coordinates=src_coords)
        src.data[:] = wavelet[:]
        src_term = src.inject(field=u.forward, expr=src.dt * rho * dt**2 / m)
        expression += src_term

    subs = model.spacing_map
    subs[u.grid.time_dim.spacing] = dt
    op = Operator(expression, subs=subs, dse='advanced', dle='advanced')
    if op_return is False:
        cf = op.cfunction
        summary = op.apply()
        if save is True and tsub_factor > 1:
            if rec_coords is not None:
                return rec.data, usave, summary
            else:
                return usave, summary
        else:
            if rec_coords is None:
                return u, summary
            else:
                return rec.data, u, summary
    else:
        if u_return is False:
            return op
        else:
            if tsub_factor > 1:
                return op, usave
            else:
                return op, u

def adjoint_modeling(model, src_coords, rec_coords, rec_data, space_order=8, free_surface=False, dt=None):
    clear_cache()

    # If wavelet is file, read it
    if isinstance(rec_data, str):
        rec_data = np.load(rec_data)

    # Parameters
    nt = rec_data.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, rho, damp = model.m, model.rho, model.damp

    # Create the adjoint wavefield
    if src_coords is not None:
        v = TimeFunction(name="v", grid=model.grid, time_order=2, space_order=space_order)
    else:
        v = TimeFunction(name="v", grid=model.grid, time_order=2, space_order=space_order, save=nt)

    # Set up PDE and rearrange
    vlaplace, rho = acoustic_laplacian(v, rho)

    # Input data is wavefield
    full_q = 0
    if isinstance(rec_data, TimeFunction):
        wf_rec = TimeFunction(name='wf_rec', grid=model.grid, time_order=2, space_order=space_order, save=nt)
        wf_rec._data = rec_data._data
        full_q = wf_rec

    stencil = damp * (2.0 * v - damp * v.forward + dt**2 * rho / m * (vlaplace + full_q))
    expression = [Eq(v.backward, stencil)]

    # Free surface
    if free_surface is True:
        expression += freesurface(v, space_order//2, model.nbpml, forward=False)

    # Adjoint source is injected at receiver locations
    if rec_coords is not None:
        rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
        rec.data[:] = rec_data[:]
        adj_src = rec.inject(field=v.backward, expr=rec * rho * dt**2 / m)
        expression += adj_src

    # Data is sampled at source locations
    if src_coords is not None:
        src = PointSource(name='src', grid=model.grid, ntime=nt, coordinates=src_coords)
        adj_rec = src.interpolate(expr=v)
        expression += adj_rec

    # Create operator and run

    subs = model.spacing_map
    subs[v.grid.time_dim.spacing] = dt
    op = Operator(expression, subs=subs, dse='advanced', dle='advanced')
    cf = op.cfunction
    summary = op.apply()
    if src_coords is None:
        return v, summary
    else:
        return src.data, summary

def forward_born(model, src_coords, wavelet, rec_coords, space_order=8, isic=False, dt=None, free_surface=False):
    clear_cache()

    # Parameters
    nt = wavelet.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, rho, dm, damp = model.m, model.rho, model.dm, model.damp

    # Create the forward and linearized wavefield
    u = TimeFunction(name="u", grid=model.grid, time_order=2, space_order=space_order)
    du = TimeFunction(name="du", grid=model.grid, time_order=2, space_order=space_order)
    if len(model.shape) == 2:
        x,y = u.space_dimensions
    else:
        x,y,z = u.space_dimensions

    # Set up PDEs and rearrange
    ulaplace, rho = acoustic_laplacian(u, rho)
    dulaplace, _ = acoustic_laplacian(du, rho)

    if isic:
        # Sum ((u.dx * d, / rho).dx for x in dimensions)
        # space_order//2  so that u.dx.dx has the same radius as u.laplace
        du_aux = sum([first_derivative(first_derivative(u, dim=d, fd_order=space_order//2) * dm / rho,
                                       fd_order=space_order//2, dim=d)
                      for d in u.space_dimensions])
        lin_source = dm /rho * u.dt2 * m - du_aux
    else:
        lin_source = dm / rho * u.dt2

    stencil_u = damp * (2.0 * u - damp * u.backward + dt**2 * rho / m * ulaplace)
    stencil_du = damp * (2.0 * du - damp * du.backward + dt**2 * rho / m * (dulaplace - lin_source))

    expression_u = [Eq(u.forward, stencil_u)]
    expression_du = [Eq(du.forward, stencil_du)]

    # Define source symbol with wavelet
    src = PointSource(name='src', grid=model.grid, ntime=nt, coordinates=src_coords)
    src.data[:] = wavelet[:]
    src_term = src.inject(field=u.forward, expr=src * rho * dt**2 / m)

    # Define receiver symbol
    rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
    rec_term = rec.interpolate(expr=du)

    # Create operator and run

    expression = expression_u + src_term + expression_du + rec_term

    # Free surface
    if free_surface is True:
        expression += freesurface(u, space_order//2, model.nbpml)
        expression += freesurface(du, space_order//2, model.nbpml)

    subs = model.spacing_map
    subs[u.grid.time_dim.spacing] = dt
    op = Operator(expression, subs=subs, dse='advanced', dle='advanced')
    cf = op.cfunction
    summary = op.apply()

    return rec.data, summary

def adjoint_born(model, rec_coords, rec_data, u=None, op_forward=None, is_residual=False,
                 space_order=8, isic=False, dt=None, n_checkpoints=None, maxmem=None,
                 free_surface=False, tsub_factor=1, checkpointing=False):
    clear_cache()

    # Parameters
    nt = rec_data.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, rho, damp = model.m, model.rho, model.damp

    # Create adjoint wavefield and gradient
    v = TimeFunction(name='v', grid=model.grid, time_order=2, space_order=space_order)
    gradient = Function(name='gradient', grid=model.grid)

    # Set up PDE and rearrange
    vlaplace, rho = acoustic_laplacian(v, rho)
    stencil = damp * (2.0 * v - damp * v.forward + dt**2 * rho / m * vlaplace)
    expression = [Eq(v.backward, stencil)]
    # Data at receiver locations as adjoint source
    rec_g = Receiver(name='rec_g', grid=model.grid, ntime=nt, coordinates=rec_coords)
    if op_forward is None:
        rec_g.data[:] = rec_data[:]
    adj_src = rec_g.inject(field=v.backward, expr=rec_g * rho * dt**2 / m)

    # Gradient update
    if u is None:
        u = TimeFunction(name='u', grid=model.grid, time_order=2, space_order=space_order)
    if isic is not True:
        gradient_update = [Inc(gradient, - dt * u.dt2 / rho * v)]
    else:
        # sum u.dx * v.dx fo x in dimensions.
        # space_order//2
        diff_u_v = sum([first_derivative(u, dim=d, fd_order=space_order//2)*
                        first_derivative(v, dim=d, fd_order=space_order//2)
                        for d in u.space_dimensions])
        gradient_update = [Inc(gradient, - tsub_factor * dt * (u * v.dt2 * m + diff_u_v) / rho)]

    # Create operator and run

    # Free surface
    if free_surface is True:
        expression += freesurface(v, space_order//2, model.nbpml, forward=False)

    expression += adj_src + gradient_update
    subs = model.spacing_map
    subs[u.grid.time_dim.spacing] = dt
    op = Operator(expression, subs=subs, dse='advanced', dle='advanced')

    # Optimal checkpointing
    summary1 = None
    summary2 = None
    if op_forward is not None and checkpointing is True:
        rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
        cp = DevitoCheckpoint([u])
        if maxmem is not None:
            n_checkpoints = int(np.floor(maxmem * 10**6 / (cp.size * u.data.itemsize)))
        wrap_fw = CheckpointOperator(op_forward, u=u, m=model.m, rec=rec)
        wrap_rev = CheckpointOperator(op, u=u, v=v, m=model.m, rec_g=rec_g)

        # Run forward
        wrp = Revolver(cp, wrap_fw, wrap_rev, n_checkpoints, nt-2)
        wrp.apply_forward()

        # Residual and gradient
        if is_residual is True:  # input data is already the residual
            rec_g.data[:] = rec_data[:]
        else:
            rec_g.data[:] = rec.data[:] - rec_data[:]   # input is observed data
            fval = .5*np.dot(rec_g.data[:].flatten(), rec_g.data[:].flatten()) * dt
        wrp.apply_reverse()

    elif op_forward is not None and checkpointing is False:

        # Compile first
        cf1 = op_forward.cfunction
        cf2 = op.cfunction

        # Run forward and adjoint
        summary1 = op_forward.apply()
        if is_residual is True:
            rec_g.data[:] = rec_data[:]
        else:
            rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
            rec_g.data[:] = rec.data[:] - rec_data[:]
            fval = .5*np.dot(rec_g.data[:].flatten(), rec_g.data[:].flatten()) * dt
        summary2 = op.apply()
    else:
        cf = op.cfunction
        summary1 = op.apply()
    clear_cache()

    if op_forward is not None and is_residual is not True:
        if summary2 is not None:
            return fval, gradient.data, summary1, summary2
        elif summary1 is not None:
            return fval, gradient.data, summary1
        else:
            return fval, gradient.data
    else:
        if summary2 is not None:
            return gradient.data, summary1, summary2
        elif summary1 is not None:
            return gradient.data, summary1
        else:
            return gradient.data

########################################################################################################################

def forward_freq_modeling(model, src_coords, wavelet, rec_coords, freq, space_order=8, dt=None, factor=None, free_surface=False):
    # Forward modeling with on-the-fly DFT of forward wavefields
    clear_cache()

    # Parameters
    nt = wavelet.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, rho, damp = model.m, model.rho, model.damp

    freq_dim = Dimension(name='freq_dim')
    time = model.grid.time_dim
    if factor is None:
        factor = int(1 / (dt*4*np.max(freq)))
        tsave = ConditionalDimension(name='tsave', parent=model.grid.time_dim, factor=factor)
    if factor==1:
        tsave = time
    else:
        tsave = ConditionalDimension(name='tsave', parent=model.grid.time_dim, factor=factor)
    print("DFT subsampling factor: ", factor)

    # Create wavefields
    nfreq = freq.shape[0]
    u = TimeFunction(name='u', grid=model.grid, time_order=2, space_order=space_order)
    f = Function(name='f', dimensions=(freq_dim,), shape=(nfreq,))
    f.data[:] = freq[:]
    ufr = Function(name='ufr', dimensions=(freq_dim,) + u.indices[1:], shape=(nfreq,) + model.shape_domain)
    ufi = Function(name='ufi', dimensions=(freq_dim,) + u.indices[1:], shape=(nfreq,) + model.shape_domain)

    ulaplace, rho = acoustic_laplacian(u, rho)

    # Set up PDE and rearrange
    stencil = damp * (2.0 * u - damp * u.backward + dt**2 * rho / m * ulaplace)
    expression = [Eq(u.forward, stencil)]
    expression += [Eq(ufr, ufr + factor*u*cos(2*np.pi*f*tsave*factor*dt))]
    expression += [Eq(ufi, ufi - factor*u*sin(2*np.pi*f*tsave*factor*dt))]

    # Source symbol with input wavelet
    src = PointSource(name='src', grid=model.grid, ntime=nt, coordinates=src_coords)
    src.data[:] = wavelet[:]
    src_term = src.inject(field=u.forward, expr=src * dt**2 / m)

    # Data is sampled at receiver locations
    rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
    rec_term = rec.interpolate(expr=u)

    # Create operator and run

    expression += src_term + rec_term
    # Free surface
    if free_surface is True:
        expression += freesurface(u, space_order//2, model.nbpml)

    subs = model.spacing_map
    subs[u.grid.time_dim.spacing] = dt
    op = Operator(expression, subs=subs, dse='advanced', dle='advanced')
    cf = op.cfunction
    op()

    return rec.data, ufr, ufi

def adjoint_freq_born(model, rec_coords, rec_data, freq, ufr, ufi, space_order=8, dt=None, isic=False, factor=None,
                      free_surface=False):
    clear_cache()

    # Parameters
    nt = rec_data.shape[0]
    if dt is None:
        dt = model.critical_dt
    m, rho, damp = model.m, model.rho, model.damp
    nfreq = ufr.shape[0]
    time = model.grid.time_dim
    if factor is None:
        factor = int(1 / (dt*4*np.max(freq)))
        tsave = ConditionalDimension(name='tsave', parent=model.grid.time_dim, factor=factor)
    if factor==1:
        tsave = time
    else:
        tsave = ConditionalDimension(name='tsave', parent=model.grid.time_dim, factor=factor)
    dtf = factor * dt
    ntf = factor / nt
    print("DFT subsampling factor: ", factor)

    # Create the forward and adjoint wavefield
    v = TimeFunction(name='v', grid=model.grid, time_order=2, space_order=space_order)
    f = Function(name='f', dimensions=(ufr.indices[0],), shape=(nfreq,))
    f.data[:] = freq[:]
    gradient = Function(name="gradient", grid=model.grid)
    vlaplace, rho = acoustic_laplacian(v, rho)

    # Set up PDE and rearrange
    stencil = damp * (2.0 * v - damp * v.forward + dt**2 * rho / m * vlaplace)
    expression = [Eq(v.backward, stencil)]

    # Data at receiver locations as adjoint source
    rec = Receiver(name='rec', grid=model.grid, ntime=nt, coordinates=rec_coords)
    rec.data[:] = rec_data[:]
    adj_src = rec.inject(field=v.backward, expr=rec * dt**2 / m)

    # Gradient update
    if isic is True:
        if len(model.shape) == 2:
            gradient_update = [Eq(gradient, gradient + (2*np.pi*f)**2*ntf*(ufr*cos(2*np.pi*f*tsave*dtf) - ufi*sin(2*np.pi*f*tsave*dtf))*v*model.m -
                                                       (ufr.dx*cos(2*np.pi*f*tsave*dtf) - ufi.dx*sin(2*np.pi*f*tsave*dtf))*v.dx*ntf -
                                                       (ufr.dy*cos(2*np.pi*f*tsave*dtf) - ufi.dy*sin(2*np.pi*f*tsave*dtf))*v.dy*ntf)]
        else:
            gradient_update = [Eq(gradient, gradient + (2*np.pi*f)**2*ntf*(ufr*cos(2*np.pi*f*tsave*dtf) - ufi*sin(2*np.pi*f*tsave*dtf))*v*model.m -
                                                       (ufr.dx*cos(2*np.pi*f*tsave*dtf) - ufi.dx*sin(2*np.pi*f*tsave*dtf))*v.dx*ntf -
                                                       (ufr.dy*cos(2*np.pi*f*tsave*dtf) - ufi.dy*sin(2*np.pi*f*tsave*dtf))*v.dy*ntf -
                                                       (ufr.dz*cos(2*np.pi*f*tsave*dtf) - ufi.dz*sin(2*np.pi*f*tsave*dtf))*v.dz*ntf)]
    else:
        gradient_update = [Eq(gradient, gradient + (2*np.pi*f)**2/nt*(ufr*cos(2*np.pi*f*tsave*dtf) - ufi*sin(2*np.pi*f*tsave*dtf))*v)]

    # Create operator and run

    # Free surface
    if free_surface is True:
        expression += freesurface(v, space_order//2, model.nbpml, forward=False)
    expression += adj_src + gradient_update
    subs = model.spacing_map
    subs[v.grid.time_dim.spacing] = dt
    op = Operator(expression, subs=subs, dse='advanced', dle='advanced')
    cf = op.cfunction
    op()
    clear_cache()
    return gradient.data

max_stars_repo_name: slimgroup/ServerlessImagingAWS

max_issues_repo_name: slimgroup/ServerlessImagingAWS

max_forks_repo_name: slimgroup/ServerlessImagingAWS

reason: import numpy,from numpy,from sympy


text:
import pandas as pd
import sqlite3
from sklearn.model_selection import train_test_split
import os

os.environ["CUDA_VISIBLE_DEVICES"]="1"
os.environ['TF_ENABLE_AUTO_MIXED_PRECISION'] = '1'

import logging
LOG_FILENAME = "modeling.log"
for handler in logging.root.handlers[:]:
    logging.root.removeHandler(handler)
logging.basicConfig(filename=LOG_FILENAME,level=logging.DEBUG)
logging.info('Modeling Started...')

#need to update input file to ESN Clean _Nostop, on my desktop

# path_input = "../data/processed/sentence_embeddings/infersent_feature_file_without_stopwords_clean.csv"

# path_input = "../data/processed/sentence_embeddings/tox_infersent_feature_file_without_stopwords_clean.csv"

path_input = "../data/processed/sentence_embeddings/universal_feature_file_without_stopwords.csv"

db_path = '../data/processed/test.db'
# db = sqlite3.connect(db_path)
con = sqlite3.connect(db_path)
cur = con.cursor()

dataset = 'hate_nostop'
# dataset = 'toxic_nostop'

# def missing_values_table(df):
#     mis_val = df.isnull().sum()
#     mis_val_percent = 100 * df.isnull().sum() / len(df)
#     mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)
#     mis_val_table_ren_columns = mis_val_table.rename(
#         columns={0: 'Missing Values', 1: '% of Total Values'})
#     mis_val_table_ren_columns = mis_val_table_ren_columns[
#         mis_val_table_ren_columns.iloc[:, 1] != 0].sort_values(
#         '% of Total Values', ascending=False).round(1)
#     print("Your selected dataframe has " + str(df.shape[1]) + " columns.\n"
#                                                               "There are " + str(mis_val_table_ren_columns.shape[0]) +
#           " columns that have missing values.")
#     return mis_val_table_ren_columns

if dataset == 'hate_nostop':
    X = pd.read_sql_query("SELECT * from hate_universal_encoder_embedding_wefat",con)

    # fill nan wefat scores
    # missing_values_table(X)
    X = X.fillna(0)
    # missing_values_table(X)
    cols = X.columns
    print(cols)

    print('hate full feature file')
    print(X['extract'].head())
    print(X.shape)
    # print(X.dtypes)

    # separate features from label
    # y_cols = ['CODE', 'CODE_0', 'CODE_1', 'CODE_2', 'CODE_3', 'CODE_4', 'CODE_5', 'CODE_6']
    y_cols = ['CODE']

    y = X.loc[:, y_cols]
    # X.drop(['id','extract','CODE','CODE_0', 'CODE_1', 'CODE_2', 'CODE_3', 'CODE_4', 'CODE_5', 'CODE_6'], axis=1, inplace = True)
    X.drop(['index','id', 'extract', 'CODE'], axis=1,
           inplace=True)

    # need to make sure i am separating a test dataset completely from the neural network training
    # https://stackoverflow.com/questions/46308374/what-is-validation-data-used-for-in-a-keras-sequential-model

    # random_seed = 42 # acc = 0.23
    # random_seed = 10 # acc = 0.28
    # random_seed = 15 # acc = 0.28
    # random_seed = 101 # 0.219
    # random_seed = 1 #0.22
    # random_seed = 11 # 0.30
    # random_seed = 111 # 0.309
    # random_seed = 1010 # 0.35
    # random_seed = 10101  #0.218

    # random_seed = 132  # 0.279
    # random_seed = 321 # 0.253
    # random_seed = 320 #0.227
    # random_seed = 404 #0.26
    # random_seed = 202 # 0.27
    # random_seed = 808
    random_seed = 123 # 0.41 #new 0.609

    # X_training, X_testing, y_training, y_testing = train_test_split(X, y, test_size=0.1, random_state=random_seed)
    X_training, X_testing, y_training, y_testing = train_test_split(X, y, test_size=0.2, random_state=random_seed)

    print('X_training')
    print(X_training.head())
    print('y_training')
    print(y_training.head())

    # output_name = 'hate_clean_nostop'

else:

    X_training = pd.read_sql_query("SELECT * from toxic_universal_encoder_embedding_features", con)

    print(X_training.head())
    print(X_training.dtypes)

    y_cols = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
    y_training = X_training.loc[:, y_cols]
    X_training.drop(['extract', "toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"],
           axis=1, inplace=True)

    print(X_training.head())
    print(y_training.head())

    # output_name = 'toxic_clean_nostop'

print('creating train and test sets')

import numpy as np
# from sklearn.model_selection import train_test_split
# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
#
# X_train_full_trial_df = pd.merge(X_train, y_train, left_index=True, right_index=True)
# X_test_full_trial_df = pd.merge(X_test, y_test, left_index=True, right_index=True)
#
# y_train = X_train_full_trial_df[['CODE_0', 'CODE_1', 'CODE_2', 'CODE_3', 'CODE_4', 'CODE_5', 'CODE_6']]
# X_train = X_train_full_trial_df.drop(
#     ['CODE', 'CODE_0', 'CODE_1', 'CODE_2', 'CODE_3', 'CODE_4', 'CODE_5', 'CODE_6'], axis=1)
#
# y_test = X_test_full_trial_df[['CODE_0', 'CODE_1', 'CODE_2', 'CODE_3', 'CODE_4', 'CODE_5', 'CODE_6']]
# X_test = X_test_full_trial_df.drop(['CODE', 'CODE_0', 'CODE_1', 'CODE_2', 'CODE_3', 'CODE_4', 'CODE_5', 'CODE_6'], axis=1)
#
# X_train_np = np.array(X_train)
# X_test_np = np.array(X_test)
# y_train_np = np.array(y_train)
# y_test_np = np.array(y_test)
#
# print("Number transactions X_train dataset: ", X_train.shape)
# print("Number transactions y_train dataset: ", y_train.shape)
# print("Number transactions X_test dataset: ", X_test.shape)
# print("Number transactions y_test dataset: ", y_test.shape)

# convert numpy stuff back into pandas dataframes
# y_cols_new = ['CODE_0', 'CODE_1', 'CODE_2', 'CODE_3', 'CODE_4', 'CODE_5', 'CODE_6']
# print("y_cols_new: " + str(y_cols_new))
#
# X_train = pd.DataFrame(X_train)
# y_train = pd.DataFrame(y_train, columns = y_cols_new)
#
#
# X_test = pd.DataFrame(X_test_np)
# y_test = pd.DataFrame(y_test_np, columns = y_cols_new)
#
# print(y_test.head())

# https://machinelearningmastery.com/reshape-input-data-long-short-term-memory-networks-keras/
# The three dimensions of LSTM input are:

# 1. Samples. One sequence is one sample. A batch is comprised of one or more samples.
# 2. Time Steps. One time step is one point of observation in the sample.
# 3. Features. One feature is one observation at a time step.

# batch_size = 1

import numpy as np

def reshape_data_model_1(X_train,X_train_wef, X_val,X_val_wef, y_train, y_val):
    X_train_reshape = np.reshape(X_train.values, ( X_train.shape[0],1, X_train.shape[1]))

    X_val_reshape = np.reshape(X_val.values, ( X_val.shape[0],1, X_val.shape[1]))

    y_train_reshape_matrix = y_train.values
    y_train_reshape_matrix = y_train_reshape_matrix.reshape(y_train.shape[0],1,y_train.shape[1])

    y_val_reshape_matrix = y_val.values
    y_val_reshape_matrix = y_val_reshape_matrix.reshape(y_val.shape[0],1,y_val.shape[1])

    X_train_wef_reshape = np.reshape(X_train_wef.values, ( X_train_wef.shape[0],1, X_train_wef.shape[1]))

    X_val_wef_reshape = np.reshape(X_val_wef.values, ( X_val_wef.shape[0],1, X_val_wef.shape[1]))

    print("X_train_reshape.shape" + str(X_train_reshape.shape))
    print("y_train_reshape_matrix.shape" + str(y_train_reshape_matrix.shape))
    print("X_val_reshape.shape" + str(X_val_reshape.shape))
    print("y_val_reshape_matrix.shape" + str(y_val_reshape_matrix.shape))
    print("X_train_wef_reshape.shape" + str(X_train_wef_reshape.shape))
    print("X_val_wef_reshape.shape" + str(X_val_wef_reshape.shape))

    return X_train_reshape,X_train_wef_reshape, y_train_reshape_matrix, X_val_reshape,X_val_wef_reshape, y_val_reshape_matrix

def reshape_data_model_2(X_train, X_val, y_train, y_val):
    X_train_reshape = np.reshape(X_train.values, ( X_train.shape[0], X_train.shape[1], 1))

    X_val_reshape = np.reshape(X_val.values, ( X_val.shape[0], X_val.shape[1],1))

    y_train_reshape_matrix = y_train.values
    y_train_reshape_matrix = y_train_reshape_matrix.reshape(y_train.shape[0],y_train.shape[1],1)

    y_val_reshape_matrix = y_val.values
    y_val_reshape_matrix = y_val_reshape_matrix.reshape(y_val.shape[0],y_val.shape[1],1)

    print(X_train_reshape.shape)

    print(y_train_reshape_matrix.shape)

    print(X_val_reshape.shape)

    print(y_val_reshape_matrix.shape)

    return X_train_reshape, y_train_reshape_matrix,X_val_reshape, y_val_reshape_matrix

# X_train_reshape, y_train_reshape_matrix,X_val_reshape, y_val_reshape_matrix = reshape_data(X_train, X_val, y_train, y_val)

from keras.models import Sequential
from keras.layers import Dense, Embedding, Bidirectional, LSTM
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.pipeline import make_pipeline

from string import punctuation
from os import listdir
from numpy import array
from numpy import asarray
from numpy import zeros
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Flatten
from keras.layers import Embedding
from keras.layers import Average
from keras.layers import concatenate
from keras.layers.convolutional import Conv1D
from keras.layers.convolutional import MaxPooling1D

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from sklearn.pipeline import TransformerMixin
from sklearn.base import BaseEstimator

from keras.models import Sequential
from keras.layers import LSTM, Dense, Dropout, Masking, Embedding
from keras.layers import Dense, Embedding, LSTM, SpatialDropout1D
from keras.models import Sequential
from keras.layers import Conv1D, Dropout, Dense, Flatten, LSTM, MaxPooling1D, Bidirectional
from keras.optimizers import Adam
from keras.callbacks import EarlyStopping, TensorBoard
from keras.models import load_model

#LSTM
def simple_LSTM(X_train_reshape,X_train_wef_reshape, y_train_reshape_matrix,X_val_reshape,X_val_wef_reshape, y_val_reshape_matrix,vector_size,model_batch_size,no_epochs):

    # We usually match up the size of the embedding layer output with the number of hidden layers in the LSTM cell.
    # https://adventuresinmachinelearning.com/keras-lstm-tutorial/
    # hidden_size = 4096
    # hidden_size = 32
    hidden_size = 512

    from keras.models import Model
    from keras.layers import Input, Dense
    from keras.layers import TimeDistributed
    from keras.layers import Conv1D, Dropout, Dense, Flatten, LSTM, MaxPooling1D, Bidirectional
    from keras.optimizers import Adam
    from keras.layers import Flatten
    DROP_RATE_DENSE = 0.1

    #perahaps i want to use a combinatinos of convolutional and lstm?
    # https://github.com/jatinmandav/Neural-Networks/blob/master/Sentiment-Analysis/universal-sentence-encoder/universal_sentence_encoder_sentiment-analysis.ipynb

    ####################### MODEL 1: ~ 30% accuracy ####################################

    # use variable length timesteps using None
    # https://datascience.stackexchange.com/questions/26366/training-an-rnn-with-examples-of-different-lengths-in-keras

    # model = Sequential()
    # timesteps = None
    # data_dim = X_train_reshape.shape[2]
    #
    # model = Sequential()
    # input = Input(shape=(timesteps, data_dim))
    # model = Bidirectional(LSTM(hidden_size, return_sequences=True, input_shape=(timesteps, data_dim)),merge_mode='concat')(input) #accuracy  = 0.316
    # model = Bidirectional(LSTM(hidden_size, return_sequences=True), merge_mode='concat')(input)  # accuracy  = 0.279
    # # model = Bidirectional(LSTM(hidden_size, return_sequences=True, input_shape=(timesteps, data_dim)),merge_mode='concat')(model) # 3) added this 0.21259
    # # model = Bidirectional(LSTM(hidden_size, return_sequences=True, input_shape=(timesteps, data_dim)), merge_mode='concat')(model)
    # model = Dropout(DROP_RATE_DENSE)(model)
    #
    # model = TimeDistributed(Dense(256, activation='relu'))(model) #accuracy after adding this line  =0.318
    # # model = Flatten(input_shape=(timesteps, data_dim))(model)
    # # model = Dense(100, activation='relu')(model)
    # model = Dropout(DROP_RATE_DENSE)(model)
    #
    # output = Dense(y_train_reshape_matrix.shape[2], activation='softmax')(model)
    # model = Model(input, output)
    # model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.0001, decay=1e-6), metrics=['accuracy'])

    ####################### MODEL 1:                 ####################################

    ####################### MODEL 2: ~ 30% accuracy ####################################

    # use variable length timesteps using None
    # https://datascience.stackexchange.com/questions/26366/training-an-rnn-with-examples-of-different-lengths-in-keras

    model = Sequential()
    timesteps = None
    data_dim = X_train_reshape.shape[2]
    vector_size = 37

    input_emb = Input(shape=(timesteps, data_dim))
    # input_wef = Input(shape=(vector_size, 1))
    # input_wef = Input(shape=(1, vector_size))
    input_wef = Input(shape=(timesteps, vector_size))

    #sentence embedding LSMT Model
    # model_emb = Conv1D(32, kernel_size=3, activation='elu', padding='same', input_shape=(vector_size, 1))(input_emb)
    # model_emb = Conv1D(32, kernel_size=3, activation='elu', padding='same')(model_emb)
    # model_emb = Conv1D(32, kernel_size=3, activation='relu', padding='same')(model_emb)
    # model_emb = MaxPooling1D(pool_size=3)(model_emb)
    model_emb = Bidirectional(LSTM(hidden_size, return_sequences=True), merge_mode='concat')(input_emb)  # accuracy  = 0.33
    model_emb = Bidirectional(LSTM(hidden_size, return_sequences=True),merge_mode='concat')(model_emb)
    model_emb = Bidirectional(LSTM(hidden_size, return_sequences=True), merge_mode='concat')(model_emb)
    model_emb_output = Dense(y_train_reshape_matrix.shape[2], activation='softmax')(model_emb)
    logging.info('model_emb_output: ' + str(model_emb_output))
    model_emb = Model(inputs=input_emb, outputs=model_emb_output)

    #sentence wefat conv model: simple fully connected network

    model_wef = Dense(512, activation='relu', input_shape=(timesteps, vector_size))(input_wef)

    model_wef = Dense(512, activation='relu')(model_wef)

    model_wef = Dense(512, activation='relu')(model_wef)

    model_wef = Dense(512, activation='relu')(model_wef)

    model_wef = Dense(512, activation='relu')(model_wef)

    model_wef = Dense(512, activation='relu')(model_wef)

    model_wef = Dense(512, activation='relu')(model_wef)
    # model_wef = Dropout(0.1)(model_wef)
    model_wef = Dense(512, activation='relu')(model_wef)
    # model_wef = Dropout(0.2)(model_wef)
    model_wef_output = Dense(y_train_reshape_matrix.shape[2], activation='softmax')(model_wef)

    # model_wef = Bidirectional(LSTM(hidden_size, return_sequences=True), merge_mode='concat')(input_wef)
    # model_wef = Bidirectional(LSTM(hidden_size, return_sequences=True),merge_mode='concat')(model_wef)
    # model_wef = Bidirectional(LSTM(hidden_size, return_sequences=True), merge_mode='concat')(model_wef)
    # model_wef_output = Dense(y_train_reshape_matrix.shape[2], activation='softmax')(model_wef)
    # logging.info('model_wef_output: ' + str(model_wef_output))

    model_wef = Model(inputs=input_wef, outputs=model_wef_output)

    # combine the output of the two branches
    combined = Average()([model_emb.output, model_wef.output])
    # combined = concatenate([model_emb.output, model_wef.output])

    # model = Model(input_emb, output)
    model = Model(inputs=[input_emb, input_wef], outputs=combined)
    # model = Model(inputs=[input_emb, input_wef], outputs=model_emb_output)
    # model = Model(inputs=[input_emb, input_wef], outputs=model_wef_output)

    model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.0001, decay=1e-6), metrics=['accuracy'])

    ####################### MODEL 2:                 ####################################

    # print(model.summary())
    # Keras detects the output_shape and automatically determines which accuracy to use when accuracy is specified. For multi-class classification, categorical_accuracy will be used internally.
    # https://stackoverflow.com/questions/43544358/categorical-crossentropy-need-to-use-categorical-accuracy-or-accuracy-as-the-met
    tensorboard = TensorBoard(log_dir='logs/', histogram_freq=0, write_graph=True, write_images=True)

    # print('data_dim shape: ' + str(data_dim))
    print('X_train_reshape shape: ' + str(X_train_reshape.shape))
    print('y_train_reshape_matrix shape: ' + str(y_train_reshape_matrix.shape))

    # model.fit(X_train_reshape,y_train_reshape_matrix, batch_size=500, epochs=15)
    model.fit([X_train_reshape,X_train_wef_reshape], y_train_reshape_matrix, verbose= 1, batch_size=model_batch_size, epochs=no_epochs,
             validation_data=([X_val_reshape,X_val_wef_reshape], y_val_reshape_matrix), callbacks=[tensorboard, EarlyStopping(min_delta=0.0001, patience=3)])

    # prevent overfitting or over training of the network, EarlyStopping() is used in callback

    print('generate final prediction accuracy metrics')
    val_lost, val_acc = model.evaluate([X_val_reshape,X_val_wef_reshape],y_val_reshape_matrix)

    print('final val_lost ' + str(val_lost))
    print('final val_acc ' + str(val_acc))

    return val_lost, val_acc, model

vector_size = 512
model_batch_size = 500
no_epochs = 25

# run LSTM
# val_lost, val_acc = simple_LSTM(X_train_reshape, y_train_reshape_matrix,X_val_reshape, y_val_reshape_matrix,vector_size,model_batch_size,no_epochs)

# result list to store accuracies across all folds ...  probably should output
# consolidated dataframe of all processed records

model_count = 0
# for index, row in df_files.iterrows():
#
#     feature_file_df = pd.read_csv(row['paths'])
#     #     valing feature files
#     #     feature_file_df = pd.read_csv(row['paths'], nrows=20)
#     #     print(feature_file_df.head())
#     X = feature_file_df
#     #     print(X.head())
#     #     X = X.drop(['Unnamed: 0','extract','tokens','CODE'], axis=1, inplace = True)
#     X = X.values
#     y = feature_file_df.iloc[:, -1].to_frame()
#     y = y.values
#     #     df.iloc[:,-1]
#     skf = StratifiedKFold(n_splits=10)
#     #     print(skf)
#     skf.get_n_splits(X, y)
#     # remove_stop_words = False

from sklearn.model_selection import KFold
from imblearn.over_sampling import SMOTE

#initialize  k fold

kf = KFold(n_splits=10, shuffle=True)
kf.get_n_splits(X_training.values)

accuracy_list = []
fold = 0

# cols = feature_file_df.columns
# print(cols)

max_accuracy = 0

for train_index, val_index in kf.split(X_training.values):
    # print("TRAIN:", train_index, "val:", val_index)
    # print("TRAIN Size:", len(train_index), "val Size:", len(val_index))

    X_train, X_val = X_training.iloc[train_index], X_training.iloc[val_index]
    y_train, y_val = y_training.iloc[train_index], y_training.iloc[val_index]

    # create val and training sets based on the cross validation indexes
    X_train_df = X_training.iloc[train_index]
    y_train_df = y_training.iloc[train_index]
    X_val_df = X_training.iloc[val_index]
    y_val_df = y_training.iloc[val_index]

    if dataset == 'hate_nostop':
         #grab original X df column headers in  order to maintain CODE column names after numpy conversion

        X_cols = list(X_train_df.columns.values) + ['CODE']
        print(X_cols)
        print(len(X_cols))
        #SMOTE#################
        #convert to numpy
        X_train = X_train_df.values
        y_train = y_train_df.values
        X_val = X_val_df.values
        y_val = y_val_df.values

        # y_train_CODE = y_train_df['CODE'].values

        print("Number transactions X_train dataset: ", X_train.shape)
        print("Number transactions y_train dataset: ", y_train.shape)
        print("Number transactions X_val dataset: ", X_val.shape)
        print("Number transactions y_val dataset: ", y_val.shape)

        print("Before OverSampling, counts of label '6': {}".format(sum(y_train == 6)))
        print("Before OverSampling, counts of label '5': {}".format(sum(y_train == 5)))
        print("Before OverSampling, counts of label '4': {}".format(sum(y_train == 4)))
        print("Before OverSampling, counts of label '3': {}".format(sum(y_train == 3)))
        print("Before OverSampling, counts of label '2': {}".format(sum(y_train == 2)))
        print("Before OverSampling, counts of label '1': {}".format(sum(y_train == 1)))
        print("Before OverSampling, counts of label '0': {} \n".format(sum(y_train == 0)))

        # print(X_train)
        # print(y_train.ravel())
        # sm = SMOTE(random_state=2)
        sm = SMOTE()
        X_train_res, y_train_res = sm.fit_sample(X_train, y_train.ravel())

        print('After OverSampling, the shape of train_X: {}'.format(X_train_res.shape))
        print('After OverSampling, the shape of train_y: {} \n'.format(y_train_res.shape))

        print("After OverSampling, counts of label '6': {}".format(sum(y_train_res == 6)))
        print("After OverSampling, counts of label '5': {}".format(sum(y_train_res == 5)))
        print("After OverSampling, counts of label '4': {}".format(sum(y_train_res == 4)))
        print("After OverSampling, counts of label '3': {}".format(sum(y_train_res == 3)))
        print("After OverSampling, counts of label '2': {}".format(sum(y_train_res == 2)))
        print("After OverSampling, counts of label '1': {}".format(sum(y_train_res == 1)))
        print("After OverSampling, counts of label '0': {}".format(sum(y_train_res == 0)))

        X_train_res_df = pd.DataFrame(X_train_res)

        y_train_res_df = pd.DataFrame(y_train_res)
        # X_train_df = pd.DataFrame(X_train)
        # y_train_df = pd.DataFrame(y_train)
        X_val_df = pd.DataFrame(X_val)

        y_val_df = pd.DataFrame(y_val)

        # Join X and y train and resplit with correct columns
        print(X_train_res_df.head())
        print(y_train_res_df.head())

        X_train_res_full_df = pd.merge(X_train_res_df, y_train_res_df, left_index=True, right_index=True)
        X_train_res_full_df.columns = X_cols
        X_val_full_df = pd.merge(X_val_df, y_val_df, left_index=True, right_index=True)
        X_val_full_df.columns = X_cols

        print(X_train_res_full_df.head())
        print(X_val_full_df.head())

        # X_train_res_full_df = X_train_res_df.join(y_train_res_df)
        # X_val_full_df = X_val_df.join(y_val_df)

        y_train_res_df = X_train_res_full_df[['CODE_0', 'CODE_1', 'CODE_2', 'CODE_3', 'CODE_4', 'CODE_5', 'CODE_6']]
        X_train_res_df = X_train_res_full_df.drop(['CODE', 'CODE_0', 'CODE_1', 'CODE_2', 'CODE_3', 'CODE_4', 'CODE_5', 'CODE_6'], axis=1)

        y_val_df = X_val_full_df[['CODE_0', 'CODE_1', 'CODE_2', 'CODE_3', 'CODE_4', 'CODE_5', 'CODE_6']]
        X_val_df = X_val_full_df.drop(['CODE', 'CODE_0', 'CODE_1', 'CODE_2', 'CODE_3', 'CODE_4', 'CODE_5', 'CODE_6'], axis=1)

        # SMOTE#################
        all_x_cols = X_train_res_df.columns
         # segment out word embedding and wefat scores
        wefat_cols = []
        embed_cols = []
        for col in all_x_cols:

             if col[-5:] == 'wefat':
                 # print(col)
                 # embed_cols.remove(col)
                 wefat_cols.append(col)
             else:
                 embed_cols.append(col)

        print('embed cols: ' + str(embed_cols))
        print('embed cols len: ' + str(len(embed_cols)))

        print('wefat cols: ' + str(wefat_cols))
        print('wefat cols len: ' + str(len(wefat_cols)))

        X_train_emb_res_df = X_train_res_df[embed_cols]
        X_val_emb_df = X_val_df[embed_cols]

        # X_train_wef_reshape, X_val_wef_reshape
        X_train_wef_res_df = X_train_res_df[wefat_cols]
        X_val_wef_df = X_val_df[wefat_cols]

        print('X and y dataframes')
        print(X_train_res_df.head())
        print(y_train_res_df.head())
        print(X_val_df.head())
        print(y_val_df.head())

         # modle 1
        X_train_reshape,X_train_wef_reshape, y_train_reshape_matrix, X_val_reshape,X_val_wef_reshape, y_val_reshape_matrix = reshape_data_model_1(X_train_emb_res_df,X_train_wef_res_df, X_val_emb_df,X_val_wef_df,
                                                                                y_train_res_df, y_val_df)

        val_lost, val_acc, model = simple_LSTM(X_train_reshape,X_train_wef_reshape, y_train_reshape_matrix,X_val_reshape,X_val_wef_reshape, y_val_reshape_matrix, vector_size,
                    model_batch_size, no_epochs)

        #model 2
        # X_train_reshape, y_train_reshape_matrix, X_val_reshape, y_val_reshape_matrix = reshape_data_model_2(
        #      X_train_res_df, X_val_df,
        #      y_train_res_df, y_val_df)
        # val_lost, val_acc, model = simple_LSTM(X_train_reshape, y_train_reshape_matrix, X_val_reshape, y_val_reshape_matrix,
        #                                vector_size,
        #                                model_batch_size, no_epochs)

    #toxic
    else:
        print('no toxic testing')
        # X_train_reshape, y_train_reshape_matrix, X_val_reshape, y_val_reshape_matrix = reshape_data(X_train_df, X_val_df,
        #                                                                                y_train_df, y_val_df)
        #
        # val_lost, val_acc = simple_LSTM(X_train_reshape, y_train_reshape_matrix, X_val_reshape, y_val_reshape_matrix, vector_size,
        #         model_batch_size, no_epochs)

    # save best performing model on validation dataset
    logging.info("val acc: " + str(val_acc))
    if val_acc > max_accuracy:
        # serialize weights to HDF5
        model.save("../models/model.h5")
        print("Saved model to disk")
        logging.info("Saved model to disk")
        max_accuracy = val_acc
    # break

    #append all accuracies
    #may want to have a model and save each model separately
    #also include option logic to control stopwords

    accuracy_list.append(val_acc)

#
#     # Classification intensity score histogram
#     #         plt.figure()
#     #         ax = sns.countplot(x = 'CODE', data=y_train_df);
#     #         ax.set_title('Code Classification Distribution')
#     #         ax.set_ylabel('Count')
#     #         handles, labels = ax.get_legend_handles_labels()
#
#     X_train_res_df = pd.DataFrame(X_train_res)
#     y_train_res_df = pd.DataFrame(y_train_res)
#     X_train_df = pd.DataFrame(X_train)
#     y_train_df = pd.DataFrame(y_train)
#     X_val_df = pd.DataFrame(X_val)
#     y_val_df = pd.DataFrame(y_val)
#
#     remove_stop_words = False
#     accuracy = rf_model(X_train_res_df, y_train_res_df, X_val_df, y_val_df, remove_stop_words, fold)
#     #     #append all accuracies
#     #     #may want to have a model and save each model separately
#     #     #also include option logic to control stopwords
#     accuracy_list.append(accuracy)
#
    fold = fold + 1
#
if (model_count == 0):
    results = pd.DataFrame(np.array(accuracy_list)).T

    # append index and text onto dataframe
    # results['embedding'] = row['paths']
    results['embedding'] = 'universal encoding'

else:
    results_2 = pd.DataFrame(np.array(accuracy_list)).T
    # results_2['embedding'] = row['paths']
    results_2['embedding'] = 'next_embedding'

    results = results.append(results_2)

print("results:")
logging.info('Accuracies: ' + str(results))
print(results)
#
# print("model_count: " + str(model_count))
# model_count += 1

# make a prediction

X_testing_full_df = pd.merge(X_testing, y_testing, left_index=True, right_index=True)
X_testing_full_df.columns = X_cols

print(X_testing_full_df.head())
# print(X_val_full_df.head())

# X_train_res_full_df = X_train_res_df.join(y_train_res_df)
# X_val_full_df = X_val_df.join(y_val_df)

y_testing_df = X_testing_full_df[['CODE_0', 'CODE_1', 'CODE_2', 'CODE_3', 'CODE_4', 'CODE_5', 'CODE_6']]
X_testing_df = X_testing_full_df[embed_cols]
X_testing_wef_df = X_testing_full_df[wefat_cols]

X_testing_reshape = np.reshape(X_testing_df.values, ( X_testing_df.shape[0],1, X_testing_df.shape[1]))
X_testing_wef_reshape = np.reshape(X_testing_wef_df.values, ( X_testing_wef_df.shape[0],1, X_testing_wef_df.shape[1]))

#load best performing validation model:
model = load_model("../models/model.h5")

#model 2
# y_pred = model.predict_classes(X_testing_reshape)

#model 1 [X_val_reshape,X_val_wef_reshape]

# y_prob = model.predict(X_testing_reshape)
# y_prob = model.predict(X_testing_wef_reshape)
y_prob = model.predict([X_testing_reshape, X_testing_wef_reshape])
y_pred = y_prob.argmax(axis=-1)

# show the inputs and predicted outputs3

# print(ynew)
for i in range(len(X_testing)):
# 	# print("X=%s, Predicted=%s" % (X_testing.values[i], ynew[i]))
    print(y_pred[i])

from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score

cm = confusion_matrix(X_testing_full_df['CODE'].values, y_pred)
print(cm)

print('accuracy on test set')

print(accuracy_score(X_testing_full_df['CODE'].values, y_pred))

ext: py

max_stars_repo_name: chrisrwhite/hate-speech-crw

max_issues_repo_name: chrisrwhite/hate-speech-crw

max_forks_repo_name: chrisrwhite/hate-speech-crw

reason: import numpy,from numpy


text:
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as mpatch
import pandas as pd
import json
import os
import sys
from Bio import AlignIO
import random

style_folder = "/code/DeeProtein/style/"
with open(style_folder + "style_seq_plotter.json", "r") as style_file, open(style_folder + "colors.json", "r") as color_file:
    plt.style.use(json.load(style_file))
    colors = json.load(color_file)
    color_list = [*colors.values()]

def plot_annotated_sequence_track(sequence=None,
                                  idx_range=None,
                                  data_dict=None,
                                  left_label="Sensitivity",
                                  title="",
                                  use_name=False,
                                  left_y_lim=None,
                                  right_y_lim=None,
                                  std_y_lim=None,
                                  xoffset=0,
                                  seqoffset=0,
                                  width=7.06,
                                  height=1.4,
                                  show_seq=True,
                                  show_pos=True,
                                  lw=2
                                  ):
    if idx_range is None:
        start_index = 1
        end_index = len(sequence)
    else:
        assert idx_range[0] < idx_range[1]
        start_index = idx_range[0]
        end_index = idx_range[1]

    width_per_aa = 0.08
    base_width = 0.5

    if not width:
        width = base_width + width_per_aa*(end_index-start_index)

    fig = plt.figure(figsize=(width, height), dpi=300)
    ax1 = fig.add_subplot(1, 1, 1)

    x_idx = np.arange(start_index, end_index + 1)
    x_slice = slice(start_index-1, end_index)
    mean_in_window = np.nanmean(np.asarray([entry["data"][x_slice] for entry in data_dict.values()]), axis=0)

    assert isinstance(data_dict, dict)
    for plot_idx, (key, entry) in enumerate(data_dict.items()):
        assert "data" in entry
        if use_name:
            assert "name" in entry
        color = entry.get("color", None)
        if color:
            color = colors[color]
        else:
            color = color_list[plot_idx]

        ax = ax1

        label = key if not 'label' in entry else entry['label']

        if entry.get("bar", False):
            ax.bar(x=x_idx, height=entry["data"][x_slice], color=color, width=0.6, label=label)
        else:
            ax.plot(x_idx, entry["data"][x_slice], color=color, lw=lw, label=label)

    # zero line
    ax1.plot([min(x_idx), max(x_idx)], [0.0, 0.0], color="black", label="Baseline", lw=1)
    if left_y_lim:
        ax1.set_ylim(left_y_lim)

    if show_seq:
        seq_letters = [' ']*seqoffset + [l for l in sequence[x_slice]]

        # TWO TICK LABELS LEVELS: (with sequence:)
        text_ticks = plt.FixedFormatter(seq_letters)
        ax1.xaxis.set_minor_locator(plt.FixedLocator(x_idx))
        x_idx_major = [x for x in x_idx if (x-x_idx[0])%10 == 0]
        if show_pos:
            x_idx_major_labels = [x + xoffset for x in x_idx_major]

        else:
            x_idx_major_labels = [None for x in x_idx_major]
        major_ticks = plt.FixedFormatter(x_idx_major_labels)
        ax1.xaxis.set_minor_formatter(text_ticks)
        ax1.xaxis.set_major_locator(plt.FixedLocator(x_idx_major))
        ax1.xaxis.set_major_formatter(major_ticks)
    else:
    # ONLY POSITION:
        x_idx_minor = [1] + [x for x in x_idx if (x-(x_idx[0]-1))%100 == 0]
        x_idx_minor_labels = [x + xoffset for x in x_idx_minor]
        ax1.set_xticks(x_idx_minor)
        ax1.set_xticklabels(x_idx_minor_labels )

    ax1.set_ylabel(left_label)
    ax1.set_title(title)

    if show_seq:
        ax1.tick_params(axis="x", which="major", length=0, pad=15)
    # resize
    fig.set_dpi(300)
    fig.set_size_inches(width, height, forward=True)

    # legend figure:
    handles, labels = ax1.get_legend_handles_labels()
    ax1.legend(bbox_to_anchor=(0.015, 1.01, 1., .101),
               columnspacing=1,
               loc='upper left',
               ncol=len(handles),
              )

    ax1.set_xlim(1, end_index)

    return fig, mean_in_window

if __name__ == '__main__':
    sen_data = sys.argv[1]
    gos = sys.argv[2].split(',')
    path_out = sys.argv[3]

    if len(sys.argv) > 4:
        xoffset = int(sys.argv[4])
    else:
        xoffset = 0

    data = pd.read_table(sen_data, sep='\t')
    sequence = data.loc[:, 'AA'].iloc[1:]

    data_dict = {}
    for go in gos:
        data_dict[go] = {"data": data[go].iloc[1:]}

    fig, _ = plot_annotated_sequence_track(sequence=sequence,
                                           data_dict=data_dict,
                                           idx_range=None,
                                           xoffset=xoffset,
                                           width=None)
    print('Writing plot to {}'.format(path_out))
    plt.savefig(path_out)

max_stars_repo_name: juzb/DeeProtein

max_issues_repo_name: juzb/DeeProtein

max_forks_repo_name: juzb/DeeProtein

reason: import numpy


text:
import pandas as pd
from sklearn import preprocessing
import numpy as np
import glob
import math

# Function to Convert the cM to probability of recombination
def probability (genaticMaps):
  prob = (1.0 - math.exp(-genaticMaps["cM"] / 50)) / 2.0
  return prob

# Read the files in the current directory that contains all the genetic map
for filename in glob.glob('*.txt'):
  genaticMaps = pd.read_csv(filename, sep="\t")

  # 1.  Convert the cM to probability of recombination
  genaticMaps["Recmb_Prob"] = genaticMaps.apply(probability, axis=1)

  # 2. Normalize the probability
  X_normalized = preprocessing.normalize(genaticMaps["Recmb_Prob"], norm='l1')
  df_normalized = pd.DataFrame(X_normalized)
  df1_transposed = df_normalized.T
  df1_transposed['chr'] = genaticMaps["chr"]
  df1_transposed['pos'] = genaticMaps["pos"]
  df1_transposed['prob'] = df1_transposed[0]
  del df1_transposed[0]

  #3. Calculate CDF
  df1_transposed['cdf'] = np.cumsum(df1_transposed["prob"])

  # Save the final result to a new file
  df1_transposed.to_csv("./genetic_map/CDF_"+filename, sep="\t", index=False)

max_stars_repo_name: azzatha/RTG-Simulation-tool

max_issues_repo_name: azzatha/rtg-tools

max_forks_repo_name: azzatha/rtg-tools


text:
import numpy as np

class ConceptEmbedder():
    def __init__(self, word_embeddings):
        self.word_embeddings = word_embeddings
        self.cache = {}
        self.errorcount = 0

    def __call__(self, words):
        """

        :param concept: str representing the concept
        :return: vectorial representation of the concept
        @type concept: list[str]
        """

        w2v = self.word_embeddings
        vector = np.zeros(w2v.embedding_size)
        if not self.cache.has_key(" ".join(words)):
            for word in words:
                if w2v.isKnown(word):
                    lookup = w2v.word2embedd(word.lower())
                    vector += lookup
                else:
                    # print("unknown word:", word)
                    self.errorcount += 1
            self.cache[" ".join(words)] = vector
        return self.cache[" ".join(words)]


text:
import os
import time
import sys
import numpy as np
from bert_serving.client import BertClient
from functools import reduce

sys.path.append("..")
from config import TOP_K, DEFAULT_TABLE
from logs import LOGGER

bc = BertClient()

def normaliz_vec(vec_list):
    for i in range(len(vec_list)):
        vec = vec_list[i]
        square_sum = reduce(lambda x,y:x+y, map(lambda x:x*x ,vec))
        sqrt_square_sum = np.sqrt(square_sum)
        coef = 1/sqrt_square_sum
        vec = list(map(lambda x:x*coef, vec))
        vec_list[i] = vec
    return vec_list

def search_in_milvus(collection_name, query_sentence,milvus_cli, mysql_cli):
    try:
        query_data = [query_sentence]
        vectors = bc.encode(query_data)
        query_list = normaliz_vec(vectors.tolist())
        LOGGER.info("Successfully insert query list")
        results = milvus_cli.search_vectors(collection_name,query_list,TOP_K)
        if results[0][0].distance < 0.8:
            LOGGER.info("Unable to search for similar text!")
        else:
            LOGGER.info("Search the similar text!")
        vids = [str(x.id) for x in results[0]]
        ids,title, text= mysql_cli.search_by_milvus_ids(vids, collection_name)
        distances = [x.distance for x in results[0]]
        return ids,title, text, distances
    except Exception as e:
        LOGGER.error(" Error with search : {}".format(e))
        sys.exit(1)

max_stars_repo_name: BeBraveBeCurious/bootcamp

max_issues_repo_name: sunshinezhihuo/bootcamp

max_forks_repo_name: sunshinezhihuo/bootcamp


text:
#
# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from collections import OrderedDict

import numpy as np
import tensorrt as trt
from polygraphy.backend.trt import util as trt_util
from polygraphy.logger.logger import G_LOGGER
from polygraphy.util import cuda

# This class always uses profile 0 binding names to refer to input and output buffers.
# Therefore, it is essentially agnostic to multiple profiles, which are instead handled in the runner.
class Buffers(object):
    # Currently, buffers are reused across profiles.
    @staticmethod
    def from_engine(engine):
        buffers = Buffers()
        bindings_per_profile = trt_util.get_bindings_per_profile(engine)
        for idx in range(bindings_per_profile):
            binding = engine[idx]
            dtype = trt.nptype(engine.get_binding_dtype(binding))
            buffers.device_buffers[binding] = cuda.DeviceBuffer(dtype=dtype)
            if not engine.binding_is_input(binding):
                buffers.outputs[binding] = np.empty(shape=tuple(), dtype=dtype)
        G_LOGGER.extra_verbose("Created device buffers: {:}".format(buffers.device_buffers))
        return buffers

    def __init__(self):
        self.device_buffers = OrderedDict()
        self.outputs = OrderedDict()

    def copy_inputs(self, feed_dict, stream=None):
        for name, buffer in feed_dict.items():
            self.device_buffers[name].copy_from(buffer, stream)

    # Resizes all device buffers to match the shapes currently set on the provided context
    def resize(self, engine, context, start_binding, end_binding):
        for binding in range(start_binding, end_binding):
            shape = tuple(context.get_binding_shape(binding))
            name = engine[binding - start_binding] # Use profile 0 binding names for all buffers.
            self.device_buffers[name].resize(shape)

    # Copies outputs from the device back to host.
    def copy_outputs(self, stream=None):
        for name, buffer in self.outputs.items():
            self.outputs[name] = self.device_buffers[name].copy_to(buffer, stream)

    def bindings(self):
        return [buf.address() for buf in self.device_buffers.values()]

    def free(self):
        [buf.free() for buf in self.device_buffers.values()]


text:
"""
    Copyright (C) 2022 Martin Ahrnbom
    This work is released under the MIT License.
    See the file LICENSE for details

    Use MOTMetrics to compute score between ground truth and a list of tracks
"""

import motmetrics as mm
from dataclasses import dataclass
import numpy as np
import shapely.geometry
import shapely.affinity
from typing import List
from pathlib import Path
import json
import argparse

from util import good_lstrip, long_str, vector_dist, print_table

@dataclass
class RoadUser:
    X:np.ndarray
    type:str
    id:int
    shape:np.ndarray
    forward:np.ndarray

def get_seqnums():
    sets_lines = [l for l in Path('sets.txt').read_text().split('\n') if l]
    seq_nums = dict()
    for line in sets_lines:
        splot = line.split(': ')
        set_name = splot[0]
        seq_nums[set_name] = [int(v) for v in splot[1].split(' ')]
    return seq_nums

def euclidean_dist3D(gt:RoadUser, ru:RoadUser):
    if gt.type == ru.type:
        dist = vector_dist(gt.X[0:2], ru.X[0:2])
        return dist
    else:
        return np.nan

def iou_dist3D(gt:RoadUser, ru:RoadUser, min_iou:float) -> float:
    if gt.type == ru.type:
        iou = iou3D(gt, ru)
        if iou < min_iou:
            return np.nan
        return 1.0 - iou # IOU "distance" is defined like this
    else:
        return np.nan

def iou3D(A:RoadUser, B:RoadUser) -> float:
    a_rect = rotated_rectangle(A)
    b_rect = rotated_rectangle(B)
    intersection = a_rect.intersection(b_rect).area
    union = a_rect.union(b_rect).area
    iou = intersection / union
    return iou

def rotated_rectangle(ru:RoadUser):
    x, y = ru.X[0:2]
    l, w = ru.shape[0:2]

    # "Native" shapely solution, more than twice as slow for some reason
    #phi = np.arctan2(ru.forward[1], ru.forward[0])
    #box = shapely.geometry.box(-l/2.0, -w/2.0, l/2.0, w/2.0)
    #box = shapely.affinity.rotate(box, 360.0*phi/(2.0*np.pi))
    #box = shapely.affinity.translate(box, x, y)
    #return box

    positions = list()
    base_pos = np.array([x, y], dtype=np.float32)
    forward = ru.forward[0:2]
    right = np.array([[0, -1], [1, 0]], dtype=np.float32) @ forward
    for il, ii in zip((-0.5, 0.5), (1.0, -1.0)):
        ll = il * l * forward
        for iw in (-0.5, 0.5):
            ww = ii*iw * w * right
            new_pos = base_pos + ll + ww
            positions.append((new_pos[0], new_pos[1]))
    box = shapely.geometry.Polygon(positions)
    return box

# Loads road users from JSON file
def load_instances(folder:Path, frame_no:int, classes=None) -> List[RoadUser]:
    file_path = folder / f"{long_str(frame_no, 6)}.json"
    file_text = file_path.read_text()
    objs = json.loads(file_text)
    rus = list()
    for obj in objs:

        # Exclude this object if not in the predefined list of included classes
        if classes is not None:
            if not obj['type'] in classes:
                continue

        X = np.array([obj['x'], obj['y'], obj['z']], dtype=np.float32)
        shape = np.array([obj['l'], obj['w'], obj['h']], dtype=np.float32)
        forward = np.array([obj['forward_x'], obj['forward_y'],
                            obj['forward_z']],
                            dtype=np.float32)
        ru =  RoadUser(X, obj['type'], obj['id'], shape, forward)
        rus.append(ru)
    return rus

def evaluate_scenario(tr_folder:Path,
                      gt_folder:Path,
                      min_iou:float=0.25,
                      name:str='some_tracker',
                      classes=None,
                      verbose=True,
                      metric='iou') -> float:

    acc = mm.MOTAccumulator(auto_id=True)

    frames = [int(f.stem) for f in gt_folder.glob('*.json')]
    frames.sort()

    for frame_no in frames:
        gt_instances = load_instances(gt_folder, frame_no, classes)
        gt_ids = [g.id for g in gt_instances]

        tr_instances = load_instances(tr_folder, frame_no, classes)
        tr_ids = [t.id for t in tr_instances]

        # Present distances to the format expected by MOTMetrics
        dists = list()
        for g in gt_instances:
            these_dists = list()
            for t in tr_instances:
                if metric == 'iou':
                    dist = iou_dist3D(g, t, min_iou)
                elif metric == 'euclidean':
                    dist = euclidean_dist3D(g, t)
                these_dists.append(dist)
            dists.append(these_dists)

        acc.update(gt_ids, tr_ids, dists)

    # Compute the metrics
    mh = mm.metrics.create()
    summary = mh.compute(acc, metrics=['mota'], name=name)

    if verbose:
        print(summary)

    return float(summary['mota'])

def main():
    args = argparse.ArgumentParser()

    args.add_argument("--folder", type=str,
                      help="Path to folder with the tracks to be evaluated, " \
                           "can also be multiple ones separated by commas. " \
                           "If you have many folders that lie next to each " \
                           "other, you only need to include the full path to " \
                           "the first, the rest only need the folder name.")
    args.add_argument("--gt_folder",  default="./output", type=str,
                      help="Path of ground truth",)
    args.add_argument("--set", type=str, default='test',
                      help="Either 'training', 'validation' or 'test'")
    args.add_argument("--classes", type=str, default="",
                      help="Set to a comma-separated list of classes to only " \
                           "evaluate those")
    args.add_argument("--iou_thresh", type=float, default=0.25,
                      help="Usually 0.25, how much the road user's rotated " \
                           "rectangles must overlap to consider it a hit, " \
                            "only relevant if metric is iou")
    args.add_argument("--metric", type=str, default="iou",
                      help="Either 'iou' or 'euclidean'")
    args = args.parse_args()

    folders = [Path(f) for f in args.folder.split(',')]
    for i in range(len(folders)):
        folder = folders[i]
        if not folder.is_dir():
            if i == 0:
                raise ValueError(f"Could not find {folder}")

            # Fallback, perhaps it's a neighbour to the first one
            alt_folder = folders[0].parent / folder.name
            if alt_folder.is_dir():
                folders[i] = alt_folder
            else:
                raise ValueError(f"Couldn't find {folder} nor {alt_folder}")

    gt_folder = Path(args.gt_folder)
    iou_thresh = args.iou_thresh
    which_set = args.set
    metric = args.metric

    assert metric in ['iou', 'euclidean']

    if args.classes:
        classes = args.classes.split(',')
    else:
        # None means all classes are included
        classes = None

    assert which_set in ['training', 'validation', 'test']
    seq_nums = get_seqnums()
    seqs = seq_nums[which_set]
    motas = np.zeros((len(folders), len(seqs)), dtype=np.float32)
    fnames = [f.name.lower() for f in folders]
    fnames = [good_lstrip(f, 'utocs') for f in fnames if f.startswith('utocs')]

    for i_seq, seq_num in enumerate(seqs):
        seq = gt_folder / 'scenarios' / long_str(seq_num, 4)
        for i_folder, folder in enumerate(folders):
            mota = evaluate_scenario(folder / seq.name,
                                    seq / 'positions',
                                    iou_thresh, name=f"{folder.name}{seq_num}",
                                    classes=classes,
                                    verbose=False,
                                    metric=metric)

            motas[i_folder, i_seq] = mota*100.0

        print_table(fnames, seqs[:i_seq+1],
                    motas[:, :i_seq+1])
        print("")

    for i_folder, folder in enumerate(folders):
        values = motas[i_folder, :]
        values = values[~np.isinf(values) & ~np.isnan(values)]
        mean_mota = np.mean(values)
        print(f"Average MOTA for method {folder.name} is {mean_mota}")

if __name__=="__main__":
    main()

max_stars_repo_name: ahrnbom/utocs

max_issues_repo_name: ahrnbom/utocs

max_forks_repo_name: ahrnbom/utocs


text:
from groot.model import (
    GrootTreeClassifier,
    GrootTreeRegressor,
    GrootRandomForestClassifier,
    GrootRandomForestRegressor,
)

from sklearn.base import clone
from sklearn.datasets import make_moons
from sklearn.utils.estimator_checks import check_estimator

import numpy as np

def assert_fit_and_predict_groot(X_train, y_train, X_test, y_test, model, kwargs):
    model = clone(model)
    model.set_params(**kwargs)
    model.fit(X_train, y_train)

    assert model.n_samples_ == X_train.shape[0]
    assert model.n_features_in_ == X_train.shape[1]

    y_pred = model.predict(X_test)
    assert y_pred.shape == y_test.shape

def test_groot_tree_parameters():
    attack_models = [
        (0, 0),
        (0.2, 0.2),
        (">", ">"),
        (0.1, "<>"),
    ]

    X_train, y_train = make_moons(
        n_samples=100, shuffle=True, noise=0.2, random_state=1
    )
    X_test, y_test = make_moons(n_samples=100, shuffle=True, noise=0.2, random_state=2)

    models = (
        GrootTreeClassifier(),
        GrootRandomForestClassifier(n_estimators=10),
        GrootTreeRegressor(),
        GrootRandomForestRegressor(n_estimators=10),
    )
    classification_models = (True, True, False, False)

    for model, is_classification in zip(models, classification_models):
        for attack_model in attack_models:
            for chen_heuristic in (False, True):
                if is_classification:
                    for one_adversarial_class in (False, True):
                        assert_fit_and_predict_groot(
                            X_train,
                            y_train,
                            X_test,
                            y_test,
                            model,
                            {
                                "attack_model": attack_model,
                                "chen_heuristic": chen_heuristic,
                                "one_adversarial_class": one_adversarial_class,
                            },
                        )
                else:
                    assert_fit_and_predict_groot(
                        X_train,
                        y_train,
                        X_test,
                        y_test,
                        model,
                        {
                            "attack_model": attack_model,
                            "chen_heuristic": chen_heuristic,
                        },
                    )

def test_groot_tree_prediction_compiling():
    attack_models = [
        (0, 0),
        (0.2, 0.2),
        (">", ">"),
        (0.1, "<>"),
    ]

    X_train, y_train = make_moons(
        n_samples=100, shuffle=True, noise=0.2, random_state=1
    )
    X_test, y_test = make_moons(n_samples=100, shuffle=True, noise=0.2, random_state=2)

    models = (
        GrootTreeClassifier(),
        GrootRandomForestClassifier(n_estimators=10),
        GrootTreeRegressor(),
        GrootRandomForestRegressor(n_estimators=10),
    )

    for model in models:
        for attack_model in attack_models:
            standard_model = clone(model)
            standard_model.set_params(
                attack_model=attack_model, compile=False, random_state=1
            )
            standard_model.fit(X_train, y_train)

            compiled_model = clone(model)
            compiled_model.set_params(
                attack_model=attack_model, compile=True, random_state=1
            )
            compiled_model.fit(X_train, y_train)

            standard_pred = standard_model.predict(X_test)
            compiled_pred = compiled_model.predict(X_test)

            assert standard_pred.shape == compiled_pred.shape
            assert np.allclose(standard_pred, compiled_pred)

def test_groot_regressors_sklearn_estimator():
    check_estimator(GrootTreeRegressor())
    check_estimator(GrootRandomForestRegressor(n_estimators=10))

def test_groot_classifiers_sklearn_estimator():
    # Skip tests that contain non-binary classification datasets for GROOT classifiers
    classifier_checks_to_skip = {
        "check_estimators_dtypes",
        "check_fit_score_takes_y",
        "check_estimators_fit_returns_self",
        "check_dtype_object",
        "check_estimators_overwrite_params",
        "check_classifier_data_not_an_array",
        "check_classifiers_classes",
        "check_classifiers_train",
        "check_supervised_y_2d",
        "check_methods_sample_order_invariance",
        "check_methods_subset_invariance",
        "check_dont_overwrite_parameters",
        "check_fit2d_predict1d",
    }
    for estimator in (
        GrootTreeClassifier(),
        GrootRandomForestClassifier(n_estimators=10),
    ):
        for _, check in check_estimator(estimator, generate_only=True):
            if check.func.__name__ in classifier_checks_to_skip:
                continue

            check(estimator)

def test_groot_tree_known_dataset():
    X = np.array(
        [
            [0, 0],
            [0, 1],
            [0, 2],
            [1, 0],
            [1, 1],
            [1, 2],
            [2, 0],
            [2, 1],
            [2, 2],
        ]
    )
    y = np.array([0, 0, 0, 1, 1, 1, 0, 0, 0])

    # Fit a tree of depth 1, it cannot be completely accurate
    tree = GrootTreeClassifier(max_depth=1, attack_model=[0.1, 0.1], random_state=1)
    tree.fit(X, y)

    y_pred = tree.predict(X)
    y_known_pred = np.zeros(9)
    assert np.array_equal(y_pred, y_known_pred)

    y_pred_proba = tree.predict_proba(X)
    y_known_proba = np.array(
        [
            [1.0, 0.0],
            [1.0, 0.0],
            [1.0, 0.0],
            [0.5, 0.5],
            [0.5, 0.5],
            [0.5, 0.5],
            [0.5, 0.5],
            [0.5, 0.5],
            [0.5, 0.5],
        ]
    )
    assert np.array_equal(y_pred_proba, y_known_proba)

    # Fit a tree of depth 2, it should be completely accurate
    tree = GrootTreeClassifier(max_depth=2, attack_model=[0.1, 0.1], random_state=1)
    tree.fit(X, y)

    y_pred = tree.predict(X)
    assert np.array_equal(y_pred, y)

    y_pred_proba = tree.predict_proba(X)
    y_known_proba = np.array(
        [
            [1.0, 0.0],
            [1.0, 0.0],
            [1.0, 0.0],
            [0.0, 1.0],
            [0.0, 1.0],
            [0.0, 1.0],
            [1.0, 0.0],
            [1.0, 0.0],
            [1.0, 0.0],
        ]
    )
    assert np.array_equal(y_pred_proba, y_known_proba)

def test_groot_tree_known_dataset_one_adversarial_class():
    X = np.array(
        [
            [0, 0],
            [0, 1],
            [0, 2],
            [1, 0],
            [1, 1],
            [1, 2],
            [2, 0],
            [2, 1],
            [2, 2],
        ]
    )
    y = np.array([0, 0, 0, 0, 0, 0, 1, 1, 1])

    # Check that when class 1 moves we get a completely correct tree
    tree = GrootTreeClassifier(
        max_depth=1, attack_model=[">", ""], one_adversarial_class=True, random_state=1
    )
    tree.fit(X, y)
    y_pred = tree.predict(X)
    assert np.array_equal(y_pred, y)

    # Check that when two classes move we get a single leaf
    tree = GrootTreeClassifier(
        max_depth=1, attack_model=[">", ""], one_adversarial_class=False, random_state=1
    )
    tree.fit(X, y)
    assert np.array_equal(tree.root_.value, [2 / 3, 1 / 3])

max_stars_repo_name: tudelft-cda-lab/GROOT

max_issues_repo_name: tudelft-cda-lab/GROOT

max_forks_repo_name: tudelft-cda-lab/GROOT


text:
from __future__ import print_function
import time
import torch

from vae_lib.optimization.loss import calculate_loss
from vae_lib.utils.visual_evaluation import plot_reconstructions
from vae_lib.utils.log_likelihood import calculate_likelihood

import numpy as np
from train_misc import count_nfe, override_divergence_fn

def train(epoch, train_loader, model, opt, args, logger):

    model.train()
    train_loss = np.zeros(len(train_loader))
    train_bpd = np.zeros(len(train_loader))

    num_data = 0

    # set warmup coefficient
    beta = min([(epoch * 1.) / max([args.warmup, 1.]), args.max_beta])
    logger.info('beta = {:5.4f}'.format(beta))
    end = time.time()
    for batch_idx, (data, _) in enumerate(train_loader):
        if args.cuda:
            data = data.cuda()

        if args.dynamic_binarization:
            data = torch.bernoulli(data)

        data = data.view(-1, *args.input_size)

        opt.zero_grad()
        x_mean, z_mu, z_var, ldj, z0, zk, lec = model(data)

        if 'cnf' in args.flow:
            f_nfe = count_nfe(model)

        loss, rec, kl, bpd = calculate_loss(x_mean, data, z_mu, z_var, z0, zk, ldj, args, beta=beta)

        loss = loss + lec * args.poly_coef if args.poly_coef is not None else loss
        loss.backward()

        if 'cnf' in args.flow:
            t_nfe = count_nfe(model)
            b_nfe = t_nfe - f_nfe

        train_loss[batch_idx] = loss.item()
        train_bpd[batch_idx] = bpd

        opt.step()

        rec = rec.item()
        kl = kl.item()

        num_data += len(data)

        batch_time = time.time() - end
        end = time.time()

        if batch_idx % args.log_interval == 0:
            if args.input_type == 'binary':
                perc = 100. * batch_idx / len(train_loader)
                log_msg = (
                    'Epoch {:3d} [{:5d}/{:5d} ({:2.0f}%)] | Time {:.3f} | Loss {:11.6f} | '
                    'Rec {:11.6f} | KL {:11.6f}'.format(
                        epoch, num_data, len(train_loader.sampler), perc, batch_time, loss.item(), rec, kl
                    )
                )
            else:
                perc = 100. * batch_idx / len(train_loader)
                tmp = 'Epoch {:3d} [{:5d}/{:5d} ({:2.0f}%)] | Time {:.3f} | Loss {:11.6f} | Bits/dim {:8.6f}'
                log_msg = tmp.format(epoch, num_data, len(train_loader.sampler), perc, batch_time, loss.item(),
                                     bpd), '\trec: {:11.3f}\tkl: {:11.6f}'.format(rec, kl)
                log_msg = "".join(log_msg)
            if 'cnf' in args.flow:
                log_msg += ' | NFE Forward {} | NFE Backward {}'.format(f_nfe, b_nfe)
            if args.poly_coef is not None:
                log_msg += " | poly loss: {:.4E}".format(lec)

            logger.info(log_msg)

    if args.input_type == 'binary':
        logger.info('====> Epoch: {:3d} Average train loss: {:.4f}'.format(epoch, train_loss.sum() / len(train_loader)))
    else:
        logger.info(
            '====> Epoch: {:3d} Average train loss: {:.4f}, average bpd: {:.4f}'.
            format(epoch, train_loss.sum() / len(train_loader), train_bpd.sum() / len(train_loader))
        )

    return train_loss

def evaluate(data_loader, model, args, logger, testing=False, epoch=0):
    model.eval()
    loss = 0.
    batch_idx = 0
    bpd = 0.

    if args.input_type == 'binary':
        loss_type = 'elbo'
    else:
        loss_type = 'bpd'

    if testing and 'cnf' in args.flow:
        override_divergence_fn(model, "brute_force")

    for data, _ in data_loader:
        batch_idx += 1

        if args.cuda:
            data = data.cuda()

        with torch.no_grad():
            data = data.view(-1, *args.input_size)

            x_mean, z_mu, z_var, ldj, z0, zk, _ = model(data)

            batch_loss, rec, kl, batch_bpd = calculate_loss(x_mean, data, z_mu, z_var, z0, zk, ldj, args)

            bpd += batch_bpd
            loss += batch_loss.item()

            # PRINT RECONSTRUCTIONS
            if batch_idx == 1 and testing is False:
                plot_reconstructions(data, x_mean, batch_loss, loss_type, epoch, args)

    loss /= len(data_loader)
    bpd /= len(data_loader)

    if testing:
        logger.info('====> Test set loss: {:.4f}'.format(loss))

    # Compute log-likelihood
    if testing and not ("cnf" in args.flow):  # don't compute log-likelihood for cnf models

        with torch.no_grad():
            test_data = data_loader.dataset.tensors[0]

            if args.cuda:
                test_data = test_data.cuda()

            logger.info('Computing log-likelihood on test set')

            model.eval()

            if args.dataset == 'caltech':
                log_likelihood, nll_bpd = calculate_likelihood(test_data, model, args, logger, S=2000, MB=500)
            else:
                log_likelihood, nll_bpd = calculate_likelihood(test_data, model, args, logger, S=5000, MB=500)

        if 'cnf' in args.flow:
            override_divergence_fn(model, args.divergence_fn)
    else:
        log_likelihood = None
        nll_bpd = None

    if args.input_type in ['multinomial']:
        bpd = loss / (np.prod(args.input_size) * np.log(2.))

    if testing and not ("cnf" in args.flow):
        logger.info('====> Test set log-likelihood: {:.4f}'.format(log_likelihood))

        if args.input_type != 'binary':
            logger.info('====> Test set bpd (elbo): {:.4f}'.format(bpd))
            logger.info(
                '====> Test set bpd (log-likelihood): {:.4f}'.
                format(log_likelihood / (np.prod(args.input_size) * np.log(2.)))
            )

    if testing and not ("cnf" in args.flow):  # don't compute log-likelihood for cnf models
        return log_likelihood, nll_bpd
    else:
        return loss, bpd

max_stars_repo_name: hanhsienhuang/CNF-TPR

max_issues_repo_name: hanhsienhuang/CNF-TPR

max_forks_repo_name: hanhsienhuang/CNF-TPR


text:
# -*- coding: utf-8 -*-
"""
Example of predicting the Van der Pol equation with a Gaussian Process

@author: Helge-André Langåker
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from sys import path
path.append(r"./../") # Add gp_mpc pagkage to path

import numpy as np
import casadi as ca
import matplotlib.pyplot as plt
from gp_mpc import Model, GP, MPC, plot_eig, lqr

def plot_van_der_pol():
    """ Plot comparison of GP prediction with exact simulation
        on a 2000 step prediction horizon
    """
    Nt = 2000
    x0 = np.array([2., .201])

    cov = np.zeros((2,2))
    x = np.zeros((Nt,2))
    x_sim = np.zeros((Nt,2))

    x[0] = x0
    x_sim[0] = x0

    gp.set_method('ME')         # Use Mean Equivalence as GP method
    for i in range(Nt-1):
        x_t, cov = gp.predict(x[i], [], cov)
        x[i + 1] = np.array(x_t).flatten()
        x_sim[i+1] = model.integrate(x0=x_sim[i], u=[], p=[])

    plt.figure()
    ax = plt.subplot(111)
    ax.plot(x_sim[:,0], x_sim[:,1], 'k-', linewidth=1.0, label='Exact')
    ax.plot(x[:,0], x[:,1], 'b-', linewidth=1.0, label='GP')
    ax.set_ylabel('y')
    ax.set_xlabel('x')
    plt.legend(loc='best')
    plt.show()

def ode(x, u, z, p):
    """ Van der Pol equation
    """
    mu = 2
    dxdt = [
            x[1],
            -x[0] + mu * (1 - x[0]**2) * x[1]
    ]
    return  ca.vertcat(*dxdt)

""" System Parameters """
dt = .01                    # Sampling time
Nx = 2                      # Number of states
Nu = 0                      # Number of inputs
R_n = np.eye(Nx) * 1e-6     # Covariance matrix of added noise

# Limits in the training data
ulb = []    # No inputs are used
uub = []    # No inputs are used
xlb = [-4., -6.]
xub = [4., 6.]

N = 40          # Number of training data
N_test = 100    # Number of test data

""" Create simulation model and generate training/test data"""
model          = Model(Nx=Nx, Nu=Nu, ode=ode, dt=dt, R=R_n, clip_negative=True)
X, Y           = model.generate_training_data(N, uub, ulb, xub, xlb, noise=True)
X_test, Y_test = model.generate_training_data(N_test, uub, ulb, xub, xlb, noise=True)

""" Create GP model and optimize hyper-parameters"""
gp = GP(X, Y, mean_func='zero', normalize=True, xlb=xlb, xub=xub, ulb=ulb,
        uub=uub, optimizer_opts=None)
gp.validate(X_test, Y_test)

""" Predict and plot the result """
plot_van_der_pol()

max_stars_repo_name: DiegoAE/GP-MPC

max_issues_repo_name: DiegoAE/GP-MPC

max_forks_repo_name: DiegoAE/GP-MPC


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import base64
from StringIO import StringIO
from PIL import Image
import numpy as np
import argparse
import facenet
import align.detect_face
import cv2
import os
import sys
import time
import math

from scipy import misc
from sklearn import metrics
from scipy.optimize import brentq
from scipy import interpolate
from scipy.misc import imread, imresize
import sklearn.preprocessing

global mx
mx = None
global globGraph
globGraph = None
global mod
global mod2
mod2 = None
global mod3
mod3 = None
DEBUG = False

DATA_RUNTIME_FOLDER = os.getenv('DATA_RUNTIME_FOLDER', '/data/runtime')
HAS_OPENCL = os.getenv('HAS_OPENCL', 'false')

def load_graph(frozen_graph_filename):
    return None

def crop(image, random_crop, image_size):
    if image.shape[1]>image_size:
        sz1 = int(image.shape[1]//2)
        sz2 = int(image_size//2)
        if random_crop:
            diff = sz1-sz2
            (h, v) = (np.random.randint(-diff, diff+1), np.random.randint(-diff, diff+1))
        else:
            (h, v) = (0,0)
        image = image[(sz1-sz2+v):(sz1+sz2+v),(sz1-sz2+h):(sz1+sz2+h),:]
    return image

def flip(image, random_flip):
    if random_flip and np.random.choice([True, False]):
        image = np.fliplr(image)
    return image

def to_rgb(img):
    w, h = img.shape
    ret = np.empty((w, h, 3), dtype=np.uint8)
    ret[:, :, 0] = ret[:, :, 1] = ret[:, :, 2] = img
    return ret

def load_image(image_path):

    img_list = [None] * 1
    img = misc.imread(os.path.expanduser(image_path))
    img_size = np.asarray(img.shape)[0:2]
    prewhitened = facenet.prewhiten(img)
    img_list[0] = prewhitened
    image = np.stack(img_list)
    return image

def init_FaceProcessing(model_path):
    global globGraph

    if globGraph != None:
        return globGraph

    globGraph = load_graph(model_path)

    previous = 'no'
    for op in globGraph.get_operations():
        try:
            firstLevel = op.name.split("/")[1]
            if firstLevel.startswith(previous) is not True:
                print(op.name)
                previous = firstLevel
        except:
            print(op.name)

    return globGraph

def InitialFaceProcessor(model_path):
    return None , None
def get_model(ctx, image_size, model_str, layer):
  _vec = model_str.split(',')
  assert len(_vec)==2
  prefix = _vec[0]
  epoch = int(_vec[1])
  print('loading',prefix, epoch)
  sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch)
  all_layers = sym.get_internals()
  sym = all_layers[layer+'_output']
  model = mx.mod.Module(symbol=sym, context=ctx, label_names = None)
  #model.bind(data_shapes=[('data', (args.batch_size, 3, image_size[0], image_size[1]))], label_shapes=[('softmax_label', (args.batch_size,))])
  model.bind(data_shapes=[('data', (1, 3, 112, 112))])
  model.set_params(arg_params, aux_params)
  return model
def init_embedding_processor():
    global mod2
    global mod3

    if HAS_OPENCL == 'false':
        global mx
        import mxnet as mx
        print('need init mxnet')

        mod2 = None
        if os.path.isfile(DATA_RUNTIME_FOLDER+'/model-0000.params'):
            ctx = mx.cpu(0)
            mod3 = get_model(ctx, [112,112], DATA_RUNTIME_FOLDER+'/model,0', 'fc1')
            print('backup model loaded')
            return mod3
        else:
            print('cant get model '+DATA_RUNTIME_FOLDER+'/model-0000.params')
    else:
        print('has opencl supporting')

    if os.path.isfile(DATA_RUNTIME_FOLDER+'/net2'):
        global __t
        global graph_runtime
        try:
            import tvm as __t
            from tvm.contrib import graph_runtime
            loaded_lib = None
            if os.path.isfile(DATA_RUNTIME_FOLDER+'/net2.tar.so'):
                loaded_lib = __t.module.load(DATA_RUNTIME_FOLDER+'/net2.tar.so')
            else:
                loaded_lib = __t.module.load(DATA_RUNTIME_FOLDER+'/net2.tar')
            loaded_json = open(DATA_RUNTIME_FOLDER+"/net2").read()
            loaded_params = bytearray(open(DATA_RUNTIME_FOLDER+"/net2.params", "rb").read())

            ctx = __t.cl(0)

            mod2 = graph_runtime.create(loaded_json, loaded_lib, ctx)
            mod2.load_params(loaded_params)
            return mod2
        except:
            print('error of loading net2')
            mod2 = None
            if os.path.isfile(DATA_RUNTIME_FOLDER+'/model-0000.params'):
                global mx
                import mxnet as mx
                ctx = mx.cpu(0)
                mod3 = get_model(ctx, [112,112], DATA_RUNTIME_FOLDER+'/model,0', 'fc1')
                print('backup model loaded')
                return mod3
    elif os.path.isfile('/root/model-r50-am-lfw/model-0000.params'):
        global mx
        import mxnet as mx
        ctx = mx.cpu(0)
        mod3 = get_model(ctx, [112,112], '/root/model-r50-am-lfw/model,0', 'fc1')
        print('backup model loaded')
        return mod3

def FaceProcessingOne(imgpath,sess,graph):
    images_placeholder = graph.get_tensor_by_name("import/input:0")
    embeddings = graph.get_tensor_by_name("import/embeddings:0")
    phase_train_placeholder = graph.get_tensor_by_name("import/phase_train:0")

    image_size = 160 #images_placeholder.get_shape()[1]
    embedding_size = 128 #embeddings.get_shape()[1]
    # Run forward pass to calculate embeddings
    print('Runnning forward pass on LFW images')
    batch_size = 1 #args.lfw_batch_size
    nrof_batches = 1

    image = load_image(imgpath)

    feed_dict = { images_placeholder:image, phase_train_placeholder:False }
    features = sess.run(embeddings, feed_dict=feed_dict)

    return features

def FaceProcessingImageData(imgData,sess,graph):
    images_placeholder = graph.get_tensor_by_name("import/input:0")
    embeddings = graph.get_tensor_by_name("import/embeddings:0")
    phase_train_placeholder = graph.get_tensor_by_name("import/phase_train:0")

    image_size = 160 #images_placeholder.get_shape()[1]
    embedding_size = 128 #embeddings.get_shape()[1]
    # Run forward pass to calculate embeddings
    print('Runnning forward pass on LFW images')
    batch_size = 1 #args.lfw_batch_size
    nrof_batches = 1

    img_list = [None] * 1
    img_list[0] = imgData
    image = np.stack(img_list)

    feed_dict = { images_placeholder:image, phase_train_placeholder:False }
    features = sess.run(embeddings, feed_dict=feed_dict)

    return features

def FaceProcessingImageData2(img_path):
    img_data = misc.imread(img_path)
    img = cv2.cvtColor(img_data, cv2.COLOR_BGR2RGB)
    return _FaceProcessingImageData2(img)

def FaceProcessingBase64ImageData2(base64_string):
    sbuf = StringIO()
    sbuf.write(base64.b64decode(base64_string))
    pimg = Image.open(sbuf)
    img = np.array(pimg)
    #img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    return _FaceProcessingImageData2(img)

def _FaceProcessingImageData2(img):
    global mod2
    global mod3

    embedding = None

    nimg = np.transpose(img, (2,0,1))
    resize_img = misc.imresize(nimg, [112, 112], interp='bilinear')
    if mod2 is not None:
        a = transform_image(resize_img).astype('float32')
        mod2.run(data=a)
        try:
            out1 = mod2.get_output(0).asnumpy()
        except TypeError as err:
            out1 = mod2.get_output(0, __t.nd.empty((512,))).asnumpy()
        embedding = sklearn.preprocessing.normalize(out1).flatten()
    elif mod3 is not None:
        aligned = resize_img.transpose((2, 0, 1))
        input_blob = np.expand_dims(aligned, axis=0)
        data = mx.nd.array(input_blob)
        db = mx.io.DataBatch(data=(data,))
        mod3.forward(db, is_train=False)
        embedding = mod3.get_outputs()[0].asnumpy()
        embedding = sklearn.preprocessing.normalize(embedding).flatten()
    return embedding

def transform_image(image):
    #image = np.array(image) - np.array([123., 117., 104.])
    #image /= np.array([58.395, 57.12, 57.375])
    image = image.transpose((2, 0, 1))
    image = image[np.newaxis, :]
    return image

max_stars_repo_name: mykiscool/DeepCamera

max_issues_repo_name: mykiscool/DeepCamera

max_forks_repo_name: mykiscool/DeepCamera

reason: import numpy,from scipy


text:
#!/usr/bin/env python

from __future__ import print_function

from sympy import symbols, sin, cos
from sympy.galgebra import xdvi, Get_Program, Print_Function
from sympy.galgebra import MV, Format

def Maxwells_Equations_in_Geometric_Calculus():
    Print_Function()
    X = symbols('t x y z')
    (g0, g1, g2, g3, grad) = MV.setup('gamma*t|x|y|z', metric='[1,-1,-1,-1]', coords=X)
    I = MV.I

    B = MV('B', 'vector', fct=True)
    E = MV('E', 'vector', fct=True)
    B.set_coef(1, 0, 0)
    E.set_coef(1, 0, 0)
    B *= g0
    E *= g0
    J = MV('J', 'vector', fct=True)
    F = E + I*B

    print(r'\text{Pseudo Scalar\;\;}I =', I)
    print('\\text{Magnetic Field Bi-Vector\\;\\;} B = \\bm{B\\gamma_{t}} =', B)
    print('\\text{Electric Field Bi-Vector\\;\\;} E = \\bm{E\\gamma_{t}} =', E)
    print('\\text{Electromagnetic Field Bi-Vector\\;\\;} F = E+IB =', F)
    print('%\\text{Four Current Density\\;\\;} J =', J)
    gradF = grad*F
    print('#Geometric Derivative of Electomagnetic Field Bi-Vector')
    gradF.Fmt(3, 'grad*F')

    print('#Maxwell Equations')
    print('grad*F = J')
    print('#Div $E$ and Curl $H$ Equations')
    (gradF.grade(1) - J).Fmt(3, '%\\grade{\\nabla F}_{1} -J = 0')
    print('#Curl $E$ and Div $B$ equations')
    (gradF.grade(3)).Fmt(3, '%\\grade{\\nabla F}_{3} = 0')
    return

def Dirac_Equation_in_Geometric_Calculus():
    Print_Function()
    vars = symbols('t x y z')
    (g0, g1, g2, g3, grad) = MV.setup('gamma*t|x|y|z', metric='[1,-1,-1,-1]', coords=vars)
    I = MV.I

    (m, e) = symbols('m e')

    psi = MV('psi', 'spinor', fct=True)
    A = MV('A', 'vector', fct=True)
    sig_z = g3*g0

    print('\\text{4-Vector Potential\\;\\;}\\bm{A} =', A)
    print('\\text{8-component real spinor\\;\\;}\\bm{\\psi} =', psi)

    dirac_eq = (grad*psi)*I*sig_z - e*A*psi - m*psi*g0
    dirac_eq.simplify()

    dirac_eq.Fmt(3, r'%\text{Dirac Equation\;\;}\nabla \bm{\psi} I \sigma_{z}-e\bm{A}\bm{\psi}-m\bm{\psi}\gamma_{t} = 0')

    return

def Lorentz_Tranformation_in_Geometric_Algebra():
    Print_Function()
    (alpha, beta, gamma) = symbols('alpha beta gamma')
    (x, t, xp, tp) = symbols("x t x' t'")
    (g0, g1) = MV.setup('gamma*t|x', metric='[1,-1]')

    from sympy import sinh, cosh

    R = cosh(alpha/2) + sinh(alpha/2)*(g0 ^ g1)
    X = t*g0 + x*g1
    Xp = tp*g0 + xp*g1
    print('R =', R)

    print(r"#%t\bm{\gamma_{t}}+x\bm{\gamma_{x}} = t'\bm{\gamma'_{t}}+x'\bm{\gamma'_{x}} = R\lp t'\bm{\gamma_{t}}+x'\bm{\gamma_{x}}\rp R^{\dagger}")

    Xpp = R*Xp*R.rev()
    Xpp = Xpp.collect([xp, tp])
    Xpp = Xpp.subs({2*sinh(alpha/2)*cosh(alpha/2): sinh(alpha), sinh(alpha/2)**2 + cosh(alpha/2)**2: cosh(alpha)})
    print(r"%t\bm{\gamma_{t}}+x\bm{\gamma_{x}} =", Xpp)
    Xpp = Xpp.subs({sinh(alpha): gamma*beta, cosh(alpha): gamma})

    print(r'%\f{\sinh}{\alpha} = \gamma\beta')
    print(r'%\f{\cosh}{\alpha} = \gamma')

    print(r"%t\bm{\gamma_{t}}+x\bm{\gamma_{x}} =", Xpp.collect(gamma))
    return

def dummy():
    return

def main():
    Get_Program()
    Format()

    Maxwells_Equations_in_Geometric_Calculus()
    Dirac_Equation_in_Geometric_Calculus()
    Lorentz_Tranformation_in_Geometric_Algebra()

    xdvi()
    return

if __name__ == "__main__":
    main()

max_stars_repo_name: dqnykamp/sympy

max_issues_repo_name: dqnykamp/sympy

max_forks_repo_name: dqnykamp/sympy

reason: from sympy