human
I am working on the file "ANTLR4-Python/LR-Parser-Generator/Shift-Reduce-Parser-Pure.ipynb".
The first task is:
The function tokenize transforms the string s into a list of tokens. See below for an example.
Can you write Python program for it?
assistant
#Here is the Python program you requested.

def tokenize(s):
    '''Transform the string s into a list of tokens.  The string s
       is supposed to represent an arithmetic expression.
    '''
    lexSpec = r'''([ \t]+)        |  # blanks and tabs
                  ([1-9][0-9]*|0) |  # number
                  ([()])          |  # parentheses 
                  ([-+*/])        |  # arithmetical operators
                  (.)                # unrecognized character
               '''
    tokenList = re.findall(lexSpec, s, re.VERBOSE)
    result    = []
    for ws, number, parenthesis, operator, error in tokenList:
        if ws:        # skip blanks and tabs
            continue
        elif number:
            result += [ 'NUMBER' ]
        elif parenthesis:
            result += [ parenthesis ]
        elif operator:
            result += [ operator ]
        else:
            result += [ f'ERROR({error})']
    return result

tokenize('1 + 2 * (3 - 4)')

class ShiftReduceParser():
    def __init__(self, actionTable, gotoTable):
        self.mActionTable = actionTable
        self.mGotoTable   = gotoTable

def parse(self, TL):
    index   = 0      # points to next token
    Symbols = []     # stack of symbols
    States  = ['s0'] # stack of states, s0 is start state
    TL     += ['$']
    while True:
        q = States[-1]
        t = TL[index]
        print('Symbols:', ' '.join(Symbols + ['|'] + TL[index:]).strip())
        p = self.mActionTable.get((q, t), 'error')
        if p == 'error': 
            return False
        elif p == 'accept':
            return True
        elif p[0] == 'shift':
            s = p[1]
            Symbols += [t]
            States  += [s]
            index   += 1
        elif p[0] == 'reduce':
            head, body = p[1]
            n       = len(body)
            Symbols = Symbols[:-n]
            States  = States [:-n] 
            Symbols = Symbols + [head]
            state   = States[-1]
            States += [ self.mGotoTable[state, head] ]
            
ShiftReduceParser.parse = parse
del parse

%run parse-table.py