How to use the moltemplate.ttree_lex.TtreeShlex function in moltemplate

def split(s, comments=False, posix=True):
    lex = TtreeShlex(s, posix=posix)
    lex.whitespace_split = True
    if not comments:
        lex.commenters = ''
    return list(lex)

expr_str += token
            paren_depth = expr_str.count(
                left_paren) - expr_str.count(right_paren)
        if (paren_depth != 0):
            raise InputError('Error somewhere between ' +
                             self.error_leader(src_loc_begin.infile,
                                               src_loc_begin.lineno)
                             + 'and ' + self.error_leader() + '\n'
                             'Invalid expression: \"' + expr_str[0:760] + '\"')
        self.wordterminators = orig_wordterm
        return expr_str


if __name__ == '__main__':
    if len(sys.argv) == 1:
        lexer = TtreeShlex()
    else:
        file = sys.argv[1]
        lexer = TtreeShlex(open(file), file)
    while 1:
        tt = lexer.get_token()
        if tt:
            sys.stderr.write("Token: " + repr(tt))
        else:
            break

def __init__(self,
                 instream=None,
                 infile=None,
                 posix=False):
        TtreeShlex.__init__(self, instream, infile, posix)
        self.line_terminators = '\n'
        self.line_extend_chars = '\\'
        self.skip_comments_during_readline = True

"""

    # Make a copy of ltmpl
    # (The workhorse function "_TableFromTemplate()" makes in-place changes to
    #  its "ltmpl" argument.  I don't want to modify "ltmpl", so I make a copy
    #  of it before I invoke "_TableFromTemplate()" on it.)

    output = [ltmpl[i] for i in range(0, len(ltmpl))]

    d = len(delimiters) - 1
    output = _TableFromTemplate(d, output, delimiters, delete_blanks)
    return output


class TemplateLexer(TtreeShlex):
    """ This class extends the standard python lexing module, shlex, adding a
    new member function (ReadTemplate()), which can read in a block of raw text,
    (halting at an (non-escaped) terminal character), and split the text into
    alternating blocks of text and variables.  (As far as this lexer is
    concerned, "variables" are simply tokens preceeded by $ or @ characters,
    and surrounded by optional curly-brackets {}.)

    """

    def __init__(self,
                 instream=None,
                 infile=None,
                 posix=False):
        TtreeShlex.__init__(self, instream, infile, posix)
        self.var_delim = '$@'  # characters which can begin a variable name
        self.var_open_paren = '{'  # optional parenthesis surround a variable

#assert(type(p) is _sre.SRE_Match)
        # I assume pattern = re.compile(some_reg_expr)
        if not pattern.search(s):
            return False
    return True


def MatchesAll(multi_string, pattern):
    assert(len(multi_string) == len(pattern))
    for i in range(0, len(pattern)):
        if not MatchesPattern(multi_string[i], pattern[i]):
            return False
    return True


class LineLex(TtreeShlex):
    """ This class extends the TtreeShlex module (a slightly modified
    version of the python 3.2.2 version of shlex).  LineLex has the
    ability to read one line at a time (in addition to one token at a time).
    (Many files and scripts must be parsed one line at a time instead of one
     token at a time.  In these cases, the whitespace position also matters.)

    Arguably, this class might not be necessary.
    I could get rid of this class completely.  That would be nice.  To do that
    we would need to augment and generalize shlex's get_token() member function
    to make it read lines, not just tokens.  Of course, you can always
    change the wordchars (or wordterminators).  Even so, there are two other
    difficulties using the current version of shlex.get_token() to read lines:
    1) File inclusion happen whenever the beginning of a line/token matches one
       of the "source_triggers" (not the whole line as required by get_token()).
    2) Lines ending in a special character (by default the backslash character)
       continue on to the next line.

if (paren_depth != 0):
            raise InputError('Error somewhere between ' +
                             self.error_leader(src_loc_begin.infile,
                                               src_loc_begin.lineno)
                             + 'and ' + self.error_leader() + '\n'
                             'Invalid expression: \"' + expr_str[0:760] + '\"')
        self.wordterminators = orig_wordterm
        return expr_str


if __name__ == '__main__':
    if len(sys.argv) == 1:
        lexer = TtreeShlex()
    else:
        file = sys.argv[1]
        lexer = TtreeShlex(open(file), file)
    while 1:
        tt = lexer.get_token()
        if tt:
            sys.stderr.write("Token: " + repr(tt))
        else:
            break

self.lineno = self.lineno + 1
                    if self.posix:
                        self.state = ' '
                        if self.token or (self.posix and quoted):
                            # Keep track of which character(s) terminated
                            # the token (including whitespace and comments).
                            self.prev_space_terminator = nextchar + comment_contents
                            break   # emit current token
                        else:
                            continue
                elif self.posix and nextchar in self.quotes:
                    self.state = nextchar
                elif self.posix and nextchar in self.escape:
                    escapedstate = 'a'
                    self.state = nextchar
                elif (TtreeShlex._belongs_to(nextchar,
                                             self.wordchars,
                                             self.wordterminators)
                      or (nextchar in self.quotes)
                      or (self.whitespace_split)):
                    self.token = self.token + nextchar
                else:
                    self.pushback.appendleft(nextchar)
                    if self.debug >= 2:
                        sys.stderr.write("TtreeShlex: I see punctuation in word state")
                    self.state = ' '
                    if self.token:
                        break   # emit current token
                    else:
                        continue
        result = self.token
        self.token = ''