How to use the ply.lex.input function in ply
To help you get started, we’ve selected a few ply examples, based on popular ways it is used in public projects.
Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
KevinGoodsell / mork-converter / src / MorkDB / morklex.py View on Github 
def print_tokens(f):
if isinstance(f, basestring):
f = open(f)
lex.input(f.read())
while True:
tok = lex.token()
if not tok:
break
print toklocEnd = i + 1
break;
else:
if c == '"' and headerFileStr[i-1] != '\\':
inQuotes = False
if locEnd:
#Strip it out but keep the linecount the same so line numbers are right
match_str = headerFileStr[locStart:locEnd]
debug_print("Striping out '%s'"%match_str)
num_newlines = len([a for a in match_str if a=="\n"])
headerFileStr = headerFileStr.replace(headerFileStr[locStart:locEnd], "\n"*num_newlines)
self.braceDepth = 0
lex.lex()
lex.input(headerFileStr)
global curLine
global curChar
curLine = 0
curChar = 0
try:
while True:
tok = lex.token()
if not tok: break
if self.anon_union_counter[0] == self.braceDepth and self.anon_union_counter[1]:
self.anon_union_counter[1] -= 1
tok.value = TagStr(tok.value, lineno=tok.lineno)
#debug_print("TOK: %s"%tok)
if tok.type == 'NAME' and tok.value in self.IGNORE_NAMES: continue
if tok.type != 'TEMPLATE_NAME':
self.stack.append( tok.value )
curLine = tok.linenodef annotate(self, doc):
"""Tokenize the document"""
# submit text to lexer
lex.input(doc.text)
# iterate through tokens
doc_tokens = []
num_tokens_seen = 0
prev_token = None
for found_token in iter(lex.token, None):
if found_token.type == "WHITESPACE":
pass
else:
# build new token if not whitespace
new_token = Token((found_token.lexpos, found_token.lexpos+len(found_token.value)),
doc, num_tokens_seen, found_token.value)
# check if preceding character was a "\n", mark this token as starting a line
if prev_token and prev_token.value[-1] == "\n":
new_token.starts_line = True
# add to list of tokens
doc_tokens.append(new_token)new_m = m.replace("\\\n", " ")
if (num_newlines > 1):
new_m += "\n"*(num_newlines)
headerFileStr = headerFileStr.replace(m, new_m)
#Filter out Extern "C" statements. These are order dependent
matches = re.findall(re.compile(r'extern[\t ]+"[Cc]"[\t \n\r]*{', re.DOTALL), headerFileStr)
for m in matches:
#Keep the newlines so that linecount doesnt break
num_newlines = len(filter(lambda a: a=="\n", m))
headerFileStr = headerFileStr.replace(m, "\n" * num_newlines)
headerFileStr = re.sub(r'extern[ ]+"[Cc]"[ ]*', "", headerFileStr)
self.braceDepth = 0
lex.lex()
lex.input(headerFileStr)
global curLine
global curChar
curLine = 0
curChar = 0
try:
while True:
tok = lex.token()
if not tok: break
if self.anon_union_counter[0] == self.braceDepth and self.anon_union_counter[1]:
self.anon_union_counter[1] -= 1
tok.value = TagStr(tok.value, lineno=tok.lineno)
#debug_print("TOK: %s"%tok)
if tok.type == 'NAME' and tok.value in self.IGNORE_NAMES: continue
self.stack.append( tok.value )
curLine = tok.lineno
curChar = tok.lexpos# this keeps the API compatible, TODO proper namespace for everything.
Resolver.CLASSES = {}
self.classes = Resolver.CLASSES
self.enums = []
self.global_enums = {}
self.nameStack = []
self.nameSpaces = []
self.curAccessSpecifier = 'private' # private is default
self._current_access = []
self.initextra() # harts hack
if (len(self.headerFileName)):
headerFileStr = "\n".join(open(self.headerFileName).readlines())
self.braceDepth = 0
lex.input(headerFileStr)
curLine = 0
curChar = 0
if 1: #try:
while True:
tok = lex.token()
if not tok: break
if tok.type == 'NAME' and tok.value in self.IGNORE_NAMES: continue
if tok.type not in ('PRECOMP_MACRO', 'PRECOMP_MACRO_CONT'): self.stack.append( tok.value )
curLine = tok.lineno
curChar = tok.lexpos
if tok.type in ('OPEN_BRACKET', 'CLOSE_BRACKET'): self.nameStack.append( tok.value )
elif (tok.type == 'OPEN_BRACE'):
_brace = True
if len(self.nameStack)>=2 and self.nameStack[0]=='extern' and self.nameStack[1]=='"C"':locEnd = i + 1
break;
else:
if c == '"' and headerFileStr[i-1] != '\\':
inQuotes = False
if locEnd:
#Strip it out but keep the linecount the same so line numbers are right
match_str = headerFileStr[locStart:locEnd]
debug_print("Striping out '%s'"%match_str)
num_newlines = len([a for a in match_str if a=="\n"])
headerFileStr = headerFileStr.replace(headerFileStr[locStart:locEnd], "\n"*num_newlines)
self.braceDepth = 0
lex.lex()
lex.input(headerFileStr)
global curLine
global curChar
curLine = 0
curChar = 0
try:
while True:
tok = lex.token()
if not tok: break
if self.anon_union_counter[0] == self.braceDepth and self.anon_union_counter[1]:
self.anon_union_counter[1] -= 1
tok.value = TagStr(tok.value, lineno=tok.lineno)
#debug_print("TOK: %s"%tok)
if tok.type == 'NAME' and tok.value in self.IGNORE_NAMES: continue
if tok.type != 'TEMPLATE_NAME':
self.stack.append( tok.value )
curLine = tok.linenor';.*'
pass
def t_newline(t):
r'\n+'
t.lexer.lineno += t.value.count('\n')
def t_error(t):
print "Illegal character '%s'" % t.value[0]
t.skip(1)
lex.lex()
gdl = open('parse_test.kif').read()
lex.input(gdl)def t_newline(t):
r'\n+'
t.lexer.lineno += t.value.count('\n')
def t_error(t):
print "Illegal character '%s'" % t.value[0]
t.skip(1)
lex.lex()
import sys
gdl = open(sys.argv[1]).read()
lex.input(gdl)return ast_expr(ast, env, edt)
# Just some test code
if __name__ == "__main__":
local_env = {
"A" : "1",
"C" : "foo",
"D" : "20",
"E" : 0x100,
"F" : "baz"
}
for line in open(sys.argv[1]).readlines():
lex.input(line)
for tok in iter(lex.token, None):
print(tok.type, tok.value)
parser = yacc.yacc()
print(parser.parse(line))
print(parse(line, local_env, None))def main(fp_in, fp_out):
lex.lex()
lex.input(fp_in.read())
tokens = iter(lex.token, None)
instructions = list(assemble(parse(tokens)))
allocate_names()
inst_stream = emit_inst_bytes(substitute_names(instructions))
data_stream = emit_data_bytes()
byte_stream = itertools.chain(inst_stream, data_stream)
write(byte_stream, fp_out)
