Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
word = ''
if self.char in self.whitespace:
while self.char in self.whitespace:
word += self.char
self.update_chars()
elif self.char in self.comment_tokens or self.group_token is None:
# Abort the iteration and build the comment token
word = line[self.idx:-1]
self.char = '\n'
elif self.char in '"\'' or self.prior_delim:
word = self.parse_string()
elif self.char in Tokenizer.punctuation:
word = self.char
self.update_chars()
else:
while (not self.char.isspace()
and self.char not in Tokenizer.punctuation):
word += self.char
self.update_chars()
tokens.append(word)
return tokens