How to use the bashlex.tokenizer.tokentype function in bashlex
To help you get started, we’ve selected a few bashlex examples, based on popular ways it is used in public projects.
Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
idank / bashlex / bashlex / tokenizer.py View on Github 
# tokenword.append(c)
# all_digit_token &= c.isdigit()
# if not dollar_present:
# dollar_present = c == '$'
# next_character
cd = self._current_delimiter()
c = self._getc(cd != "'" and not d['pass_next_character'])
# got_token
self._recordpos()
tokenword = ''.join(tokenword)
if d['all_digit_token'] and (c in '<>' or self._last_read_token.ttype in (tokentype.LESS_AND, tokentype.GREATER_AND)) and shutils.legal_number(tokenword):
return self._createtoken(tokentype.NUMBER, int(tokenword))
# bashlex/parse.y L4811
specialtokentype = self._specialcasetokens(tokenword)
if specialtokentype:
return self._createtoken(specialtokentype, tokenword)
if not d['dollar_present'] and not d['quoted'] and self._reserved_word_acceptable(self._last_read_token):
if tokenword in valid_reserved_first_command:
ttype = valid_reserved_first_command[tokenword]
ps = self._parserstate
if ps & parserflags.CASEPAT and ttype != tokentype.ESAC:
pass
elif ttype == tokentype.TIME and not self._time_command_acceptable():
pass
elif ttype == tokentype.ESAC:
ps.discard(parserflags.CASEPAT)"esac" : tokentype.ESAC,
"for" : tokentype.FOR,
"select" : tokentype.SELECT,
"while" : tokentype.WHILE,
"until" : tokentype.UNTIL,
"do" : tokentype.DO,
"done" : tokentype.DONE,
"in" : tokentype.IN,
"function" : tokentype.FUNCTION,
"time" : tokentype.TIME,
"{" : tokentype.LEFT_CURLY,
"}" : tokentype.RIGHT_CURLY,
"!" : tokentype.BANG,
"[[" : tokentype.COND_START,
"]]" : tokentype.COND_END,
"coproc" : tokentype.COPROC
}
class MatchedPairError(errors.ParsingError):
def __init__(self, startline, message, tokenizer):
# TODO use startline?
super(MatchedPairError, self).__init__(message,
tokenizer.source,
tokenizer._shell_input_line_index - 1)
wordflags = flags.word
parserflags = flags.parser
class token(object):
def __init__(self, type_, value, pos=None, flags=None):
if type_ is not None:
assert isinstance(type_, tokentype)self._parserstate.discard(parserflags.ASSIGNOK)
peek_char = self._getc(True)
both = character
if peek_char:
both += peek_char
if character == peek_char:
if character == '<':
peek_char = self._getc()
if peek_char == '-':
return tokentype.LESS_LESS_MINUS
elif peek_char == '<':
return tokentype.LESS_LESS_LESS
else:
self._ungetc(peek_char)
return tokentype.LESS_LESS
elif character == '>':
return tokentype.GREATER_GREATER
elif character == ';':
self._parserstate |= parserflags.CASEPAT
# bashlex/parse.y L3085 ALIAS
peek_char = self._getc()
if peek_char == '&':
return tokentype.SEMI_SEMI_AND
else:
self._ungetc(peek_char)
return tokentype.SEMI_SEMI
elif character == '&':
return tokentype.AND_AND
elif character == '|':
return tokentype.OR_OR
# bashlex/parse.y L3105def _specialcasetokens(self, tokstr):
if (self._last_read_token.ttype == tokentype.WORD and
self._token_before_that.ttype in (tokentype.FOR,
tokentype.CASE,
tokentype.SELECT) and
tokstr == 'in'):
if self._token_before_that.ttype == tokentype.CASE:
self._parserstate.add(parserflags.CASEPAT)
self._esacs_needed_count += 1
return tokentype.IN
if (self._last_read_token.ttype == tokentype.WORD and
self._token_before_that.ttype in (tokentype.FOR, tokentype.SELECT) and
tokstr == 'do'):
return tokentype.DO
if self._esacs_needed_count:
self._esacs_needed_count -= 1
if tokstr == 'esac':
self._parserstate.discard(parserflags.CASEPAT)
return tokentype.ESAC
if self._parserstate & parserflags.ALLOWOPNBRC:
self._parserstate.discard(parserflags.ALLOWOPNBRC)
if tokstr == '{':
self._open_brace_count += 1
# bash/parse.y L2887
return tokentype.LEFT_CURLY
if (self._last_read_token.ttype == tokentype.ARITH_FOR_EXPRS and
tokstr == 'do'):self._token_before_that.ttype in (tokentype.FOR, tokentype.SELECT) and
tokstr == 'do'):
return tokentype.DO
if self._esacs_needed_count:
self._esacs_needed_count -= 1
if tokstr == 'esac':
self._parserstate.discard(parserflags.CASEPAT)
return tokentype.ESAC
if self._parserstate & parserflags.ALLOWOPNBRC:
self._parserstate.discard(parserflags.ALLOWOPNBRC)
if tokstr == '{':
self._open_brace_count += 1
# bash/parse.y L2887
return tokentype.LEFT_CURLY
if (self._last_read_token.ttype == tokentype.ARITH_FOR_EXPRS and
tokstr == 'do'):
return tokentype.DO
if (self._last_read_token.ttype == tokentype.ARITH_FOR_EXPRS and
tokstr == '{'):
self._open_brace_count += 1
return tokentype.LEFT_CURLY
if (self._open_brace_count and
self._reserved_word_acceptable(self._last_read_token) and
tokstr == '}'):
self._open_brace_count -= 1
return tokentype.RIGHT_CURLYtokstr == 'in'):
if self._token_before_that.ttype == tokentype.CASE:
self._parserstate.add(parserflags.CASEPAT)
self._esacs_needed_count += 1
return tokentype.IN
if (self._last_read_token.ttype == tokentype.WORD and
self._token_before_that.ttype in (tokentype.FOR, tokentype.SELECT) and
tokstr == 'do'):
return tokentype.DO
if self._esacs_needed_count:
self._esacs_needed_count -= 1
if tokstr == 'esac':
self._parserstate.discard(parserflags.CASEPAT)
return tokentype.ESAC
if self._parserstate & parserflags.ALLOWOPNBRC:
self._parserstate.discard(parserflags.ALLOWOPNBRC)
if tokstr == '{':
self._open_brace_count += 1
# bash/parse.y L2887
return tokentype.LEFT_CURLY
if (self._last_read_token.ttype == tokentype.ARITH_FOR_EXPRS and
tokstr == 'do'):
return tokentype.DO
if (self._last_read_token.ttype == tokentype.ARITH_FOR_EXPRS and
tokstr == '{'):
self._open_brace_count += 1
return tokentype.LEFT_CURLYelif peek_char == '<':
return tokentype.LESS_LESS_LESS
else:
self._ungetc(peek_char)
return tokentype.LESS_LESS
elif character == '>':
return tokentype.GREATER_GREATER
elif character == ';':
self._parserstate |= parserflags.CASEPAT
# bashlex/parse.y L3085 ALIAS
peek_char = self._getc()
if peek_char == '&':
return tokentype.SEMI_SEMI_AND
else:
self._ungetc(peek_char)
return tokentype.SEMI_SEMI
elif character == '&':
return tokentype.AND_AND
elif character == '|':
return tokentype.OR_OR
# bashlex/parse.y L3105
elif both == '<&':
return tokentype.LESS_AND
elif both == '>&':
return tokentype.GREATER_AND
elif both == '<>':
return tokentype.LESS_GREATER
elif both == '>|':
return tokentype.GREATER_BAR
elif both == '&>':
peek_char = self._getc()
if peek_char == '>':return self._readtokenword(character)
if _shellmeta(character) and not (self._parserstate & parserflags.DBLPAREN):
self._parserstate.discard(parserflags.ASSIGNOK)
peek_char = self._getc(True)
both = character
if peek_char:
both += peek_char
if character == peek_char:
if character == '<':
peek_char = self._getc()
if peek_char == '-':
return tokentype.LESS_LESS_MINUS
elif peek_char == '<':
return tokentype.LESS_LESS_LESS
else:
self._ungetc(peek_char)
return tokentype.LESS_LESS
elif character == '>':
return tokentype.GREATER_GREATER
elif character == ';':
self._parserstate |= parserflags.CASEPAT
# bashlex/parse.y L3085 ALIAS
peek_char = self._getc()
if peek_char == '&':
return tokentype.SEMI_SEMI_AND
else:
self._ungetc(peek_char)
return tokentype.SEMI_SEMI
elif character == '&':
return tokentype.AND_ANDif character is None:
return eoftoken
if character == '#':
self._discard_until('\n')
self._getc(False)
character = '\n'
self._recordpos(1)
if character == '\n':
# bashlex/parse.y L3034 ALIAS
heredoc.gatherheredocuments(self)
self._parserstate.discard(parserflags.ASSIGNOK)
return tokentype(character)
if self._parserstate & parserflags.REGEXP:
return self._readtokenword(character)
if _shellmeta(character) and not (self._parserstate & parserflags.DBLPAREN):
self._parserstate.discard(parserflags.ASSIGNOK)
peek_char = self._getc(True)
both = character
if peek_char:
both += peek_char
if character == peek_char:
if character == '<':
peek_char = self._getc()
if peek_char == '-':
return tokentype.LESS_LESS_MINUSdef _specialcasetokens(self, tokstr):
if (self._last_read_token.ttype == tokentype.WORD and
self._token_before_that.ttype in (tokentype.FOR,
tokentype.CASE,
tokentype.SELECT) and
tokstr == 'in'):
if self._token_before_that.ttype == tokentype.CASE:
self._parserstate.add(parserflags.CASEPAT)
self._esacs_needed_count += 1
return tokentype.IN
if (self._last_read_token.ttype == tokentype.WORD and
self._token_before_that.ttype in (tokentype.FOR, tokentype.SELECT) and
tokstr == 'do'):
return tokentype.DO
if self._esacs_needed_count:
self._esacs_needed_count -= 1
if tokstr == 'esac':
self._parserstate.discard(parserflags.CASEPAT)
Popular bashlex functions
- bashlex.ast
- bashlex.ast.findfirstkind
- bashlex.ast.node
- bashlex.ast.nodevisitor
- bashlex.errors
- bashlex.errors.ParsingError
- bashlex.flags
- bashlex.flags.parser
- bashlex.flags.word
- bashlex.parser
- bashlex.parser._parser
- bashlex.parser.parse
- bashlex.parsesingle
- bashlex.tokenizer
- bashlex.tokenizer.MatchedPairError
- bashlex.tokenizer.token
- bashlex.tokenizer.tokentype
- bashlex.tokenizer.tokentype.ASSIGNMENT_WORD
- bashlex.tokenizer.tokentype.WORD
- bashlex.yacc.YaccSymbol