1. Advisor
  2. bashlex
  3. functions
  4. bashlex.tokenizer.token
View all bashlex analysis

How to use the bashlex.tokenizer.token function in bashlex

To help you get started, we’ve selected a few bashlex examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github idank / bashlex / tests / test-tokenizer.py View on Github external
def test_tokenize(self):
        s = 'bar -x'
        self.assertTokens(s, [
                          t(tt.WORD, 'bar', [0, 3]),
                          t(tt.WORD, '-x', [4, 6])])

        s = 'wx    y =z '
        self.assertTokens(s, [
                          t(tt.WORD, 'wx', [0, 2]),
                          t(tt.WORD, 'y', [6, 7]),
                          t(tt.WORD, '=z', [8, 10])])

        s = "a 'b' c"
        self.assertTokens(s, [
                          t(tt.WORD, 'a', [0, 1]),
                          t(tt.WORD, "'b'", [2, 5], set([flags.word.QUOTED])),
                          t(tt.WORD, 'c', [6, 7])])

        s = "a 'b  ' c"
        self.assertTokens(s, [
                          t(tt.WORD, 'a', [0, 1]),
                          t(tt.WORD, "'b  '", [2, 7], set([flags.word.QUOTED])),
                          t(tt.WORD, 'c', [8, 9])])
github idank / bashlex / tests / test-tokenizer.py View on Github external
def test_parsematchedpair(self):
        s = '"`foo`"'
        self.assertTokens(s, [
                          t(tt.WORD, '"`foo`"', [0, len(s)], set([flags.word.QUOTED]))])

        s = '"${a}"'
        self.assertTokens(s, [
                          t(tt.WORD, '"${a}"', [0, len(s)], set([flags.word.HASDOLLAR,
                                                                flags.word.QUOTED]))])

        s = '${\'a\'}'
        self.assertTokens(s, [
                          t(tt.WORD, '${\'a\'}', [0, len(s)], hasdollarset)])

        s = '${$\'a\'}'
        self.assertTokens(s, [
                          t(tt.WORD, '${$\'a\'}', [0, len(s)], hasdollarset)])

        s = "'a\\'"
        self.assertTokens(s, [
                          t(tt.WORD, "'a\\'", [0, len(s)], set([flags.word.QUOTED]))])
github idank / bashlex / tests / test-tokenizer.py View on Github external
t(tt.LESS_GREATER, '<>', [4, 6]),
                          t(tt.SEMI_AND, ';&', [6, 8]),
                          t(tt.SEMI_SEMI_AND, ';;&', [8, 11]),
                          t(tt.SEMI_SEMI, ';;', [11, 13]),
                          t(tt.BAR, '|', [14, 15]),
                          t(tt.LESS_LESS_MINUS, '<<-', [15, 18]),
                          t(tt.LESS_LESS, '<<', [18, 20]),
                          t(tt.LESS_LESS_LESS, '<<<', [21, 24]),
                          t(tt.GREATER_GREATER, '>>', [24, 26]),
                          t(tt.AND_AND, '&&', [26, 28]),
                          t(tt.OR_OR, '||', [28, 30]),
                          t(tt.LESS_AND, '<&', [30, 32]),
                          t(tt.GREATER_AND, '>&', [32, 34]),
                          t(tt.LESS_GREATER, '<>', [34, 36]),
                          t(tt.GREATER_BAR, '>|', [36, 38]),
                          t(tt.AND_GREATER, '&>', [38, 40]),
                          t(tt.AND_GREATER_GREATER, '&>>', [41, 44]),
                          t(tt.BAR_AND, '|&', [44, 46])])

        s = '<&-'
        self.assertTokens(s, [
                          t(tt.LESS_AND, '<&', [0, 2]),
                          t(tt.DASH, '-', [2, 3])])
github idank / bashlex / tests / test-tokenizer.py View on Github external
s = '"${a}"'
        self.assertTokens(s, [
                          t(tt.WORD, '"${a}"', [0, len(s)], set([flags.word.HASDOLLAR,
                                                                flags.word.QUOTED]))])

        s = '${\'a\'}'
        self.assertTokens(s, [
                          t(tt.WORD, '${\'a\'}', [0, len(s)], hasdollarset)])

        s = '${$\'a\'}'
        self.assertTokens(s, [
                          t(tt.WORD, '${$\'a\'}', [0, len(s)], hasdollarset)])

        s = "'a\\'"
        self.assertTokens(s, [
                          t(tt.WORD, "'a\\'", [0, len(s)], set([flags.word.QUOTED]))])
github idank / bashlex / tests / test-tokenizer.py View on Github external
def test_meta(self):
        s = '!&()<>;&;;&;; |<<-<< <<<>>&&||<&>&<>>|&> &>>|&'
        self.assertTokens(s, [
                          t(tt.BANG, '!', [0, 1]),
                          t(tt.AMPERSAND, '&', [1, 2]),
                          t(tt.LEFT_PAREN, '(', [2, 3]),
                          t(tt.RIGHT_PAREN, ')', [3, 4]),
                          t(tt.LESS_GREATER, '<>', [4, 6]),
                          t(tt.SEMI_AND, ';&', [6, 8]),
                          t(tt.SEMI_SEMI_AND, ';;&', [8, 11]),
                          t(tt.SEMI_SEMI, ';;', [11, 13]),
                          t(tt.BAR, '|', [14, 15]),
                          t(tt.LESS_LESS_MINUS, '<<-', [15, 18]),
                          t(tt.LESS_LESS, '<<', [18, 20]),
                          t(tt.LESS_LESS_LESS, '<<<', [21, 24]),
                          t(tt.GREATER_GREATER, '>>', [24, 26]),
                          t(tt.AND_AND, '&&', [26, 28]),
                          t(tt.OR_OR, '||', [28, 30]),
                          t(tt.LESS_AND, '<&', [30, 32]),
                          t(tt.GREATER_AND, '>&', [32, 34]),
                          t(tt.LESS_GREATER, '<>', [34, 36]),
                          t(tt.GREATER_BAR, '>|', [36, 38]),
                          t(tt.AND_GREATER, '&>', [38, 40]),
                          t(tt.AND_GREATER_GREATER, '&>>', [41, 44]),
                          t(tt.BAR_AND, '|&', [44, 46])])

        s = '<&-'
        self.assertTokens(s, [
                          t(tt.LESS_AND, '<&', [0, 2]),
github idank / bashlex / tests / test-tokenizer.py View on Github external
self.assertTokens(s, [
                          t(tt.WORD, '$(a ${b})', [0, 9], hasdollarset)])

        s = '$(a $[b])'
        self.assertTokens(s, [
                          t(tt.WORD, '$(a $[b])', [0, 9], hasdollarset)])

        s = '"$(a)"'
        self.assertTokens(s, [
                          t(tt.WORD, '"$(a)"', [0, 6], set([flags.word.HASDOLLAR,
                                                            flags.word.QUOTED]))])

        s = 'a $(! b)'
        self.assertTokens(s, [
                          t(tt.WORD, 'a', [0, 1]),
                          t(tt.WORD, '$(! b)', [2, 8], hasdollarset)])

        s = '$(!|!||)'
        self.assertTokens(s, [
                          t(tt.WORD, '$(!|!||)', [0, 8], hasdollarset)])

        s = '$(a <
github idank / bashlex / tests / test-tokenizer.py View on Github external
def test_meta(self):
        s = '!&()<>;&;;&;; |<<-<< <<<>>&&||<&>&<>>|&> &>>|&'
        self.assertTokens(s, [
                          t(tt.BANG, '!', [0, 1]),
                          t(tt.AMPERSAND, '&', [1, 2]),
                          t(tt.LEFT_PAREN, '(', [2, 3]),
                          t(tt.RIGHT_PAREN, ')', [3, 4]),
                          t(tt.LESS_GREATER, '<>', [4, 6]),
                          t(tt.SEMI_AND, ';&', [6, 8]),
                          t(tt.SEMI_SEMI_AND, ';;&', [8, 11]),
                          t(tt.SEMI_SEMI, ';;', [11, 13]),
                          t(tt.BAR, '|', [14, 15]),
                          t(tt.LESS_LESS_MINUS, '<<-', [15, 18]),
                          t(tt.LESS_LESS, '<<', [18, 20]),
                          t(tt.LESS_LESS_LESS, '<<<', [21, 24]),
                          t(tt.GREATER_GREATER, '>>', [24, 26]),
                          t(tt.AND_AND, '&&', [26, 28]),
                          t(tt.OR_OR, '||', [28, 30]),
                          t(tt.LESS_AND, '<&', [30, 32]),
                          t(tt.GREATER_AND, '>&', [32, 34]),
                          t(tt.LESS_GREATER, '<>', [34, 36]),
                          t(tt.GREATER_BAR, '>|', [36, 38]),
                          t(tt.AND_GREATER, '&>', [38, 40]),
github idank / bashlex / tests / test-tokenizer.py View on Github external
def test_heredoc(self):
        s = 'a <
github idank / bashlex / tests / test-tokenizer.py View on Github external
t(tt.LEFT_PAREN, '(', [2, 3]),
                          t(tt.RIGHT_PAREN, ')', [3, 4]),
                          t(tt.LESS_GREATER, '<>', [4, 6]),
                          t(tt.SEMI_AND, ';&', [6, 8]),
                          t(tt.SEMI_SEMI_AND, ';;&', [8, 11]),
                          t(tt.SEMI_SEMI, ';;', [11, 13]),
                          t(tt.BAR, '|', [14, 15]),
                          t(tt.LESS_LESS_MINUS, '<<-', [15, 18]),
                          t(tt.LESS_LESS, '<<', [18, 20]),
                          t(tt.LESS_LESS_LESS, '<<<', [21, 24]),
                          t(tt.GREATER_GREATER, '>>', [24, 26]),
                          t(tt.AND_AND, '&&', [26, 28]),
                          t(tt.OR_OR, '||', [28, 30]),
                          t(tt.LESS_AND, '<&', [30, 32]),
                          t(tt.GREATER_AND, '>&', [32, 34]),
                          t(tt.LESS_GREATER, '<>', [34, 36]),
                          t(tt.GREATER_BAR, '>|', [36, 38]),
                          t(tt.AND_GREATER, '&>', [38, 40]),
                          t(tt.AND_GREATER_GREATER, '&>>', [41, 44]),
                          t(tt.BAR_AND, '|&', [44, 46])])

        s = '<&-'
        self.assertTokens(s, [
                          t(tt.LESS_AND, '<&', [0, 2]),
                          t(tt.DASH, '-', [2, 3])])
github idank / bashlex / tests / test-tokenizer.py View on Github external
def test_meta(self):
        s = '!&()<>;&;;&;; |<<-<< <<<>>&&||<&>&<>>|&> &>>|&'
        self.assertTokens(s, [
                          t(tt.BANG, '!', [0, 1]),
                          t(tt.AMPERSAND, '&', [1, 2]),
                          t(tt.LEFT_PAREN, '(', [2, 3]),
                          t(tt.RIGHT_PAREN, ')', [3, 4]),
                          t(tt.LESS_GREATER, '<>', [4, 6]),
                          t(tt.SEMI_AND, ';&', [6, 8]),
                          t(tt.SEMI_SEMI_AND, ';;&', [8, 11]),
                          t(tt.SEMI_SEMI, ';;', [11, 13]),
                          t(tt.BAR, '|', [14, 15]),
                          t(tt.LESS_LESS_MINUS, '<<-', [15, 18]),
                          t(tt.LESS_LESS, '<<', [18, 20]),
                          t(tt.LESS_LESS_LESS, '<<<', [21, 24]),
                          t(tt.GREATER_GREATER, '>>', [24, 26]),
                          t(tt.AND_AND, '&&', [26, 28]),
                          t(tt.OR_OR, '||', [28, 30]),
                          t(tt.LESS_AND, '<&', [30, 32]),
                          t(tt.GREATER_AND, '>&', [32, 34]),
                          t(tt.LESS_GREATER, '<>', [34, 36]),
                          t(tt.GREATER_BAR, '>|', [36, 38]),
                          t(tt.AND_GREATER, '&>', [38, 40]),
                          t(tt.AND_GREATER_GREATER, '&>>', [41, 44]),
                          t(tt.BAR_AND, '|&', [44, 46])])

bashlex

Python parser for bash

GitHub
GPL-3.0
Latest version published 2 years ago

Package Health Score

64 / 100
Full package analysis

Popular bashlex functions

Similar packages