How to use the lark.Token function in lark

To help you get started, we’ve selected a few lark examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github simon816 / Command-Block-Assembly / cmd_ir / reader.py View on Github external
def extern_func(self, node):
        name, params, returns, newline = self.visit_children(node)
        name = name.value
        if isinstance(params, Token):
            params = self.token_to_val(params)
        if isinstance(returns, Token):
            returns = self.token_to_val(returns)
        from .variables import VarType
        params = [(VarType._init_from_parser(params[i*2]), params[(i*2)+1]) \
            for i in range(len(params or []) // 2)]
        returns = [VarType._init_from_parser(r) for r in returns or []]
        existing = self.top.lookup_func(name)
        extern = ExternFunction(name, params, returns)
        if existing:
            existing.expect_signature(extern)
            if not existing.finished and \
               not isinstance(existing, ExternFunction):
                # This is a forward-referenced function, replace references
                # with our extern
                mapping = {existing: extern}
github GoogleCloudPlatform / oozie-to-airflow / o2a / o2a_libs / el_parser.py View on Github external
def _translate_el(tree: Union[Tree, Token], functions_module: str = "") -> str:
    """
    Translates el expression to jinja equivalent.
    """

    if isinstance(tree, Token):
        return _translate_token(tree)

    if tree.data == "binary_op":
        return _translate_binary_operator(tree)

    if tree.data == "function_invocation":
        return _translate_function(tree, functions_module)

    ternary = _translate_ternary(tree, functions_module)
    if ternary is not None:
        return ternary

    return _translate_tail(tree, functions_module)
github terrencepreilly / darglint / bin / bnf_to_cnf / bnf_to_cnf / node.py View on Github external
def from_lark_tree(self, tree: Union[Tree, Token]) -> 'Node':
        if isinstance(tree, Token):
            return Node(
                NodeType.TERMINAL,
                value=tree.value,
            )

        assert isinstance(tree, Tree)
        if tree.data == 'start':
            return Node.from_lark_tree(tree.children[0])
        elif tree.data == 'grammar':
            # Don't include an import node if there were no
            # imports.  Imports will also be stripped out later.
            return Node(
                NodeType.GRAMMAR,
                children=list(map(Node.from_lark_tree, tree.children)),
            )
        elif tree.data == 'production':
github ros2 / launch / launch / launch / frontend / parse_substitution.py View on Github external
def part(self, content):
        assert(len(content) == 1)
        content = content[0]
        if isinstance(content, Token):
            assert content.type.endswith('_RSTRING')
            return TextSubstitution(text=replace_escaped_characters(content.value))
        return content
github Scony / godot-gdscript-toolkit / gdtoolkit / formatter / expression_utils.py View on Github external
def is_expression_forcing_multiple_lines(expression: Node) -> bool:
    if has_trailing_comma(expression):
        return True
    if _is_multiline_string(expression):
        return True
    if isinstance(expression, Token):
        return False
    for child in expression.children:
        if is_expression_forcing_multiple_lines(child):
            return True
    return False
github Scony / godot-gdscript-toolkit / gdtoolkit / formatter / expression_utils.py View on Github external
def has_leading_dot(expression: Node) -> bool:
    return (
        isinstance(expression.children[0], Token)
        and expression.children[0].type == "DOT"
    )
github chanzuckerberg / miniwdl / WDL / _parser.py View on Github external
return ans

        if param or param2:
            raise Error.InvalidType(self._sp(meta), "Unexpected type parameter(s)")

        ans = Type.StructInstance(items[0].value, "optional" in quantifiers)
        ans.pos = self._sp(meta)
        return ans


class _DocTransformer(_ExprTransformer, _TypeTransformer):
    # pylint: disable=no-self-use,unused-argument

    _keywords: Set[str]
    _source_text: str
    _comments: List[lark.Token]
    _version: Optional[str]
    _declared_version: Optional[str]

    def __init__(
        self,
        source_text: str,
        keywords: Set[str],
        comments: List[lark.Token],
        version: str,
        declared_version: Optional[str],
        *args,
        **kwargs,
    ):
        super().__init__(*args, **kwargs)
        self._source_text = source_text
        self._keywords = keywords
github GoogleCloudPlatform / oozie-to-airflow / o2a / o2a_libs / el_parser.py View on Github external
def _unpack_identifier(node: Union[Tree, Token]) -> str:
    if isinstance(node, Token):
        return str(node.value)

    return str(node.children[0].value)
github Scony / godot-gdscript-toolkit / gdtoolkit / formatter / types.py View on Github external
from typing import Union, Tuple, List, Optional

from lark import Tree, Token


Node = Union[Tree, Token]
PreviouslyProcessedLineNumber = int
FormattedLines = List[Tuple[Optional[int], str]]
Outcome = Tuple[FormattedLines, PreviouslyProcessedLineNumber]
github Scony / godot-gdscript-toolkit / gdtoolkit / linter / basic_checks.py View on Github external
and func_def.children[1].data == "func_args"
        ):
            argument_definitions = {}  # type: Dict[str, int]
            argument_tokens = {}
            func_args = func_def.children[1]
            for func_arg in func_args.children:
                arg_name_token = find_name_token_among_children(func_arg)
                arg_name = arg_name_token.value
                argument_definitions[arg_name] = (
                    argument_definitions.get(arg_name, 0) + 1
                )
                argument_tokens[arg_name] = arg_name_token
            name_occurances = {}  # type: Dict[str, int]
            for xnode in func_def.iter_subtrees():
                for node in xnode.children:
                    if isinstance(node, Token) and node.type == "NAME":
                        name = node.value
                        name_occurances[name] = name_occurances.get(name, 0) + 1
            for argument in argument_definitions:
                if argument_definitions[argument] == name_occurances[
                    argument
                ] and not argument.startswith("_"):
                    problems.append(
                        Problem(
                            name="unused-argument",
                            description="unused function argument '{}'".format(
                                argument
                            ),
                            line=argument_tokens[argument].line,
                            column=argument_tokens[argument].column,
                        )
                    )