How to use the flynt.lexer.PyToken.PyToken function in flynt

To help you get started, we’ve selected a few flynt examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github ikamensh / flynt / src / flynt / format.py View on Github external
def get_quote_type(code: str):
    from flynt.lexer.PyToken import PyToken

    g = tokenize.tokenize(io.BytesIO(code.encode("utf-8")).readline)
    next(g)
    token = PyToken(next(g))

    return token.get_quote_type()
github ikamensh / flynt / src / flynt / lexer / split.py View on Github external
def get_chunks(code) -> Generator[Chunk, None, None]:
    g = tokenize.tokenize(io.BytesIO(code.encode("utf-8")).readline)
    chunk = Chunk()

    try:
        for item in g:
            t = PyToken(item)
            reuse = chunk.append(t)

            if chunk.complete:

                yield chunk
                chunk = Chunk()
                if reuse:
                    reuse = chunk.append(t)
                    # assert not reuse
                    if chunk.complete:
                        yield chunk
                        chunk = Chunk()

        yield chunk
    except tokenize.TokenError as e:
        if state.verbose:
github ikamensh / flynt / src / flynt / lexer / Chunk.py View on Github external
def __init__(self, tokens=()):
        self.tokens: Deque[PyToken] = deque(tokens)
        self.complete = False

        self.is_percent_chunk = False
        self.percent_ongoing = False

        self.is_call_chunk = False
        self.successful = False

        self.string_in_string = False