How to use the lkml.Lexer function in lkml

def test_scan_expression_block_with_complex_sql_block():
    text = "concat(${orders.order_id}, '|',\n${orders__items.primary_key}) ;;"
    token = lkml.Lexer(text).scan_expression_block()
    token == tokens.ExpressionBlockToken(
        "concat(${orders.order_id}, '|', ${orders__items.primary_key})", 1
    )

def test_scan_literal():
    text = "unquoted_literal"
    token = lkml.Lexer(text).scan_literal()
    assert token == tokens.LiteralToken("unquoted_literal", 1)

def test_scan_with_non_expression_block_starting_with_html():
    text = "html_not_reserved_field: yes"
    output = lkml.Lexer(text).scan()
    assert output == (
        tokens.StreamStartToken(1),
        tokens.LiteralToken("html_not_reserved_field", 1),
        tokens.ValueToken(1),
        tokens.LiteralToken("yes", 1),
        tokens.StreamEndToken(1),
    )

def lexer():
    text = "Some sample text for testing."
    return lkml.Lexer(text)

def test_scan_literal_with_following_whitespace():
    text = "unquoted_literal \n and text following whitespace"
    token = lkml.Lexer(text).scan_literal()
    assert token == tokens.LiteralToken("unquoted_literal", 1)

def test_scan_until_token_skips_whitespace():
    text = "\n\t   \n  Start here"
    lexer = lkml.Lexer(text)
    lexer.scan_until_token()
    result = lexer.peek()
    assert result == "S"

def test_scan_quoted_literal_with_otherwise_illegal_chars():
    text = '"This: is {quoted} \n text."'
    lexer = lkml.Lexer(text)
    lexer.index = 1
    token = lexer.scan_quoted_literal()
    assert token == tokens.QuotedLiteralToken("This: is {quoted} \n text.", 1)

def test_scan_quoted_literal():
    text = '"This is quoted text."'
    lexer = lkml.Lexer(text)
    lexer.index = 1
    token = lexer.scan_quoted_literal()
    assert token == tokens.QuotedLiteralToken("This is quoted text.", 1)

def test_scan_quoted_literal_with_escaped_quotes():
    text = r'"#.### \"M\""'
    lexer = lkml.Lexer(text)
    lexer.index = 1
    token = lexer.scan_quoted_literal()
    assert token == tokens.QuotedLiteralToken(r"#.### \"M\"", 1)

def test_scan_with_complex_sql_block():
    text = (
        "sql_distinct_key: concat(${orders.order_id}, '|', "
        "${orders__items.primary_key}) ;;"
    )
    output = lkml.Lexer(text).scan()
    assert output == (
        tokens.StreamStartToken(1),
        tokens.LiteralToken("sql_distinct_key", 1),
        tokens.ValueToken(1),
        tokens.ExpressionBlockToken(
            "concat(${orders.order_id}, '|', ${orders__items.primary_key})", 1
        ),
        tokens.ExpressionBlockEndToken(1),
        tokens.StreamEndToken(1),
    )