How to use the codespell.tokenize function in codespell
To help you get started, we’ve selected a few codespell examples, based on popular ways it is used in public projects.
Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
shinglyu / vim-codespell / plugin / test_codespell.py View on Github 
def test_tokenize_camel_case():
assert ["camel", "Case"] == codespell.tokenize("camelCase")
assert ["Title", "Case"] == codespell.tokenize("TitleCase")
assert ["Title", "Case", "mixed"] == codespell.tokenize("TitleCase mixed")def test_tokenize_brackets():
assert ["Hello", "World"] == codespell.tokenize("Hello(World)")
assert ["Hello", "World"] == codespell.tokenize("Hello[World]")
assert ["Hello", "World"] == codespell.tokenize("Hello{World}")
assert ["Hello", "World"] == codespell.tokenize("Hello")def test_tokenize_alphanumeric():
assert ["uzer"] == codespell.tokenize("uzer42")def test_tokenize_brackets():
assert ["Hello", "World"] == codespell.tokenize("Hello(World)")
assert ["Hello", "World"] == codespell.tokenize("Hello[World]")
assert ["Hello", "World"] == codespell.tokenize("Hello{World}")
assert ["Hello", "World"] == codespell.tokenize("Hello")def test_tokenize_pointers():
assert ["pointer"] == codespell.tokenize("*pointer")
assert ["pointer"] == codespell.tokenize("&pointer")
assert ["pointer"] == codespell.tokenize("&&*pointer")def test_tokenize_pointers():
assert ["pointer"] == codespell.tokenize("*pointer")
assert ["pointer"] == codespell.tokenize("&pointer")
assert ["pointer"] == codespell.tokenize("&&*pointer")def test_tokenize_quotes():
assert ["Hello", "World"] == codespell.tokenize("Hello\"World\"")
assert ["Hello", "World"] == codespell.tokenize("Hello\'World\'")
assert ["Hello", "World"] == codespell.tokenize("Hello \'World\'")def test_tokenize_brackets():
assert ["Hello", "World"] == codespell.tokenize("Hello(World)")
assert ["Hello", "World"] == codespell.tokenize("Hello[World]")
assert ["Hello", "World"] == codespell.tokenize("Hello{World}")
assert ["Hello", "World"] == codespell.tokenize("Hello")def test_tokenize_quotes():
assert ["Hello", "World"] == codespell.tokenize("Hello\"World\"")
assert ["Hello", "World"] == codespell.tokenize("Hello\'World\'")
assert ["Hello", "World"] == codespell.tokenize("Hello \'World\'")def test_tokenize_camel_case():
assert ["camel", "Case"] == codespell.tokenize("camelCase")
assert ["Title", "Case"] == codespell.tokenize("TitleCase")
assert ["Title", "Case", "mixed"] == codespell.tokenize("TitleCase mixed")
