Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def setUp(self):
self.tokenizer = tokenizers.JapaneseTokenizer()
def __init__(
self,
lang: str = 'en',
tokenizer: Tokenizer = None,
word_embedder: WordEmbedding = None) -> None:
tokenizer = tokenizer or {"en": SimpleTokenizer(),
"fr": SimpleTokenizer(),
"ja": JapaneseTokenizer()}[lang]
word_embedder = word_embedder or FasttextEmbedding(lang)
super().__init__(tokenizer, word_embedder)