Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
system_dictionary_path=self.system_dictionary_path,
with_postag=self.with_postag,
dictionary_format=self.dictionary_format,
)
if self._tokenizer == "janome":
self.tokenizer = word_tokenizers.JanomeTokenizer(
user_dictionary_path=self.user_dictionary_path,
with_postag=self.with_postag,
)
if self._tokenizer == "sudachi":
if self.mode is None:
raise ValueError("`mode` must be specified for sudachi.")
self.tokenizer = word_tokenizers.SudachiTokenizer(
mode=self.mode,
with_postag=self.with_postag,
)