Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
tokenizer: str = 'MeCab',
with_postag: bool = False,
user_dictionary_path: Optional[str] = None,
system_dictionary_path: Optional[str] = None,
model_path: Optional[str] = None,
mode: Optional[str] = None,
dictionary_format: Optional[str] = None,
) -> None:
"""Create tokenizer.
Keyword Arguments:
tokenizer {str or None} -- specify the type of tokenizer (default: {None}) # NOQA
flags {str} -- option passing to tokenizer (default: {''})
"""
user_dictionary = resource.Resource(user_dictionary_path)
system_dictionary = resource.Resource(system_dictionary_path)
model = resource.Resource(model_path)
self._tokenizer = tokenizer.lower()
self.with_postag = with_postag
self.user_dictionary_path = user_dictionary.path
self.system_dictionary_path = system_dictionary.path
self.model_path = model.path
self.mode = mode.lower() if mode is not None else None
self.dictionary_format = dictionary_format
self.tokenizer = None # type: Any
self._setup_tokenizer()