How to use the pyknp.Jumanpp function in pyknp
To help you get started, we’ve selected a few pyknp examples, based on popular ways it is used in public projects.
Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
Kensuke-Mitsuzawa / JapaneseTokenizers / JapaneseTokenizer / jumanpp_wrapper / __jumanpp_wrapper_python3.py View on Github 
def call_juman_interface(self, input_str):
"""* What you can do
- You call Juman tokenizer interface.
* Output
- pyknp.MList
"""
# type: (six.text_type)->MList
if isinstance(self.jumanpp_obj, Jumanpp):
ml_token_object = self.jumanpp_obj.analysis(input_str=input_str)
elif isinstance(self.jumanpp_obj, JumanppHnadler):
try:
result_token = self.jumanpp_obj.query(input_string=input_str)
except UnicodeDecodeError:
logger.warning(msg="Process is down by some reason. It restarts process automatically.")
self.jumanpp_obj.restart_process()
result_token = self.jumanpp_obj.query(input_string=input_str)
ml_token_object = MList(result_token)
elif isinstance(self.jumanpp_obj, JumanppClient):
server_response = self.jumanpp_obj.query(sentence=input_str, pattern=self.eos_pattern)
ml_token_object = MList(server_response)
else:
raise Exception('Not defined')
return ml_token_objectelse:
pass
if os.name=='nt':
"""It forces to use pyknp if it runs on Windows."""
if not self.is_use_pyknp:
logger.warning(msg="You're not able to use pexpect in Windows. It forced to set is_use_pyknp = True")
else:
pass
self.is_use_pyknp = True
else:
pass
if server is None and self.is_use_pyknp:
# jumanpp-pexpect #
self.jumanpp_obj = Jumanpp(
command=command,
timeout=timeout,
pattern=pattern,
**args)
elif server is None:
# jumanpp-pexpect #
self.jumanpp_obj = JumanppHnadler(jumanpp_command=command, timeout_second=timeout, pattern=pattern)
else:
# jumanpp-server #
self.jumanpp_obj = JumanppClient(hostname=server, port=port, timeout=timeout)def call_juman_interface(self, input_str):
# type: (unicode) -> MList
"""* What you can do
- You call Juman tokenizer interface.
* Output
- pyknp.MList
"""
if isinstance(self.jumanpp_obj, Jumanpp):
ml_token_object = self.jumanpp_obj.analysis(input_str=input_str)
elif isinstance(self.jumanpp_obj, JumanppHnadler):
try:
result_token = self.jumanpp_obj.query(input_string=input_str)
except UnicodeDecodeError:
logger.warning(msg="Process is down by some reason. It restarts process automatically.")
self.jumanpp_obj.restart_process()
result_token = self.jumanpp_obj.query(input_string=input_str)
ml_token_object = MList(result_token)
elif isinstance(self.jumanpp_obj, JumanppClient):
server_response = self.jumanpp_obj.query(sentence=input_str, pattern=self.eos_pattern)
ml_token_object = MList(server_response)
else:
raise Exception('Not defined')
return ml_token_objectelse:
pass
if os.name == 'nt':
"""It forces to use pyknp if it runs on Windows."""
if not self.is_use_pyknp:
logger.warning(msg="You're not able to use pexpect in Windows. It forced to set is_use_pyknp = True")
else:
pass
self.is_use_pyknp = True
else:
pass
if server is None and self.is_use_pyknp:
# jumanpp-pexpect #
self.jumanpp_obj = Jumanpp(
command=command,
timeout=timeout,
pattern=pattern,
**args)
elif server is None:
# jumanpp-pexpect #
self.jumanpp_obj = JumanppHnadler(jumanpp_command=command, timeout_second=timeout, pattern=pattern)
else:
# jumanpp-server #
self.jumanpp_obj = JumanppClient(hostname=server, port=port, timeout=timeout)
