Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
pattern = re.compile(pattern) # type: ignore
document = re.sub(pattern, self.conv_period, document)
result = []
for line in document.split("\n"):
line = line.rstrip()
line = line.replace("\n", "")
line = line.replace("\r", "")
line = line.replace("ใ", "ใ\n")
sentences = line.split("\n")
for sentence in sentences:
if not sentence:
continue
period_special = SentenceTokenizer.PERIOD_SPECIAL
period = SentenceTokenizer.PERIOD
sentence = sentence.replace(period_special, period)
result.append(sentence)
return result
def conv_period(item) -> str:
return item.group(0).replace(
SentenceTokenizer.PERIOD, SentenceTokenizer.PERIOD_SPECIAL
)