Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def sentencepiece_vocab(extra_ids=0):
return sentencepiece_vocabulary.SentencePieceVocabulary(
os.path.join(TEST_DATA_DIR, "sentencepiece", "sentencepiece.model"),
extra_ids=extra_ids)
def get_vocabulary(self):
"""Returns a SentencePieceVocabulary object using the Task's model."""
return sentencepiece_vocabulary.SentencePieceVocabulary(
self.sentencepiece_model_path)