Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
self.config = config
# Build the spacy pipeline
self.nlp = SpacyPipe(spacy_split_all)
#self.nlp.add_punct_tagger(tagger=spacy_tag_punct)
self.nlp.add_punct_tagger(tagger=partial(spacy_tag_punct,
skip_stopwords=skip_stopwords,
keep_punct=self.config.get("keep_punct", [':', '.'])))
# Add spell checker
self.spell_checker = CustomSpellChecker(cdb_vocab=self.cdb.vocab, data_vocab=self.vocab)
self.nlp.add_spell_checker(spell_checker=self.spell_checker)
# Add them cat class that does entity detection
self.spacy_cat = SpacyCat(cdb=self.cdb, vocab=self.vocab, tokenizer=tokenizer)
self.nlp.add_cat(spacy_cat=self.spacy_cat)
# Add meta_annotaiton classes if they exist
self._meta_annotations = False
for meta_cat in meta_cats:
self.nlp.add_meta_cat(meta_cat, meta_cat.category_name)
self._meta_annotations = True
def __init__(self, cdb, vocab=None, skip_stopwords=True, meta_cats=[]):
self.cdb = cdb
self.vocab = vocab
# Build the required spacy pipeline
self.nlp = SpacyPipe(spacy_split_all)
#self.nlp.add_punct_tagger(tagger=spacy_tag_punct)
self.nlp.add_punct_tagger(tagger=partial(spacy_tag_punct,
skip_stopwords=skip_stopwords,
keep_punct=self.KEEP_PUNCT))
# Add spell checker pipe
self.spell_checker = CustomSpellChecker(cdb_vocab=self.cdb.vocab, data_vocab=self.vocab)
self.nlp.add_spell_checker(spell_checker=self.spell_checker)
# Add cat
self.spacy_cat = SpacyCat(cdb=self.cdb, vocab=self.vocab)
self.nlp.add_cat(spacy_cat=self.spacy_cat)
for meta_cat in meta_cats:
self.nlp.add_meta_cat(meta_cat, meta_cat.category_name)