Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_embeddings_with_gensim(self):
for emb in self.embeddings_for_testing:
embeddings = load_wv_with_gensim(emb)
self.assertEqual(MODELS[emb]['vocab_size'], len(embeddings.vocab))
import spacy
# spaCy does not support subwords
word_embeddings_available(pretrained_embedding, can_use_subword=False)
spacy_model_dir = os.path.join(cache_dir, pretrained_embedding + ".spacy")
if os.path.isdir(spacy_model_dir):
# Return spaCy model if spaCy model dir exists
return spacy.load(spacy_model_dir)
bin_file_path = os.path.join(cache_dir, pretrained_embedding + ".bin")
if os.path.isfile(bin_file_path):
# Then we do not need to download the model
model_info = MODELS[pretrained_embedding]
model_info['name'] = pretrained_embedding
_process_embeddings_for_spacy(bin_file_path[:-4] + ".tmp", model_info)
else:
download_model(pretrained_embedding, cache_dir,
_process_embeddings_for_spacy, verbose=True,
file_extension='.spacy')
return spacy.load(spacy_model_dir)