Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def setUp(self):
# Download pre-built CACM index; append a random value to avoid filename clashes.
r = randint(0, 10000000)
self.collection_url = 'https://github.com/castorini/anserini-data/raw/master/CACM/lucene-index.cacm.tar.gz'
self.tarball_name = 'lucene-index.cacm-{}.tar.gz'.format(r)
self.index_dir = 'index{}/'.format(r)
_, _ = urlretrieve(self.collection_url, self.tarball_name)
tarball = tarfile.open(self.tarball_name)
tarball.extractall(self.index_dir)
tarball.close()
self.searcher = search.SimpleSearcher(f'{self.index_dir}lucene-index.cacm')
self.index_utils = index.IndexReader(f'{self.index_dir}lucene-index.cacm')
def setUp(self):
# Download pre-built CACM index; append a random value to avoid filename clashes.
r = randint(0, 10000000)
self.collection_url = 'https://github.com/castorini/anserini-data/raw/master/CACM/lucene-index.cacm.tar.gz'
self.tarball_name = 'lucene-index.cacm-{}.tar.gz'.format(r)
self.index_dir = 'index{}/'.format(r)
_, _ = urlretrieve(self.collection_url, self.tarball_name)
tarball = tarfile.open(self.tarball_name)
tarball.extractall(self.index_dir)
tarball.close()
self.index_path = os.path.join(self.index_dir, 'lucene-index.cacm')
self.searcher = search.SimpleSearcher(self.index_path)
self.index_reader = index.IndexReader(self.index_path)
def test_cacm(self):
# We're going to append a random value to downloaded files:
r = randint(0, 10000000)
url = 'https://github.com/castorini/anserini/blob/master/src/main/resources/cacm/cacm.tar.gz?raw=true'
tarball_name = 'cacm{}.tar.gz'.format(r)
directory = 'collection{}/'.format(r)
_, _ = urlretrieve(url, tarball_name)
tarball = tarfile.open(tarball_name)
tarball.extractall(directory)
tarball.close()
cacm = collection.Collection('HtmlCollection', directory)
generator = index.Generator('DefaultLuceneDocumentGenerator')
cnt = 0
for (i, fs) in enumerate(cacm):
for (j, doc) in enumerate(fs):
self.assertTrue(isinstance(doc, collection.SourceDocument))
self.assertTrue(doc.raw is not None)
self.assertTrue(doc.raw != '')
self.assertTrue('' in doc.raw)
self.assertTrue(doc.contents is not None)
self.assertTrue(doc.contents != '')
self.assertTrue('' not in doc.contents)
parsed = generator.create_document(doc)
docid = parsed.get('id') # FIELD_ID
raw = parsed.get('raw') # FIELD_RAW
contents = parsed.get('contents') # FIELD_BODY
def __init__(self, lucene_index_path: str, min_df: int = 1, verbose: bool = False):
self.min_df: int = min_df
self.verbose: bool = verbose
self.index_reader = index.IndexReader(lucene_index_path)
self.searcher = search.SimpleSearcher(lucene_index_path)
self.num_docs: int = self.searcher.num_docs
# build vocabulary
self.vocabulary_ = set()
for term in self.index_reader.terms():
if term.df > self.min_df:
self.vocabulary_.add(term.term)
# build term to index mapping
self.term_to_index = {}
for i, term in enumerate(self.vocabulary_):
self.term_to_index[term] = i
self.vocabulary_size = len(self.vocabulary_)
if self.verbose: