How to use the pyserini.index function in pyserini

To help you get started, we’ve selected a few pyserini examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github castorini / pyserini / tests / test_analysis.py View on Github external
def setUp(self):
        # Download pre-built CACM index; append a random value to avoid filename clashes.
        r = randint(0, 10000000)
        self.collection_url = 'https://github.com/castorini/anserini-data/raw/master/CACM/lucene-index.cacm.tar.gz'
        self.tarball_name = 'lucene-index.cacm-{}.tar.gz'.format(r)
        self.index_dir = 'index{}/'.format(r)

        _, _ = urlretrieve(self.collection_url, self.tarball_name)

        tarball = tarfile.open(self.tarball_name)
        tarball.extractall(self.index_dir)
        tarball.close()
        self.searcher = search.SimpleSearcher(f'{self.index_dir}lucene-index.cacm')
        self.index_utils = index.IndexReader(f'{self.index_dir}lucene-index.cacm')
github castorini / pyserini / tests / test_index_reader.py View on Github external
def setUp(self):
        # Download pre-built CACM index; append a random value to avoid filename clashes.
        r = randint(0, 10000000)
        self.collection_url = 'https://github.com/castorini/anserini-data/raw/master/CACM/lucene-index.cacm.tar.gz'
        self.tarball_name = 'lucene-index.cacm-{}.tar.gz'.format(r)
        self.index_dir = 'index{}/'.format(r)

        _, _ = urlretrieve(self.collection_url, self.tarball_name)

        tarball = tarfile.open(self.tarball_name)
        tarball.extractall(self.index_dir)
        tarball.close()

        self.index_path = os.path.join(self.index_dir, 'lucene-index.cacm')
        self.searcher = search.SimpleSearcher(self.index_path)
        self.index_reader = index.IndexReader(self.index_path)
github castorini / pyserini / tests / test_collection.py View on Github external
def test_cacm(self):
        # We're going to append a random value to downloaded files:
        r = randint(0, 10000000)
        url = 'https://github.com/castorini/anserini/blob/master/src/main/resources/cacm/cacm.tar.gz?raw=true'
        tarball_name = 'cacm{}.tar.gz'.format(r)
        directory = 'collection{}/'.format(r)

        _, _ = urlretrieve(url, tarball_name)

        tarball = tarfile.open(tarball_name)
        tarball.extractall(directory)
        tarball.close()

        cacm = collection.Collection('HtmlCollection', directory)
        generator = index.Generator('DefaultLuceneDocumentGenerator')

        cnt = 0
        for (i, fs) in enumerate(cacm):
            for (j, doc) in enumerate(fs):
                self.assertTrue(isinstance(doc, collection.SourceDocument))
                self.assertTrue(doc.raw is not None)
                self.assertTrue(doc.raw != '')
                self.assertTrue('' in doc.raw)
                self.assertTrue(doc.contents is not None)
                self.assertTrue(doc.contents != '')
                self.assertTrue('' not in doc.contents)

                parsed = generator.create_document(doc)
                docid = parsed.get('id')            # FIELD_ID
                raw = parsed.get('raw')             # FIELD_RAW
                contents = parsed.get('contents')   # FIELD_BODY
github castorini / pyserini / pyserini / vectorizer / _base.py View on Github external
def __init__(self, lucene_index_path: str, min_df: int = 1, verbose: bool = False):
        self.min_df: int = min_df
        self.verbose: bool = verbose
        self.index_reader = index.IndexReader(lucene_index_path)
        self.searcher = search.SimpleSearcher(lucene_index_path)
        self.num_docs: int = self.searcher.num_docs

        # build vocabulary
        self.vocabulary_ = set()
        for term in self.index_reader.terms():
            if term.df > self.min_df:
                self.vocabulary_.add(term.term)

        # build term to index mapping
        self.term_to_index = {}
        for i, term in enumerate(self.vocabulary_):
            self.term_to_index[term] = i
        self.vocabulary_size = len(self.vocabulary_)

        if self.verbose: