Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
req.useExtendedUniqueKey = true;
req.keepUrlFragment = this.input.keepUrlFragments;
return req;
});
this.requestList = await Apify.openRequestList('CHEERIO_SCRAPER', startUrls);
// RequestQueue if selected
if (this.input.useRequestQueue) this.requestQueue = await Apify.openRequestQueue();
// Dataset
this.dataset = await Apify.openDataset();
const { itemsCount } = await this.dataset.getInfo();
this.pagesOutputted = itemsCount || 0;
// KeyValueStore
this.keyValueStore = await Apify.openKeyValueStore();
}
req.useExtendedUniqueKey = true;
req.keepUrlFragment = this.input.keepUrlFragments;
return req;
});
this.requestList = await Apify.openRequestList('WEB_SCRAPER', startUrls);
// RequestQueue if selected
if (this.input.useRequestQueue) this.requestQueue = await Apify.openRequestQueue();
// Dataset
this.dataset = await Apify.openDataset();
const { itemsCount } = await this.dataset.getInfo();
this.pagesOutputted = itemsCount || 0;
// KeyValueStore
this.keyValueStore = await Apify.openKeyValueStore();
}
req.useExtendedUniqueKey = true;
req.keepUrlFragment = this.input.keepUrlFragments;
return req;
});
this.requestList = await Apify.openRequestList('PUPPETEER_SCRAPER', startUrls);
// RequestQueue if selected
if (this.input.useRequestQueue) this.requestQueue = await Apify.openRequestQueue();
// Dataset
this.dataset = await Apify.openDataset();
const { itemsCount } = await this.dataset.getInfo();
this.pagesOutputted = itemsCount || 0;
// KeyValueStore
this.keyValueStore = await Apify.openKeyValueStore();
}