Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
async _initializeAsync() {
// RequestList
const startUrls = this.input.startUrls.map((req) => {
req.useExtendedUniqueKey = true;
req.keepUrlFragment = this.input.keepUrlFragments;
return req;
});
this.requestList = await Apify.openRequestList('CHEERIO_SCRAPER', startUrls);
// RequestQueue if selected
if (this.input.useRequestQueue) this.requestQueue = await Apify.openRequestQueue();
// Dataset
this.dataset = await Apify.openDataset();
const { itemsCount } = await this.dataset.getInfo();
this.pagesOutputted = itemsCount || 0;
// KeyValueStore
this.keyValueStore = await Apify.openKeyValueStore();
}
async _initializeAsync() {
// RequestList
const startUrls = this.input.startUrls.map((req) => {
req.useExtendedUniqueKey = true;
req.keepUrlFragment = this.input.keepUrlFragments;
return req;
});
this.requestList = await Apify.openRequestList('PUPPETEER_SCRAPER', startUrls);
// RequestQueue if selected
if (this.input.useRequestQueue) this.requestQueue = await Apify.openRequestQueue();
// Dataset
this.dataset = await Apify.openDataset();
const { itemsCount } = await this.dataset.getInfo();
this.pagesOutputted = itemsCount || 0;
// KeyValueStore
this.keyValueStore = await Apify.openKeyValueStore();
}
async _initializeAsync() {
// RequestList
const startUrls = this.input.startUrls.map((req) => {
req.useExtendedUniqueKey = true;
req.keepUrlFragment = this.input.keepUrlFragments;
return req;
});
this.requestList = await Apify.openRequestList('WEB_SCRAPER', startUrls);
// RequestQueue if selected
if (this.input.useRequestQueue) this.requestQueue = await Apify.openRequestQueue();
// Dataset
this.dataset = await Apify.openDataset();
const { itemsCount } = await this.dataset.getInfo();
this.pagesOutputted = itemsCount || 0;
// KeyValueStore
this.keyValueStore = await Apify.openKeyValueStore();
}
async function checkSaveCount(datasetId, maxResults) {
const dataset = await Apify.openDataset(datasetId);
const { itemCount } = await dataset.getInfo();
if (maxResults === null || maxResults === 0) {
return true;
}
if (itemCount < maxResults) {
return true;
}
return false;
}