Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
constructor(input) {
this.name = 'Web Scraper';
// Set log level early to prevent missed messages.
if (input.debugLog) log.setLevel(log.LEVELS.DEBUG);
// Keep this as string to be immutable.
this.rawInput = JSON.stringify(input);
// Attempt to load page function from disk if not present on input.
tools.maybeLoadPageFunctionFromDisk(input, __dirname);
// Validate INPUT if not running on Apify Cloud Platform.
if (!Apify.isAtHome()) tools.checkInputOrThrow(input, SCHEMA);
/**
* @type {Input}
*/
this.input = input;
this.env = Apify.getEnv();
// Validations
if (this.input.pseudoUrls.length && !this.input.useRequestQueue) {
throw new Error('Cannot enqueue links using Pseudo-URLs without using a request queue. '
+ 'Either enable the "Use request queue" option or '
+ 'remove your Pseudo-URLs.');
}
this.input.pseudoUrls.forEach((purl) => {
if (!tools.isPlainObject(purl)) throw new Error('The pseudoUrls Array must only contain Objects.');
if (purl.userData && !tools.isPlainObject(purl.userData)) throw new Error('The userData property of a pseudoUrl must be an Object.');
constructor(input) {
this.name = 'Cheerio Scraper';
// Set log level early to prevent missed messages.
if (input.debugLog) log.setLevel(log.LEVELS.DEBUG);
// Keep this as string to be immutable.
this.rawInput = JSON.stringify(input);
// Attempt to load page function from disk if not present on input.
tools.maybeLoadPageFunctionFromDisk(input, __dirname);
// Validate INPUT if not running on Apify Cloud Platform.
if (!Apify.isAtHome()) tools.checkInputOrThrow(input, SCHEMA);
/**
* @type {Input}
*/
this.input = input;
this.env = Apify.getEnv();
// Validations
if (this.input.pseudoUrls.length && !this.input.useRequestQueue) {
throw new Error('Cannot enqueue links using Pseudo-URLs without using a request queue. '
+ 'Either enable the "Use request queue" option or '
+ 'remove your Pseudo-URLs.');
}
this.input.pseudoUrls.forEach((purl) => {
if (!tools.isPlainObject(purl)) throw new Error('The pseudoUrls Array must only contain Objects.');
if (purl.userData && !tools.isPlainObject(purl.userData)) throw new Error('The userData property of a pseudoUrl must be an Object.');
constructor(input) {
this.name = 'Puppeteer Scraper';
// Set log level early to prevent missed messages.
if (input.debugLog) log.setLevel(log.LEVELS.DEBUG);
// Keep this as string to be immutable.
this.rawInput = JSON.stringify(input);
// Attempt to load page function from disk if not present on input.
tools.maybeLoadPageFunctionFromDisk(input, __dirname);
// Validate INPUT if not running on Apify Cloud Platform.
if (!Apify.isAtHome()) tools.checkInputOrThrow(input, SCHEMA);
/**
* @type {Input}
*/
this.input = input;
this.env = Apify.getEnv();
// Validations
if (this.input.pseudoUrls.length && !this.input.useRequestQueue) {
throw new Error('Cannot enqueue links using Pseudo-URLs without using a request queue. '
+ 'Either enable the "Use request queue" option or '
+ 'remove your Pseudo-URLs.');
}
this.input.pseudoUrls.forEach((purl) => {
if (!tools.isPlainObject(purl)) throw new Error('The pseudoUrls Array must only contain Objects.');
if (purl.userData && !tools.isPlainObject(purl.userData)) throw new Error('The userData property of a pseudoUrl must be an Object.');