Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
function createCrawler(base: string, options: Object = {}): Object {
if (typeof base !== "string") {
throw new Error(`Base must be a string, not ${typeof base}`);
}
const config: Object = {
interval: 250,
concurrent: 10,
...options
};
// Will be used by retry-request:
const requestJar = request.jar();
const requestObj = request.defaults({ jar: requestJar, ...config });
setPrototypeOf(requestObj, requestJar);
// Crawler base:
const crawler: Object = {
base: base,
opts: config,
req: SpiderRequest,
res: SpiderResponse,
request: requestObj
};
// Glues all the components together:
mixin(crawler, SpiderQueue, false);
mixin(crawler, SpiderRouter, false);
mixin(crawler, EventEmitter.prototype);
return crawler;
retryRequest(uri, { request: this.request }, (error, response) => {
if (error || response.statusCode !== 200) {
return reject(error || uri);
}
const req: Object = {};
const res: Object = response;
// Set circular references:
res.req = req;
req.res = res;
// Alter the prototypes:
setPrototypeOf(req, this.req);
setPrototypeOf(res, this.res);
resolve({ req, res, uri });
});
});
retryRequest(uri, { request: this.request }, (error, response) => {
if (error || response.statusCode !== 200) {
return reject(error || uri);
}
const req: Object = {};
const res: Object = response;
// Set circular references:
res.req = req;
req.res = res;
// Alter the prototypes:
setPrototypeOf(req, this.req);
setPrototypeOf(res, this.res);
resolve({ req, res, uri });
});
});