Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
return;
}
// Update finalHref to point to the final URL.
this.finalHref = this._targetNetworkData.response.url;
debug(`HTML for ${this.finalHref} downloaded`);
const fetchEnd: FetchEnd = {
element: null,
request: this._targetNetworkData.request,
resource: this.finalHref,
response: this._targetNetworkData.response
};
const { charset, mediaType } = getContentTypeData(fetchEnd.element, fetchEnd.resource, fetchEnd.response.headers, fetchEnd.response.body.rawContent);
fetchEnd.response.mediaType = mediaType!;
fetchEnd.response.charset = charset!;
// Event is also emitted when status code in response is not 200.
await this.server.emitAsync(`fetch::end::${getType(mediaType!)}` as 'fetch::end::*', fetchEnd);
/*
* If the target is not an HTML we don't need to
* traverse it.
*/
/* istanbul ignore if */
if (!isHTMLDocument(this.finalHref, this.headers)) {
await this.server.emitAsync('scan::end', { resource: this.finalHref });
resolve();
const [content, rawContent] = await Promise.all([
response.text(),
response.buffer()
])
.catch((e) => {
return ['', Buffer.alloc(0)];
});
const body = {
content,
rawContent: rawContent /* istanbul ignore next */ || Buffer.alloc(0),
rawResponse: getRawResponse(response, fetchContent)
};
const responseHeaders = normalizeHeaders(response.headers() as any) as HttpHeaders;
const { charset, mediaType } = getContentTypeData(element, originalUrl, responseHeaders, body.rawContent);
const networkResponse = {
body,
charset: charset!,
headers: responseHeaders,
hops,
mediaType: mediaType!,
statusCode: response.status(),
url: response.url()
};
const data = {
element,
request: networkRequest,
resource: resourceUrl,
response: networkResponse
public async fetchContent(target: string, headers?: object, options?: IFetchOptions): Promise {
/*
* target can have one of these forms:
* - /path/to/file
* - C:/path/to/file
* - file:///path/to/file
* - file:///C:/path/to/file
*
* That's why we need to parse it to an URL
* and then get the path string.
*/
const uri = getAsUri(target);
const filePath: string = uri ? asPathString(uri).replace('%20', ' ') : '';
const rawContent: Buffer = options && options.content ? Buffer.from(options.content) : await readFileAsBuffer(filePath);
const contentType = getContentTypeData(null as any, filePath, null, rawContent);
let content = '';
if (isTextMediaType(contentType.mediaType || '')) {
content = rawContent.toString(contentType.charset || undefined);
}
// Need to do some magic to create a fetch::end::*
return {
request: {} as any,
response: {
body: {
content,
rawContent,
rawResponse() {
/* istanbul ignore next */
return Promise.resolve(rawContent);
await this._connector.server.emitAsync('fetch::start', { resource: resourceUrl });
try {
const resourceNetworkData: NetworkData = await this._connector.fetchContent(resourceUrl);
debug(`resource ${resourceUrl} fetched`);
const fetchEndEvent: FetchEnd = {
element,
request: resourceNetworkData.request,
resource: resourceNetworkData.response.url,
response: resourceNetworkData.response
};
const { charset, mediaType } = getContentTypeData(element, fetchEndEvent.resource, fetchEndEvent.response.headers, fetchEndEvent.response.body.rawContent);
const type = mediaType ? getType(mediaType) : /* istanbul ignore next */ 'unknown';
fetchEndEvent.response.mediaType = mediaType!;
fetchEndEvent.response.charset = charset!;
/*
* TODO: Replace `null` with `resource` once it
* can be converted to `JSDOMAsyncHTMLElement`.
* Event is also emitted when status code in response is not 200.
*/
await this._connector.server.emitAsync(`fetch::end::${type}` as 'fetch::end::*', fetchEndEvent);
return resolve(resourceNetworkData.response.body.rawContent);
} catch (err) {
const hops: string[] = this._connector.request.getRedirects(err.uri);
const fetchError: FetchError = {
return reject(`The number of redirects(${currentRedirectNumber}) exceeds the limit(${this._maxRedirects}).`);
}
try {
debug(`Redirect found for ${uriString}`);
const results = await getUri(newUri);
return resolve(results);
} catch (e) {
return reject(e);
}
}
const contentEncoding: string | null = normalizeHeaderValue(response.headers as HttpHeaders, 'content-encoding');
const rawBody: Buffer | null = await this.decompressResponse(contentEncoding, rawBodyResponse);
const contentTypeData = getContentTypeData(null, uri, response.headers as HttpHeaders, rawBody as Buffer);
const charset = contentTypeData.charset || '';
const mediaType = contentTypeData.mediaType || '';
const hops: string[] = this._redirects.calculate(uriString);
const body: string | null = rawBody && iconv.encodingExists(charset) ? iconv.decode(rawBody, charset) : null;
const networkData: NetworkData = {
request: {
headers: response.request.headers,
url: hops[0] || uriString
},
response: {
body: {
content: body as string,
rawContent: rawBody as Buffer,
rawResponse: () => {
return Promise.resolve(rawBodyResponse);