Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
export async function makeRequest (url: string): Promise {
const req = request({
url,
headers: {
'User-Agent': 'Scrappy-LinkExpanding 1.0 (+https://github.com/blakeembrey/node-scrappy)'
},
use: [],
transport: createTransport({ type: 'stream', jar: jar() })
})
return req.then((res) => {
// Abort wrapper to ignore streaming errors from aborting (e.g. unzipping).
function abort () {
res.body.on('error', () => undefined)
req.abort()
}
return {
stream: res.body,
headers: res.headers,
status: res.status,
abort: abort,
url: res.url
}
function makeRequest (url) {
const req = popsicle.get({
url,
headers: {
'User-Agent': 'Scrappy-LinkExpanding 1.0 (+https://github.com/blakeembrey/node-scrappy)'
},
use: [cache.handle],
transport: popsicle.createTransport({ type: 'stream', jar: popsicle.jar() })
})
return req.then(function (res) {
return {
stream: res.body,
headers: res.headers,
status: res.status,
abort: () => req.abort(),
url: res.url
}
})
}
function fetch () {
console.log('Fetching "' + fixtureUrl + '"...')
return popsicle.request({
url: fixtureUrl,
transport: popsicle.createTransport({
type: 'buffer',
jar: popsicle.jar(),
maxBufferSize: 20 * 1000 * 1000
})
})
.then(function (res) {
return mkdir(dir)
.then(function () {
console.log('Writing "' + filename + '"...')
const meta = {
originalUrl: fixtureUrl,
url: res.url,
headers: res.headers,
status: res.status,
statusText: res.statusText
}