How to use the popsicle.jar function in popsicle

To help you get started, we’ve selected a few popsicle examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github blakeembrey / node-scrappy / src / scrape / support.ts View on Github external
export async function makeRequest (url: string): Promise {
  const req = request({
    url,
    headers: {
      'User-Agent': 'Scrappy-LinkExpanding 1.0 (+https://github.com/blakeembrey/node-scrappy)'
    },
    use: [],
    transport: createTransport({ type: 'stream', jar: jar() })
  })

  return req.then((res) => {
    // Abort wrapper to ignore streaming errors from aborting (e.g. unzipping).
    function abort () {
      res.body.on('error', () => undefined)
      req.abort()
    }

    return {
      stream: res.body,
      headers: res.headers,
      status: res.status,
      abort: abort,
      url: res.url
    }
github blakeembrey / node-scrappy / test / scrape.spec.js View on Github external
function makeRequest (url) {
    const req = popsicle.get({
      url,
      headers: {
        'User-Agent': 'Scrappy-LinkExpanding 1.0 (+https://github.com/blakeembrey/node-scrappy)'
      },
      use: [cache.handle],
      transport: popsicle.createTransport({ type: 'stream', jar: popsicle.jar() })
    })

    return req.then(function (res) {
      return {
        stream: res.body,
        headers: res.headers,
        status: res.status,
        abort: () => req.abort(),
        url: res.url
      }
    })
  }
github blakeembrey / node-htmlmetaparser / scripts / fixtures.js View on Github external
function fetch () {
    console.log('Fetching "' + fixtureUrl + '"...')

    return popsicle.request({
      url: fixtureUrl,
      transport: popsicle.createTransport({
        type: 'buffer',
        jar: popsicle.jar(),
        maxBufferSize: 20 * 1000 * 1000
      })
    })
      .then(function (res) {
        return mkdir(dir)
          .then(function () {
            console.log('Writing "' + filename + '"...')

            const meta = {
              originalUrl: fixtureUrl,
              url: res.url,
              headers: res.headers,
              status: res.status,
              statusText: res.statusText
            }