How to use the urllib.requestThunk function in urllib

To help you get started, we’ve selected a few urllib examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github cnpm / mirrors / sync / node.js View on Github external
proto.listdir = function* (fullname) {
  var url = this.disturl + fullname;

  // if is doc path
  var isDocPath = this.DOC_API_RE.test(fullname);
  if (isDocPath) {
    url += 'index.html';

    if (!this._syncDocument) {
      return [];
    }
  }

  var res = yield urllib.requestThunk(url, {
    timeout: 60 * 1000,
    dataType: 'text',
    followRedirect: true,
  });
  debug('listdir %s got %s, %j', url, res.status, res.headers);

  return isDocPath
    ? this.parseDocHtml(res, fullname)
    : this.parseDistHtml(res, fullname);
};
github cnpm / mirrors / sync / mirrors.js View on Github external
proto.listdir = function* (fullname, dirIndex) {
  var alwayNewDir = false;
  if (typeof this._alwayNewDirIndex === 'number' && this._alwayNewDirIndex === dirIndex) {
    alwayNewDir = true;
  } else if (Array.isArray(this._alwayNewDirIndex) && this._alwayNewDirIndex.indexOf(dirIndex) >= 0) {
    // node-inspector `alwayNewDirIndex = [0, 1]`
    alwayNewDir = true;
  }
  var url = this.disturl + fullname;

  var res = yield urllib.requestThunk(url, {
    timeout: 60000,
    dataType: 'json',
    followRedirect: true,
  });
  debug('listdir %s got %s, %j', url, res.status, res.headers);
  if (res.status !== 200) {
    var msg = util.format('request %s error, got %s', url, res.status);
    throw new Error(msg);
  }

  return res.data.map(function (file) {
    var item = {
      isNew: null,
      name: file.name,
      size: file.size || '-',
      date: file.date,

urllib

Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, timeout and more. Base undici API.

MIT
Latest version published 2 days ago

Package Health Score

86 / 100
Full package analysis