How to use the scrapelib.__init__.Scraper function in scrapelib

To help you get started, we’ve selected a few scrapelib examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github jamesturk / scrapelib / scrapelib / __init__.py View on Github external
result = self.request(method, url, data=body, **kwargs)
        result.code = result.status_code    # backwards compat

        if not filename:
            fd, filename = tempfile.mkstemp(dir=dir)
            f = os.fdopen(fd, 'wb')
        else:
            f = open(filename, 'wb')

        f.write(result.content)
        f.close()

        return filename, result


_default_scraper = Scraper(requests_per_minute=0)


def urlopen(url, method='GET', body=None, **kwargs):  # pragma: no cover
    return _default_scraper.urlopen(url, method, body, **kwargs)
github jamesturk / scrapelib / scrapelib / __init__.py View on Github external
def __init__(self, raise_errors=True, requests_per_minute=60, retry_attempts=0,
                 retry_wait_seconds=5, verify=True, header_func=None):

        super(Scraper, self).__init__()
        self.mount('ftp://', FTPAdapter())

        # added by this class
        self.raise_errors = raise_errors

        # added by ThrottledSession
        self.requests_per_minute = requests_per_minute

        # added by RetrySession
        self.retry_attempts = retry_attempts
        self.retry_wait_seconds = retry_wait_seconds

        # added by this class
        self._header_func = header_func

        # added by CachingSession