Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
headers, self.headers,
dict_class=requests.structures.CaseInsensitiveDict)
headers = requests.sessions.merge_setting(
kwarg_headers, headers,
dict_class=requests.structures.CaseInsensitiveDict)
_start_time = time.time()
resp = super(Scraper, self).request(method, url, timeout=timeout, headers=headers,
**kwargs)
self.stats['total_requests'] += 1
self.stats['total_time'] += (time.time() - _start_time)
self.stats['average_time'] = self.stats['total_time'] / self.stats['total_requests']
if self.raise_errors and not self.accept_response(resp):
raise HTTPError(resp)
return resp
def __init__(self, response, body=None):
message = '%s while retrieving %s' % (response.status_code, response.url)
super(HTTPError, self).__init__(message)
self.response = response
self.body = body or self.response.text