Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# The `retrieve()` method follows any eventual redirects, so the
# initial url might be different from the final one
try:
response, final_url = http_retrieve(self.opener, url)
except urllib3.exceptions.MaxRetryError:
return
content_type = response.headers.get('content-type', '')
if locators.HTML_CONTENT_TYPE.match(content_type):
data = response.data
encoding = response.headers.get('content-encoding')
if encoding:
decoder = self.decoders[encoding] # fail if not found
data = decoder(data)
encoding = 'utf-8'
m = locators.CHARSET.search(content_type)
if m:
encoding = m.group(1)
try:
data = data.decode(encoding)
except UnicodeError:
data = data.decode('latin-1') # fallback
return locators.Page(data, final_url)
# The `retrieve()` method follows any eventual redirects, so the
# initial url might be different from the final one
try:
response, final_url = self.opener.retrieve(url)
except urllib3.exceptions.MaxRetryError:
return
content_type = response.headers.get('content-type', '')
if locators.HTML_CONTENT_TYPE.match(content_type):
data = response.data
encoding = response.headers.get('content-encoding')
if encoding:
decoder = self.decoders[encoding] # fail if not found
data = decoder(data)
encoding = 'utf-8'
m = locators.CHARSET.search(content_type)
if m:
encoding = m.group(1)
try:
data = data.decode(encoding)
except UnicodeError:
data = data.decode('latin-1') # fallback
return locators.Page(data, final_url)