Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def scrape_url(self, url, headers={}, cookies={}, timeout=10):
"""
Scrape the target URL and collects all the data that will be filtered afterwards
"""
if BURP:
# Burp flag is set when requests is not installed.
# When using Burp we shouldn't end up in this function so we are in a Python CLI env without requests
raise ImportError("Missing Requests module")
# By default we don't verify SSL certificates, we are only performing some useless GETs
try:
response = get(url, headers=headers, cookies=cookies, verify=False, allow_redirects=True, timeout=timeout)
except RequestException as e:
raise ConnectionException(e)
# print("status: {}".format(response.status_code))
# TODO: switch-case for various response.status_code
self.data['url'] = url
self.data['html'] = response.text
self.data['headers'] = dict_from_caseinsensitivedict(response.headers)
self.data['cookies'] = dict_from_cookiejar(response.cookies)
self.parse_html_page()
def scrape_url(self, url, headers={}, cookies={}, timeout=10):
"""
Scrape the target URL and collects all the data that will be filtered afterwards
"""
if BURP:
# Burp flag is set when requests is not installed.
# When using Burp we shouldn't end up in this function so we are in a Python CLI env without requests
raise ImportError("Missing Requests module")
# By default we don't verify SSL certificates, we are only performing some useless GETs
try:
response = get(url, headers=headers, cookies=cookies, verify=False, allow_redirects=True, timeout=timeout)
except RequestException as e:
raise ConnectionException(e)
# print("status: {}".format(response.status_code))
# TODO: switch-case for various response.status_code
self.data['url'] = url
self.data['html'] = response.text
self.data['headers'] = dict_from_caseinsensitivedict(response.headers)
self.data['cookies'] = dict_from_cookiejar(response.cookies)
self.parse_html_page()
def start(self):
"""
Start the engine, fetch an URL and report the findings
"""
self.output = {}
for url in self.urls or []:
try:
temp_output = self.start_from_url(url)
except (FileNotFoundException, ValueError) as e:
print(e)
continue
except ConnectionException as e:
print("Connection error while scanning {}".format(url))
continue
if self.output_format == Format['text']:
print(temp_output)
else:
self.output[url] = temp_output
if self.output_format == Format['json']:
print(json.dumps(self.output))
else:
for o in self.output.values():
print(o)
def start(self):
"""
Start the engine, fetch an URL and report the findings
"""
self.output = {}
for url in self.urls or []:
try:
temp_output = self.start_from_url(url)
except (FileNotFoundException, ValueError) as e:
print(e)
continue
except ConnectionException as e:
print("Connection error while scanning {}".format(url))
continue
if self.output_format == Format['text']:
print(temp_output)
else:
self.output[url] = temp_output
if self.output_format == Format['json']:
print(json.dumps(self.output))
else:
for o in self.output.values():
print(o)