Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def scrape_url(self, url, headers={}, cookies={}, timeout=10):
"""
Scrape the target URL and collects all the data that will be filtered afterwards
"""
if BURP:
# Burp flag is set when requests is not installed.
# When using Burp we shouldn't end up in this function so we are in a Python CLI env without requests
raise ImportError("Missing Requests module")
# By default we don't verify SSL certificates, we are only performing some useless GETs
try:
response = get(url, headers=headers, cookies=cookies, verify=False, allow_redirects=True, timeout=timeout)
except RequestException as e:
raise ConnectionException(e)
# print("status: {}".format(response.status_code))
# TODO: switch-case for various response.status_code
self.data['url'] = url
self.data['html'] = response.text
self.data['headers'] = dict_from_caseinsensitivedict(response.headers)
self.data['cookies'] = dict_from_cookiejar(response.cookies)
def __init__(self, options=None):
if not BURP:
update = False if options is None else options.get('update_db', False)
database.update_database(force=update)
with open(database.WAPPALYZER_DATABASE_FILE) as f:
self.db = json.load(f)
with open(database.DATABASE_FILE) as f:
self.db = database.merge_databases(self.db, json.load(f))
# Output text only
self.output_format = Format['text']
# Default user agent
self.USER_AGENT = default_user_agent()
if options is None:
return
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os.path
import time
from .__burp__ import BURP
if not BURP:
try:
from urllib.request import urlopen
from urllib.error import URLError
except ImportError as e:
from urllib2 import urlopen, URLError
INSTALLATION_DIR = os.path.realpath(os.path.dirname(__file__))
DATABASE_FILE = os.path.join(INSTALLATION_DIR, "webtech.json")
WAPPALYZER_DATABASE_FILE = os.path.join(INSTALLATION_DIR, "apps.json")
WAPPALYZER_DATABASE_URL = "https://raw.githubusercontent.com/AliasIO/Wappalyzer/master/src/apps.json"
WEBTECH_DATABASE_URL = "https://raw.githubusercontent.com/ShielderSec/webtech/master/webtech/webtech.json"
DAYS = 60 * 60 * 24
def download_database_file(url, target_file):
def __init__(self, options=None):
if not BURP:
update = False if options is None else options.get('update_db', False)
database.update_database(force=update)
with open(database.WAPPALYZER_DATABASE_FILE) as f:
self.db = json.load(f)
with open(database.DATABASE_FILE) as f:
self.db = database.merge_databases(self.db, json.load(f))
# Output text only
self.output_format = Format['text']
# Default user agent
self.USER_AGENT = default_user_agent()
if options is None:
return