Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
args = get_args()
logging.basicConfig(
level=logging.INFO, stream=sys.stderr,
format='%(asctime)s - %(levelname)s (%(name)s): %(message)s')
logger = logging.getLogger('urlextract')
try:
urlextract = URLExtract()
if args.ignore_file:
urlextract.load_ignore_list(args.ignore_file)
urlextract.update_when_older(30)
content = args.input_file.read()
for url in urlextract.find_urls(content, args.unique):
print(url)
except CacheFileError as e:
logger.error(str(e))
sys.exit(-1)
finally:
args.input_file.close()
if not os.path.exists(dir_path_user):
try:
os.makedirs(dir_path_user, exist_ok=True)
except PermissionError:
# if PermissionError exception is raised we should continue
# and try to set the last fallback dir
pass
if os.access(dir_path_user, os.W_OK):
return dir_path_user
dir_path_temp = tempfile.gettempdir()
if os.access(dir_path_temp, os.W_OK):
return dir_path_temp
raise CacheFileError("Cache directories are not writable.")
def _get_default_cache_file_path(self):
"""
Returns default cache file path
:return: default cache file path (to data directory)
:rtype: str
"""
default_list_path = os.path.join(
self._get_default_cache_dir(), self._CACHE_FILE_NAME)
if not os.access(default_list_path, os.F_OK):
raise CacheFileError(
"Default cache file does not exist "
"'{}'!".format(default_list_path)
)
return default_list_path
def _load_cached_tlds(self):
"""
Loads TLDs from cached file to set.
:return: Set of current TLDs
:rtype: set
"""
# check if cached file is readable
if not os.access(self._tld_list_path, os.R_OK):
self._logger.error("Cached file is not readable for current "
"user. ({})".format(self._tld_list_path))
raise CacheFileError(
"Cached file is not readable for current user."
)
set_of_tlds = set()
with open(self._tld_list_path, 'r') as f_cache_tld:
for line in f_cache_tld:
tld = line.strip().lower()
# skip empty lines
if not tld:
continue
# skip comments
if tld[0] == '#':
continue
set_of_tlds.add("." + tld)
set_of_tlds.add("." + idna.decode(tld))
def _get_cache_file_path(self, cache_dir=None):
"""
Get path for cache file
:param str cache_dir: base path for TLD cache, defaults to data dir
:raises: CacheFileError when cached directory is not writable for user
:return: Full path to cached file with TLDs
:rtype: str
"""
if cache_dir is None:
# Tries to get writable cache dir with fallback to users data dir
# and temp directory
cache_dir = self._get_writable_cache_dir()
else:
if not os.access(cache_dir, os.W_OK):
raise CacheFileError("None of cache directories is writable.")
# get directory for cached file
return os.path.join(cache_dir, self._CACHE_FILE_NAME)