How to use the urlextract.cachefile.CacheFileError function in urlextract

To help you get started, we’ve selected a few urlextract examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github lipoja / URLExtract / urlextract / urlextract_core.py View on Github external
args = get_args()
    logging.basicConfig(
        level=logging.INFO, stream=sys.stderr,
        format='%(asctime)s - %(levelname)s (%(name)s): %(message)s')
    logger = logging.getLogger('urlextract')

    try:
        urlextract = URLExtract()
        if args.ignore_file:
            urlextract.load_ignore_list(args.ignore_file)
        urlextract.update_when_older(30)
        content = args.input_file.read()
        for url in urlextract.find_urls(content, args.unique):
            print(url)
    except CacheFileError as e:
        logger.error(str(e))
        sys.exit(-1)
    finally:
        args.input_file.close()
github lipoja / URLExtract / urlextract / cachefile.py View on Github external
if not os.path.exists(dir_path_user):
            try:
                os.makedirs(dir_path_user, exist_ok=True)
            except PermissionError:
                # if PermissionError exception is raised we should continue
                # and try to set the last fallback dir
                pass

        if os.access(dir_path_user, os.W_OK):
            return dir_path_user

        dir_path_temp = tempfile.gettempdir()
        if os.access(dir_path_temp, os.W_OK):
            return dir_path_temp

        raise CacheFileError("Cache directories are not writable.")
github lipoja / URLExtract / urlextract / cachefile.py View on Github external
def _get_default_cache_file_path(self):
        """
        Returns default cache file path

        :return: default cache file path (to data directory)
        :rtype: str
        """

        default_list_path = os.path.join(
            self._get_default_cache_dir(), self._CACHE_FILE_NAME)

        if not os.access(default_list_path, os.F_OK):
            raise CacheFileError(
                "Default cache file does not exist "
                "'{}'!".format(default_list_path)
            )

        return default_list_path
github lipoja / URLExtract / urlextract / cachefile.py View on Github external
def _load_cached_tlds(self):
        """
        Loads TLDs from cached file to set.

        :return: Set of current TLDs
        :rtype: set
        """

        # check if cached file is readable
        if not os.access(self._tld_list_path, os.R_OK):
            self._logger.error("Cached file is not readable for current "
                               "user. ({})".format(self._tld_list_path))
            raise CacheFileError(
                "Cached file is not readable for current user."
            )

        set_of_tlds = set()
        with open(self._tld_list_path, 'r') as f_cache_tld:
            for line in f_cache_tld:
                tld = line.strip().lower()
                # skip empty lines
                if not tld:
                    continue
                # skip comments
                if tld[0] == '#':
                    continue

                set_of_tlds.add("." + tld)
                set_of_tlds.add("." + idna.decode(tld))
github lipoja / URLExtract / urlextract / cachefile.py View on Github external
def _get_cache_file_path(self, cache_dir=None):
        """
        Get path for cache file

        :param str cache_dir: base path for TLD cache, defaults to data dir
        :raises: CacheFileError when cached directory is not writable for user
        :return: Full path to cached file with TLDs
        :rtype: str
        """
        if cache_dir is None:
            # Tries to get writable cache dir with fallback to users data dir
            # and temp directory
            cache_dir = self._get_writable_cache_dir()
        else:
            if not os.access(cache_dir, os.W_OK):
                raise CacheFileError("None of cache directories is writable.")

        # get directory for cached file
        return os.path.join(cache_dir, self._CACHE_FILE_NAME)

urlextract

Collects and extracts URLs from given text.

MIT
Latest version published 8 months ago

Package Health Score

68 / 100
Full package analysis

Similar packages