How to use the niworkflows.NIWORKFLOWS_LOG.warn function in niworkflows

To help you get started, we’ve selected a few niworkflows examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github poldracklab / niworkflows / niworkflows / data / utils.py View on Github external
# Download has been interrupted, we try to resume it.
        local_file_size = op.getsize(temp_part_name)
        # If the file exists, then only download the remainder
        request.add_header("Range", "bytes={}-".format(local_file_size))
        try:
            data = urlopen(request)
            content_range = data.info().get('Content-Range')
            if (content_range is None or not content_range.startswith(
                    'bytes {}-'.format(local_file_size))):
                raise IOError('Server does not support resuming')
        except Exception:
            # A wide number of errors can be raised here. HTTPError,
            # URLError... I prefer to catch them all and rerun without
            # resuming.
            if verbose > 0:
                NIWORKFLOWS_LOG.warn(
                    'Resuming failed, try to download the whole file.')
            return fetch_file(
                dataset_name, url, dataset_dir,
                resume=False, overwrite=overwrite,
                md5sum=md5sum, username=username, password=password,
                verbose=verbose)
        local_file = open(temp_part_name, "ab")
        initial_size = local_file_size
    else:
        try:
            data = urlopen(request)
        except (HTTPError, URLError):
            if retry < MAX_RETRIES:
                if verbose > 0:
                    NIWORKFLOWS_LOG.warn('Download failed, retrying (attempt %d)',
                                         retry + 1)
github poldracklab / niworkflows / niworkflows / data / utils.py View on Github external
NIWORKFLOWS_LOG.warn(
                    'Resuming failed, try to download the whole file.')
            return fetch_file(
                dataset_name, url, dataset_dir,
                resume=False, overwrite=overwrite,
                md5sum=md5sum, username=username, password=password,
                verbose=verbose)
        local_file = open(temp_part_name, "ab")
        initial_size = local_file_size
    else:
        try:
            data = urlopen(request)
        except (HTTPError, URLError):
            if retry < MAX_RETRIES:
                if verbose > 0:
                    NIWORKFLOWS_LOG.warn('Download failed, retrying (attempt %d)',
                                         retry + 1)
                time.sleep(5)
                return fetch_file(
                    dataset_name, url, dataset_dir, resume=False, overwrite=overwrite,
                    md5sum=md5sum, username=username, password=password,
                    verbose=verbose, retry=retry + 1)
            else:
                raise

        local_file = open(temp_part_name, "wb")

    _chunk_read_(data, local_file, report_hook=(verbose > 0),
                 initial_size=initial_size, verbose=verbose)
    # temp file must be closed prior to the move
    if not local_file.closed:
        local_file.close()
github poldracklab / niworkflows / niworkflows / data / utils.py View on Github external
:param bool report_hook: whether or not to show downloading advancement
    :param int initial_size: if resuming, indicate the initial size of the file
    :param int total_size: Expected final size of download (None means it
        is unknown).
    :param int verbose: verbosity level (0 means no message).
    :returns: the downloaded file path.
    :rtype: string

    """
    try:
        if total_size is None:
            total_size = response.info().get('Content-Length').strip()
        total_size = int(total_size) + initial_size
    except Exception as exc:
        if verbose > 2:
            NIWORKFLOWS_LOG.warn('Total size of chunk could not be determined')
            if verbose > 3:
                NIWORKFLOWS_LOG.warn("Full stack trace: %s", str(exc))
        total_size = None
    bytes_so_far = initial_size

    t_0 = time_last_display = time.time()
    while True:
        chunk = response.read(chunk_size)
        bytes_so_far += len(chunk)
        time_last_read = time.time()
        if (report_hook and
                # Refresh report every half second or when download is
                # finished.
                (time_last_read > time_last_display + 0.5 or not chunk)):
            _chunk_report_(bytes_so_far,
                           total_size, initial_size, t_0)
github poldracklab / niworkflows / niworkflows / data / utils.py View on Github external
:param int total_size: Expected final size of download (None means it
        is unknown).
    :param int verbose: verbosity level (0 means no message).
    :returns: the downloaded file path.
    :rtype: string

    """
    try:
        if total_size is None:
            total_size = response.info().get('Content-Length').strip()
        total_size = int(total_size) + initial_size
    except Exception as exc:
        if verbose > 2:
            NIWORKFLOWS_LOG.warn('Total size of chunk could not be determined')
            if verbose > 3:
                NIWORKFLOWS_LOG.warn("Full stack trace: %s", str(exc))
        total_size = None
    bytes_so_far = initial_size

    t_0 = time_last_display = time.time()
    while True:
        chunk = response.read(chunk_size)
        bytes_so_far += len(chunk)
        time_last_read = time.time()
        if (report_hook and
                # Refresh report every half second or when download is
                # finished.
                (time_last_read > time_last_display + 0.5 or not chunk)):
            _chunk_report_(bytes_so_far,
                           total_size, initial_size, t_0)
            time_last_display = time_last_read
        if chunk: