Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# Download has been interrupted, we try to resume it.
local_file_size = op.getsize(temp_part_name)
# If the file exists, then only download the remainder
request.add_header("Range", "bytes={}-".format(local_file_size))
try:
data = urlopen(request)
content_range = data.info().get('Content-Range')
if (content_range is None or not content_range.startswith(
'bytes {}-'.format(local_file_size))):
raise IOError('Server does not support resuming')
except Exception:
# A wide number of errors can be raised here. HTTPError,
# URLError... I prefer to catch them all and rerun without
# resuming.
if verbose > 0:
NIWORKFLOWS_LOG.warn(
'Resuming failed, try to download the whole file.')
return fetch_file(
dataset_name, url, dataset_dir,
resume=False, overwrite=overwrite,
md5sum=md5sum, username=username, password=password,
verbose=verbose)
local_file = open(temp_part_name, "ab")
initial_size = local_file_size
else:
try:
data = urlopen(request)
except (HTTPError, URLError):
if retry < MAX_RETRIES:
if verbose > 0:
NIWORKFLOWS_LOG.warn('Download failed, retrying (attempt %d)',
retry + 1)
NIWORKFLOWS_LOG.warn(
'Resuming failed, try to download the whole file.')
return fetch_file(
dataset_name, url, dataset_dir,
resume=False, overwrite=overwrite,
md5sum=md5sum, username=username, password=password,
verbose=verbose)
local_file = open(temp_part_name, "ab")
initial_size = local_file_size
else:
try:
data = urlopen(request)
except (HTTPError, URLError):
if retry < MAX_RETRIES:
if verbose > 0:
NIWORKFLOWS_LOG.warn('Download failed, retrying (attempt %d)',
retry + 1)
time.sleep(5)
return fetch_file(
dataset_name, url, dataset_dir, resume=False, overwrite=overwrite,
md5sum=md5sum, username=username, password=password,
verbose=verbose, retry=retry + 1)
else:
raise
local_file = open(temp_part_name, "wb")
_chunk_read_(data, local_file, report_hook=(verbose > 0),
initial_size=initial_size, verbose=verbose)
# temp file must be closed prior to the move
if not local_file.closed:
local_file.close()
:param bool report_hook: whether or not to show downloading advancement
:param int initial_size: if resuming, indicate the initial size of the file
:param int total_size: Expected final size of download (None means it
is unknown).
:param int verbose: verbosity level (0 means no message).
:returns: the downloaded file path.
:rtype: string
"""
try:
if total_size is None:
total_size = response.info().get('Content-Length').strip()
total_size = int(total_size) + initial_size
except Exception as exc:
if verbose > 2:
NIWORKFLOWS_LOG.warn('Total size of chunk could not be determined')
if verbose > 3:
NIWORKFLOWS_LOG.warn("Full stack trace: %s", str(exc))
total_size = None
bytes_so_far = initial_size
t_0 = time_last_display = time.time()
while True:
chunk = response.read(chunk_size)
bytes_so_far += len(chunk)
time_last_read = time.time()
if (report_hook and
# Refresh report every half second or when download is
# finished.
(time_last_read > time_last_display + 0.5 or not chunk)):
_chunk_report_(bytes_so_far,
total_size, initial_size, t_0)
:param int total_size: Expected final size of download (None means it
is unknown).
:param int verbose: verbosity level (0 means no message).
:returns: the downloaded file path.
:rtype: string
"""
try:
if total_size is None:
total_size = response.info().get('Content-Length').strip()
total_size = int(total_size) + initial_size
except Exception as exc:
if verbose > 2:
NIWORKFLOWS_LOG.warn('Total size of chunk could not be determined')
if verbose > 3:
NIWORKFLOWS_LOG.warn("Full stack trace: %s", str(exc))
total_size = None
bytes_so_far = initial_size
t_0 = time_last_display = time.time()
while True:
chunk = response.read(chunk_size)
bytes_so_far += len(chunk)
time_last_read = time.time()
if (report_hook and
# Refresh report every half second or when download is
# finished.
(time_last_read > time_last_display + 0.5 or not chunk)):
_chunk_report_(bytes_so_far,
total_size, initial_size, t_0)
time_last_display = time_last_read
if chunk: