Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# uses nose test generators
def check(error_type, args_dict):
nt.assert_raises(error_type, BloscpackHeader, **args_dict)
for error_type, args_dict in [
(ValueError, {'format_version': -1}),
(ValueError, {'format_version': MAX_FORMAT_VERSION+1}),
(TypeError, {'format_version': 'foo'}),
(ValueError, {'checksum': -1}),
(ValueError, {'checksum': len(checksums.CHECKSUMS)+1}),
(exceptions.NoSuchChecksum, {'checksum': 'foo'}),
(ValueError, {'typesize': -1}),
(ValueError, {'typesize': blosc.BLOSC_MAX_TYPESIZE+1}),
(TypeError, {'typesize': 'foo'}),
(ValueError, {'chunk_size': blosc.BLOSC_MAX_BUFFERSIZE+1}),
(ValueError, {'chunk_size': -2}),
(TypeError, {'chunk_size': 'foo'}),
(ValueError, {'last_chunk': blosc.BLOSC_MAX_BUFFERSIZE+1}),
(ValueError, {'last_chunk': -2}),
(TypeError, {'last_chunk': 'foo'}),
(ValueError, {'nchunks': MAX_CHUNKS+1}),
(ValueError, {'nchunks': -2}),
(TypeError, {'nchunks': 'foo'}),
(ValueError, {'max_app_chunks': MAX_CHUNKS+1}),
(ValueError, {'max_app_chunks': -1}),
(TypeError, {'max_app_chunks': 'foo'}),
# sum of nchunks and max_app_chunks
(ValueError, {'nchunks': MAX_CHUNKS//2+1,
'max_app_chunks': MAX_CHUNKS//2+1}),
# check that max_app_chunks is zero
(ValueError, {'nchunks': -1,
def pack_unpack_extreme():
""" Test on somewhat larer arrays, uses loads of memory. """
# this will create a huge array, and then use the
# blosc.BLOSC_MAX_BUFFERSIZE as chunk-szie
pack_unpack(300, chunk_size=blosc.BLOSC_MAX_BUFFERSIZE,
progress=simple_progress)
def __call__(self, parser, namespace, value, option_string=None):
if value == 'max':
value = blosc.BLOSC_MAX_BUFFERSIZE
else:
try:
# try to get the value as bytes
if value[-1] in SUFFIXES.keys():
value = reverse_pretty(value)
# seems to be intended to be a naked int
else:
value = int(value)
except ValueError as ve:
error('%s error: %s' % (option_string, str(ve) +
" or 'max'"))
if value < 0:
error('%s must be > 0 ' % option_string)
setattr(namespace, self.dest, value)
for p in [compress_parser, c_parser]:
Raises
------
ChunkingException
if the resulting nchunks is larger than MAX_CHUNKS
"""
if in_file_size < 0:
raise ValueError("'in_file_size' must be strictly positive, not %d"
% in_file_size)
elif in_file_size == 0:
return (1, 0, 0)
log.verbose("Input was length zero, ignoring 'chunk_size'")
# convert a human readable description to an int
if isinstance(chunk_size, string_types):
chunk_size = reverse_pretty(chunk_size)
check_range('chunk_size', chunk_size, 1, blosc.BLOSC_MAX_BUFFERSIZE)
# downcast
if chunk_size > in_file_size:
log.verbose(
"Input was smaller than the given 'chunk_size': %s using: %s"
% (double_pretty_size(chunk_size),
double_pretty_size(in_file_size)))
chunk_size = in_file_size
quotient, remainder = divmod(in_file_size, chunk_size)
# the user wants a single chunk
if chunk_size == in_file_size:
nchunks = 1
chunk_size = in_file_size
last_chunk_size = in_file_size
# no remainder, perfect fit
elif remainder == 0:
nchunks = quotient
def __call__(self, parser, namespace, value, option_string=None):
if value == 'max':
value = blosc.BLOSC_MAX_BUFFERSIZE
else:
try:
# try to get the value as bytes
if value[-1] in SUFFIXES.keys():
value = reverse_pretty(value)
# seems to be intended to be a naked int
else:
value = int(value)
except ValueError as ve:
log.error('%s error: %s' % (option_string, str(ve)))
if value < 0:
log.error('%s must be > 0' % option_string)
setattr(namespace, self.dest, value)
for p in [compress_parser, c_parser]:
def __init__(self,
format_version=FORMAT_VERSION,
offsets=False,
metadata=False,
checksum='None',
typesize=0,
chunk_size=-1,
last_chunk=-1,
nchunks=-1,
max_app_chunks=0):
check_range('format_version', format_version, 0, MAX_FORMAT_VERSION)
check_valid_checksum(checksum)
check_range('typesize', typesize, 0, blosc.BLOSC_MAX_TYPESIZE)
check_range('chunk_size', chunk_size, -1, blosc.BLOSC_MAX_BUFFERSIZE)
check_range('last_chunk', last_chunk, -1, blosc.BLOSC_MAX_BUFFERSIZE)
check_range('nchunks', nchunks, -1, MAX_CHUNKS)
check_range('max_app_chunks', max_app_chunks, 0, MAX_CHUNKS)
if nchunks != -1:
check_range('nchunks + max_app_chunks',
nchunks + max_app_chunks, 0, MAX_CHUNKS)
elif max_app_chunks != 0:
raise ValueError(
"'max_app_chunks' can not be non '0' if 'nchunks' is '-1'")
if chunk_size != -1 and last_chunk != -1 and last_chunk > chunk_size:
raise ValueError(
"'last_chunk' (%d) is larger than 'chunk_size' (%d)"
% (last_chunk, chunk_size))
self._attrs = ['format_version',
'offsets',
'metadata',
def __init__(self,
format_version=FORMAT_VERSION,
offsets=False,
metadata=False,
checksum='None',
typesize=0,
chunk_size=-1,
last_chunk=-1,
nchunks=-1,
max_app_chunks=0):
check_range('format_version', format_version, 0, MAX_FORMAT_VERSION)
check_valid_checksum(checksum)
check_range('typesize', typesize, 0, blosc.BLOSC_MAX_TYPESIZE)
check_range('chunk_size', chunk_size, -1, blosc.BLOSC_MAX_BUFFERSIZE)
check_range('last_chunk', last_chunk, -1, blosc.BLOSC_MAX_BUFFERSIZE)
check_range('nchunks', nchunks, -1, MAX_CHUNKS)
check_range('max_app_chunks', max_app_chunks, 0, MAX_CHUNKS)
if nchunks != -1:
check_range('nchunks + max_app_chunks',
nchunks + max_app_chunks, 0, MAX_CHUNKS)
elif max_app_chunks != 0:
raise ValueError(
"'max_app_chunks' can not be non '0' if 'nchunks' is '-1'")
if chunk_size != -1 and last_chunk != -1 and last_chunk > chunk_size:
raise ValueError(
"'last_chunk' (%d) is larger than 'chunk_size' (%d)"
% (last_chunk, chunk_size))
self._attrs = ['format_version',
'offsets',
last_chunk_size : int
the size of the last chunk in bytes
Raises
------
ChunkingException
if the resulting nchunks is larger than MAX_CHUNKS
"""
if in_file_size <= 0:
raise ValueError("'in_file_size' must be strictly positive, not %d"
% in_file_size)
# convert a human readable description to an int
if isinstance(chunk_size, basestring):
chunk_size = reverse_pretty(chunk_size)
check_range('chunk_size', chunk_size, 1, blosc.BLOSC_MAX_BUFFERSIZE)
# downcast
if chunk_size > in_file_size:
print_verbose(
"Input was smaller than the given 'chunk_size': %s using: %s"
% (double_pretty_size(chunk_size),
double_pretty_size(in_file_size)))
chunk_size = in_file_size
quotient, remainder = divmod(in_file_size, chunk_size)
# the user wants a single chunk
if chunk_size == in_file_size:
nchunks = 1
chunk_size = in_file_size
last_chunk_size = in_file_size
# no remainder, perfect fit
elif remainder == 0:
nchunks = quotient