How to use the blosc.BLOSC_MAX_BUFFERSIZE function in blosc

To help you get started, we’ve selected a few blosc examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github Blosc / bloscpack / test / test_headers.py View on Github external
# uses nose test generators

    def check(error_type, args_dict):
        nt.assert_raises(error_type, BloscpackHeader, **args_dict)

    for error_type, args_dict in [
            (ValueError, {'format_version': -1}),
            (ValueError, {'format_version': MAX_FORMAT_VERSION+1}),
            (TypeError,  {'format_version': 'foo'}),
            (ValueError, {'checksum': -1}),
            (ValueError, {'checksum': len(checksums.CHECKSUMS)+1}),
            (exceptions.NoSuchChecksum, {'checksum': 'foo'}),
            (ValueError, {'typesize': -1}),
            (ValueError, {'typesize': blosc.BLOSC_MAX_TYPESIZE+1}),
            (TypeError,  {'typesize': 'foo'}),
            (ValueError, {'chunk_size': blosc.BLOSC_MAX_BUFFERSIZE+1}),
            (ValueError, {'chunk_size': -2}),
            (TypeError,  {'chunk_size': 'foo'}),
            (ValueError, {'last_chunk': blosc.BLOSC_MAX_BUFFERSIZE+1}),
            (ValueError, {'last_chunk': -2}),
            (TypeError,  {'last_chunk': 'foo'}),
            (ValueError, {'nchunks': MAX_CHUNKS+1}),
            (ValueError, {'nchunks': -2}),
            (TypeError,  {'nchunks': 'foo'}),
            (ValueError, {'max_app_chunks': MAX_CHUNKS+1}),
            (ValueError, {'max_app_chunks': -1}),
            (TypeError,  {'max_app_chunks': 'foo'}),
            # sum of nchunks and max_app_chunks
            (ValueError, {'nchunks': MAX_CHUNKS//2+1,
                          'max_app_chunks': MAX_CHUNKS//2+1}),
            # check that max_app_chunks is zero
            (ValueError, {'nchunks': -1,
github Blosc / bloscpack / test / test_file_io.py View on Github external
def pack_unpack_extreme():
    """ Test on somewhat larer arrays, uses loads of memory. """
    # this will create a huge array, and then use the
    # blosc.BLOSC_MAX_BUFFERSIZE as chunk-szie
    pack_unpack(300, chunk_size=blosc.BLOSC_MAX_BUFFERSIZE,
                progress=simple_progress)
github Blosc / bloscpack / bloscpack / __init__.py View on Github external
def __call__(self, parser, namespace, value, option_string=None):
            if value == 'max':
                value = blosc.BLOSC_MAX_BUFFERSIZE
            else:
                try:
                    # try to get the value as bytes
                    if value[-1] in SUFFIXES.keys():
                        value = reverse_pretty(value)
                    # seems to be intended to be a naked int
                    else:
                        value = int(value)
                except ValueError as ve:
                    error('%s error: %s' % (option_string, str(ve) +
                        " or 'max'"))
                if value < 0:
                    error('%s must be > 0 ' % option_string)
            setattr(namespace, self.dest, value)
    for p in [compress_parser, c_parser]:
github Blosc / bloscpack / bloscpack / args.py View on Github external
Raises
    ------
    ChunkingException
        if the resulting nchunks is larger than MAX_CHUNKS

    """
    if in_file_size < 0:
            raise ValueError("'in_file_size' must be strictly positive, not %d"
                             % in_file_size)
    elif in_file_size == 0:
        return (1, 0, 0)
        log.verbose("Input was length zero, ignoring 'chunk_size'")
    # convert a human readable description to an int
    if isinstance(chunk_size, string_types):
        chunk_size = reverse_pretty(chunk_size)
    check_range('chunk_size', chunk_size, 1, blosc.BLOSC_MAX_BUFFERSIZE)
    # downcast
    if chunk_size > in_file_size:
        log.verbose(
            "Input was smaller than the given 'chunk_size': %s using: %s"
            % (double_pretty_size(chunk_size),
               double_pretty_size(in_file_size)))
        chunk_size = in_file_size
    quotient, remainder = divmod(in_file_size, chunk_size)
    # the user wants a single chunk
    if chunk_size == in_file_size:
        nchunks = 1
        chunk_size = in_file_size
        last_chunk_size = in_file_size
    # no remainder, perfect fit
    elif remainder == 0:
        nchunks = quotient
github Blosc / bloscpack / bloscpack / cli.py View on Github external
def __call__(self, parser, namespace, value, option_string=None):
            if value == 'max':
                value = blosc.BLOSC_MAX_BUFFERSIZE
            else:
                try:
                    # try to get the value as bytes
                    if value[-1] in SUFFIXES.keys():
                        value = reverse_pretty(value)
                    # seems to be intended to be a naked int
                    else:
                        value = int(value)
                except ValueError as ve:
                    log.error('%s error: %s' % (option_string, str(ve)))
                if value < 0:
                    log.error('%s must be > 0' % option_string)
            setattr(namespace, self.dest, value)
    for p in [compress_parser, c_parser]:
github Blosc / bloscpack / bloscpack / headers.py View on Github external
def __init__(self,
                 format_version=FORMAT_VERSION,
                 offsets=False,
                 metadata=False,
                 checksum='None',
                 typesize=0,
                 chunk_size=-1,
                 last_chunk=-1,
                 nchunks=-1,
                 max_app_chunks=0):

        check_range('format_version', format_version, 0, MAX_FORMAT_VERSION)
        check_valid_checksum(checksum)
        check_range('typesize',   typesize,    0, blosc.BLOSC_MAX_TYPESIZE)
        check_range('chunk_size', chunk_size, -1, blosc.BLOSC_MAX_BUFFERSIZE)
        check_range('last_chunk', last_chunk, -1, blosc.BLOSC_MAX_BUFFERSIZE)
        check_range('nchunks',    nchunks,    -1, MAX_CHUNKS)
        check_range('max_app_chunks', max_app_chunks, 0, MAX_CHUNKS)
        if nchunks != -1:
            check_range('nchunks + max_app_chunks',
                        nchunks + max_app_chunks, 0, MAX_CHUNKS)
        elif max_app_chunks != 0:
            raise ValueError(
                "'max_app_chunks' can not be non '0' if 'nchunks' is '-1'")
        if chunk_size != -1 and last_chunk != -1 and last_chunk > chunk_size:
            raise ValueError(
                "'last_chunk' (%d) is larger than 'chunk_size' (%d)"
                % (last_chunk, chunk_size))

        self._attrs = ['format_version',
                       'offsets',
                       'metadata',
github Blosc / bloscpack / bloscpack / headers.py View on Github external
def __init__(self,
                 format_version=FORMAT_VERSION,
                 offsets=False,
                 metadata=False,
                 checksum='None',
                 typesize=0,
                 chunk_size=-1,
                 last_chunk=-1,
                 nchunks=-1,
                 max_app_chunks=0):

        check_range('format_version', format_version, 0, MAX_FORMAT_VERSION)
        check_valid_checksum(checksum)
        check_range('typesize',   typesize,    0, blosc.BLOSC_MAX_TYPESIZE)
        check_range('chunk_size', chunk_size, -1, blosc.BLOSC_MAX_BUFFERSIZE)
        check_range('last_chunk', last_chunk, -1, blosc.BLOSC_MAX_BUFFERSIZE)
        check_range('nchunks',    nchunks,    -1, MAX_CHUNKS)
        check_range('max_app_chunks', max_app_chunks, 0, MAX_CHUNKS)
        if nchunks != -1:
            check_range('nchunks + max_app_chunks',
                        nchunks + max_app_chunks, 0, MAX_CHUNKS)
        elif max_app_chunks != 0:
            raise ValueError(
                "'max_app_chunks' can not be non '0' if 'nchunks' is '-1'")
        if chunk_size != -1 and last_chunk != -1 and last_chunk > chunk_size:
            raise ValueError(
                "'last_chunk' (%d) is larger than 'chunk_size' (%d)"
                % (last_chunk, chunk_size))

        self._attrs = ['format_version',
                       'offsets',
github Blosc / bloscpack / bloscpack / __init__.py View on Github external
last_chunk_size : int
        the size of the last chunk in bytes

    Raises
    ------
    ChunkingException
        if the resulting nchunks is larger than MAX_CHUNKS

    """
    if in_file_size <= 0:
            raise ValueError("'in_file_size' must be strictly positive, not %d"
                    % in_file_size)
    # convert a human readable description to an int
    if isinstance(chunk_size, basestring):
        chunk_size = reverse_pretty(chunk_size)
    check_range('chunk_size', chunk_size, 1, blosc.BLOSC_MAX_BUFFERSIZE)
    # downcast
    if chunk_size > in_file_size:
        print_verbose(
                "Input was smaller than the given 'chunk_size': %s using: %s"
                % (double_pretty_size(chunk_size),
                double_pretty_size(in_file_size)))
        chunk_size = in_file_size
    quotient, remainder = divmod(in_file_size, chunk_size)
    # the user wants a single chunk
    if chunk_size == in_file_size:
        nchunks = 1
        chunk_size = in_file_size
        last_chunk_size = in_file_size
    # no remainder, perfect fit
    elif remainder == 0:
        nchunks = quotient