Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
assert np.issubdtype(batch_like.dtype, np.number), "invalid non-flattened array subtype"
auto_chunker = False
if chunk_size is None:
auto_chunker = True
chunk_size = (1, *batch_like.shape[1:])
chunk_byte_size = np.multiply.reduce(chunk_size) * batch_like.dtype.itemsize
assert auto_chunker or 10 * (2 ** 10) <= chunk_byte_size < 2 ** 20, \
f"unrecommended chunk byte size ({chunk_byte_size}) should be in [10KiB,1MiB];" \
" see http://docs.h5py.org/en/stable/high/dataset.html#chunked-storage"
if compression == "chunk_lz4":
dset = fd.create_dataset(
name=name,
shape=(max_len, *batch_like.shape[1:]),
chunks=chunk_size,
dtype=batch_like.dtype,
**hdf5plugin.LZ4(nbytes=0)
)
else:
assert compression not in no_compression_flags or len(compression_args) == 0
dset = fd.create_dataset(
name=name,
shape=(max_len, *batch_like.shape[1:]),
chunks=chunk_size,
dtype=batch_like.dtype,
compression=compression if compression not in no_compression_flags else None,
**compression_args
)
dset.attrs["orig_shape"] = batch_like.shape[1:] # removes batch dim
else:
assert thelper.utils.is_scalar(batch_like[0])
if np.issubdtype(batch_like.dtype, np.number):
assert compression in no_compression_flags, "cannot compress scalar elements"