Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
for cname in blosc.compressor_list():
print("Using *** %s *** compressor::" % cname)
ctic = time.time()
c = blosc.pack_array(in_, clevel=clevel, shuffle=True, cname=cname)
ctoc = time.time()
dtic = time.time()
out = blosc.unpack_array(c)
dtoc = time.time()
assert((in_ == out).all())
print(" Time for pack_array/unpack_array: %.3f/%.3f s." % \
(ctoc-ctic, dtoc-dtic), end='')
print("\tCompr ratio: %.2f" % (in_.size*in_.dtype.itemsize*1. / len(c)))
ctic = time.time()
c = blosc.compress_ptr(in_.__array_interface__['data'][0],
in_.size, in_.dtype.itemsize,
clevel=clevel, shuffle=True, cname=cname)
ctoc = time.time()
out = np.empty(in_.size, dtype=in_.dtype)
dtic = time.time()
blosc.decompress_ptr(c, out.__array_interface__['data'][0])
dtoc = time.time()
assert((in_ == out).all())
print(" Time for compress_ptr/decompress_ptr: %.3f/%.3f s." % \
(ctoc-ctic, dtoc-dtic), end='')
print("\tCompr ratio: %.2f" % (in_.size*in_.dtype.itemsize*1. / len(c)))
def compressSlice( args ):
"""
args = (numpy array address, array_size, item_size, bytesList, bytesIndex)
"""
args[3][args[4]] = blosc.compress_ptr( args[0], args[1], args[2], \
clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR )
# Let's try instead just pool threads...
#poolThreads = np.arange( 1, maxThreads+1 )
#bloscThreads = np.ones_like( poolThreads )
solo_times = np.zeros_like( poolThreads, dtype='float64' )
solo_unlocked_times = np.zeros_like( poolThreads, dtype='float64' )
locked_times = np.zeros_like( poolThreads, dtype='float64' )
unlocked_times = np.zeros_like( poolThreads, dtype='float64' )
for J in np.arange(nRuns):
print( "Run %d of %d" % (J+1, nRuns) )
blosc.set_releasegil(False)
for I in np.arange( len(poolThreads) ):
t1 = time.time()
blosc.set_nthreads( bloscThreads[I] )
blosc.compress_ptr( stack.__array_interface__['data'][0], stack.size, stack.dtype.itemsize, \
clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR )
solo_times[I] += time.time() - t1
blosc.set_releasegil(True)
for I in np.arange( len(poolThreads) ):
t2 = time.time()
blosc.set_nthreads( bloscThreads[I] )
blosc.compress_ptr( stack.__array_interface__['data'][0], stack.size, stack.dtype.itemsize, \
clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR )
solo_unlocked_times[I] += time.time() - t2
blosc.set_releasegil(True)
for I in np.arange( len(poolThreads) ):
t3 = time.time()
compressStack( stack, blosc_threads=bloscThreads[I], pool_threads=poolThreads[I] )
unlocked_times[I] += time.time() - t3
compress = lambda in_: (in_.size, in_.dtype, blosc.compress_ptr(in_.__array_interface__['data'][0], in_.size, in_.dtype.itemsize, clevel = 1, shuffle = blosc.BITSHUFFLE, cname = 'lz4'))
for J in np.arange(nRuns):
print( "Run %d of %d" % (J+1, nRuns) )
blosc.set_releasegil(False)
for I in np.arange( len(poolThreads) ):
t1 = time.time()
blosc.set_nthreads( bloscThreads[I] )
blosc.compress_ptr( stack.__array_interface__['data'][0], stack.size, stack.dtype.itemsize, \
clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR )
solo_times[I] += time.time() - t1
blosc.set_releasegil(True)
for I in np.arange( len(poolThreads) ):
t2 = time.time()
blosc.set_nthreads( bloscThreads[I] )
blosc.compress_ptr( stack.__array_interface__['data'][0], stack.size, stack.dtype.itemsize, \
clevel=CLEVEL, shuffle=SHUFFLE, cname=COMPRESSOR )
solo_unlocked_times[I] += time.time() - t2
blosc.set_releasegil(True)
for I in np.arange( len(poolThreads) ):
t3 = time.time()
compressStack( stack, blosc_threads=bloscThreads[I], pool_threads=poolThreads[I] )
unlocked_times[I] += time.time() - t3
blosc.set_releasegil(False)
for I in np.arange( len(poolThreads) ):
t4 = time.time()
compressStack( stack, blosc_threads=bloscThreads[I], pool_threads=poolThreads[I] )
locked_times[I] += time.time() - t4
def _compress_chunk_ptr(chunk, blosc_args):
ptr, size = chunk
return blosc.compress_ptr(ptr, size, **blosc_args)
def _compress_chunk_ptr(chunk, blosc_args):
ptr, size = chunk
return blosc.compress_ptr(ptr, size, **blosc_args)