Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
#in_ = np.random.random_integers(0, 100, N) # random distribution
print(" ", in_)
tic = time.time()
out_ = np.copy(in_)
toc = time.time()
print(" Time for copying array with np.copy(): %.3f s" % (toc-tic,))
print()
for cname in blosc.compressor_list():
print("Using *** %s *** compressor::" % cname)
ctic = time.time()
c = blosc.pack_array(in_, clevel=clevel, shuffle=True, cname=cname)
ctoc = time.time()
dtic = time.time()
out = blosc.unpack_array(c)
dtoc = time.time()
assert((in_ == out).all())
print(" Time for pack_array/unpack_array: %.3f/%.3f s." % \
(ctoc-ctic, dtoc-dtic), end='')
print("\tCompr ratio: %.2f" % (in_.size*in_.dtype.itemsize*1. / len(c)))
ctic = time.time()
c = blosc.compress_ptr(in_.__array_interface__['data'][0],
in_.size, in_.dtype.itemsize,
clevel=clevel, shuffle=True, cname=cname)
ctoc = time.time()
out = np.empty(in_.size, dtype=in_.dtype)
dtic = time.time()
blosc.decompress_ptr(c, out.__array_interface__['data'][0])
dtoc = time.time()
assert((in_ == out).all())
hdf5_ds_values = []
blosc_ds_values = []
numpy_ss_values = []
hdf5_ss_values = []
blosc_ss_values = []
x_axis_values = []
print "-----SERIALIZATION TIME-----"
for i in range(0, ITERATIONS, 1):
CUBE_VALUE = int(BASE_SIZE*math.pow(2,i))
x_axis_values.append(CUBE_VALUE)
print "SIZE:{}".format(CUBE_VALUE)
cutout_args = (OFFSET[0], OFFSET[0]+CUBE_VALUE, OFFSET[1], OFFSET[1]+CUBE_VALUE, OFFSET[2], OFFSET[2]+Z_SIZE)
data = blosc.unpack_array(getURL(generateURLBlosc(HOST, TOKEN, CHANNELS, RESOLUTION, cutout_args)))
# data = np.asarray(range(CUBE_VALUE*CUBE_VALUE*Z_SIZE), dtype=np.uint32).reshape(CUBE_VALUE,CUBE_VALUE,Z_SIZE)
# NUMPY
start = time.time()
fileobj = cStringIO.StringIO()
np.save(fileobj, data)
test = zlib.compress(fileobj.getvalue())
numpy_ts_values.append(time.time()-start)
# numpy_ss_values.append(sys.getsizeof(data)/sys.getsizeof(test))
numpy_ss_values.append(sys.getsizeof(test))
start = time.time()
test = np.load ( cStringIO.StringIO ( zlib.decompress ( test ) ) )
numpy_ds_values.append(time.time()-start)
# HDF5
hdf5_ss_values.append(sys.getsizeof(tmpfile.read()))
tmpfile.seek(0)
start = time.time()
fh5in = h5py.File(tmpfile.name, driver='core', backing_store=True)
test = np.array(fh5in['TEST']['CUTOUT'])
hdf5_ds_values.append(time.time()-start)
fh5in.close()
tmpfile.close()
# BLOSC
start = time.time()
test = blosc.pack_array(data)
blosc_ts_values.append(time.time()-start)
blosc_ss_values.append(sys.getsizeof(test))
start = time.time()
test = blosc.unpack_array(test)
blosc_ds_values.append(time.time()-start)
# opening a pdf file
pp = PdfPages('time_serialization.pdf')
# Time Serlization Graph
# plot values
plt.figure(figsize=(10,10))
plt.plot(x_axis_values, numpy_ts_values, color='green', marker='o')
plt.plot(x_axis_values, hdf5_ts_values, color='blue', marker='^')
plt.plot(x_axis_values, blosc_ts_values, color='red', marker='s')
# configure x-axis
plt.xlim(0,5500)
'-pix_fmt': 'yuv420p',
'-r': str(self.fps),
'-preset': 'ultrafast',
},
verbosity=1
)
try:
for input in iter(self.q.get, 'END'):
try:
if self.given_timestamps:
self.timestamps.append(input[0])
if self.blosc:
vid_out.writeFrame(blosc.unpack_array(input[1]))
else:
vid_out.writeFrame(input[1])
else:
self.timestamps.append(datetime.now().isoformat())
if self.blosc:
vid_out.writeFrame(blosc.unpack_array(input))
else:
vid_out.writeFrame(input)
except Exception as e:
print(e)
traceback.print_tb()
# TODO: Too general
break
finally:
def g_decompress(msg):
if sys.version_info[0] < 3:
# Python 2.x implementation
assert isinstance(msg, str)
else:
# Python 3.x implementation
assert isinstance(msg, bytes)
grad = blosc.unpack_array(msg)
return grad
def setup(self, worker):
self.cache = Buffer(
fast={},
slow=Func(
dump=blosc.pack_array,
load=blosc.unpack_array,
d=Buffer(
fast={},
slow=LRU(
n=self._maxdisk,
d=File(os.path.join(worker.local_directory, 'cache')),
weight=lambda k, v: len(v),
),
n=self._maxcompressed,
weight=lambda k, v: len(v),
),
),
n=self._maxmem,
weight=lambda k, v: v.nbytes,
)
self.lock = Lock()
self.hits = 0
if has_label:
sitk_pair_list[2],_ = self.resize_img(sitk_pair_list[2], is_label=True)
sitk_pair_list[3],_ = self.resize_img(sitk_pair_list[3], is_label=True)
pair_list = [sitk.GetArrayFromImage(sitk_pair) for sitk_pair in sitk_pair_list]
pair_list = [item.astype(np.float32) for item in pair_list]
img_after_resize = self.img_after_resize if self.img_after_resize is not None else original_sz
new_spacing= original_spacing*(original_sz-1)/(np.array(img_after_resize)-1)
spacing = self._normalize_spacing(new_spacing,img_after_resize, silent_mode=True)
else:
zipnp_pair_list = self.pair_list[idx]
spacing = self.spacing_list[idx]
original_spacing = self.original_spacing_list[idx]
original_sz = self.original_sz_list[idx]
pair_list = [blosc.unpack_array(item) for item in zipnp_pair_list]
sample = {'image': np.asarray([self.normalize_intensity(pair_list[0]),self.normalize_intensity(pair_list[1])])}
sample['pair_path'] = pair_path
if self.load_init_weight:
sample['init_weight']=self.init_weight_list[idx]
if has_label:
try:
sample ['label']= np.asarray([pair_list[2], pair_list[3]]).astype(np.float32)
except:
print(pair_list[2].shape,pair_list[3].shape)
print(filename)
# else:
# sample['label'] = None
if self.transform:
sample['image'] = self.transform(sample['image'])
sample = {}
########################### channel sel
if self.transform:
index = [0] if not self.is_brats else [0,1,2,3]
if self.add_loc:
index = index + [index[-1]+1 for _ in range(3)]
if self.is_train:
sample['image'] = self.transform(data['img'][index].copy())*2-1
else:
if not self.use_org_size:
sample['image'] = self.transform(data['img'][:,index].copy())*2-1
else:
sample['image'] = self.transform(data['img'][index].copy()) * 2 - 1
if self.add_resampled:
sample['resampled_img'] = self.transform(blosc.unpack_array((dic['resampled_img'])))
if self.detect_et: #self.use_org_size or self.detect_et:
sample['checked_label']=self.transform(data['checked_label'].astype(np.int32))
#sample['image'] = self.transform(data['img'].copy())
if 'seg'in data and data['seg'] is not None:
sample['label'] = self.transform(data['seg'].copy())
else:
sample['label'] = self.transform(np.array([-1]).astype((np.int32)))
# global count
# count[1] +=1
# print(count)
return sample,fname
Raises:
requests.HTTPError
"""
req = self.get_cutout_request(
resource, 'GET', 'application/blosc-python',
url_prefix, auth, resolution, x_range, y_range, z_range, time_range)
prep = session.prepare_request(req)
# Hack in Accept header for now.
prep.headers['Accept'] = 'application/blosc-python'
#resp = session.send(prep, stream = True, **send_opts)
resp = session.send(prep, **send_opts)
if resp.status_code == 200:
return blosc.unpack_array(resp.content)
msg = ('Get cutout failed on {}, got HTTP response: ({}) - {}'.format(
resource.name, resp.status_code, resp.text))
raise HTTPError(msg, request = req, response = resp)