Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def getNullReference(self):
tmpGrp = None
if "{tmp}" not in self.dbGrp:
tmpGrp = self.dbGrp.create_group("{tmp}")
else:
tmpGrp = self.dbGrp["{tmp}"]
if 'nullref' not in tmpGrp:
dt = h5py.special_dtype(ref=h5py.Reference)
tmpGrp.create_dataset('nullref', (1,), dtype=dt)
nullref_dset = tmpGrp['nullref']
return nullref_dset[0]
dset_a = grp2.create_dataset('a', data=np.uint64([0, 0]))
set_attribute_string(dset_a, 'MATLAB_class',
'canonical empty')
set_attribute(dset_a, 'MATLAB_empty',
np.uint8(1))
except:
dset_a = grp2.create_dataset('a', data=np.uint64([0, 0]))
set_attribute_string(dset_a, 'MATLAB_class',
'canonical empty')
set_attribute(dset_a, 'MATLAB_empty',
np.uint8(1))
# We need to grab the special reference dtype and make an empty
# array to store all the references in.
data_refs = np.full(data.shape, dset_a.ref,
dtype=h5py.special_dtype(ref=h5py.Reference))
# Go through all the elements of data and write them, gabbing their
# references and putting them in data_refs. They will be put in
# group_for_references, which is also what the H5PATH needs to be
# set to if we are doing MATLAB compatibility (otherwise, the
# attribute needs to be deleted). If an element can't be written
# (doing matlab compatibility, but it isn't compatible with matlab
# and action_for_matlab_incompatible option is True), the reference
# to the canonical empty will be used for the reference array to
# point to.
grp2name = grp2.name
data_refs_flat = data_refs.reshape(-1)
for index, x in enumerate(data.flat):
name_for_ref = next_unused_name_in_group(grp2, 16)
obj = write_data(f, grp2, name_for_ref, x, None, options)
if obj is not None:
dataset_size = len(names_li) - err_cnt
stdout.write(' done\n'); stdout.flush()
return dataset_size
train_size = add_subset('train', train_names_li)
valid_size = add_subset('valid', valid_names_li)
test_size = add_subset('test', test_names_li)
split_array = np.empty(
3, dtype=np.dtype([
('split', 'a', 5),
('source', 'a', 15),
('start', np.int64),
('stop', np.int64),
('indices', h5py.special_dtype(ref=h5py.Reference)),
('available', np.bool, 1),
('comment', 'a', 1)]))
split_array[0]['split'] = 'train'.encode('utf8')
split_array[1]['split'] = 'valid'.encode('utf8')
split_array[2]['split'] = 'test'.encode('utf8')
split_array[0]['source'] = 'train_spectra'.encode('utf8')
split_array[1]['source'] = 'valid_spectra'.encode('utf8')
split_array[2]['source'] = 'test_spectra'.encode('utf8')
split_array[:]['start'] = 0
split_array[0]['stop'] = train_size
split_array[1]['stop'] = valid_size
split_array[2]['stop'] = test_size
split_array[:]['indices'] = h5py.Reference()
split_array[:]['available'] = True
sources |= set(split.keys())
for val in split.values():
if len(val) == 4:
comment_len = max([comment_len, len(val[-1])])
sources = sorted(list(sources))
source_len = max(len(source) for source in sources)
# Instantiate empty split array
split_array = numpy.empty(
len(split_dict) * len(sources),
dtype=numpy.dtype([
('split', 'a', split_len),
('source', 'a', source_len),
('start', numpy.int64, 1),
('stop', numpy.int64, 1),
('indices', h5py.special_dtype(ref=h5py.Reference)),
('available', numpy.bool, 1),
('comment', 'a', comment_len)]))
# Fill split array
for i, (split, source) in enumerate(product(split_dict, sources)):
if source in split_dict[split]:
start, stop = split_dict[split][source][:2]
available = True
indices = h5py.Reference()
# Workaround for bug when pickling an empty string
comment = '.'
if len(split_dict[split][source]) > 2:
indices = split_dict[split][source][2]
if len(split_dict[split][source]) > 3:
comment = split_dict[split][source][3]
if not comment:
for i, (split, source) in enumerate(product(split_dict, sources)):
if source in split_dict[split]:
start, stop = split_dict[split][source][:2]
available = True
indices = h5py.Reference()
# Workaround for bug when pickling an empty string
comment = '.'
if len(split_dict[split][source]) > 2:
indices = split_dict[split][source][2]
if len(split_dict[split][source]) > 3:
comment = split_dict[split][source][3]
if not comment:
comment = '.'
else:
(start, stop, indices, available, comment) = (
0, 0, h5py.Reference(), False, '.')
# Workaround for H5PY being unable to store unicode type
split_array[i]['split'] = split.encode('utf8')
split_array[i]['source'] = source.encode('utf8')
split_array[i]['start'] = start
split_array[i]['stop'] = stop
split_array[i]['indices'] = indices
split_array[i]['available'] = available
split_array[i]['comment'] = comment.encode('utf8')
return split_array
def WriteModelVariables(self):
scalarVariables = self.modelVariable
# Get maximum length of string vectors
maxLenName = self._getMaxLength(scalarVariables.keys())
maxLenDescription = self._getMaxLength([x.description for x in scalarVariables.values()])
#Create dtype object
numpyDataType = numpy.dtype({'names': ['name', 'simpleTypeRow',
'causality', 'variability',
'description', 'objectId', 'column', 'negated'],
'formats': ['S' + str(max(maxLenName, 1)),
'uint32',
'uint8', # h5py.special_dtype(enum=(numpy.uint8, CausalityType)),
'uint8', # h5py.special_dtype(enum=(numpy.uint8, VariabilityType)),
'S' + str(max(maxLenDescription, 1)),
h5py.special_dtype(ref=h5py.Reference),
'uint32',
'uint8']}) # h5py.special_dtype(enum=(numpy.uint8, {'false':0, 'true':1}))]})
self.description = self.file.create_group("ModelDescription")
#Write information on Simulation group
description = self.modelDescription
self.description.attrs['modelName'] = description.modelName
self.description.attrs['description'] = description.description
self.description.attrs['author'] = description.author
self.description.attrs['version'] = description.version
self.description.attrs['generationTool'] = description.generationTool
self.description.attrs['generationDateAndTime'] = description.generationDateAndTime
self.description.attrs['variableNamingConvention'] = description.variableNamingConvention
dataset = self.description.create_dataset('Variables', (len(scalarVariables), 1), dtype=numpyDataType, maxshape=(len(scalarVariables), 1), compression='gzip')
# Sort Variables by names
nameList = [x for x in scalarVariables.keys()]
nameList.sort()
for i, (split, source) in enumerate(product(split_dict, sources)):
if source in split_dict[split]:
start, stop = split_dict[split][source][:2]
available = True
indices = h5py.Reference()
# Workaround for bug when pickling an empty string
comment = '.'
if len(split_dict[split][source]) > 2:
indices = split_dict[split][source][2]
if len(split_dict[split][source]) > 3:
comment = split_dict[split][source][3]
if not comment:
comment = '.'
else:
(start, stop, indices, available, comment) = (
0, 0, h5py.Reference(), False, '.')
# Workaround for H5PY being unable to store unicode type
split_array[i]['split'] = split.encode('utf8')
split_array[i]['source'] = source.encode('utf8')
split_array[i]['start'] = start
split_array[i]['stop'] = stop
split_array[i]['indices'] = indices
split_array[i]['available'] = available
split_array[i]['comment'] = comment.encode('utf8')
return split_array
print ref_data
subset_data = ds[ref_data]
subset_data_dark = ds[ref_data_dark]
subset_data_white = ds[ref_data_white]
subset_data = ds[ref_data]
subset_data_dark = ds[ref_data_dark]
subset_data_white = ds[ref_data_white]
print subset_data.shape
print subset_data_dark.shape
print subset_data_white.shape
ref_dtype = h5py.special_dtype(ref=h5py.Reference)
ref_dataset = f.create_dataset("exchange/data", (1,), dtype=ref_dtype)
ref_dataset_dark = f.create_dataset("exchange/data_dark", (1,), dtype=ref_dtype)
ref_dataset_white = f.create_dataset("exchange/data_white", (1,), dtype=ref_dtype)
# set the reference to the object
ref_dataset[...] = ref_data
ref_dataset_dark[...] = ref_data_dark
ref_dataset_white[...] = ref_data_white
## f["exchange"] = h5py.SoftLink('/entry/instrument/detector')
print ref_data
print ref_dataset[0]
print ref_data_dark
print ref_dataset_dark[0]
from ...data_utils import AbstractDataChunkIterator, get_shape
from ...build import Builder, GroupBuilder, DatasetBuilder, LinkBuilder, BuildManager,\
RegionBuilder, ReferenceBuilder, TypeMap
from ...spec import RefSpec, DtypeSpec, NamespaceCatalog, GroupSpec
from ...spec import NamespaceBuilder
from .h5_utils import H5ReferenceDataset, H5RegionDataset, H5TableDataset,\
H5DataIO, H5SpecReader, H5SpecWriter
from ..io import FORMIO
ROOT_NAME = 'root'
SPEC_LOC_ATTR = '.specloc'
H5_TEXT = special_dtype(vlen=text_type)
H5_BINARY = special_dtype(vlen=binary_type)
H5_REF = special_dtype(ref=Reference)
H5_REGREF = special_dtype(ref=RegionReference)
class HDF5IO(FORMIO):
@docval({'name': 'path', 'type': str, 'doc': 'the path to the HDF5 file'},
{'name': 'manager', 'type': BuildManager, 'doc': 'the BuildManager to use for I/O', 'default': None},
{'name': 'mode', 'type': str,
'doc': 'the mode to open the HDF5 file with, one of ("w", "r", "r+", "a", "w-")'},
{'name': 'comm', 'type': 'Intracomm',
'doc': 'the MPI communicator to use for parallel I/O', 'default': None},
{'name': 'file', 'type': File, 'doc': 'a pre-existing h5py.File object', 'default': None})
def __init__(self, **kwargs):
'''Open an HDF5 file for IO
For `mode`, see `h5py.File _`.
np_shape = tuple(np_shape) # for comparison with ndarray shape
self.log.info("selection shape:" + str(np_shape))
# need some special conversion for compound types --
# each element must be a tuple, but the JSON decoder
# gives us a list instead.
if format != "binary" and len(dset.dtype) > 1 and type(data) in (list, tuple):
data = self.toTuple(rank, data)
#for i in range(len(data)):
# converted_data.append(self.toTuple(data[i]))
#data = converted_data
else:
h5t_check = h5py.check_dtype(ref=dset.dtype)
if h5t_check in (h5py.Reference, h5py.RegionReference):
# convert data to data refs
if format == "binary":
msg = "Only JSON is supported for for this data type"
self.log.info(msg)
raise IOError(errno.EINVAL, msg)
data = self.listToRef(data)
if format == "binary":
if npoints*itemSize != len(data):
msg = "Expected: " + str(npoints*itemSize) + " bytes, but got: " + str(len(data))
self.log.info(msg)
raise IOError(errno.EINVAL, msg)
if dset.dtype.shape == ():
arr = np.fromstring(data, dtype=dset.dtype)
arr = arr.reshape(np_shape) # conform to selection shape
else: