How to use the h5py.Reference function in h5py

To help you get started, we’ve selected a few h5py examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github HDFGroup / hdf5-json / h5json / hdf5db.py View on Github external
def getNullReference(self):
        tmpGrp = None
        if "{tmp}" not in self.dbGrp:
            tmpGrp = self.dbGrp.create_group("{tmp}")
        else:
            tmpGrp = self.dbGrp["{tmp}"]
        if 'nullref' not in tmpGrp:
            dt = h5py.special_dtype(ref=h5py.Reference)
            tmpGrp.create_dataset('nullref', (1,), dtype=dt)
        nullref_dset = tmpGrp['nullref']
        return nullref_dset[0]
github frejanordsiek / hdf5storage / hdf5storage / utilities.py View on Github external
dset_a = grp2.create_dataset('a', data=np.uint64([0, 0]))
            set_attribute_string(dset_a, 'MATLAB_class',
                                 'canonical empty')
            set_attribute(dset_a, 'MATLAB_empty',
                          np.uint8(1))
    except:
        dset_a = grp2.create_dataset('a', data=np.uint64([0, 0]))
        set_attribute_string(dset_a, 'MATLAB_class',
                             'canonical empty')
        set_attribute(dset_a, 'MATLAB_empty',
                      np.uint8(1))

    # We need to grab the special reference dtype and make an empty
    # array to store all the references in.
    data_refs = np.full(data.shape, dset_a.ref,
                        dtype=h5py.special_dtype(ref=h5py.Reference))

    # Go through all the elements of data and write them, gabbing their
    # references and putting them in data_refs. They will be put in
    # group_for_references, which is also what the H5PATH needs to be
    # set to if we are doing MATLAB compatibility (otherwise, the
    # attribute needs to be deleted). If an element can't be written
    # (doing matlab compatibility, but it isn't compatible with matlab
    # and action_for_matlab_incompatible option is True), the reference
    # to the canonical empty will be used for the reference array to
    # point to.
    grp2name = grp2.name
    data_refs_flat = data_refs.reshape(-1)
    for index, x in enumerate(data.flat):
        name_for_ref = next_unused_name_in_group(grp2, 16)
        obj = write_data(f, grp2, name_for_ref, x, None, options)
        if obj is not None:
github khaotik / DaNet-Tensorflow / app / datasets / WSJ0 / process.py View on Github external
dataset_size = len(names_li) - err_cnt
    stdout.write(' done\n'); stdout.flush()
    return dataset_size


train_size = add_subset('train', train_names_li)
valid_size = add_subset('valid', valid_names_li)
test_size = add_subset('test', test_names_li)

split_array = np.empty(
    3, dtype=np.dtype([
        ('split', 'a', 5),
        ('source', 'a', 15),
        ('start', np.int64),
        ('stop', np.int64),
        ('indices', h5py.special_dtype(ref=h5py.Reference)),
        ('available', np.bool, 1),
        ('comment', 'a', 1)]))

split_array[0]['split'] = 'train'.encode('utf8')
split_array[1]['split'] = 'valid'.encode('utf8')
split_array[2]['split'] = 'test'.encode('utf8')

split_array[0]['source'] = 'train_spectra'.encode('utf8')
split_array[1]['source'] = 'valid_spectra'.encode('utf8')
split_array[2]['source'] = 'test_spectra'.encode('utf8')
split_array[:]['start'] = 0
split_array[0]['stop'] = train_size
split_array[1]['stop'] = valid_size
split_array[2]['stop'] = test_size
split_array[:]['indices'] = h5py.Reference()
split_array[:]['available'] = True
github mila-iqia / fuel / fuel / datasets / hdf5.py View on Github external
sources |= set(split.keys())
            for val in split.values():
                if len(val) == 4:
                    comment_len = max([comment_len, len(val[-1])])
        sources = sorted(list(sources))
        source_len = max(len(source) for source in sources)

        # Instantiate empty split array
        split_array = numpy.empty(
            len(split_dict) * len(sources),
            dtype=numpy.dtype([
                ('split', 'a', split_len),
                ('source', 'a', source_len),
                ('start', numpy.int64, 1),
                ('stop', numpy.int64, 1),
                ('indices', h5py.special_dtype(ref=h5py.Reference)),
                ('available', numpy.bool, 1),
                ('comment', 'a', comment_len)]))

        # Fill split array
        for i, (split, source) in enumerate(product(split_dict, sources)):
            if source in split_dict[split]:
                start, stop = split_dict[split][source][:2]
                available = True
                indices = h5py.Reference()
                # Workaround for bug when pickling an empty string
                comment = '.'
                if len(split_dict[split][source]) > 2:
                    indices = split_dict[split][source][2]
                if len(split_dict[split][source]) > 3:
                    comment = split_dict[split][source][3]
                    if not comment:
github mila-iqia / fuel / fuel / datasets / hdf5.py View on Github external
for i, (split, source) in enumerate(product(split_dict, sources)):
            if source in split_dict[split]:
                start, stop = split_dict[split][source][:2]
                available = True
                indices = h5py.Reference()
                # Workaround for bug when pickling an empty string
                comment = '.'
                if len(split_dict[split][source]) > 2:
                    indices = split_dict[split][source][2]
                if len(split_dict[split][source]) > 3:
                    comment = split_dict[split][source][3]
                    if not comment:
                        comment = '.'
            else:
                (start, stop, indices, available, comment) = (
                    0, 0, h5py.Reference(), False, '.')
            # Workaround for H5PY being unable to store unicode type
            split_array[i]['split'] = split.encode('utf8')
            split_array[i]['source'] = source.encode('utf8')
            split_array[i]['start'] = start
            split_array[i]['stop'] = stop
            split_array[i]['indices'] = indices
            split_array[i]['available'] = available
            split_array[i]['comment'] = comment.encode('utf8')

        return split_array
github OpenModelica / OMPython / PythonInterface / PySimulator / Plugins / SimulationResult / Mtsf / pyMtsf.py View on Github external
def WriteModelVariables(self):
        scalarVariables = self.modelVariable
        # Get maximum length of string vectors
        maxLenName = self._getMaxLength(scalarVariables.keys())
        maxLenDescription = self._getMaxLength([x.description for x in scalarVariables.values()])
        #Create dtype object
        numpyDataType = numpy.dtype({'names': ['name', 'simpleTypeRow',
                                              'causality', 'variability',
                                              'description', 'objectId', 'column', 'negated'],
                               'formats': ['S' + str(max(maxLenName, 1)),
                                          'uint32',
                                          'uint8',  # h5py.special_dtype(enum=(numpy.uint8, CausalityType)),
                                          'uint8',  # h5py.special_dtype(enum=(numpy.uint8, VariabilityType)),
                                          'S' + str(max(maxLenDescription, 1)),
                                          h5py.special_dtype(ref=h5py.Reference),
                                          'uint32',
                                          'uint8']})  # h5py.special_dtype(enum=(numpy.uint8, {'false':0, 'true':1}))]})
        self.description = self.file.create_group("ModelDescription")
        #Write information on Simulation group
        description = self.modelDescription
        self.description.attrs['modelName'] = description.modelName
        self.description.attrs['description'] = description.description
        self.description.attrs['author'] = description.author
        self.description.attrs['version'] = description.version
        self.description.attrs['generationTool'] = description.generationTool
        self.description.attrs['generationDateAndTime'] = description.generationDateAndTime
        self.description.attrs['variableNamingConvention'] = description.variableNamingConvention
        dataset = self.description.create_dataset('Variables', (len(scalarVariables), 1), dtype=numpyDataType, maxshape=(len(scalarVariables), 1), compression='gzip')
        # Sort Variables by names
        nameList = [x for x in scalarVariables.keys()]
        nameList.sort()
github dmitriy-serdyuk / twinnet-asr / fuel / datasets / hdf5.py View on Github external
for i, (split, source) in enumerate(product(split_dict, sources)):
            if source in split_dict[split]:
                start, stop = split_dict[split][source][:2]
                available = True
                indices = h5py.Reference()
                # Workaround for bug when pickling an empty string
                comment = '.'
                if len(split_dict[split][source]) > 2:
                    indices = split_dict[split][source][2]
                if len(split_dict[split][source]) > 3:
                    comment = split_dict[split][source][3]
                    if not comment:
                        comment = '.'
            else:
                (start, stop, indices, available, comment) = (
                    0, 0, h5py.Reference(), False, '.')
            # Workaround for H5PY being unable to store unicode type
            split_array[i]['split'] = split.encode('utf8')
            split_array[i]['source'] = source.encode('utf8')
            split_array[i]['start'] = start
            split_array[i]['stop'] = stop
            split_array[i]['indices'] = indices
            split_array[i]['available'] = available
            split_array[i]['comment'] = comment.encode('utf8')

        return split_array
github data-exchange / dxchange / demo / convert_Diamond_2.py View on Github external
print ref_data
    
    subset_data = ds[ref_data]
    subset_data_dark = ds[ref_data_dark]
    subset_data_white = ds[ref_data_white]
    
    subset_data = ds[ref_data]
    subset_data_dark = ds[ref_data_dark]
    subset_data_white = ds[ref_data_white]

    print subset_data.shape
    print subset_data_dark.shape
    print subset_data_white.shape


    ref_dtype = h5py.special_dtype(ref=h5py.Reference)
    ref_dataset = f.create_dataset("exchange/data", (1,), dtype=ref_dtype)
    ref_dataset_dark = f.create_dataset("exchange/data_dark", (1,), dtype=ref_dtype)
    ref_dataset_white = f.create_dataset("exchange/data_white", (1,), dtype=ref_dtype)

    # set the reference to the object
    ref_dataset[...] = ref_data
    ref_dataset_dark[...] = ref_data_dark
    ref_dataset_white[...] = ref_data_white

##    f["exchange"] = h5py.SoftLink('/entry/instrument/detector')


    print ref_data
    print ref_dataset[0]
    print ref_data_dark
    print ref_dataset_dark[0]
github NeurodataWithoutBorders / pynwb / src / pynwb / form / backends / hdf5 / h5tools.py View on Github external
from ...data_utils import AbstractDataChunkIterator, get_shape
from ...build import Builder, GroupBuilder, DatasetBuilder, LinkBuilder, BuildManager,\
                     RegionBuilder, ReferenceBuilder, TypeMap
from ...spec import RefSpec, DtypeSpec, NamespaceCatalog, GroupSpec
from ...spec import NamespaceBuilder

from .h5_utils import H5ReferenceDataset, H5RegionDataset, H5TableDataset,\
                      H5DataIO, H5SpecReader, H5SpecWriter

from ..io import FORMIO

ROOT_NAME = 'root'
SPEC_LOC_ATTR = '.specloc'
H5_TEXT = special_dtype(vlen=text_type)
H5_BINARY = special_dtype(vlen=binary_type)
H5_REF = special_dtype(ref=Reference)
H5_REGREF = special_dtype(ref=RegionReference)


class HDF5IO(FORMIO):

    @docval({'name': 'path', 'type': str, 'doc': 'the path to the HDF5 file'},
            {'name': 'manager', 'type': BuildManager, 'doc': 'the BuildManager to use for I/O', 'default': None},
            {'name': 'mode', 'type': str,
             'doc': 'the mode to open the HDF5 file with, one of ("w", "r", "r+", "a", "w-")'},
            {'name': 'comm', 'type': 'Intracomm',
             'doc': 'the MPI communicator to use for parallel I/O', 'default': None},
            {'name': 'file', 'type': File, 'doc': 'a pre-existing h5py.File object', 'default': None})
    def __init__(self, **kwargs):
        '''Open an HDF5 file for IO

        For `mode`, see `h5py.File _`.
github HDFGroup / hdf5-json / h5json / hdf5db.py View on Github external
np_shape = tuple(np_shape)  # for comparison with ndarray shape
                
        self.log.info("selection shape:" + str(np_shape))


        # need some special conversion for compound types --
        # each element must be a tuple, but the JSON decoder
        # gives us a list instead.
        if format != "binary" and len(dset.dtype) > 1 and type(data) in (list, tuple):
            data = self.toTuple(rank, data)
            #for i in range(len(data)):
            #    converted_data.append(self.toTuple(data[i]))
            #data = converted_data
        else:
            h5t_check = h5py.check_dtype(ref=dset.dtype)
            if h5t_check in (h5py.Reference, h5py.RegionReference):
                # convert data to data refs
                if format == "binary":
                    msg = "Only JSON is supported for for this data type"
                    self.log.info(msg)
                    raise IOError(errno.EINVAL, msg)  
                data = self.listToRef(data)
                    
        if format == "binary":
            if npoints*itemSize != len(data):
                msg = "Expected: " + str(npoints*itemSize) + " bytes, but got: " + str(len(data))
                self.log.info(msg)
                raise IOError(errno.EINVAL, msg)
            if dset.dtype.shape == ():
                arr = np.fromstring(data, dtype=dset.dtype)
                arr = arr.reshape(np_shape)  # conform to selection shape
            else: