How to use the disdat.hyperframe.FrameRecord function in disdat

To help you get started, we’ve selected a few disdat examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github kyocum / disdat / tests / non_pipelines / test_hframe.py View on Github external
if True:
        for test_name, nda in test_data.items():
            frames.append(hyperframe.FrameRecord.from_ndarray(hfid, test_name, nda))
            if 'int' in test_name or 'float' in test_name:
                test_series = nda.byteswap().newbyteorder()
                frames.append(hyperframe.FrameRecord.from_ndarray(hfid, test_name+"_swapped", test_series))

    file_link = hyperframe.FileLinkRecord(hfid, None, BUNDLE_URI_SCHEME+'Users/someuser/somefile.txt')

    frames.append(hyperframe.FrameRecord(name='links', hframe_uuid=hfid,
                                         type='LINK',
                                         shape=(1,),
                                         links=[file_link,]))

    if hframes is not None:
        frames.append(hyperframe.FrameRecord(name='hframes', hframe_uuid=hfid,
                               type='HFRAME',
                               shape=(len(hframes),),
                               hframes=hframes))

    lr = _make_lineage_record(name, hfid)

    hf = hyperframe.HyperFrameRecord(owner='vklartho', human_name=name, uuid=hfid, frames=frames, lin_obj=lr, tags=tags)

    return hf
github kyocum / disdat / tests / non_pipelines / test_hframe.py View on Github external
frames.append(hyperframe.FrameRecord(name='bytes_data', hframe_uuid=hfid,
                                         type='INT32',
                                         shape=(len(bytes_data),),
                                         data=bytes_data))

    # This code tests our ability to turn ndarrays into pb messages and back
    if True:
        for test_name, nda in test_data.items():
            frames.append(hyperframe.FrameRecord.from_ndarray(hfid, test_name, nda))
            if 'int' in test_name or 'float' in test_name:
                test_series = nda.byteswap().newbyteorder()
                frames.append(hyperframe.FrameRecord.from_ndarray(hfid, test_name+"_swapped", test_series))

    file_link = hyperframe.FileLinkRecord(hfid, None, BUNDLE_URI_SCHEME+'Users/someuser/somefile.txt')

    frames.append(hyperframe.FrameRecord(name='links', hframe_uuid=hfid,
                                         type='LINK',
                                         shape=(1,),
                                         links=[file_link,]))

    if hframes is not None:
        frames.append(hyperframe.FrameRecord(name='hframes', hframe_uuid=hfid,
                               type='HFRAME',
                               shape=(len(hframes),),
                               hframes=hframes))

    lr = _make_lineage_record(name, hfid)

    hf = hyperframe.HyperFrameRecord(owner='vklartho', human_name=name, uuid=hfid, frames=frames, lin_obj=lr, tags=tags)

    return hf
github kyocum / disdat / tests / non_pipelines / test_hframe.py View on Github external
hframes (`HyperFrameRecords`): Optional hframes to attach

    Returns:
        (`HyperFrameRecord`)

    """

    hfid = str(uuid.uuid1())

    if tags is None:
        tags = {'datagroup':'lab', 'description':'regress the covmat'}

    frames = []

    # Raw bytes
    frames.append(hyperframe.FrameRecord(name='bytes_data', hframe_uuid=hfid,
                                         type='INT32',
                                         shape=(len(bytes_data),),
                                         data=bytes_data))

    # This code tests our ability to turn ndarrays into pb messages and back
    if True:
        for test_name, nda in test_data.items():
            frames.append(hyperframe.FrameRecord.from_ndarray(hfid, test_name, nda))
            if 'int' in test_name or 'float' in test_name:
                test_series = nda.byteswap().newbyteorder()
                frames.append(hyperframe.FrameRecord.from_ndarray(hfid, test_name+"_swapped", test_series))

    file_link = hyperframe.FileLinkRecord(hfid, None, BUNDLE_URI_SCHEME+'Users/someuser/somefile.txt')

    frames.append(hyperframe.FrameRecord(name='links', hframe_uuid=hfid,
                                         type='LINK',
github kyocum / disdat / disdat / hyperframe.py View on Github external
link_type = FileLinkRecord
        elif file_paths[0].startswith('s3://'):
            link_type = S3LinkRecord
        elif file_paths[0].startswith('db://'):
            _logger.error("Found string-based database reference[{}], use DBLink object instead.".format(file_paths[0]))
            raise Exception("hyperframe:make_link_frame: error trying to copy in string-based database reference.")
        else:
            raise ValueError("Bad file paths -- cannot determine link type: example path {}".format(file_paths[0]))

        if link_type is FileLinkRecord:
            to_remove = "file://" + local_managed_path
        elif link_type is S3LinkRecord:
            assert remote_managed_path.startswith('s3://')
            to_remove = remote_managed_path

        frame = FrameRecord(name=name,
                            hframe_uuid=hfid,
                            type='LINK')

        frame_uuid = frame.get_uuid()

        file_paths = [common.BUNDLE_URI_SCHEME + os.path.relpath(fn, to_remove) for fn in file_paths]
        links = [link_type(frame_uuid, None, fn) for fn in file_paths]

        return frame.add_links(links)
github kyocum / disdat / disdat / hyperframe.py View on Github external
def mod_frames(self, new_frames):
        """
        Replace all frames of in-memory HFR

        NOTE: HFRs are immutable.  Never save this HFR unless you also (or will) call mod_uuid().

        Args:
            new_frames:

        Returns:
            hyperframe.HyperFrameRecord

        """
        # reset the internal frame_cache and frame_dict
        # these will be rebuilt on add_frames
        self.frame_cache = defaultdict(FrameRecord)
        self.frame_dict  = {}

        self.pb.ClearField('frames')
        self.add_frames(new_frames)

        return self._mod_finish()
github kyocum / disdat / disdat / hyperframe.py View on Github external
def make_numpy_array(self):
        """
        Create a np ndarray from native bytes in frame

        Returns:
            (`numpy.ndarray`)
        """

        assert (self.pb.type != hyperframe_pb2.LINK)
        assert (self.pb.type != hyperframe_pb2.HFRAME)
        assert (self.pb.type != hyperframe_pb2.STRING)

        dtype = np.dtype(FrameRecord.get_numpy_type(self.pb.type))
        dtype = dtype.newbyteorder(FrameRecord.get_numpy_byteorder(self.pb.byteorder))

        nda = np.frombuffer(self.pb.data, dtype=dtype)
        nda = nda.reshape(self.pb.shape)

        return nda
github kyocum / disdat / disdat / hyperframe.py View on Github external
# print("Series_data {}".format(series_data))
                # print("frame_type {}".format(frame_type))
                # raise Exception("make_native_frame does not yet support non-string objects")

        elif nda.dtype.type == np.unicode_ or nda.dtype.type == np.string_:
            # If it's an ndarray containing scalar string types
            frame_type = FrameRecord.get_proto_type(nda.dtype)
            if len(nda.shape) == 0:
                series_data = [nda.item()]
            else:
                series_data = nda
        else:
            frame_type = FrameRecord.get_proto_type(nda.dtype)
            series_data = nda.tobytes()

        frame = FrameRecord(name=name,
                            hframe_uuid=hfid,
                            type=frame_type,
                            byteorder=nda.dtype.byteorder,
                            shape=nda.shape,
                            data=series_data)

        return frame
github kyocum / disdat / disdat / hyperframe.py View on Github external
Add frames to the PB.  If UUID, just add the UUID.  If a FrameRecord,
        then add both the UUID in the pb.frames map and the FrameRecord in the frame_cache.

        Args:
            frames (:list:'FrameRecord' or :list:tuple:str,str): List of Frames to append or list of tuples(str,uuid)

        Returns:
            Nothing
        """

        for f in frames:
            if isinstance(f, tuple):
                k = f[0]
                v = f[1]
            elif isinstance(f, FrameRecord):
                # TODO: REMOVE THIS DEPENDENCY -- means we need to be very careful about FR objects.
                f.hframe_uuid = self.pb.uuid
                k = f.pb.name
                v = f.pb.uuid
                self.frame_cache[f.pb.name] = f
            else:
                print("Unable to add frame with type {}: Data {}".format(type(f), f))
                assert False

            st = self.pb.frames.add()
            st.k = k
            st.v = v
            self.frame_dict[k] = v
github kyocum / disdat / disdat / data_context.py View on Github external
(`hyperframe.FrameRecord`)

        """
        # Force everything to be ndarrays.
        try:
            if not isinstance(series_like, np.ndarray):
                series_like = np.array(series_like[0:])
        except TypeError:
            series_like = np.array(series_like)

        local_files_series = hyperframe.detect_local_fs_path(series_like)

        if local_files_series is not None:
            series_like = local_files_series

        if hyperframe.FrameRecord.is_link_series(series_like):
            """ 
            If src is s3 file
              If s3 file not managed
                if have remote: 
                  copy in to remote
                else:
                  copy in to local
              if s3 file managed:
                if have remote:
                  do nothing
                else:
                  error
            if src is local:
              If file not managed
                copy in to local
              if file managed: