Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
if True:
for test_name, nda in test_data.items():
frames.append(hyperframe.FrameRecord.from_ndarray(hfid, test_name, nda))
if 'int' in test_name or 'float' in test_name:
test_series = nda.byteswap().newbyteorder()
frames.append(hyperframe.FrameRecord.from_ndarray(hfid, test_name+"_swapped", test_series))
file_link = hyperframe.FileLinkRecord(hfid, None, BUNDLE_URI_SCHEME+'Users/someuser/somefile.txt')
frames.append(hyperframe.FrameRecord(name='links', hframe_uuid=hfid,
type='LINK',
shape=(1,),
links=[file_link,]))
if hframes is not None:
frames.append(hyperframe.FrameRecord(name='hframes', hframe_uuid=hfid,
type='HFRAME',
shape=(len(hframes),),
hframes=hframes))
lr = _make_lineage_record(name, hfid)
hf = hyperframe.HyperFrameRecord(owner='vklartho', human_name=name, uuid=hfid, frames=frames, lin_obj=lr, tags=tags)
return hf
frames.append(hyperframe.FrameRecord(name='bytes_data', hframe_uuid=hfid,
type='INT32',
shape=(len(bytes_data),),
data=bytes_data))
# This code tests our ability to turn ndarrays into pb messages and back
if True:
for test_name, nda in test_data.items():
frames.append(hyperframe.FrameRecord.from_ndarray(hfid, test_name, nda))
if 'int' in test_name or 'float' in test_name:
test_series = nda.byteswap().newbyteorder()
frames.append(hyperframe.FrameRecord.from_ndarray(hfid, test_name+"_swapped", test_series))
file_link = hyperframe.FileLinkRecord(hfid, None, BUNDLE_URI_SCHEME+'Users/someuser/somefile.txt')
frames.append(hyperframe.FrameRecord(name='links', hframe_uuid=hfid,
type='LINK',
shape=(1,),
links=[file_link,]))
if hframes is not None:
frames.append(hyperframe.FrameRecord(name='hframes', hframe_uuid=hfid,
type='HFRAME',
shape=(len(hframes),),
hframes=hframes))
lr = _make_lineage_record(name, hfid)
hf = hyperframe.HyperFrameRecord(owner='vklartho', human_name=name, uuid=hfid, frames=frames, lin_obj=lr, tags=tags)
return hf
hframes (`HyperFrameRecords`): Optional hframes to attach
Returns:
(`HyperFrameRecord`)
"""
hfid = str(uuid.uuid1())
if tags is None:
tags = {'datagroup':'lab', 'description':'regress the covmat'}
frames = []
# Raw bytes
frames.append(hyperframe.FrameRecord(name='bytes_data', hframe_uuid=hfid,
type='INT32',
shape=(len(bytes_data),),
data=bytes_data))
# This code tests our ability to turn ndarrays into pb messages and back
if True:
for test_name, nda in test_data.items():
frames.append(hyperframe.FrameRecord.from_ndarray(hfid, test_name, nda))
if 'int' in test_name or 'float' in test_name:
test_series = nda.byteswap().newbyteorder()
frames.append(hyperframe.FrameRecord.from_ndarray(hfid, test_name+"_swapped", test_series))
file_link = hyperframe.FileLinkRecord(hfid, None, BUNDLE_URI_SCHEME+'Users/someuser/somefile.txt')
frames.append(hyperframe.FrameRecord(name='links', hframe_uuid=hfid,
type='LINK',
link_type = FileLinkRecord
elif file_paths[0].startswith('s3://'):
link_type = S3LinkRecord
elif file_paths[0].startswith('db://'):
_logger.error("Found string-based database reference[{}], use DBLink object instead.".format(file_paths[0]))
raise Exception("hyperframe:make_link_frame: error trying to copy in string-based database reference.")
else:
raise ValueError("Bad file paths -- cannot determine link type: example path {}".format(file_paths[0]))
if link_type is FileLinkRecord:
to_remove = "file://" + local_managed_path
elif link_type is S3LinkRecord:
assert remote_managed_path.startswith('s3://')
to_remove = remote_managed_path
frame = FrameRecord(name=name,
hframe_uuid=hfid,
type='LINK')
frame_uuid = frame.get_uuid()
file_paths = [common.BUNDLE_URI_SCHEME + os.path.relpath(fn, to_remove) for fn in file_paths]
links = [link_type(frame_uuid, None, fn) for fn in file_paths]
return frame.add_links(links)
def mod_frames(self, new_frames):
"""
Replace all frames of in-memory HFR
NOTE: HFRs are immutable. Never save this HFR unless you also (or will) call mod_uuid().
Args:
new_frames:
Returns:
hyperframe.HyperFrameRecord
"""
# reset the internal frame_cache and frame_dict
# these will be rebuilt on add_frames
self.frame_cache = defaultdict(FrameRecord)
self.frame_dict = {}
self.pb.ClearField('frames')
self.add_frames(new_frames)
return self._mod_finish()
def make_numpy_array(self):
"""
Create a np ndarray from native bytes in frame
Returns:
(`numpy.ndarray`)
"""
assert (self.pb.type != hyperframe_pb2.LINK)
assert (self.pb.type != hyperframe_pb2.HFRAME)
assert (self.pb.type != hyperframe_pb2.STRING)
dtype = np.dtype(FrameRecord.get_numpy_type(self.pb.type))
dtype = dtype.newbyteorder(FrameRecord.get_numpy_byteorder(self.pb.byteorder))
nda = np.frombuffer(self.pb.data, dtype=dtype)
nda = nda.reshape(self.pb.shape)
return nda
# print("Series_data {}".format(series_data))
# print("frame_type {}".format(frame_type))
# raise Exception("make_native_frame does not yet support non-string objects")
elif nda.dtype.type == np.unicode_ or nda.dtype.type == np.string_:
# If it's an ndarray containing scalar string types
frame_type = FrameRecord.get_proto_type(nda.dtype)
if len(nda.shape) == 0:
series_data = [nda.item()]
else:
series_data = nda
else:
frame_type = FrameRecord.get_proto_type(nda.dtype)
series_data = nda.tobytes()
frame = FrameRecord(name=name,
hframe_uuid=hfid,
type=frame_type,
byteorder=nda.dtype.byteorder,
shape=nda.shape,
data=series_data)
return frame
Add frames to the PB. If UUID, just add the UUID. If a FrameRecord,
then add both the UUID in the pb.frames map and the FrameRecord in the frame_cache.
Args:
frames (:list:'FrameRecord' or :list:tuple:str,str): List of Frames to append or list of tuples(str,uuid)
Returns:
Nothing
"""
for f in frames:
if isinstance(f, tuple):
k = f[0]
v = f[1]
elif isinstance(f, FrameRecord):
# TODO: REMOVE THIS DEPENDENCY -- means we need to be very careful about FR objects.
f.hframe_uuid = self.pb.uuid
k = f.pb.name
v = f.pb.uuid
self.frame_cache[f.pb.name] = f
else:
print("Unable to add frame with type {}: Data {}".format(type(f), f))
assert False
st = self.pb.frames.add()
st.k = k
st.v = v
self.frame_dict[k] = v
(`hyperframe.FrameRecord`)
"""
# Force everything to be ndarrays.
try:
if not isinstance(series_like, np.ndarray):
series_like = np.array(series_like[0:])
except TypeError:
series_like = np.array(series_like)
local_files_series = hyperframe.detect_local_fs_path(series_like)
if local_files_series is not None:
series_like = local_files_series
if hyperframe.FrameRecord.is_link_series(series_like):
"""
If src is s3 file
If s3 file not managed
if have remote:
copy in to remote
else:
copy in to local
if s3 file managed:
if have remote:
do nothing
else:
error
if src is local:
If file not managed
copy in to local
if file managed: