Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# access database
db = client[database]
# access collection
coll = db[collection]
# find all the matching records
found = coll.find(find)
if limit is not None:
found = found.limit(limit)
# populate ODSs
results = {}
for record in found:
ods = ODS(consistency_check=consistency_check, imas_version=imas_version)
_id = record['_id']
del record['_id']
ods.from_structure(record)
results[_id] = ods
return results
:param info: output of omas_info_node
:param consistency_check: True, False, 'warn'
:param imas_version: IMAS version
:return: value
'''
# force type consistent with data dictionary
txt = ''
if numpy.atleast_1d(is_uncertain(value)).any() or 'data_type' not in info:
pass
elif isinstance(value, numpy.ndarray):
if 'STRUCT_ARRAY' in info['data_type'] and not len(value):
value = ODS()
value.omas_data = []
elif 'FLT' in info['data_type']:
value = value.astype(float)
elif 'INT' in info['data_type']:
value = value.astype(int)
elif 'STR' in info['data_type']:
value = value.astype(str)
elif isinstance(value, (int, float, numpy.integer, numpy.floating)):
if 'FLT' in info['data_type']:
value = float(value)
elif 'INT' in info['data_type']:
value = int(value)
elif 'STR' in info['data_type']:
value = str(value)
elif isinstance(value, bytes):
if 'STR' in info['data_type']:
if not omas_rcparams['allow_fake_imas_fallback']:
raise
filename = os.sep.join([omas_rcparams['fake_imas_dir'], '%s_%s_%d_%d_v%s.pkl' % (user, machine, pulse, run, imas_versions.get(imas_version, imas_version))])
printe('Overloaded load_omas_imas: %s' % filename)
from . import load_omas_pkl
ods = load_omas_pkl(filename, consistency_check=False)
else:
try:
# see what paths have data
# NOTE: this is where the IDS.get operation occurs
fetch_paths, joined_fetch_paths = infer_fetch_paths(ids, occurrence=occurrence, paths=paths, time=time,
imas_version=imas_version, verbose=verbose)
# build omas data structure
ods = ODS(imas_version=imas_version, consistency_check=False)
for k, path in enumerate(fetch_paths):
if path[-1].endswith('_error_upper') or path[-1].endswith('_error_lower') or path[-1].endswith('_error_index'):
continue
if verbose and (k % int(numpy.ceil(len(fetch_paths) / 10)) == 0 or k == len(fetch_paths) - 1):
print('Loading {0:3.1f}%'.format(100 * float(k) / (len(fetch_paths) - 1)))
# get data from IDS
data = imas_get(ids, path, None)
# continue for empty data
if data is None:
continue
# add uncertainty
if not skip_uncertainties and l2i(path[:-1] + [path[-1] + '_error_upper']) in joined_fetch_paths:
stdata = imas_get(ids, path[:-1] + [path[-1] + '_error_upper'], None)
if stdata is not None:
try:
data = uarray(data, stdata)
requested_paths = [[structure] for structure in list_structures(imas_version=imas_version)]
else:
requested_paths = map(p2l, paths)
available_ds = []
for ds in numpy.unique([p[0] for p in requested_paths]):
if uda_get(client, [ds, 'ids_properties', 'homogeneous_time'], pulse, run) is None:
if verbose:
print('- ', ds)
continue
if verbose:
print('* ', ds)
available_ds.append(ds)
ods = ODS(consistency_check=False)
for k, ds in enumerate(available_ds):
filled_paths_in_uda(ods, client, pulse, run, load_structure(ds, imas_version=imas_version)[1],
path=[], paths=[], requested_paths=requested_paths, skip_uncertainties=skip_uncertainties,
perc=[float(k) / len(available_ds) * 100, float(k + 1) / len(available_ds) * 100, float(k) / len(available_ds) * 100])
ods.consistency_check = True
ods.prune()
if verbose:
print()
return ods
:param recursive: traverse the dictionary
:param lists_as_dicts: convert lists to dictionaries with integer strings
:param compression: gzip compression level
'''
import h5py
if isinstance(filename, str):
with h5py.File(filename, 'w') as g:
dict2hdf5(g, dictin, recursive=recursive, lists_as_dicts=lists_as_dicts, compression=compression)
return
else:
parent = filename
if isinstance(dictin, ODS):
dictin = dictin.omas_data
if groupname:
g = parent.create_group(groupname)
else:
g = parent
for key, item in list(dictin.items()):
if isinstance(item, ODS):
item = item.omas_data
if isinstance(item, dict):
if recursive:
dict2hdf5(g, item, key, recursive=recursive, lists_as_dicts=lists_as_dicts, compression=compression)
def load_omas_h5(filename, consistency_check=True):
"""
Load OMAS data set from HDF5
:param filename: filename or file descriptor to load from
:param consistency_check: verify that data is consistent with IMAS schema
:return: OMAS data set
"""
import h5py
ods = ODS(consistency_check=False)
with h5py.File(filename, 'r') as data:
convertDataset(ods, data)
ods.set_child_locations()
ods.consistency_check = consistency_check
return ods
Test save and load OMAS MongoDB
:param ods: ods
:return: ods
"""
if method == 'function':
_id = save_omas_mongo(ods, collection='test', database='test')
results = load_omas_mongo({'_id': _id}, collection='test', database='test')
if len(results) != 1:
raise Exception('through_omas_mongo failed')
ods1 = list(results.values())[0]
return ods1
else:
_id = ods.save('mongo', collection='test', database='test')
ods1 = ODS().load('mongo', {'_id': _id}, collection='test', database='test')
return ods1
propagate_requested_paths = [p[1:] for p in requested_paths if len(p) > 1 and (kid == p[0] or p[0] == ':')]
else:
continue
# recursive call
pp0 = perc[0] + k / n * (perc[1] - perc[0])
pp1 = perc[0] + (k + 1) / n * (perc[1] - perc[0])
pp2 = perc[2]
paths = filled_paths_in_uda(ods[kid], client, pulse, run, ds[kkid], propagate_path, [], propagate_requested_paths, skip_uncertainties, [pp0, pp1, pp2])
# generate uncertain data
if not skip_uncertainties and isinstance(ods.omas_data, dict):
for kid in list(ods.omas_data.keys()):
if kid.endswith('_error_upper') and kid[:-len('_error_upper')] in ods.omas_data:
try:
if isinstance(ods[kid], ODS):
pass
elif isinstance(ods[kid], float):
ods[kid[:-len('_error_upper')]] = ufloat(ods[kid[:-len('_error_upper')]], ods[kid])
else:
ods[kid[:-len('_error_upper')]] = uarray(ods[kid[:-len('_error_upper')]], ods[kid])
del ods[kid]
except Exception as _excp:
printe('Error loading uncertain data: %s' % kid)
return paths