Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def _write_audit(self, user, message, changed_version):
"""
Creates an audit entry, which is much like a snapshot in that
it references versions and provides some history of the changes made.
"""
audit = {'_id': bson.ObjectId(),
'user': user,
'message': message,
'symbol': changed_version.symbol
}
orig_version = changed_version.orig_version.version
new_version = changed_version.new_version.version
audit['orig_v'] = orig_version
audit['new_v'] = new_version
# Update the versions to contain the audit
mongo_retry(self._versions.update_many)({'symbol': changed_version.symbol,
'version': {'$in': [orig_version, new_version]}
},
{'$addToSet': {'parent': audit['_id']}})
# Create the audit entry
mongo_retry(self._audit.insert_one)(audit)
SEGMENT: chunk[SEGMENT]},
{'$set': chunk}, upsert=True))
else:
# already exists, dont need to update in mongo
previous_shas.remove(chunk[SHA])
if ops:
self._collection.bulk_write(ops, ordered=False)
if meta_ops:
self._mdata.bulk_write(meta_ops, ordered=False)
doc[CHUNK_COUNT] = chunk_count
doc[APPEND_COUNT] = 0
if previous_shas:
mongo_retry(self._collection.delete_many)({SYMBOL: symbol, SHA: {'$in': list(previous_shas)}})
mongo_retry(self._symbols.update_one)({SYMBOL: symbol},
{'$set': doc},
upsert=True)
if audit is not None:
audit['symbol'] = symbol
audit['action'] = 'write'
audit['chunks'] = chunk_count
self._audit.insert_one(audit)
@mongo_retry
def read_history(self, symbol):
"""
Return all metadata saved for `symbol`
Parameters
----------
symbol : `str`
symbol name for the item
Returns
-------
pandas.DateFrame containing timestamps and metadata entries
"""
find = self.find({'symbol': symbol}, sort=[('start_time', pymongo.ASCENDING)])
times = []
entries = []
if old_metadata['start_time'] <= start_time:
raise ValueError('start_time={} is later than the first metadata @{}'.format(start_time,
old_metadata['start_time']))
if old_metadata['metadata'] == metadata:
self.find_one_and_update({'symbol': symbol}, {'$set': {'start_time': start_time}},
sort=[('start_time', pymongo.ASCENDING)])
old_metadata['start_time'] = start_time
return old_metadata
end_time = old_metadata.get('start_time')
else:
end_time = None
document = {'_id': bson.ObjectId(), 'symbol': symbol, 'metadata': metadata, 'start_time': start_time}
if end_time is not None:
document['end_time'] = end_time
mongo_retry(self.insert_one)(document)
logger.debug('Finished writing metadata for %s', symbol)
return document
def initialize_library(cls, arctic_lib, **kwargs):
logger.info("Creating BSONStore without sharding. Use BSONStore.enable_sharding to "
"enable sharding for large amounts of data.")
c = arctic_lib.get_top_level_collection()
if c.name not in mongo_retry(c.database.list_collection_names)():
mongo_retry(c.database.create_collection)(c.name)
else:
logger.warning("Collection %s already exists", c.name)
@mongo_retry
def find_one_and_update(self, filter, update, **kwargs):
"""
See http://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.find_one_and_update
"""
self._arctic_lib.check_quota()
return self._collection.find_one_and_update(filter, update, **kwargs)
audit: dict
audit information
"""
sym = self._get_symbol_info(from_symbol)
if not sym:
raise NoDataFoundException('No data found for %s' % (from_symbol))
if self._get_symbol_info(to_symbol) is not None:
raise Exception('Symbol %s already exists' % (to_symbol))
mongo_retry(self._collection.update_many)({SYMBOL: from_symbol},
{'$set': {SYMBOL: to_symbol}})
mongo_retry(self._symbols.update_one)({SYMBOL: from_symbol},
{'$set': {SYMBOL: to_symbol}})
mongo_retry(self._mdata.update_many)({SYMBOL: from_symbol},
{'$set': {SYMBOL: to_symbol}})
mongo_retry(self._audit.update_many)({'symbol': from_symbol},
{'$set': {'symbol': to_symbol}})
if audit is not None:
audit['symbol'] = to_symbol
audit['action'] = 'symbol rename'
audit['old_symbol'] = from_symbol
self._audit.insert_one(audit)
@mongo_retry
def insert_one(self, document, **kwargs):
"""
See http://api.mongodb.com/python/current/api/pymongo/collection.html#pymongo.collection.Collection.insert_one
"""
self._arctic_lib.check_quota()
return self._collection.insert_one(document, **kwargs)
@mongo_retry
def set_library_metadata(self, field, value):
self._library_coll[self.arctic.METADATA_COLL].update_one({'_id': self.arctic.METADATA_DOC_ID},
{'$set': {field: value}}, upsert=True)
# and the new version's "base_version_id" won't be referenced by any segments.
# Do a naive check for concurrent mods
lastv_seqn = self._last_version_seqnum(symbol)
if lastv_seqn != new_version['version']:
raise OperationFailure("The symbol {} has been modified concurrently ({} != {})".format(
symbol, lastv_seqn, new_version['version']))
# Insert the new version into the version DB
# (must come before the pruning, otherwise base version won't be preserved)
self._insert_version(new_version)
# Check if in the meanwhile the reference version (based on which we updated incrementally) has been removed
last_look = self._versions.find_one({'_id': reference_version['_id']})
if last_look is None or last_look.get('deleted'):
# Revert the change
mongo_retry(self._versions.delete_one)({'_id': new_version['_id']})
# Indicate the failure
raise OperationFailure("Failed to write metadata for symbol %s. "
"The previous version (%s, %d) has been removed during the update" %
(symbol, str(reference_version['_id']), reference_version['version']))
if prune_previous_version and reference_version:
self._prune_previous_versions(symbol, new_version_shas=new_version.get(FW_POINTERS_REFS_KEY))
logger.debug('Finished updating versions with new metadata for %s', symbol)
return VersionedItem(symbol=symbol, library=self._arctic_lib.get_name(), version=new_version['version'],
metadata=new_version.get('metadata'), data=None,
host=self._arctic_lib.arctic.mongo_host)