Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def _encValKey(v):
'''
Encode a value as used in a key.
Non-negative numbers are msgpack encoded. Negative numbers are encoded as a marker, then the
encoded negative of that value, so that the ordering of the encodings is easily mapped to the
ordering of the negative numbers. Strings too long are hashed. Note that this scheme prevents
interleaving of value types: all string encodings compare larger than all negative number
encodings compare larger than all nonnegative encodings.
'''
if isinstance(v, int):
if v >= 0:
return s_msgpack.en(v)
else:
return NEGATIVE_VAL_MARKER_ENC + s_msgpack.en(-v)
else:
if len(v) >= LARGE_STRING_SIZE:
return (HASH_VAL_MARKER_ENC + s_msgpack.en(xxhash.xxh64(v).intdigest()))
else:
return STRING_VAL_MARKER_ENC + s_msgpack.en(v)
for layr in self.layers:
async with self.getTempSlab() as slab:
i = 0
async for buid, valu in layr.iterPropRows(prop.form.name, prop.name):
i = i + 1
if i % modulus == 0:
logger.debug(f'Consumed {prop.form.name} count: {i}')
norm, info = prop.type.norm(valu)
if norm == valu:
continue
slab.put(buid, s_msgpack.en(norm))
if i:
logger.debug(f'Total {prop.form.name} count: {i}')
i = 0
for buid, byts in slab.scanByFull():
norm = s_msgpack.un(byts)
await layr.storPropSet(buid, prop, norm)
i = i + 1
if i % modulus == 0:
logger.debug(f'Set prop count: {i}')
if i:
logger.debug(f'Total propset count: {i}')
logger.debug(f'Done norming: {prop.form.name}')
def _storPropSetCommon(self, buid, penc, bpkey, pvpref, univ, valu, indx):
bpval = s_msgpack.en((valu, indx))
pvvalu = s_msgpack.en((buid,))
byts = self.layrslab.replace(bpkey, bpval, db=self.bybuid)
if byts is not None:
oldv, oldi = s_msgpack.un(byts)
if oldi is not None:
if isinstance(oldi, bytes):
self.layrslab.delete(pvpref + oldi, pvvalu, db=self.byprop)
if univ:
self.layrslab.delete(penc + oldi, pvvalu, db=self.byuniv)
else:
for oldibyts in oldi:
self.layrslab.delete(pvpref + oldibyts, pvvalu, db=self.byprop)
def encodeMsg(self, mesg):
'''Get byts for a message'''
fmt = self.locs.get('log:fmt')
if fmt == 'jsonl':
s = json.dumps(mesg, sort_keys=True) + '\n'
buf = s.encode()
return buf
elif fmt == 'mpk':
buf = s_msgpack.en(mesg)
return buf
mesg = f'Unknown encoding format: {fmt}'
raise s_exc.SynErr(mesg=mesg)
def set(self, prop, valu):
'''
Set a property in the KvLook.
Args:
prop (str): The property name to set.
valu (obj): A msgpack compatible value.
Returns:
None
'''
lkey = self.iden + prop.encode('utf8')
self.stor.setKvProp(lkey, s_msgpack.en(valu))
dependent. In purely memory backed cortexes, this KV store may not be
persistent, even if the tufo-layer is persistent, through something
such as the savefile mechanism.
Notes:
Data which is stored in the KV store is msgpacked, so caveats with
that apply.
Args:
key (str): Name of the value to store.
valu: Value to store in the KV store.
Returns:
The input value, unchanged.
'''
buf = s_msgpack.en(valu)
self._setBlobValu(key, buf)
self.savebus.fire('syn:core:blob:set', key=key, valu=buf)
return valu
if not curs.set_key(formname.encode('utf8')):
continue
logger.info('Stage 2a: (%2d/%2d) processing nodes from %s', i + 1, len(self.first_forms), formname)
for enc_iden in curs.iternext_dup():
props = None
try:
with txn.cursor(self.iden_tbl) as curs:
if not curs.set_key(enc_iden):
raise ConsistencyError('missing iden', hexlify(s_msgpack.un(enc_iden)))
props = self._get_props_from_cursor(txn, curs)
node = self.convert_props(props)
if node is None:
logger.debug('cannot convert %s', props)
continue
if formname == 'file:bytes':
if not txn.put(props['node:ndef'].encode('utf8'), s_msgpack.en(node[0]), db=self.comp_tbl):
raise ConsistencyError('put failure')
if not txn.put(props[formname].encode('utf8'), s_msgpack.en(node[0]), db=self.comp_tbl):
raise ConsistencyError('put failure')
else:
txn.put(props[formname].encode('utf8'), s_msgpack.en(node[0]), db=self.comp_tbl)
self.write_node_to_file(node)
except Exception:
logger.debug('Failed on processing node of form %s', formname, exc_info=True)
if props is not None and self.rejects_fh is not None:
self.rejects_fh.write(s_msgpack.en(props))
comp_node_time = time.time()
logger.debug('Stage 2a complete in %.1fs', comp_node_time - start_time)
def _calcFirstLastKeys(prop, valu, mintime, maxtime):
'''
Returns the encoded bytes for the start and end keys to the pt or pvt
index. Helper function for _{get,del}RowsByProp
'''
p_enc = _encProp(prop)
v_key_enc = b'' if valu is None else _encValKey(valu)
v_is_hashed = valu is not None and (v_key_enc[0] == HASH_VAL_MARKER_ENC)
if mintime is None and maxtime is None:
return (p_enc + v_key_enc, None, v_is_hashed, True)
mintime_enc = b'' if mintime is None else s_msgpack.en(mintime)
maxtime_enc = MAX_TIME_ENC if maxtime is None else s_msgpack.en(maxtime)
first_key = p_enc + v_key_enc + mintime_enc
last_key = p_enc + v_key_enc + maxtime_enc
return (first_key, last_key, v_is_hashed, False)