Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def __open_index(self, index_name, index_config=None):
start_time = time.time()
index = None
try:
# open the index
index = self.__indices.get(index_name)
if index is None:
self.__logger.debug('opening {0}'.format(index_name))
if index_config is None:
# set saved index config
with open(os.path.join(self.__file_storage.folder, self.get_index_config_file(index_name)),
'rb') as f:
self.__index_configs[index_name] = pickle.loads(f.read())
else:
# set given index config
self.__index_configs[index_name] = index_config
if self.__index_configs[index_name].get_storage_type() == 'ram':
index = self.__ram_storage.open_index(indexname=index_name,
schema=self.__index_configs[index_name].get_schema())
else:
index = self.__file_storage.open_index(indexname=index_name,
schema=self.__index_configs[index_name].get_schema())
self.__indices[index_name] = index
self.__logger.info('{0} has opened'.format(index_name))
# open the index writer
self.__open_writer(index_name)
with zipfile.ZipFile(filename, 'r') as zf:
# get file names in snapshot file
filenames = list(zf.namelist())
# get index names in snapshot file
index_names = []
pattern_toc = re.compile(r'^_(.+)_\d+\.toc$')
for f in filenames:
match = pattern_toc.search(f)
if match and match.group(1) not in index_names:
index_names.append(match.group(1))
for index_name in index_names:
# extract the index config first
zf.extract(self.get_index_config_file(index_name), path=self.__file_storage.folder)
index_config = pickle.loads(zf.read(self.get_index_config_file(index_name)))
# get index files
pattern_toc = re.compile(r'^_{0}_(\d+)\..+$'.format(index_name)) # ex) _myindex_0.toc
pattern_seg = re.compile(
r'^{0}_([a-z0-9]+)\..+$'.format(index_name)) # ex) myindex_zseabukc2nbpvh0u.seg
pattern_lock = re.compile(r'^{0}_WRITELOCK$'.format(index_name)) # ex) myindex_WRITELOCK
index_files = []
for file_name in filenames:
if re.match(pattern_toc, file_name):
index_files.append(file_name)
elif re.match(pattern_seg, file_name):
index_files.append(file_name)
elif re.match(pattern_lock, file_name):
index_files.append(file_name)
# extract the index files
def __parseChangeClusterRequest(self, command):
commandType = ord(command[:1])
if commandType != _COMMAND_TYPE.MEMBERSHIP:
return None
return pickle.loads(command[1:])
def __doApplyCommand(self, command):
commandType = ord(command[:1])
# Skip no-op and membership change commands
if commandType == _COMMAND_TYPE.VERSION:
ver = pickle.loads(command[1:])
if self.__selfCodeVersion < ver:
raise SyncObjExceptionWrongVer(ver)
oldVer = self.__enabledCodeVersion
self.__enabledCodeVersion = ver
callback = self.__conf.onCodeVersionChanged
self.__onSetCodeVersion(ver)
if callback is not None:
callback(oldVer, ver)
return
if commandType != _COMMAND_TYPE.REGULAR:
return
command = pickle.loads(command[1:])
args = []
kwargs = {
'_doApply': True,
}
def __deserialize(self, filename):
raft_data = None
with self.__lock:
try:
self.__logger.info('deserializer has started')
with zipfile.ZipFile(filename, 'r') as zf:
# extract the federation data
zf.extract('federation.bin', path=self.__data_dir)
self.__data = pickle.loads(zf.read('federation.bin'))
self.__logger.info('federation.bin has restored')
# restore the raft data
raft_data = pickle.loads(zf.read(RAFT_DATA_FILE))
self.__logger.info('raft.{0} has restored'.format(RAFT_DATA_FILE))
self.__logger.info('snapshot has restored')
except Exception as ex:
self.__logger.error('failed to restore indices: {0}'.format(ex))
finally:
self.__logger.info('deserializer has stopped')
return raft_data