Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def _recover_from_lastwriting(self, truncate_last_failure=True):
"""
recovery from last log writing
:raise Exception:
IOError, if any error happened
"""
folder = self._get_storage_dir()
files = self._get_ordered_logfiles(folder)
need_finish_file = False
if len(files) < 1:
log.info('no need recovery. Does not contain any files')
return
file_start_logid = -1
seek_file = None
if files[-1].find('writing') < 0:
# does not need recovery
log.info('does not have unfinished logfile, return')
file_start_logid = int(files[-1].split('.')[-1]) + 1
seek_file = files[-1]
log.info('next logid will be {0}'.format(self._logid))
else:
# need recovery, checkup, must <= 0 writing log file
count = 0
for fname in files:
if fname.find('writing') >= 0:
count += 1
if count > 1:
:raise:
cup.err.ConfigError if there's any config item missing
"""
ObjectInterface.__init__(self, config)
required_keys = ['uri', 'user', 'passwords']
if not self._validate_config(self._config, required_keys):
raise err.ConfigError(str(required_keys))
self._uri = self._config['uri']
self._user = self._config['user']
self._passwd = self._config['passwords']
self._extra = self._config['extra']
self._dufault_timeout = 30
if self._extra is not None and isinstance(self._config['extra'], int):
self._dufault_timeout = self._extra
log.info('to connect to ftp server')
self._ftp_con = ftplib.FTP()
self._host = self._uri.split(':')[1][2:]
self._port = ftplib.FTP_PORT
if len(self._uri.split(':')[2]) > 0:
self._port = int(self._uri.split(':')[2])
self._ftp_con.connect(self._host, self._port, self._dufault_timeout)
self._ftp_con.login(self._user, self._passwd)
self._last_optime = time.time()
self._timeout = 15 # idle time for ftp
def _get_ordered_logfiles(self, folder):
"""get log files in order"""
try:
files = sorted(os.listdir(folder), cmp=self.__cmp_logfile_id)
except TypeError:
import functools
files = sorted(os.listdir(folder), key=functools.cmp_to_key(
self.__cmp_logfile_id)
)
retfiles = []
for item in files:
if any([
len(item.split('.')) != 2,
item.find('done.') < 0 and item.find('writing.') < 0
]):
log.info('file name {0} invalid, will skip'.format(item))
continue
retfiles.append(item)
return retfiles
def stop(self):
"""stop the master"""
log.info('to stop the arrow master')
log.info('to stop control service')
self._control_service.stop()
log.info('arrow master stopped')
context info %s, errno:%s' %
(str(error), context.get_context_info(), err)
)
context.to_destroy()
break
except Exception as error:
log.error(
'Socket error happend, error:%s, context info %s, trace:%s' %
(str(error), context.get_context_info(), traceback.format_exc())
)
context.to_destroy()
break
finally:
del data
if msg.is_msg_already_sent():
log.info(
'sent out a msg uniqid:{0}'.format(
async_msg.netmsg_tostring(msg))
)
# if we have successfully send out a msg. Then move to next one
msg = context.try_move2next_sending_msg()
if msg is None:
break
return context
def lscp(
src, hostname, username, passwd, dst,
timeout=800, b_print_stdout=True
):
"""
copy [localhost]:src to [hostname]:[dst]
:return:
return a dict with keys ('exitstatus' 'remote_exitstatus' 'result')
"""
cmd = 'scp -r %s %s@%s:%s' % (src, username, hostname, dst)
log.info('{0}'.format(cmd))
return _do_expect_ex(passwd, cmd, timeout, b_print_stdout)
def cleanup_oldlost(self, dump_file=None):
"""
cleanup old lost devices.
:param dump_file:
if dump_file is not None, we will store devices info into dump_file
Otherwise, we will cleanup the lost devices only.
"""
self._lock.acquire()
log.info('start - empty_lost devices, dump_file:%s' % dump_file)
if self._lost_devices is None:
log.info('end - does not keep_lost devices, return')
self._lock.release()
return
if dump_file is None:
self._lost_devices = {}
log.info('end - does not have dump_file, return')
self._lock.release()
return
info_dict = {}
info_dict['devices'] = {}
if len(self._lost_devices) != 0:
info_dict['devices']['lost'] = []
info_dict['devices']['lost_num'] = len(self._lost_devices)
else:
info_dict['devices']['lost_num'] = 0
for dkey in self._lost_devices.keys():
try:
def handle(self, msg):
"""
handle function which should be implemented by
sub-class.
"""
log.info('handle in msgcenter')
def cleanup_oldlost(self, dump_file=None):
"""
cleanup old lost devices.
:param dump_file:
if dump_file is not None, we will store devices info into dump_file
Otherwise, we will cleanup the lost devices only.
"""
self._lock.acquire()
log.info('start - empty_lost devices, dump_file:%s' % dump_file)
if self._lost_devices is None:
log.info('end - does not keep_lost devices, return')
self._lock.release()
return
if dump_file is None:
self._lost_devices = {}
log.info('end - does not have dump_file, return')
self._lock.release()
return
info_dict = {}
info_dict['devices'] = {}
if len(self._lost_devices) != 0:
info_dict['devices']['lost'] = []
info_dict['devices']['lost_num'] = len(self._lost_devices)
else:
info_dict['devices']['lost_num'] = 0
def go_ex(
hostname, username, passwd, command='', timeout=800, b_print_stdout=True
):
"""
Run [command] on remote [hostname] and return result. If you have a lot
of escape sign in the command, recommand using go_with_scp
:param timeout:
execution timeout, by default 800 seconds
:return:
return a dict with keys ('exitstatus' 'remote_exitstatus' 'result')
"""
cmd = """ssh %s@%s '%s'""" % (username, hostname, command)
log.info('go_ex {0}'.format(cmd))
ret = _do_expect_ex(passwd, cmd, timeout, b_print_stdout)
return ret