Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
'00000001000000000000000A.00000020.backup')
assert xlog.is_any_xlog_file(
'test2/00000001000000000000000A.00000020.backup')
assert xlog.is_any_xlog_file(
'00000001000000000000000A.partial')
assert xlog.is_any_xlog_file(
'test2/00000001000000000000000A.partial')
assert xlog.is_any_xlog_file('00000002.history')
assert xlog.is_any_xlog_file('test3/00000002.history')
assert not xlog.is_any_xlog_file('00000000000000000000000')
assert not xlog.is_any_xlog_file('0000000000000000000000000')
assert not xlog.is_any_xlog_file('000000000000X00000000000')
assert not xlog.is_any_xlog_file('00000001000000000000000A.backup')
assert not xlog.is_any_xlog_file(
'test.00000001000000000000000A.00000020.backup')
assert not xlog.is_any_xlog_file(
'test.00000001000000000000000A.00000020.partial')
assert not xlog.is_any_xlog_file('00000001000000000000000A.history')
def test_partial_file(self):
assert not xlog.is_partial_file('000000000000000200000001')
assert xlog.is_partial_file('00000001000000000000000A.partial')
assert xlog.is_partial_file('test/00000001000000000000000A.partial')
assert not xlog.is_partial_file('00000002.history')
assert not xlog.is_partial_file('00000000000000000000000.partial')
assert not xlog.is_partial_file('0000000000000000000000000.partial')
assert not xlog.is_partial_file('000000000000X00000000000.partial')
assert not xlog.is_partial_file(
'00000001000000000000000A.00000020.partial')
assert not xlog.is_any_xlog_file(
'test.00000001000000000000000A.partial')
assert not xlog.is_partial_file('00000001.partial')
def test_is_any_xlog_file(self):
assert xlog.is_any_xlog_file('000000000000000200000001')
assert xlog.is_any_xlog_file('test1/000000000000000200000001')
assert xlog.is_any_xlog_file(
'00000001000000000000000A.00000020.backup')
assert xlog.is_any_xlog_file(
'test2/00000001000000000000000A.00000020.backup')
assert xlog.is_any_xlog_file(
'00000001000000000000000A.partial')
assert xlog.is_any_xlog_file(
'test2/00000001000000000000000A.partial')
assert xlog.is_any_xlog_file('00000002.history')
assert xlog.is_any_xlog_file('test3/00000002.history')
assert not xlog.is_any_xlog_file('00000000000000000000000')
assert not xlog.is_any_xlog_file('0000000000000000000000000')
assert not xlog.is_any_xlog_file('000000000000X00000000000')
assert not xlog.is_any_xlog_file('00000001000000000000000A.backup')
assert not xlog.is_any_xlog_file(
'test.00000001000000000000000A.00000020.backup')
assert not xlog.is_any_xlog_file(
'test.00000001000000000000000A.00000020.partial')
assert not xlog.is_any_xlog_file('00000001000000000000000A.history')
def test_is_any_xlog_file(self):
assert xlog.is_any_xlog_file('000000000000000200000001')
assert xlog.is_any_xlog_file('test1/000000000000000200000001')
assert xlog.is_any_xlog_file(
'00000001000000000000000A.00000020.backup')
assert xlog.is_any_xlog_file(
'test2/00000001000000000000000A.00000020.backup')
assert xlog.is_any_xlog_file(
'00000001000000000000000A.partial')
assert xlog.is_any_xlog_file(
'test2/00000001000000000000000A.partial')
assert xlog.is_any_xlog_file('00000002.history')
assert xlog.is_any_xlog_file('test3/00000002.history')
assert not xlog.is_any_xlog_file('00000000000000000000000')
assert not xlog.is_any_xlog_file('0000000000000000000000000')
assert not xlog.is_any_xlog_file('000000000000X00000000000')
assert not xlog.is_any_xlog_file('00000001000000000000000A.backup')
assert not xlog.is_any_xlog_file(
'test.00000001000000000000000A.00000020.backup')
assert not xlog.is_any_xlog_file(
'test.00000001000000000000000A.00000020.partial')
assert not xlog.is_any_xlog_file('00000001000000000000000A.history')
def test_is_any_xlog_file(self):
assert xlog.is_any_xlog_file('000000000000000200000001')
assert xlog.is_any_xlog_file('test1/000000000000000200000001')
assert xlog.is_any_xlog_file(
'00000001000000000000000A.00000020.backup')
assert xlog.is_any_xlog_file(
'test2/00000001000000000000000A.00000020.backup')
assert xlog.is_any_xlog_file(
'00000001000000000000000A.partial')
assert xlog.is_any_xlog_file(
'test2/00000001000000000000000A.partial')
assert xlog.is_any_xlog_file('00000002.history')
assert xlog.is_any_xlog_file('test3/00000002.history')
assert not xlog.is_any_xlog_file('00000000000000000000000')
assert not xlog.is_any_xlog_file('0000000000000000000000000')
assert not xlog.is_any_xlog_file('000000000000X00000000000')
assert not xlog.is_any_xlog_file('00000001000000000000000A.backup')
assert not xlog.is_any_xlog_file(
'test.00000001000000000000000A.00000020.backup')
assert not xlog.is_any_xlog_file(
'test.00000001000000000000000A.00000020.partial')
assert not xlog.is_any_xlog_file('00000001000000000000000A.history')
:param str|None output_directory: directory where to deposit the
WAL file
:param int|None peek: if defined list the next N WAL file
"""
# If used through SSH identify the client to add it to logs
source_suffix = ''
ssh_connection = os.environ.get('SSH_CONNECTION')
if ssh_connection:
# The client IP is the first value contained in `SSH_CONNECTION`
# which contains four space-separated values: client IP address,
# client port number, server IP address, and server port number.
source_suffix = ' (SSH host: %s)' % (ssh_connection.split()[0],)
# Sanity check
if not xlog.is_any_xlog_file(wal_name):
output.error("'%s' is not a valid wal file name%s",
wal_name, source_suffix)
return
# If peek is requested we only output a list of files
if peek:
# Get the next ``peek`` files following the provided ``wal_name``.
# If ``wal_name`` is not a simple wal file,
# we cannot guess the names of the following WAL files.
# So ``wal_name`` is the only possible result, if exists.
if xlog.is_wal_file(wal_name):
# We can't know what was the segment size of PostgreSQL WAL
# files at backup time. Because of this, we generate all
# the possible names for a WAL segment, and then we check
# if the requested one is included.
wal_peek_list = xlog.generate_segment_names(wal_name)
def main(args=None):
"""
The main script entry point
:param list[str] args: the raw arguments list. When not provided
it defaults to sys.args[1:]
"""
config = parse_arguments(args)
configure_logging(config)
# Validate the WAL file name before uploading it
if not is_any_xlog_file(config.wal_path):
logging.error('%s is an invalid name for a WAL file' % config.wal_path)
raise SystemExit(1)
try:
cloud_interface = CloudInterface(
url=config.destination_url,
encryption=config.encryption,
profile_name=config.profile,
endpoint_url=config.endpoint_url)
with closing(cloud_interface):
uploader = S3WalUploader(
cloud_interface=cloud_interface,
server_name=config.server_name,
compression=config.compression)
files = []
skip = []
errors = []
for file_name in file_names:
# Ignore temporary files
if file_name.endswith('.tmp'):
continue
# If the file doesn't exist, it has been renamed/removed while
# we were reading the directory. Ignore it.
if not os.path.exists(file_name):
continue
if not os.path.isfile(file_name):
errors.append(file_name)
elif xlog.is_partial_file(file_name):
skip.append(file_name)
elif xlog.is_any_xlog_file(file_name):
files.append(file_name)
else:
errors.append(file_name)
# In case of more than a partial file, keep the last
# and treat the rest as normal files
if len(skip) > 1:
partials = skip[:-1]
_logger.info('Archiving partial files for server %s: %s' %
(self.config.name,
", ".join([os.path.basename(f) for f in partials])))
files.extend(partials)
skip = skip[-1:]
# Keep the last full WAL file in case no partial file is present
elif len(skip) == 0 and files:
skip.append(files.pop())
# IMPORTANT: the list is sorted, and this allows us to know that the
# WAL stream we have is monotonically increasing. That allows us to
# verify that a backup has all the WALs required for the restore.
file_names = glob(os.path.join(
self.config.incoming_wals_directory, '*'))
file_names.sort()
# Process anything that looks like a valid WAL file. Anything
# else is treated like an error/anomaly
files = []
errors = []
for file_name in file_names:
# Ignore temporary files
if file_name.endswith('.tmp'):
continue
if xlog.is_any_xlog_file(file_name) and os.path.isfile(file_name):
files.append(file_name)
else:
errors.append(file_name)
# Build the list of WalFileInfo
wal_files = [WalFileInfo.from_file(f) for f in files]
return WalArchiverQueue(wal_files,
batch_size=batch_size,
errors=errors)
If timelines_to_protect list is passed, never remove a wal in one of
these timelines.
:param BackupInfo|None backup_info: the backup information structure
:param set timelines_to_protect: optional list of timelines
to protect
:return list: a list of removed WAL files
"""
removed = []
with self.server.xlogdb() as fxlogdb:
xlogdb_new = fxlogdb.name + ".new"
with open(xlogdb_new, 'w') as fxlogdb_new:
for line in fxlogdb:
wal_info = WalFileInfo.from_xlogdb_line(line)
if not xlog.is_any_xlog_file(wal_info.name):
output.error(
"invalid WAL segment name %r\n"
"HINT: Please run \"barman rebuild-xlogdb %s\" "
"to solve this issue",
wal_info.name, self.config.name)
continue
# Keeps the WAL segment if it is a history file
keep = xlog.is_history_file(wal_info.name)
# Keeps the WAL segment if its timeline is in
# `timelines_to_protect`
if timelines_to_protect:
tli, _, _ = xlog.decode_segment_name(wal_info.name)
keep |= tli in timelines_to_protect