Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_recover_xlog(self, rsync_pg_mock, cm_mock, tmpdir):
"""
Test the recovery of the xlogs of a backup
:param rsync_pg_mock: Mock rsync object for the purpose if this test
"""
# Build basic folders/files structure
dest = tmpdir.mkdir('destination')
wals = tmpdir.mkdir('wals')
# Create 3 WAL files with different compressions
xlog_dir = wals.mkdir(xlog.hash_dir('000000000000000000000002'))
xlog_plain = xlog_dir.join('000000000000000000000001')
xlog_gz = xlog_dir.join('000000000000000000000002')
xlog_bz2 = xlog_dir.join('000000000000000000000003')
xlog_plain.write('dummy content')
xlog_gz.write('dummy content gz')
xlog_bz2.write('dummy content bz2')
server = testing_helpers.build_real_server(
main_conf={'wals_directory': wals.strpath})
# Prepare compressors mock
c = {
'gzip': mock.Mock(name='gzip'),
'bzip2': mock.Mock(name='bzip2'),
}
cm_mock.return_value.get_compressor = \
lambda compression=None: c[compression]
# touch destination files to avoid errors on cleanup
wal_file = incoming_dir.join(wal_name)
wal_file.ensure()
archive_dir.ensure(dir=True)
xlog_db.ensure()
backup_manager.server.xlogdb.return_value.__enter__.return_value = \
xlog_db.open(mode='a')
backup_manager.server.archivers = [FileWalArchiver(backup_manager)]
backup_manager.archive_wal()
# Check that the WAL file is present inside the wal catalog
with xlog_db.open() as f:
line = str(f.readline())
assert wal_name in line
wal_path = os.path.join(archive_dir.strpath,
barman.xlog.hash_dir(wal_name),
wal_name)
# Check that the wal file have been archived
assert os.path.exists(wal_path)
# Check the output for the archival of the wal file
out, err = capsys.readouterr()
assert ("\t%s\n" % wal_name) in out
def test_hash_dir(self):
assert xlog.hash_dir(
'000000000000000200000001') == '0000000000000002'
assert xlog.hash_dir(
'000000010000000000000002') == '0000000100000000'
assert xlog.hash_dir(
'test/000000020000000100000000') == '0000000200000001'
assert xlog.hash_dir(
'00000001.history') == ''
assert xlog.hash_dir(
'00000002.history') == ''
assert xlog.hash_dir(
'00000001000000000000000A.00000020.backup') == '0000000100000000'
assert xlog.hash_dir(
'00000002000000050000000A.00000020.backup') == '0000000200000005'
with pytest.raises(barman.exceptions.BadXlogSegmentName):
xlog.hash_dir('00000000000000000000000')
with pytest.raises(barman.exceptions.BadXlogSegmentName):
xlog.hash_dir('0000000000000000000000000')
with pytest.raises(barman.exceptions.BadXlogSegmentName):
xlog.hash_dir('000000000000X00000000000')
def get_wal_full_path(self, wal_name):
"""
Build the full path of a WAL for a server given the name
:param wal_name: WAL file name
"""
# Build the path which contains the file
hash_dir = os.path.join(self.config.wals_directory,
xlog.hash_dir(wal_name))
# Build the WAL file full path
full_path = os.path.join(hash_dir, wal_name)
return full_path
try:
timeline = name[0:8]
int(timeline, 16)
except ValueError:
continue
# If this timeline already has a file, skip this directory
if timeline in timelines:
continue
hash_dir = fullname
# Inspect contained files in reverse order
for wal_name in sorted(os.listdir(hash_dir), reverse=True):
fullname = join(hash_dir, wal_name)
# Return the first file that has the correct name
if not isdir(fullname) and xlog.is_wal_file(fullname):
timelines[timeline] = comp_manager.get_wal_file_info(
fullname)
break
# Return the timeline map
return timelines
removed = []
with self.server.xlogdb() as fxlogdb:
xlogdb_new = fxlogdb.name + ".new"
with open(xlogdb_new, 'w') as fxlogdb_new:
for line in fxlogdb:
wal_info = WalFileInfo.from_xlogdb_line(line)
if not xlog.is_any_xlog_file(wal_info.name):
output.error(
"invalid WAL segment name %r\n"
"HINT: Please run \"barman rebuild-xlogdb %s\" "
"to solve this issue",
wal_info.name, self.config.name)
continue
# Keeps the WAL segment if it is a history file
keep = xlog.is_history_file(wal_info.name)
# Keeps the WAL segment if its timeline is in
# `timelines_to_protect`
if timelines_to_protect:
tli, _, _ = xlog.decode_segment_name(wal_info.name)
keep |= tli in timelines_to_protect
# Keeps the WAL segment if it is a newer
# than the given backup (the first available)
if backup_info and backup_info.begin_wal is not None:
keep |= wal_info.name >= backup_info.begin_wal
# If the file has to be kept write it in the new xlogdb
# otherwise delete it and record it in the removed list
if keep:
fxlogdb_new.write(wal_info.to_xlogdb_line())
# ignore the xlogdb and its lockfile
if name.startswith(self.server.XLOG_DB):
continue
fullname = join(root, name)
if isdir(fullname):
# all relevant files are in subdirectories
hash_dir = fullname
for wal_name in sorted(os.listdir(hash_dir)):
fullname = join(hash_dir, wal_name)
if isdir(fullname):
_logger.warning(
'unexpected directory '
'rebuilding the wal database: %s',
fullname)
else:
if xlog.is_wal_file(fullname):
wal_count += 1
elif xlog.is_backup_file(fullname):
label_count += 1
elif fullname.endswith('.tmp'):
_logger.warning(
'temporary file found '
'rebuilding the wal database: %s',
fullname)
continue
else:
_logger.warning(
'unexpected file '
'rebuilding the wal database: %s',
fullname)
continue
wal_info = comp_manager.get_wal_file_info(
if self.server.postgres.get_setting('data_directory') and \
self.server.postgres.get_setting('archive_command'):
cmd = UnixRemoteCommand(self.ssh_command,
self.ssh_options,
path=self.server.path)
# Here the name of the PostgreSQL WALs directory is
# hardcoded, but that doesn't represent a problem as
# this code runs only for PostgreSQL < 9.4
archive_dir = os.path.join(
self.server.postgres.get_setting('data_directory'),
'pg_xlog', 'archive_status')
out = str(cmd.list_dir_content(archive_dir, ['-t']))
for line in out.splitlines():
if line.endswith('.done'):
name = line[:-5]
if xlog.is_any_xlog_file(name):
remote_status['last_archived_wal'] = name
break
except (PostgresConnectionError, FsOperationFailed) as e:
_logger.warning("Error retrieving PostgreSQL status: %s", e)
return remote_status
def get_children_timelines(self, tli, forked_after=None):
"""
Get a list of the children of the passed timeline
:param int tli: Id of the timeline to check
:param str forked_after: XLog location after which the timeline
must have been created
:return List[xlog.HistoryFileData]: the list of timelines that
have the timeline with id 'tli' as parent
"""
comp_manager = self.backup_manager.compression_manager
if forked_after:
forked_after = xlog.parse_lsn(forked_after)
children = []
# Search all the history files after the passed timeline
children_tli = tli
while True:
children_tli += 1
history_path = os.path.join(self.config.wals_directory,
"%08X.history" % children_tli)
# If the file doesn't exists, stop searching
if not os.path.exists(history_path):
break
# Create the WalFileInfo object using the file
wal_info = comp_manager.get_wal_file_info(history_path)
# Get content of the file. We need to pass a compressor manager
# here to handle an eventual compression of the history file
children = []
# Search all the history files after the passed timeline
children_tli = tli
while True:
children_tli += 1
history_path = os.path.join(self.config.wals_directory,
"%08X.history" % children_tli)
# If the file doesn't exists, stop searching
if not os.path.exists(history_path):
break
# Create the WalFileInfo object using the file
wal_info = comp_manager.get_wal_file_info(history_path)
# Get content of the file. We need to pass a compressor manager
# here to handle an eventual compression of the history file
history_info = xlog.decode_history_file(
wal_info,
self.backup_manager.compression_manager)
# Save the history only if is reachable from this timeline.
for tinfo in history_info:
# The history file contains the full genealogy
# but we keep only the line with `tli` timeline as parent.
if tinfo.parent_tli != tli:
continue
# We need to return this history info only if this timeline
# has been forked after the passed LSN
if forked_after and tinfo.switchpoint < forked_after:
continue
children.append(tinfo)