How to use the barman.xlog function in barman

To help you get started, we’ve selected a few barman examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github 2ndquadrant-it / barman / tests / test_recovery_executor.py View on Github external
def test_recover_xlog(self, rsync_pg_mock, cm_mock, tmpdir):
        """
        Test the recovery of the xlogs of a backup
        :param rsync_pg_mock: Mock rsync object for the purpose if this test
        """
        # Build basic folders/files structure
        dest = tmpdir.mkdir('destination')
        wals = tmpdir.mkdir('wals')
        # Create 3 WAL files with different compressions
        xlog_dir = wals.mkdir(xlog.hash_dir('000000000000000000000002'))
        xlog_plain = xlog_dir.join('000000000000000000000001')
        xlog_gz = xlog_dir.join('000000000000000000000002')
        xlog_bz2 = xlog_dir.join('000000000000000000000003')
        xlog_plain.write('dummy content')
        xlog_gz.write('dummy content gz')
        xlog_bz2.write('dummy content bz2')
        server = testing_helpers.build_real_server(
            main_conf={'wals_directory': wals.strpath})
        # Prepare compressors mock
        c = {
            'gzip': mock.Mock(name='gzip'),
            'bzip2': mock.Mock(name='bzip2'),
        }
        cm_mock.return_value.get_compressor = \
            lambda compression=None: c[compression]
        # touch destination files to avoid errors on cleanup
github 2ndquadrant-it / barman / tests / test_wal_archiver.py View on Github external
wal_file = incoming_dir.join(wal_name)
        wal_file.ensure()
        archive_dir.ensure(dir=True)
        xlog_db.ensure()
        backup_manager.server.xlogdb.return_value.__enter__.return_value = \
            xlog_db.open(mode='a')
        backup_manager.server.archivers = [FileWalArchiver(backup_manager)]

        backup_manager.archive_wal()

        # Check that the WAL file is present inside the wal catalog
        with xlog_db.open() as f:
            line = str(f.readline())
            assert wal_name in line
        wal_path = os.path.join(archive_dir.strpath,
                                barman.xlog.hash_dir(wal_name),
                                wal_name)
        # Check that the wal file have been archived
        assert os.path.exists(wal_path)
        # Check the output for the archival of the wal file
        out, err = capsys.readouterr()
        assert ("\t%s\n" % wal_name) in out
github 2ndquadrant-it / barman / tests / test_xlog.py View on Github external
def test_hash_dir(self):
        assert xlog.hash_dir(
            '000000000000000200000001') == '0000000000000002'
        assert xlog.hash_dir(
            '000000010000000000000002') == '0000000100000000'
        assert xlog.hash_dir(
            'test/000000020000000100000000') == '0000000200000001'
        assert xlog.hash_dir(
            '00000001.history') == ''
        assert xlog.hash_dir(
            '00000002.history') == ''
        assert xlog.hash_dir(
            '00000001000000000000000A.00000020.backup') == '0000000100000000'
        assert xlog.hash_dir(
            '00000002000000050000000A.00000020.backup') == '0000000200000005'
        with pytest.raises(barman.exceptions.BadXlogSegmentName):
            xlog.hash_dir('00000000000000000000000')
        with pytest.raises(barman.exceptions.BadXlogSegmentName):
            xlog.hash_dir('0000000000000000000000000')
        with pytest.raises(barman.exceptions.BadXlogSegmentName):
            xlog.hash_dir('000000000000X00000000000')
github 2ndquadrant-it / barman / barman / server.py View on Github external
def get_wal_full_path(self, wal_name):
        """
        Build the full path of a WAL for a server given the name

        :param wal_name: WAL file name
        """
        # Build the path which contains the file
        hash_dir = os.path.join(self.config.wals_directory,
                                xlog.hash_dir(wal_name))
        # Build the WAL file full path
        full_path = os.path.join(hash_dir, wal_name)
        return full_path
github 2ndquadrant-it / barman / barman / backup.py View on Github external
try:
                    timeline = name[0:8]
                    int(timeline, 16)
                except ValueError:
                    continue

                # If this timeline already has a file, skip this directory
                if timeline in timelines:
                    continue

                hash_dir = fullname
                # Inspect contained files in reverse order
                for wal_name in sorted(os.listdir(hash_dir), reverse=True):
                    fullname = join(hash_dir, wal_name)
                    # Return the first file that has the correct name
                    if not isdir(fullname) and xlog.is_wal_file(fullname):
                        timelines[timeline] = comp_manager.get_wal_file_info(
                            fullname)
                        break

        # Return the timeline map
        return timelines
github 2ndquadrant-it / barman / barman / backup.py View on Github external
removed = []
        with self.server.xlogdb() as fxlogdb:
            xlogdb_new = fxlogdb.name + ".new"
            with open(xlogdb_new, 'w') as fxlogdb_new:
                for line in fxlogdb:
                    wal_info = WalFileInfo.from_xlogdb_line(line)
                    if not xlog.is_any_xlog_file(wal_info.name):
                        output.error(
                            "invalid WAL segment name %r\n"
                            "HINT: Please run \"barman rebuild-xlogdb %s\" "
                            "to solve this issue",
                            wal_info.name, self.config.name)
                        continue

                    # Keeps the WAL segment if it is a history file
                    keep = xlog.is_history_file(wal_info.name)

                    # Keeps the WAL segment if its timeline is in
                    # `timelines_to_protect`
                    if timelines_to_protect:
                        tli, _, _ = xlog.decode_segment_name(wal_info.name)
                        keep |= tli in timelines_to_protect

                    # Keeps the WAL segment if it is a newer
                    # than the given backup (the first available)
                    if backup_info and backup_info.begin_wal is not None:
                        keep |= wal_info.name >= backup_info.begin_wal

                    # If the file has to be kept write it in the new xlogdb
                    # otherwise delete it  and record it in the removed list
                    if keep:
                        fxlogdb_new.write(wal_info.to_xlogdb_line())
github 2ndquadrant-it / barman / barman / backup.py View on Github external
# ignore the xlogdb and its lockfile
                    if name.startswith(self.server.XLOG_DB):
                        continue
                    fullname = join(root, name)
                    if isdir(fullname):
                        # all relevant files are in subdirectories
                        hash_dir = fullname
                        for wal_name in sorted(os.listdir(hash_dir)):
                            fullname = join(hash_dir, wal_name)
                            if isdir(fullname):
                                _logger.warning(
                                    'unexpected directory '
                                    'rebuilding the wal database: %s',
                                    fullname)
                            else:
                                if xlog.is_wal_file(fullname):
                                    wal_count += 1
                                elif xlog.is_backup_file(fullname):
                                    label_count += 1
                                elif fullname.endswith('.tmp'):
                                    _logger.warning(
                                        'temporary file found '
                                        'rebuilding the wal database: %s',
                                        fullname)
                                    continue
                                else:
                                    _logger.warning(
                                        'unexpected file '
                                        'rebuilding the wal database: %s',
                                        fullname)
                                    continue
                                wal_info = comp_manager.get_wal_file_info(
github 2ndquadrant-it / barman / barman / backup_executor.py View on Github external
if self.server.postgres.get_setting('data_directory') and \
                        self.server.postgres.get_setting('archive_command'):
                    cmd = UnixRemoteCommand(self.ssh_command,
                                            self.ssh_options,
                                            path=self.server.path)
                    # Here the name of the PostgreSQL WALs directory is
                    # hardcoded, but that doesn't represent a problem as
                    # this code runs only for PostgreSQL < 9.4
                    archive_dir = os.path.join(
                        self.server.postgres.get_setting('data_directory'),
                        'pg_xlog', 'archive_status')
                    out = str(cmd.list_dir_content(archive_dir, ['-t']))
                    for line in out.splitlines():
                        if line.endswith('.done'):
                            name = line[:-5]
                            if xlog.is_any_xlog_file(name):
                                remote_status['last_archived_wal'] = name
                                break
        except (PostgresConnectionError, FsOperationFailed) as e:
            _logger.warning("Error retrieving PostgreSQL status: %s", e)
        return remote_status
github 2ndquadrant-it / barman / barman / server.py View on Github external
def get_children_timelines(self, tli, forked_after=None):
        """
        Get a list of the children of the passed timeline

        :param int tli: Id of the timeline to check
        :param str forked_after: XLog location after which the timeline
          must have been created
        :return List[xlog.HistoryFileData]: the list of timelines that
          have the timeline with id 'tli' as parent
        """
        comp_manager = self.backup_manager.compression_manager

        if forked_after:
            forked_after = xlog.parse_lsn(forked_after)

        children = []
        # Search all the history files after the passed timeline
        children_tli = tli
        while True:
            children_tli += 1
            history_path = os.path.join(self.config.wals_directory,
                                        "%08X.history" % children_tli)
            # If the file doesn't exists, stop searching
            if not os.path.exists(history_path):
                break

            # Create the WalFileInfo object using the file
            wal_info = comp_manager.get_wal_file_info(history_path)
            # Get content of the file. We need to pass a compressor manager
            # here to handle an eventual compression of the history file
github 2ndquadrant-it / barman / barman / server.py View on Github external
children = []
        # Search all the history files after the passed timeline
        children_tli = tli
        while True:
            children_tli += 1
            history_path = os.path.join(self.config.wals_directory,
                                        "%08X.history" % children_tli)
            # If the file doesn't exists, stop searching
            if not os.path.exists(history_path):
                break

            # Create the WalFileInfo object using the file
            wal_info = comp_manager.get_wal_file_info(history_path)
            # Get content of the file. We need to pass a compressor manager
            # here to handle an eventual compression of the history file
            history_info = xlog.decode_history_file(
                wal_info,
                self.backup_manager.compression_manager)

            # Save the history only if is reachable from this timeline.
            for tinfo in history_info:
                # The history file contains the full genealogy
                # but we keep only the line with `tli` timeline as parent.
                if tinfo.parent_tli != tli:
                    continue

                # We need to return this history info only if this timeline
                # has been forked after the passed LSN
                if forked_after and tinfo.switchpoint < forked_after:
                    continue

                children.append(tinfo)