ipatests: add test for ipa-restore in multi-master configuration

Test ensures that after ipa-restore on the master, the replica can be
re-synchronized and a new replica can be created.

https://pagure.io/freeipa/issue/7455

Reviewed-By: Christian Heimes <cheimes@redhat.com>
Reviewed-By: Tibor Dudlak <tdudlak@redhat.com>

ipatests/pytest_ipa/integration/tasks.py

@@ -1121,19 +1121,26 @@ def _entries_to_ldif(entries):
     return io.getvalue()
 
 
-def wait_for_replication(ldap, timeout=30):
+def wait_for_replication(ldap, timeout=30,
-    """Wait until updates on all replication agreements are done (or failed)
+                         target_status_re=r'^0 |^Error \(0\) ',
+                         raise_on_timeout=False):
+    """Wait for all replication agreements to reach desired state
 
+    With defaults waits until updates on all replication agreements are
+    done (or failed) and exits without exception
     :param ldap: LDAP client
         autenticated with necessary rights to read the mapping tree
     :param timeout: Maximum time to wait, in seconds
+    :param target_status_re: Regexp of status to wait for
+    :param raise_on_timeout: if True, raises AssertionError if status not
+        reached in specified time
 
     Note that this waits for updates originating on this host, not those
     coming from other hosts.
     """
     logger.debug('Waiting for replication to finish')
-    for i in range(timeout):
+    start = time.time()
-        time.sleep(1)
+    while True:
         status_attr = 'nsds5replicaLastUpdateStatus'
         progress_attr = 'nsds5replicaUpdateInProgress'
         entries = ldap.get_entries(
@@ -1141,25 +1148,24 @@ def wait_for_replication(ldap, timeout=30):
             filter='(objectclass=nsds5replicationagreement)',
             attrs_list=[status_attr, progress_attr])
         logger.debug('Replication agreements: \n%s', _entries_to_ldif(entries))
-        if any(
+        statuses = [entry.single_value[status_attr] for entry in entries]
-                not (
+        wrong_statuses = [s for s in statuses
-                    # older DS format
+                          if not re.match(target_status_re, s)]
-                    e.single_value[status_attr].startswith('0 ') or
-                    # newer DS format
-                    e.single_value[status_attr].startswith('Error (0) ')
-                )
-            for e in entries
-        ):
-            logger.error('Replication error')
-            continue
         if any(e.single_value[progress_attr] == 'TRUE' for e in entries):
-            logger.debug('Replication in progress (waited %s/%ss)',
+            msg = 'Replication not finished'
-                         i, timeout)
+            logger.debug(msg)
+        elif wrong_statuses:
+            msg = 'Unexpected replication status: %s' % wrong_statuses[0]
+            logger.debug(msg)
         else:
             logger.debug('Replication finished')
-            break
+            return
-    else:
+        if time.time() - start > timeout:
             logger.error('Giving up wait for replication to finish')
+            if raise_on_timeout:
+                raise AssertionError(msg)
+            break
+        time.sleep(1)
 
 
 def wait_for_cleanallruv_tasks(ldap, timeout=30):
@@ -1555,3 +1561,11 @@ def strip_cert_header(pem):
         return s.group(1)
     else:
         return pem
+
+
+def user_add(host, login):
+    host.run_command([
+        "ipa", "user-add", login,
+        "--first", "test",
+        "--last", "user"
+    ])

ipatests/test_integration/test_backup_and_restore.py

@@ -24,6 +24,7 @@ import os
 import re
 import contextlib
 from tempfile import NamedTemporaryFile
+import pytest
 
 from ipaplatform.constants import constants
 from ipaplatform.paths import paths
@@ -137,6 +138,8 @@ def restore_checker(host):
 
     yield
 
+    tasks.kinit_admin(host)
+
     for (check, assert_func), expected in zip(CHECKS, results):
         logger.info('Checking result for %s', check.__name__)
         got = check(host)
@@ -164,6 +167,24 @@ def backup(host):
         raise AssertionError('Backup directory not found in output')
 
 
+@pytest.yield_fixture(scope="function")
+def cert_sign_request(request):
+    master = request.instance.master
+    hosts = [master] + request.instance.replicas
+    csrs = {}
+    for host in hosts:
+        request_path = host.run_command(['mktemp']).stdout_text.strip()
+        openssl_command = [
+            'openssl', 'req', '-new', '-nodes', '-out', request_path,
+            '-subj', '/CN=' + master.hostname
+        ]
+        host.run_command(openssl_command)
+        csrs[host.hostname] = request_path
+    yield csrs
+    for host in hosts:
+        host.run_command(['rm', csrs[host.hostname]])
+
+
 class TestBackupAndRestore(IntegrationTest):
     topology = 'star'
 
@@ -446,36 +467,92 @@ class TestBackupReinstallRestoreWithKRA(BaseBackupAndRestoreWithKRA):
 
 
 class TestBackupAndRestoreWithReplica(IntegrationTest):
-    """Regression test for https://pagure.io/freeipa/issue/7234"""
+    """Regression tests for issues 7234 and 7455
-    num_replicas = 1
+
+    https://pagure.io/freeipa/issue/7234
+        - check that oddjobd service is started after restore
+        - check new replica  setup after restore
+    https://pagure.io/freeipa/issue/7455
+        check that after restore and replication reinitialization
+            - users and CA data are at state before backup
+            - CA can be installed on existing replica
+            - new replica with CA can be setup
+    """
+    num_replicas = 2
     topology = "star"
 
     @classmethod
     def install(cls, mh):
+        cls.replica1 = cls.replicas[0]
+        cls.replica2 = cls.replicas[1]
         if cls.domain_level is None:
             domain_level = cls.master.config.domain_level
         else:
             domain_level = cls.domain_level
-
+        # Configure only master and one replica.
-        if cls.topology is None:
+        # Replica is configured without CA
-            return
-        else:
         tasks.install_topo(
-                cls.topology, cls.master, [],
+            cls.topology, cls.master, [cls.replica1],
-                cls.clients, domain_level
+            cls.clients, domain_level,
+            setup_replica_cas=False
         )
 
-    def test_full_backup_and_restore_with_replica(self):
+    def get_users(self, host):
-        replica = self.replicas[0]
+        res = host.run_command(['ipa', 'user-find'])
+        users = set()
+        for line in res.stdout_text.splitlines():
+            k, _unused, v = line.strip().partition(': ')
+            if k == 'User login':
+                users.add(v)
+        return users
+
+    def check_replication_error(self, host):
+        status = r'Error \(19\) Replication error acquiring replica: ' \
+                 'Replica has different database generation ID'
+        tasks.wait_for_replication(
+            host.ldap_connect(), target_status_re=status,
+            raise_on_timeout=True)
+
+    def check_replication_success(self, host):
+        status = r'Error \(0\) Replica acquired successfully: ' \
+                 'Incremental update succeeded'
+        tasks.wait_for_replication(
+            host.ldap_connect(), target_status_re=status,
+            raise_on_timeout=True)
+
+    def request_test_service_cert(self, host, request_path,
+                                  expect_connection_error=False):
+        res = host.run_command([
+            'ipa', 'cert-request', '--principal=TEST/' + self.master.hostname,
+            request_path
+        ], raiseonerr=not expect_connection_error)
+        if expect_connection_error:
+            assert (1 == res.returncode and
+                    '[Errno 111] Connection refused' in res.stderr_text)
+
+    def test_full_backup_and_restore_with_replica(self, cert_sign_request):
+        # check prerequisites
+        self.check_replication_success(self.master)
+        self.check_replication_success(self.replica1)
+
+        self.master.run_command(
+            ['ipa', 'service-add', 'TEST/' + self.master.hostname])
+
+        tasks.user_add(self.master, 'test1_master')
+        tasks.user_add(self.replica1, 'test1_replica')
 
         with restore_checker(self.master):
             backup_path = backup(self.master)
 
-            logger.info("Backup path for %s is %s", self.master, backup_path)
+            # change data after backup
+            self.master.run_command(['ipa', 'user-del', 'test1_master'])
+            self.replica1.run_command(['ipa', 'user-del', 'test1_replica'])
+            tasks.user_add(self.master, 'test2_master')
+            tasks.user_add(self.replica1, 'test2_replica')
 
-            self.master.run_command([
+            # simulate master crash
-                "ipa-server-install", "--uninstall", "-U"
+            self.master.run_command(['ipactl', 'stop'])
-            ])
+            tasks.uninstall_master(self.master, clean=False)
 
             logger.info("Stopping and disabling oddjobd service")
             self.master.run_command([
@@ -485,18 +562,76 @@ class TestBackupAndRestoreWithReplica(IntegrationTest):
                 "systemctl", "disable", "oddjobd"
             ])
 
-            self.master.run_command(
+            self.master.run_command(['ipa-restore', '-U', backup_path])
-                ["ipa-restore", backup_path],
-                stdin_text='yes'
-            )
 
         status = self.master.run_command([
             "systemctl", "status", "oddjobd"
         ])
         assert "active (running)" in status.stdout_text
 
-        tasks.install_replica(self.master, replica)
+        # replication should not work after restoration
-        check_replication(self.master, replica, "testuser1")
+        # create users to force master and replica to try to replicate
+        tasks.user_add(self.master, 'test3_master')
+        tasks.user_add(self.replica1, 'test3_replica')
+        self.check_replication_error(self.master)
+        self.check_replication_error(self.replica1)
+        assert {'admin', 'test1_master', 'test1_replica', 'test3_master'} == \
+            self.get_users(self.master)
+        assert {'admin', 'test2_master', 'test2_replica', 'test3_replica'} == \
+            self.get_users(self.replica1)
+
+        # reestablish and check replication
+        self.replica1.run_command(['ipa-replica-manage', 're-initialize',
+                                  '--from', self.master.hostname])
+        # create users to force master and replica to try to replicate
+        tasks.user_add(self.master, 'test4_master')
+        tasks.user_add(self.replica1, 'test4_replica')
+        self.check_replication_success(self.master)
+        self.check_replication_success(self.replica1)
+        assert {'admin', 'test1_master', 'test1_replica',
+                'test3_master', 'test4_master', 'test4_replica'} == \
+            self.get_users(self.master)
+        assert {'admin', 'test1_master', 'test1_replica',
+                'test3_master', 'test4_master', 'test4_replica'} == \
+            self.get_users(self.replica1)
+
+        # CA on master should be accesible from master and replica
+        self.request_test_service_cert(
+            self.master, cert_sign_request[self.master.hostname])
+        self.request_test_service_cert(
+            self.replica1, cert_sign_request[self.replica1.hostname])
+
+        # replica should not be able to sign certificates without CA on master
+        self.master.run_command(['ipactl', 'stop'])
+        try:
+            self.request_test_service_cert(
+                self.replica1, cert_sign_request[self.replica1.hostname],
+                expect_connection_error=True)
+        finally:
+            self.master.run_command(['ipactl', 'start'])
+
+        tasks.install_ca(self.replica1)
+
+        # now replica should be able to sign certificates without CA on master
+        self.master.run_command(['ipactl', 'stop'])
+        self.request_test_service_cert(
+            self.replica1, cert_sign_request[self.replica1.hostname])
+        self.master.run_command(['ipactl', 'start'])
+
+        # check installation of new replica
+        tasks.install_replica(self.master, self.replica2, setup_ca=True)
+        check_replication(self.master, self.replica2, "testuser")
+
+        # new replica should be able to sign certificates without CA on master
+        # and old replica
+        self.master.run_command(['ipactl', 'stop'])
+        self.replica1.run_command(['ipactl', 'stop'])
+        try:
+            self.request_test_service_cert(
+                self.replica2, cert_sign_request[self.replica2.hostname])
+        finally:
+            self.replica1.run_command(['ipactl', 'start'])
+            self.master.run_command(['ipactl', 'start'])
 
 
 class TestUserRootFilesOwnershipPermission(IntegrationTest):