ipatests: add utility for managing domain name resolvers

Many test scenarios need to configure resolvers on test machines. Most
notable patterns are:

* using IPA master as DNS resolver on clients and replicas
* intentionally breaking name resolution

Now it is done by directly editing /etc/resolv.conf file. While being
simple this approach has following issues:

* NetworkManager restores this file periodically and on specific events
* This is not how users are expected to manage resolvers on modern
  systems with NetworkManager and systemd-resolved.

This patch introduces three classes for main types of resolvers management:
* plain file
* NetworkManager
* systemd-resolved

For each resolver manager the native way of configuring of nameserves is
used: direct editing for /etc/resolv.conf or drop-in config files for
NM and resolved.

The type of resolver is automatically detected for each host and an
appropriate instance is added to Host object.

The Resolver class (and it's subclasses) provide convenience functions
for changing nameservers and restoring the original config.
During all operations (backup, modify, restore) it checks that resolver
configuration has not been altered unexpectedly and raises exception if it
was. This helps to detect unexpected changes in resolvers.

Related to https://pagure.io/freeipa/issue/8703

Reviewed-By: Florence Blanc-Renaud <flo@redhat.com>

ipatests/pytest_ipa/integration/host.py

@@ -32,6 +32,7 @@ from .fips import (
     is_fips_enabled, enable_userspace_fips, disable_userspace_fips
 )
 from .transport import IPAOpenSSHTransport
+from .resolver import resolver
 
 FIPS_NOISE_RE = re.compile(br"FIPS mode initialized\r?\n?")
 
@@ -78,6 +79,7 @@ class Host(pytest_multihost.host.Host):
         )
         self._fips_mode = None
         self._userspace_fips = False
+        self.resolver = resolver(self)
 
     @property
     def is_fips_mode(self):

ipatests/pytest_ipa/integration/resolver.py

@@ -0,0 +1,329 @@
+import os
+import abc
+import logging
+import textwrap
+import time
+
+from ipaplatform.paths import paths
+from . import tasks
+
+
+logger = logging.getLogger(__name__)
+
+
+class Resolver(abc.ABC):
+    def __init__(self, host):
+        self.host = host
+        self.backups = []
+        self.current_state = self._get_state()
+        logger.info('Obtained initial resolver state for host %s: %s',
+                    self.host, self.current_state)
+
+    def setup_resolver(self, nameservers, searchdomains=None):
+        """Configure DNS resolver
+
+        :param nameservers: IP address of nameserver or a list of addresses
+        :param searchdomains: searchdomain or list of searchdomains.
+               None - do not configure
+
+        Resolver.backup() must be called prior to using this method.
+
+        Raises exception if configuration was changed externally since last call
+        to any method of Resolver class.
+        """
+        if len(self.backups) == 0:
+            raise Exception(
+                'Changing resolver state without backup is forbidden')
+        self.check_state_expected()
+        if isinstance(nameservers, str):
+            nameservers = [nameservers]
+        if isinstance(searchdomains, str):
+            searchdomains = [searchdomains]
+        if searchdomains is None:
+            searchdomains = []
+        logger.info(
+            'Setting up resolver for host %s: nameservers=%s, searchdomains=%s',
+            self.host, nameservers, searchdomains
+        )
+        state = self._make_state_from_args(nameservers, searchdomains)
+        self._set_state(state)
+
+    def backup(self):
+        """Saves current configuration to stack
+
+        Raises exception if configuration was changed externally since last call
+        to any method of Resolver class.
+        """
+        self.check_state_expected()
+        self.backups.append(self._get_state())
+        logger.info(
+            'Saved resolver state for host %s, number of saved states: %s',
+            self.host, len(self.backups)
+        )
+
+    def restore(self):
+        """Restore configuration from stack of backups.
+
+        Raises exception if configuration was changed externally since last call
+        to any method of Resolver class.
+        """
+
+        if len(self.backups) == 0:
+            raise Exception('No resolver backups found for host {}'.format(
+                self.host))
+        self.check_state_expected()
+        self._set_state(self.backups.pop())
+        logger.info(
+            'Restored resolver state for host %s, number of saved states: %s',
+            self.host, len(self.backups)
+        )
+
+    def has_backups(self):
+        """Checks if stack of backups is not empty"""
+        return bool(self.backups)
+
+    def check_state_expected(self):
+        """Checks if resolver configuration has not changed.
+
+        Raises AssertionError if actual configuration has changed since last
+        call to any method of Resolver
+        """
+        assert self._get_state() == self.current_state, (
+            'Resolver state changed unexpectedly at host {}'.format(self.host))
+
+    def __enter__(self):
+        self.backup()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.restore()
+
+    def _set_state(self, state):
+        self._apply_state(state)
+        logger.info('Applying resolver state for host %s: %s', self.host, state)
+        self.current_state = state
+
+    @abc.abstractclassmethod
+    def is_our_resolver(cls, host):
+        """Checks if the class is appropriate for managing resolver on the host.
+        """
+
+    @abc.abstractmethod
+    def _make_state_from_args(self, nameservers, searchdomains):
+        """
+
+        :param nameservers: list of ip addresses of nameservers
+        :param searchdomains: list of searchdomain, can be an empty list
+        :return: internal state object specific to subclass implementaion
+        """
+
+    @abc.abstractmethod
+    def _get_state(self):
+        """Acquire actual host configuration.
+
+        :return: internal state object  specific to subclass implementaion
+        """
+
+    @abc.abstractmethod
+    def _apply_state(self, state):
+        """Apply configuration to host.
+
+        :param state: internal state object  specific to subclass implementaion
+        """
+
+
+class ResolvedResolver(Resolver):
+    RESOLVED_RESOLV_CONF = {
+        "/run/systemd/resolve/stub-resolv.conf",
+        "/run/systemd/resolve/resolv.conf",
+        "/lib/systemd/resolv.conf",
+        "/usr/lib/systemd/resolv.conf",
+    }
+    RESOLVED_CONF_FILE = (
+        '/etc/systemd/resolved.conf.d/zzzz-ipatests-nameservers.conf')
+    RESOLVED_CONF = textwrap.dedent('''
+        # generated by IPA tests
+        [Resolve]
+        DNS={nameservers}
+        Domains=~. {searchdomains}
+    ''')
+
+    @classmethod
+    def is_our_resolver(cls, host):
+        res = host.run_command(
+            ['stat', '--format', '%F', paths.RESOLV_CONF])
+        filetype = res.stdout_text.strip()
+        if filetype == 'symbolic link':
+            res = host.run_command(['realpath', paths.RESOLV_CONF])
+            return (res.stdout_text.strip() in cls.RESOLVED_RESOLV_CONF)
+        return False
+
+    def _restart_resolved(self):
+        # Restarting service at rapid pace (which is what happens in some test
+        # scenarios) can exceed the threshold configured in systemd option
+        # StartLimitIntervalSec. In that case restart fails, but we can simply
+        # continue trying until it succeeds
+        tasks.run_repeatedly(
+            self.host, ['systemctl', 'restart', 'systemd-resolved.service'],
+            timeout=15)
+
+    def _make_state_from_args(self, nameservers, searchdomains):
+        return {
+            'resolved_config': self.RESOLVED_CONF.format(
+                nameservers=' '.join(nameservers),
+                searchdomains=' '.join(searchdomains))
+        }
+
+    def _get_state(self):
+        exists = self.host.transport.file_exists(self.RESOLVED_CONF_FILE)
+        return {
+            'resolved_config':
+                self.host.get_file_contents(self.RESOLVED_CONF_FILE, 'utf-8')
+                if exists else None
+        }
+
+    def _apply_state(self, state):
+        if state['resolved_config'] is None:
+            self.host.run_command(['rm', '-f', self.RESOLVED_CONF_FILE])
+        else:
+            self.host.run_command(
+                ['mkdir', '-p', os.path.dirname(self.RESOLVED_CONF_FILE)])
+            self.host.put_file_contents(
+                self.RESOLVED_CONF_FILE, state['resolved_config'])
+        self._restart_resolved()
+
+
+class PlainFileResolver(Resolver):
+    IPATESTS_RESOLVER_COMMENT = '# created by ipatests'
+
+    @classmethod
+    def is_our_resolver(cls, host):
+        res = host.run_command(
+            ['stat', '--format', '%F', paths.RESOLV_CONF])
+        filetype = res.stdout_text.strip()
+        if filetype == 'regular file':
+            # We want to be sure that /etc/resolv.conf is not generated
+            # by NetworkManager or systemd-resolved. When it is then
+            # the first line of the file is a comment of the form:
+            #
+            # Generated by NetworkManager
+            #
+            # or
+            #
+            # This file is managed by man:systemd-resolved(8). Do not edit.
+            #
+            # So we check that either first line of resolv.conf
+            # is not a comment or the comment does not mention NM or
+            # systemd-resolved
+            resolv_conf = host.get_file_contents(paths.RESOLV_CONF, 'utf-8')
+            line = resolv_conf.splitlines()[0].strip()
+            return not line.startswith('#') or all([
+                'resolved' not in line,
+                'NetworkManager' not in line
+            ])
+        return False
+
+    def _make_state_from_args(self, nameservers, searchdomains):
+        contents_lines = [self.IPATESTS_RESOLVER_COMMENT]
+        contents_lines.extend('nameserver {}'.format(r) for r in nameservers)
+        if searchdomains:
+            contents_lines.append('search {}'.format(' '.join(searchdomains)))
+        contents = '\n'.join(contents_lines)
+        return {'resolv_conf': contents}
+
+    def _get_state(self):
+        return {
+            'resolv_conf': self.host.get_file_contents(
+                paths.RESOLV_CONF, 'utf-8')
+        }
+
+    def _apply_state(self, state):
+        self.host.put_file_contents(paths.RESOLV_CONF, state['resolv_conf'])
+
+
+class NetworkManagerResolver(Resolver):
+    NM_CONF_FILE = '/etc/NetworkManager/conf.d/zzzz-ipatests.conf'
+    NM_CONF = textwrap.dedent('''
+        # generated by IPA tests
+        [main]
+        dns=default
+
+        [global-dns]
+        searches={searchdomains}
+
+        [global-dns-domain-*]
+        servers={nameservers}
+    ''')
+
+    @classmethod
+    def is_our_resolver(cls, host):
+        res = host.run_command(
+            ['stat', '--format', '%F', paths.RESOLV_CONF])
+        filetype = res.stdout_text.strip()
+        if filetype == 'regular file':
+            resolv_conf = host.get_file_contents(paths.RESOLV_CONF, 'utf-8')
+            return resolv_conf.startswith('# Generated by NetworkManager')
+        return False
+
+    def _restart_network_manager(self):
+        # Restarting service at rapid pace (which is what happens in some test
+        # scenarios) can exceed the threshold configured in systemd option
+        # StartLimitIntervalSec. In that case restart fails, but we can simply
+        # continue trying until it succeeds
+        tasks.run_repeatedly(
+            self.host, ['systemctl', 'restart', 'NetworkManager.service'],
+            timeout=15)
+
+    def _make_state_from_args(self, nameservers, searchdomains):
+        return {'nm_config': self.NM_CONF.format(
+            nameservers=','.join(nameservers),
+            searchdomains=','.join(searchdomains))}
+
+    def _get_state(self):
+        exists = self.host.transport.file_exists(self.NM_CONF_FILE)
+        return {
+            'nm_config':
+                self.host.get_file_contents(self.NM_CONF_FILE, 'utf-8')
+                if exists else None
+        }
+
+    def _apply_state(self, state):
+        def get_resolv_conf_mtime():
+            """Get mtime of /etc/resolv.conf.
+
+            Returns mtime with sub-second precision as a string with format
+            "2020-08-25 14:35:05.980503425 +0200"
+            """
+            return self.host.run_command(
+                ['stat', '-c', '%y', paths.RESOLV_CONF]).stdout_text.strip()
+
+        if state['nm_config'] is None:
+            self.host.run_command(['rm', '-f', self.NM_CONF_FILE])
+        else:
+            self.host.run_command(
+                ['mkdir', '-p', os.path.dirname(self.NM_CONF_FILE)])
+            self.host.put_file_contents(
+                self.NM_CONF_FILE, state['nm_config'])
+        # NetworkManager writes /etc/resolv.conf few moments after
+        # `systemctl restart` returns so we need to wait until the file is
+        # updated
+        mtime_before = get_resolv_conf_mtime()
+        self._restart_network_manager()
+        wait_until = time.time() + 10
+        while time.time() < wait_until:
+            if get_resolv_conf_mtime() != mtime_before:
+                break
+            time.sleep(1)
+        else:
+            raise Exception('NetworkManager did not update /etc/resolv.conf '
+                            'in 10 seconds after restart')
+
+
+def resolver(host):
+    for cls in [ResolvedResolver, NetworkManagerResolver,
+                PlainFileResolver]:
+        if cls.is_our_resolver(host):
+            logger.info('Detected DNS resolver manager for host %s is %s',
+                        host.hostname, cls)
+            return cls(host)
+    raise Exception('Resolver manager could not be detected')

ipatests/test_integration/test_resolvers_manager.py

@@ -0,0 +1,140 @@
+import pytest
+import re
+from contextlib import contextmanager
+
+from ipatests.pytest_ipa.integration import tasks
+from ipatests.test_integration.base import IntegrationTest
+
+
+class TestResolverManager(IntegrationTest):
+    topology = 'line'
+    num_clients = 1
+    invalid_resolver = '2.3.4.5'
+
+    @classmethod
+    def install(cls, mh):
+        test_record = 'test1234'
+        cls.client = cls.clients[0]
+        cls.test_record = '{}.{}'.format(test_record, cls.master.domain.name)
+        cls.test_record_address = '1.2.3.4'
+
+        if cls.domain_level is not None:
+            domain_level = cls.domain_level
+        else:
+            domain_level = cls.master.config.domain_level
+        tasks.install_topo(cls.topology, cls.master, [], [], domain_level)
+        tasks.kinit_admin(cls.master)
+        cls.master.run_command([
+            'ipa', 'dnsrecord-add', cls.master.domain.name,
+            test_record,
+            '--a-ip-address={}'.format(cls.test_record_address)])
+
+    def is_resolver_operational(self):
+        res = self.client.run_command(['dig', '+short', 'redhat.com'])
+        return re.match(r'\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}',
+                        res.stdout_text.strip())
+
+    def is_ipa_dns_used(self):
+        res = self.client.run_command(['dig', '+short', self.test_record])
+        output = res.stdout_text.strip()
+        error = res.stderr_text.strip()
+        if output == self.test_record_address:
+            return True
+        if output == '' and error == '':
+            return False
+        raise Exception('Unexpected result of dig command')
+
+    def check_ipa_name_resolution_fails(self):
+        res = self.client.run_command(['dig', '+short', self.test_record],
+                                      ok_returncode=9)
+        assert 'connection timed out' in res.stdout_text
+
+    @contextmanager
+    def start_end_checks(self):
+        assert not self.client.resolver.has_backups()
+        assert self.is_resolver_operational()
+        assert not self.is_ipa_dns_used()
+        yield
+        assert self.is_resolver_operational()
+        assert not self.is_ipa_dns_used()
+        assert not self.client.resolver.has_backups()
+
+    def test_ipa_dns_not_used_by_default(self):
+        assert self.is_resolver_operational()
+        assert not self.is_ipa_dns_used()
+
+    def test_changing_config_without_backup_not_allowed(self):
+        with pytest.raises(Exception, match='without backup'):
+            self.client.resolver.setup_resolver(self.master.ip)
+
+    def test_change_resolver(self):
+        with self.start_end_checks():
+            self.client.resolver.backup()
+            self.client.resolver.setup_resolver(self.master.ip)
+            assert self.is_resolver_operational()
+            assert self.is_ipa_dns_used()
+            self.client.resolver.restore()
+
+    def test_nested_change_resolver(self):
+        with self.start_end_checks():
+            self.client.resolver.backup()
+            self.client.resolver.setup_resolver(self.master.ip)
+
+            assert self.is_resolver_operational()
+            assert self.is_ipa_dns_used()
+
+            self.client.resolver.backup()
+            self.client.resolver.setup_resolver(self.invalid_resolver)
+            self.check_ipa_name_resolution_fails()
+            self.client.resolver.restore()
+
+            assert self.is_resolver_operational()
+            assert self.is_ipa_dns_used()
+
+            self.client.resolver.restore()
+
+    def test_nested_change_resolver_with_context(self):
+        with self.start_end_checks():
+            self.client.resolver.backup()
+            self.client.resolver.setup_resolver(self.master.ip)
+            assert self.is_resolver_operational()
+            assert self.is_ipa_dns_used()
+
+            with self.client.resolver:
+                self.client.resolver.setup_resolver(self.invalid_resolver)
+                self.check_ipa_name_resolution_fails()
+
+            self.client.resolver.restore()
+
+    def test_repeated_changing_resolver(self):
+        with self.start_end_checks():
+            self.client.resolver.backup()
+            self.client.resolver.setup_resolver(self.master.ip)
+            assert self.is_resolver_operational()
+            assert self.is_ipa_dns_used()
+
+            self.client.resolver.setup_resolver(self.invalid_resolver)
+            self.check_ipa_name_resolution_fails()
+
+            self.client.resolver.setup_resolver(self.master.ip)
+            assert self.is_resolver_operational()
+            assert self.is_ipa_dns_used()
+
+            self.client.resolver.restore()
+
+    @pytest.mark.parametrize('reverse', [True, False])
+    def test_multiple_resolvers(self, reverse):
+        resolvers = [self.invalid_resolver, self.master.ip]
+        if reverse:
+            resolvers.reverse()
+
+        with self.start_end_checks():
+            self.client.resolver.backup()
+            self.client.resolver.setup_resolver(resolvers)
+            assert self.is_resolver_operational()
+            assert self.is_ipa_dns_used()
+            self.client.resolver.restore()
+
+    @classmethod
+    def uninstall(cls, mh):
+        tasks.uninstall_master(cls.master)