Azure: Populate containers with self-AAAA records

IPA server's AAAA records at embedded DNS mode depend on result of
`get_server_ip_address` function(`ipaserver.install.installutils`),
which in turn, relies on NSS.

In case of Azure Pipelines, there are neither IPv6 records in
'/etc/hosts' nor external DNS, which may provide such. This leads to
the missing AAAA records for master and missing AAAA records for `ipa-ca`
pointing to master in embedded DNS.

In particular, tests `test_ipa_healthcheck_no_errors`,
`test_ipa_dns_systemrecords_check` fail with:
```
[
  {
    "source": "ipahealthcheck.ipa.idns",
    "check": "IPADNSSystemRecordsCheck",
    "result": "WARNING",
    "uuid": "b979a88a-6373-4990-bc83-ce724e9730b4",
    "when": "20210120055054Z",
    "duration": "0.032740",
    "kw": {
      "msg": "Got {count} ipa-ca AAAA records, expected {expected}",
      "count": 1,
      "expected": 2
    }
  }
]
```
where `ipa-ca` record exists only for replica.

Note: since the most of the code in setup_containers was touched it has
been reformatted.

Fixes: https://pagure.io/freeipa/issue/8683
Signed-off-by: Stanislav Levin <slev@altlinux.org>
Reviewed-By: Alexander Bokovoy <abokovoy@redhat.com>

ipatests/azure/scripts/setup_containers.py

@@ -5,20 +5,21 @@ import subprocess
 import docker
 from jinja2 import Template
 
-logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
 
-IPA_TESTS_ENV_WORKING_DIR = os.environ.get('IPA_TESTS_ENV_WORKING_DIR')
+IPA_TESTS_ENV_WORKING_DIR = os.environ.get("IPA_TESTS_ENV_WORKING_DIR")
-IPA_TESTS_ENV_NAME = os.environ.get('IPA_TESTS_ENV_NAME')
+IPA_TESTS_ENV_NAME = os.environ.get("IPA_TESTS_ENV_NAME")
-IPA_TESTS_ENV_ID = os.environ.get('IPA_TESTS_ENV_ID', '1')
+IPA_TESTS_ENV_ID = os.environ.get("IPA_TESTS_ENV_ID", "1")
-IPA_TESTS_CLIENTS = int(os.environ.get('IPA_TESTS_CLIENTS', 0))
+IPA_TESTS_CLIENTS = int(os.environ.get("IPA_TESTS_CLIENTS", 0))
-IPA_TESTS_REPLICAS = int(os.environ.get('IPA_TESTS_REPLICAS', 0))
+IPA_TESTS_REPLICAS = int(os.environ.get("IPA_TESTS_REPLICAS", 0))
-IPA_TESTS_DOMAIN = os.environ.get('IPA_TESTS_DOMAIN', 'ipa.test')
+IPA_TESTS_DOMAIN = os.environ.get("IPA_TESTS_DOMAIN", "ipa.test")
-IPA_SSH_PRIV_KEY = os.environ.get('IPA_SSH_PRIV_KEY', '/root/.ssh/id_rsa')
+IPA_SSH_PRIV_KEY = os.environ.get("IPA_SSH_PRIV_KEY", "/root/.ssh/id_rsa")
-IPA_DNS_FORWARDER = os.environ.get('IPA_DNS_FORWARDER', '8.8.8.8')
+IPA_DNS_FORWARDER = os.environ.get("IPA_DNS_FORWARDER", "8.8.8.8")
-IPA_NETWORK = os.environ.get('IPA_NETWORK', 'ipanet')
+IPA_NETWORK = os.environ.get("IPA_NETWORK", "ipanet")
-IPA_CONTROLLER_TYPE = os.environ.get('IPA_CONTROLLER_TYPE', 'master')
+IPA_CONTROLLER_TYPE = os.environ.get("IPA_CONTROLLER_TYPE", "master")
 IPA_TEST_CONFIG_TEMPLATE = os.environ.get(
-    'IPA_TEST_CONFIG_TEMPLATE', './templates/ipa-test-config-template.yaml')
+    "IPA_TEST_CONFIG_TEMPLATE", "./templates/ipa-test-config-template.yaml"
+)
 
 IPA_TESTS_ENV_DIR = os.path.join(IPA_TESTS_ENV_WORKING_DIR, IPA_TESTS_ENV_NAME)
 IPA_TEST_CONFIG = "ipa-test-config.yaml"
@@ -26,61 +27,107 @@ IPA_TEST_CONFIG = "ipa-test-config.yaml"
 
 class Container:
     """
-    Represents group of Docker container
+    Represents Docker container
     """
-    def __init__(self, role, dns=IPA_DNS_FORWARDER, num=1,
+
-                 prefix=IPA_TESTS_ENV_ID, domain=IPA_TESTS_DOMAIN):
+    def __init__(self, name, hostname, network):
-        self.role = role
+        self.name = name
-        self.num = num
+        self.hostname = hostname
-        self.prefix = prefix
+        self.network = network
-        self.dns = dns
-        self.domain = domain
         self.dclient = docker.from_env()
 
     @property
-    def hostnames(self):
+    def ip(self):
-        """
-        hostnames of containers within group
-        """
-        if not hasattr(self, '_hostnames'):
-            self._hostnames = ['{}{}.{}'.format(self.role, c, self.domain)
-                               for c in range(1, self.num + 1)]
-        return self._hostnames
-
-    @property
-    def names(self):
-        """
-        names of containers within group
-        """
-        if not hasattr(self, '_names'):
-            self._names = ['{}_{}_{}'.format(self.prefix, self.role, c)
-                           for c in range(1, self.num + 1)]
-        return self._names
-
-    def ip(self, name):
         """
         ipv4 address of container
         """
-        ipanet = '{}_{}'.format(IPA_TESTS_ENV_ID, IPA_NETWORK)
+        if not hasattr(self, "_ip"):
-        dcont = self.dclient.containers.get(name)
+            dcont = self.dclient.containers.get(self.name)
-        return dcont.attrs['NetworkSettings']['Networks'][ipanet]['IPAddress']
+            self._ip = dcont.attrs["NetworkSettings"]["Networks"][
+                self.network
+            ]["IPAddress"]
+
+        return self._ip
 
     @property
+    def ipv6(self):
+        """
+        ipv6 address of container
+        """
+        if not hasattr(self, "_ipv6"):
+            dcont = self.dclient.containers.get(self.name)
+            self._ipv6 = dcont.attrs["NetworkSettings"]["Networks"][
+                self.network
+            ]["GlobalIPv6Address"]
+
+        return self._ipv6
+
+    def execute(self, args):
+        """
+        Exec an arbitrary command within container
+        """
+        dcont = self.dclient.containers.get(self.name)
+        logging.info("%s: run: %s", dcont.name, args)
+        result = dcont.exec_run(args, demux=True)
+        if result.output[0] is not None:
+            logging.info("%s: %s", dcont.name, result.output[0])
+        logging.info("%s: result: %s", dcont.name, result.exit_code)
+        if result.exit_code:
+            logging.error("stderr: %s", result.output[1].decode())
+            raise subprocess.CalledProcessError(
+                result.exit_code, args, result.output[1]
+            )
+        return result
+
+
+class ContainersGroup:
+    """
+    Represents group of Docker containers
+    """
+
+    HOME_SSH_DIR = "/root/.ssh"
+
+    def __init__(
+        self,
+        role,
+        nameservers=[IPA_DNS_FORWARDER],
+        scale=1,
+        prefix=IPA_TESTS_ENV_ID,
+        domain=IPA_TESTS_DOMAIN,
+    ):
+        self.role = role
+        self.scale = scale
+        self.prefix = prefix
+        self.nameservers = nameservers
+        self.domain = domain
+
+        # initialize containers
+        self.containers = [
+            Container(
+                name=f"{self.prefix}_{self.role}_{c}",
+                hostname=f"{self.role}{c}.{self.domain}",
+                network=f"{IPA_TESTS_ENV_ID}_{IPA_NETWORK}",
+            )
+            for c in range(1, self.scale + 1)
+        ]
+
+    def execute_all(self, args):
+        """
+        Sequentially exec an arbitrary command within every container of group
+        """
+        results = []
+        for cont in self.containers:
+            results.append(cont.execute(args))
+        return results
+
     def ips(self):
-        """
+        return [cont.ip for cont in self.containers]
-        ipv4 addresses of containers within group
-        """
-        if not hasattr(self, '_ips'):
-            self._ips = [self.ip(n) for n in self.names]
-        return self._ips
 
     def umount_docker_resource(self, path):
         """
         Umount resource by its path
         """
-        cmd = [
+        cmd = ["/bin/umount", path]
-            "/bin/umount", path
-        ]
         self.execute_all(cmd)
 
         cmd = [
@@ -90,47 +137,21 @@ class Container:
         ]
         self.execute_all(cmd)
 
-    def execute(self, name, args):
-        """
-        Exec an arbitrary command within container
-        """
-        dcont = self.dclient.containers.get(name)
-        logging.info("%s: run: %s", dcont.name, args)
-        result = dcont.exec_run(args, demux=True)
-        if result.output[0] is not None:
-            logging.info("%s: %s", dcont.name, result.output[0])
-        logging.info("%s: result: %s", dcont.name, result.exit_code)
-        if result.exit_code:
-            logging.error("stderr: %s", result.output[1].decode())
-            raise subprocess.CalledProcessError(
-                result.exit_code, args,
-                result.output[1]
-            )
-        return result
-
-    def execute_all(self, args):
-        """
-        Exec an arbitrary command within every container of group
-        """
-        results = []
-        for n in self.names:
-            results.append(self.execute(n, args))
-        return results
-
     def add_ssh_pubkey(self, key):
         """
         Add ssh public key into every container of group
         """
-        home_ssh_dir = "/root/.ssh"
+        auth_keys = os.path.join(self.HOME_SSH_DIR, "authorized_keys")
-        auth_keys = os.path.join(home_ssh_dir, "authorized_keys")
         cmd = [
-            "/bin/bash", "-c",
+            "/bin/bash",
-            (f"mkdir {home_ssh_dir} "
+            "-c",
-             f"; chmod 0700 {home_ssh_dir} "
+            (
-             f"&& touch {auth_keys} "
+                f"mkdir {self.HOME_SSH_DIR} "
-             f"&& chmod 0600 {auth_keys} "
+                f"; chmod 0700 {self.HOME_SSH_DIR} "
-             f"&& echo {key} >> {auth_keys}"
+                f"&& touch {auth_keys} "
-             )
+                f"&& chmod 0600 {auth_keys} "
+                f"&& echo {key} >> {auth_keys}"
+            ),
         ]
         self.execute_all(cmd)
 
@@ -139,40 +160,43 @@ class Container:
         Overwrite hosts within every container of group
         """
         self.umount_docker_resource("/etc/hosts")
-        for n, i, h in zip(self.names, self.ips, self.hostnames):
+        for cont in self.containers:
-            hosts = "127.0.0.1 localhost\n::1 localhost\n{ip} {host}".format(
+            hosts = "\n".join(
-                ip=i, host=h,
+                [
+                    "127.0.0.1 localhost",
+                    "::1 localhost",
+                    f"{cont.ip} {cont.hostname}",
+                    f"{cont.ipv6} {cont.hostname}",
+                ]
             )
-            cmd = [
+            cmd = ["/bin/bash", "-c", f"echo -e '{hosts}' > /etc/hosts"]
-                "/bin/bash", "-c",
+            cont.execute(cmd)
-                "echo -e '{hosts}' > /etc/hosts".format(hosts=hosts),
-            ]
-            self.execute(name=n, args=cmd)
 
     def setup_hostname(self):
         self.umount_docker_resource("/etc/hostname")
-        for n, h in zip(self.names, self.hostnames):
+        for cont in self.containers:
             cmd = [
-                "/bin/bash", "-c",
+                "/bin/bash",
-                "echo -e '{hostname}' > /etc/hostname".format(hostname=h),
+                "-c",
+                f"echo -e '{cont.hostname}' > /etc/hostname",
             ]
-            self.execute(name=n, args=cmd)
+            cont.execute(cmd)
 
-            cmd = [
+            cmd = ["hostnamectl", "set-hostname", cont.hostname]
-                "hostnamectl",
+            cont.execute(cmd)
-                "set-hostname", h,
-            ]
-            self.execute(name=n, args=cmd)
 
     def setup_resolvconf(self):
         """
         Overwrite resolv conf within every container of group
         """
         self.umount_docker_resource("/etc/resolv.conf")
-        ns = "nameserver {dns}".format(dns=self.dns)
+        nameservers = "\n".join(
+            [f"nameserver {ns}" for ns in self.nameservers]
+        )
         cmd = [
-            "/bin/bash", "-c",
+            "/bin/bash",
-            "echo {ns} > /etc/resolv.conf".format(ns=ns),
+            "-c",
+            f"echo -e '{nameservers}' > /etc/resolv.conf",
         ]
         self.execute_all(cmd)
 
@@ -181,88 +205,120 @@ class Container:
         Amend systemd service configuration to be ignored in a container
         """
         service_dir = os.path.join(
-            "/etc/systemd/system", "{}.service.d".format(service))
+            "/etc/systemd/system", "{}.service.d".format(service)
+        )
         override_file = os.path.join(service_dir, "ipa-override.conf")
         cmds = [
-            "/bin/bash", "-c",
+            "/bin/bash",
-            (f"mkdir -p {service_dir};"
+            "-c",
-             f"echo '[Unit]' > {override_file};"
+            (
-             f"echo 'ConditionVirtualization=!container' >> {override_file}")
+                f"mkdir -p {service_dir};"
+                f"echo '[Unit]' > {override_file};"
+                f"echo 'ConditionVirtualization=!container' >> {override_file}"
+            ),
         ]
-        self.execute_all(args=cmds)
+        self.execute_all(cmds)
 
     def setup_container_overrides(self):
         """
         Set services known to not work in containers to be ignored
         """
-        for service in ['nis-domainname', 'chronyd']:
+        for service in [
+            "nis-domainname",
+            "chronyd",
+        ]:
             self.ignore_service_in_container(service)
 
-        self.execute_all(args=["systemctl", "daemon-reload"])
+        self.execute_all(["systemctl", "daemon-reload"])
 
 
 class Controller(Container):
     """
-    Manages groups of containers
+    Represents Controller, which manages groups of containers groups
     """
-    def __init__(self, contr_type=IPA_CONTROLLER_TYPE):
-        self.containers = []
-        self.contr_type = contr_type
-        if self.contr_type == 'master':
-            self.master = None
 
-    def append(self, container):
+    def __init__(self, contr_type=IPA_CONTROLLER_TYPE):
-        self.containers.append(container)
+        self.containers_groups = []
+        self.contr_type = contr_type
+
+    def append(self, containers_group):
+        self.containers_groups.append(containers_group)
 
     def setup_ssh(self):
         """
         Generate ssh key pair and copy public part to all containers
         """
         cmd = ["rm", "-f", IPA_SSH_PRIV_KEY]
-        self.execute(args=cmd)
+        self.execute(cmd)
 
         cmd = [
-            "ssh-keygen", "-q",
+            "ssh-keygen",
-            "-f", IPA_SSH_PRIV_KEY,
+            "-q",
-            "-t", "rsa",
+            "-f",
-            "-m", "PEM",
+            IPA_SSH_PRIV_KEY,
-            "-N", "",
+            "-t",
+            "rsa",
+            "-m",
+            "PEM",
+            "-N",
+            "",
         ]
-        self.execute(args=cmd)
+        self.execute(cmd)
 
         cmd = ["/bin/bash", "-c", "cat {}.pub".format(IPA_SSH_PRIV_KEY)]
         key = self.execute(cmd).output[0].decode().rstrip()
-        for container in self.containers:
+        for containers_group in self.containers_groups:
-            container.add_ssh_pubkey(key)
+            containers_group.add_ssh_pubkey(key)
+
+    @property
+    def master_container(self):
+        if not hasattr(self, "_master_container"):
+            master_containers_group = None
+            for containers_group in self.containers_groups:
+                if containers_group.role == "master":
+                    master_containers_group = containers_group
+                    break
+            if master_containers_group is None:
+                raise ValueError(
+                    "There must be container group with master role"
+                )
+            # assume the only master
+            self._master_container = master_containers_group.containers[0]
+
+        return self._master_container
 
     def execute(self, args):
         """
         Execute a command on controller (either master or local machine)
         """
-        if self.contr_type == 'master':
+        if self.contr_type != "master":
-            if self.master is None:
+            proc = subprocess.run(args, check=True, capture_output=True)
-                for container in self.containers:
+            return [proc.stdout.decode().rstrip().strip("'")]
-                    if container.role == "master":
-                        self.master = container
-                        break
-            return self.master.execute(name=master.names[0], args=args)
 
-        proc = subprocess.run(args, check=True, capture_output=True)
+        return self.master_container.execute(args)
-        return [proc.stdout.decode().rstrip().strip("'")]
 
     def setup_hosts(self):
         """
         Overwrite Docker's hosts
         """
         hosts = []
-        for container in self.containers:
+        for containers_group in self.containers_groups:
-            container.setup_hosts()
+            containers_group.setup_hosts()
-            for i, h in zip(container.ips, container.hostnames):
+            # prevent duplication of master entries
-                hosts.append("{} {}".format(i, h))
+            if (
+                self.contr_type == "master"
+                and containers_group.role == "master"
+            ):
+                continue
+
+            for container in containers_group.containers:
+                hosts.append(f"{container.ip} {container.hostname}")
+                hosts.append(f"{container.ipv6} {container.hostname}")
 
         cmd = [
-            "/bin/bash", "-c",
+            "/bin/bash",
-            "echo -e '{hosts}' >> /etc/hosts".format(hosts='\n'.join(hosts)),
+            "-c",
+            "echo -e '{hosts}' >> /etc/hosts".format(hosts="\n".join(hosts)),
         ]
         self.execute(cmd)
 
@@ -270,38 +326,44 @@ class Controller(Container):
         """
         Overwrite Docker's hostname
         """
-        for container in self.containers:
+        for containers_group in self.containers_groups:
-            container.setup_hostname()
+            containers_group.setup_hostname()
 
     def setup_resolvconf(self):
         """
         Overwrite Docker's embedded DNS ns
         """
-        for container in self.containers:
+        for containers_group in self.containers_groups:
-            container.setup_resolvconf()
+            containers_group.setup_resolvconf()
 
     def generate_ipa_test_config(self, config):
-        with open(IPA_TEST_CONFIG_TEMPLATE, 'r') as f:
+        with open(IPA_TEST_CONFIG_TEMPLATE, "r") as f:
-            # assert foobar
             template = Template(f.read(), trim_blocks=True, lstrip_blocks=True)
 
-        print(template.render(config))
+        logging.info(template.render(config))
 
-        with open(os.path.join(IPA_TESTS_ENV_DIR, IPA_TEST_CONFIG), 'w') as f:
+        with open(os.path.join(IPA_TESTS_ENV_DIR, IPA_TEST_CONFIG), "w") as f:
             f.write(template.render(config))
 
     def setup_container_overrides(self):
         """
         Override services known to not work in containers
         """
-        for container in self.containers:
+        for containers_group in self.containers_groups:
-            container.setup_container_overrides()
+            containers_group.setup_container_overrides()
 
 
 controller = Controller()
-master = Container(role='master')
+master = ContainersGroup(role="master")
-clients = Container(role='client', num=IPA_TESTS_CLIENTS, dns=master.ips[0])
+
-replicas = Container(role='replica', num=IPA_TESTS_REPLICAS, dns=master.ips[0])
+# assume the only master
+master_ips = [master.containers[0].ip, master.containers[0].ipv6]
+clients = ContainersGroup(
+    role="client", scale=IPA_TESTS_CLIENTS, nameservers=master_ips
+)
+replicas = ContainersGroup(
+    role="replica", scale=IPA_TESTS_REPLICAS, nameservers=master_ips
+)
 
 controller.append(master)
 controller.append(clients)
@@ -314,11 +376,11 @@ controller.setup_resolvconf()
 controller.setup_container_overrides()
 
 config = {
-    'dns_forwarder': IPA_DNS_FORWARDER,
+    "dns_forwarder": IPA_DNS_FORWARDER,
-    'ssh_private_key': IPA_SSH_PRIV_KEY,
+    "ssh_private_key": IPA_SSH_PRIV_KEY,
-    'domain_name': IPA_TESTS_DOMAIN,
+    "domain_name": IPA_TESTS_DOMAIN,
-    'master': master.ips,
+    "master": master.ips(),
-    'replicas': replicas.ips,
+    "replicas": replicas.ips(),
-    'clients': clients.ips,
+    "clients": clients.ips(),
 }
 controller.generate_ipa_test_config(config)