diff --git a/ipatests/azure/scripts/setup_containers.py b/ipatests/azure/scripts/setup_containers.py index d52b432c5..fe9919132 100644 --- a/ipatests/azure/scripts/setup_containers.py +++ b/ipatests/azure/scripts/setup_containers.py @@ -5,20 +5,21 @@ import subprocess import docker from jinja2 import Template -logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") -IPA_TESTS_ENV_WORKING_DIR = os.environ.get('IPA_TESTS_ENV_WORKING_DIR') -IPA_TESTS_ENV_NAME = os.environ.get('IPA_TESTS_ENV_NAME') -IPA_TESTS_ENV_ID = os.environ.get('IPA_TESTS_ENV_ID', '1') -IPA_TESTS_CLIENTS = int(os.environ.get('IPA_TESTS_CLIENTS', 0)) -IPA_TESTS_REPLICAS = int(os.environ.get('IPA_TESTS_REPLICAS', 0)) -IPA_TESTS_DOMAIN = os.environ.get('IPA_TESTS_DOMAIN', 'ipa.test') -IPA_SSH_PRIV_KEY = os.environ.get('IPA_SSH_PRIV_KEY', '/root/.ssh/id_rsa') -IPA_DNS_FORWARDER = os.environ.get('IPA_DNS_FORWARDER', '8.8.8.8') -IPA_NETWORK = os.environ.get('IPA_NETWORK', 'ipanet') -IPA_CONTROLLER_TYPE = os.environ.get('IPA_CONTROLLER_TYPE', 'master') +IPA_TESTS_ENV_WORKING_DIR = os.environ.get("IPA_TESTS_ENV_WORKING_DIR") +IPA_TESTS_ENV_NAME = os.environ.get("IPA_TESTS_ENV_NAME") +IPA_TESTS_ENV_ID = os.environ.get("IPA_TESTS_ENV_ID", "1") +IPA_TESTS_CLIENTS = int(os.environ.get("IPA_TESTS_CLIENTS", 0)) +IPA_TESTS_REPLICAS = int(os.environ.get("IPA_TESTS_REPLICAS", 0)) +IPA_TESTS_DOMAIN = os.environ.get("IPA_TESTS_DOMAIN", "ipa.test") +IPA_SSH_PRIV_KEY = os.environ.get("IPA_SSH_PRIV_KEY", "/root/.ssh/id_rsa") +IPA_DNS_FORWARDER = os.environ.get("IPA_DNS_FORWARDER", "8.8.8.8") +IPA_NETWORK = os.environ.get("IPA_NETWORK", "ipanet") +IPA_CONTROLLER_TYPE = os.environ.get("IPA_CONTROLLER_TYPE", "master") IPA_TEST_CONFIG_TEMPLATE = os.environ.get( - 'IPA_TEST_CONFIG_TEMPLATE', './templates/ipa-test-config-template.yaml') + "IPA_TEST_CONFIG_TEMPLATE", "./templates/ipa-test-config-template.yaml" +) IPA_TESTS_ENV_DIR = os.path.join(IPA_TESTS_ENV_WORKING_DIR, IPA_TESTS_ENV_NAME) IPA_TEST_CONFIG = "ipa-test-config.yaml" @@ -26,61 +27,107 @@ IPA_TEST_CONFIG = "ipa-test-config.yaml" class Container: """ - Represents group of Docker container + Represents Docker container """ - def __init__(self, role, dns=IPA_DNS_FORWARDER, num=1, - prefix=IPA_TESTS_ENV_ID, domain=IPA_TESTS_DOMAIN): - self.role = role - self.num = num - self.prefix = prefix - self.dns = dns - self.domain = domain + + def __init__(self, name, hostname, network): + self.name = name + self.hostname = hostname + self.network = network self.dclient = docker.from_env() @property - def hostnames(self): - """ - hostnames of containers within group - """ - if not hasattr(self, '_hostnames'): - self._hostnames = ['{}{}.{}'.format(self.role, c, self.domain) - for c in range(1, self.num + 1)] - return self._hostnames - - @property - def names(self): - """ - names of containers within group - """ - if not hasattr(self, '_names'): - self._names = ['{}_{}_{}'.format(self.prefix, self.role, c) - for c in range(1, self.num + 1)] - return self._names - - def ip(self, name): + def ip(self): """ ipv4 address of container """ - ipanet = '{}_{}'.format(IPA_TESTS_ENV_ID, IPA_NETWORK) - dcont = self.dclient.containers.get(name) - return dcont.attrs['NetworkSettings']['Networks'][ipanet]['IPAddress'] + if not hasattr(self, "_ip"): + dcont = self.dclient.containers.get(self.name) + self._ip = dcont.attrs["NetworkSettings"]["Networks"][ + self.network + ]["IPAddress"] + + return self._ip @property + def ipv6(self): + """ + ipv6 address of container + """ + if not hasattr(self, "_ipv6"): + dcont = self.dclient.containers.get(self.name) + self._ipv6 = dcont.attrs["NetworkSettings"]["Networks"][ + self.network + ]["GlobalIPv6Address"] + + return self._ipv6 + + def execute(self, args): + """ + Exec an arbitrary command within container + """ + dcont = self.dclient.containers.get(self.name) + logging.info("%s: run: %s", dcont.name, args) + result = dcont.exec_run(args, demux=True) + if result.output[0] is not None: + logging.info("%s: %s", dcont.name, result.output[0]) + logging.info("%s: result: %s", dcont.name, result.exit_code) + if result.exit_code: + logging.error("stderr: %s", result.output[1].decode()) + raise subprocess.CalledProcessError( + result.exit_code, args, result.output[1] + ) + return result + + +class ContainersGroup: + """ + Represents group of Docker containers + """ + + HOME_SSH_DIR = "/root/.ssh" + + def __init__( + self, + role, + nameservers=[IPA_DNS_FORWARDER], + scale=1, + prefix=IPA_TESTS_ENV_ID, + domain=IPA_TESTS_DOMAIN, + ): + self.role = role + self.scale = scale + self.prefix = prefix + self.nameservers = nameservers + self.domain = domain + + # initialize containers + self.containers = [ + Container( + name=f"{self.prefix}_{self.role}_{c}", + hostname=f"{self.role}{c}.{self.domain}", + network=f"{IPA_TESTS_ENV_ID}_{IPA_NETWORK}", + ) + for c in range(1, self.scale + 1) + ] + + def execute_all(self, args): + """ + Sequentially exec an arbitrary command within every container of group + """ + results = [] + for cont in self.containers: + results.append(cont.execute(args)) + return results + def ips(self): - """ - ipv4 addresses of containers within group - """ - if not hasattr(self, '_ips'): - self._ips = [self.ip(n) for n in self.names] - return self._ips + return [cont.ip for cont in self.containers] def umount_docker_resource(self, path): """ Umount resource by its path """ - cmd = [ - "/bin/umount", path - ] + cmd = ["/bin/umount", path] self.execute_all(cmd) cmd = [ @@ -90,47 +137,21 @@ class Container: ] self.execute_all(cmd) - def execute(self, name, args): - """ - Exec an arbitrary command within container - """ - dcont = self.dclient.containers.get(name) - logging.info("%s: run: %s", dcont.name, args) - result = dcont.exec_run(args, demux=True) - if result.output[0] is not None: - logging.info("%s: %s", dcont.name, result.output[0]) - logging.info("%s: result: %s", dcont.name, result.exit_code) - if result.exit_code: - logging.error("stderr: %s", result.output[1].decode()) - raise subprocess.CalledProcessError( - result.exit_code, args, - result.output[1] - ) - return result - - def execute_all(self, args): - """ - Exec an arbitrary command within every container of group - """ - results = [] - for n in self.names: - results.append(self.execute(n, args)) - return results - def add_ssh_pubkey(self, key): """ Add ssh public key into every container of group """ - home_ssh_dir = "/root/.ssh" - auth_keys = os.path.join(home_ssh_dir, "authorized_keys") + auth_keys = os.path.join(self.HOME_SSH_DIR, "authorized_keys") cmd = [ - "/bin/bash", "-c", - (f"mkdir {home_ssh_dir} " - f"; chmod 0700 {home_ssh_dir} " - f"&& touch {auth_keys} " - f"&& chmod 0600 {auth_keys} " - f"&& echo {key} >> {auth_keys}" - ) + "/bin/bash", + "-c", + ( + f"mkdir {self.HOME_SSH_DIR} " + f"; chmod 0700 {self.HOME_SSH_DIR} " + f"&& touch {auth_keys} " + f"&& chmod 0600 {auth_keys} " + f"&& echo {key} >> {auth_keys}" + ), ] self.execute_all(cmd) @@ -139,40 +160,43 @@ class Container: Overwrite hosts within every container of group """ self.umount_docker_resource("/etc/hosts") - for n, i, h in zip(self.names, self.ips, self.hostnames): - hosts = "127.0.0.1 localhost\n::1 localhost\n{ip} {host}".format( - ip=i, host=h, + for cont in self.containers: + hosts = "\n".join( + [ + "127.0.0.1 localhost", + "::1 localhost", + f"{cont.ip} {cont.hostname}", + f"{cont.ipv6} {cont.hostname}", + ] ) - cmd = [ - "/bin/bash", "-c", - "echo -e '{hosts}' > /etc/hosts".format(hosts=hosts), - ] - self.execute(name=n, args=cmd) + cmd = ["/bin/bash", "-c", f"echo -e '{hosts}' > /etc/hosts"] + cont.execute(cmd) def setup_hostname(self): self.umount_docker_resource("/etc/hostname") - for n, h in zip(self.names, self.hostnames): + for cont in self.containers: cmd = [ - "/bin/bash", "-c", - "echo -e '{hostname}' > /etc/hostname".format(hostname=h), + "/bin/bash", + "-c", + f"echo -e '{cont.hostname}' > /etc/hostname", ] - self.execute(name=n, args=cmd) + cont.execute(cmd) - cmd = [ - "hostnamectl", - "set-hostname", h, - ] - self.execute(name=n, args=cmd) + cmd = ["hostnamectl", "set-hostname", cont.hostname] + cont.execute(cmd) def setup_resolvconf(self): """ Overwrite resolv conf within every container of group """ self.umount_docker_resource("/etc/resolv.conf") - ns = "nameserver {dns}".format(dns=self.dns) + nameservers = "\n".join( + [f"nameserver {ns}" for ns in self.nameservers] + ) cmd = [ - "/bin/bash", "-c", - "echo {ns} > /etc/resolv.conf".format(ns=ns), + "/bin/bash", + "-c", + f"echo -e '{nameservers}' > /etc/resolv.conf", ] self.execute_all(cmd) @@ -181,88 +205,120 @@ class Container: Amend systemd service configuration to be ignored in a container """ service_dir = os.path.join( - "/etc/systemd/system", "{}.service.d".format(service)) + "/etc/systemd/system", "{}.service.d".format(service) + ) override_file = os.path.join(service_dir, "ipa-override.conf") cmds = [ - "/bin/bash", "-c", - (f"mkdir -p {service_dir};" - f"echo '[Unit]' > {override_file};" - f"echo 'ConditionVirtualization=!container' >> {override_file}") + "/bin/bash", + "-c", + ( + f"mkdir -p {service_dir};" + f"echo '[Unit]' > {override_file};" + f"echo 'ConditionVirtualization=!container' >> {override_file}" + ), ] - self.execute_all(args=cmds) + self.execute_all(cmds) def setup_container_overrides(self): """ Set services known to not work in containers to be ignored """ - for service in ['nis-domainname', 'chronyd']: + for service in [ + "nis-domainname", + "chronyd", + ]: self.ignore_service_in_container(service) - self.execute_all(args=["systemctl", "daemon-reload"]) + self.execute_all(["systemctl", "daemon-reload"]) class Controller(Container): """ - Manages groups of containers + Represents Controller, which manages groups of containers groups """ - def __init__(self, contr_type=IPA_CONTROLLER_TYPE): - self.containers = [] - self.contr_type = contr_type - if self.contr_type == 'master': - self.master = None - def append(self, container): - self.containers.append(container) + def __init__(self, contr_type=IPA_CONTROLLER_TYPE): + self.containers_groups = [] + self.contr_type = contr_type + + def append(self, containers_group): + self.containers_groups.append(containers_group) def setup_ssh(self): """ Generate ssh key pair and copy public part to all containers """ cmd = ["rm", "-f", IPA_SSH_PRIV_KEY] - self.execute(args=cmd) + self.execute(cmd) cmd = [ - "ssh-keygen", "-q", - "-f", IPA_SSH_PRIV_KEY, - "-t", "rsa", - "-m", "PEM", - "-N", "", + "ssh-keygen", + "-q", + "-f", + IPA_SSH_PRIV_KEY, + "-t", + "rsa", + "-m", + "PEM", + "-N", + "", ] - self.execute(args=cmd) + self.execute(cmd) cmd = ["/bin/bash", "-c", "cat {}.pub".format(IPA_SSH_PRIV_KEY)] key = self.execute(cmd).output[0].decode().rstrip() - for container in self.containers: - container.add_ssh_pubkey(key) + for containers_group in self.containers_groups: + containers_group.add_ssh_pubkey(key) + + @property + def master_container(self): + if not hasattr(self, "_master_container"): + master_containers_group = None + for containers_group in self.containers_groups: + if containers_group.role == "master": + master_containers_group = containers_group + break + if master_containers_group is None: + raise ValueError( + "There must be container group with master role" + ) + # assume the only master + self._master_container = master_containers_group.containers[0] + + return self._master_container def execute(self, args): """ Execute a command on controller (either master or local machine) """ - if self.contr_type == 'master': - if self.master is None: - for container in self.containers: - if container.role == "master": - self.master = container - break - return self.master.execute(name=master.names[0], args=args) + if self.contr_type != "master": + proc = subprocess.run(args, check=True, capture_output=True) + return [proc.stdout.decode().rstrip().strip("'")] - proc = subprocess.run(args, check=True, capture_output=True) - return [proc.stdout.decode().rstrip().strip("'")] + return self.master_container.execute(args) def setup_hosts(self): """ Overwrite Docker's hosts """ hosts = [] - for container in self.containers: - container.setup_hosts() - for i, h in zip(container.ips, container.hostnames): - hosts.append("{} {}".format(i, h)) + for containers_group in self.containers_groups: + containers_group.setup_hosts() + # prevent duplication of master entries + if ( + self.contr_type == "master" + and containers_group.role == "master" + ): + continue + + for container in containers_group.containers: + hosts.append(f"{container.ip} {container.hostname}") + hosts.append(f"{container.ipv6} {container.hostname}") cmd = [ - "/bin/bash", "-c", - "echo -e '{hosts}' >> /etc/hosts".format(hosts='\n'.join(hosts)), + "/bin/bash", + "-c", + "echo -e '{hosts}' >> /etc/hosts".format(hosts="\n".join(hosts)), ] self.execute(cmd) @@ -270,38 +326,44 @@ class Controller(Container): """ Overwrite Docker's hostname """ - for container in self.containers: - container.setup_hostname() + for containers_group in self.containers_groups: + containers_group.setup_hostname() def setup_resolvconf(self): """ Overwrite Docker's embedded DNS ns """ - for container in self.containers: - container.setup_resolvconf() + for containers_group in self.containers_groups: + containers_group.setup_resolvconf() def generate_ipa_test_config(self, config): - with open(IPA_TEST_CONFIG_TEMPLATE, 'r') as f: - # assert foobar + with open(IPA_TEST_CONFIG_TEMPLATE, "r") as f: template = Template(f.read(), trim_blocks=True, lstrip_blocks=True) - print(template.render(config)) + logging.info(template.render(config)) - with open(os.path.join(IPA_TESTS_ENV_DIR, IPA_TEST_CONFIG), 'w') as f: + with open(os.path.join(IPA_TESTS_ENV_DIR, IPA_TEST_CONFIG), "w") as f: f.write(template.render(config)) def setup_container_overrides(self): """ Override services known to not work in containers """ - for container in self.containers: - container.setup_container_overrides() + for containers_group in self.containers_groups: + containers_group.setup_container_overrides() controller = Controller() -master = Container(role='master') -clients = Container(role='client', num=IPA_TESTS_CLIENTS, dns=master.ips[0]) -replicas = Container(role='replica', num=IPA_TESTS_REPLICAS, dns=master.ips[0]) +master = ContainersGroup(role="master") + +# assume the only master +master_ips = [master.containers[0].ip, master.containers[0].ipv6] +clients = ContainersGroup( + role="client", scale=IPA_TESTS_CLIENTS, nameservers=master_ips +) +replicas = ContainersGroup( + role="replica", scale=IPA_TESTS_REPLICAS, nameservers=master_ips +) controller.append(master) controller.append(clients) @@ -314,11 +376,11 @@ controller.setup_resolvconf() controller.setup_container_overrides() config = { - 'dns_forwarder': IPA_DNS_FORWARDER, - 'ssh_private_key': IPA_SSH_PRIV_KEY, - 'domain_name': IPA_TESTS_DOMAIN, - 'master': master.ips, - 'replicas': replicas.ips, - 'clients': clients.ips, + "dns_forwarder": IPA_DNS_FORWARDER, + "ssh_private_key": IPA_SSH_PRIV_KEY, + "domain_name": IPA_TESTS_DOMAIN, + "master": master.ips(), + "replicas": replicas.ips(), + "clients": clients.ips(), } controller.generate_ipa_test_config(config)