installer: Upload unattended and cloudinit data to remote connections

* rename kernelupload.py to volumeupload.py and make the entrypoint more generic * move all upload invocation to the Installer class * use it with cloudinit and unattended ISO generation if required Signed-off-by: Cole Robinson <crobinso@redhat.com>
2026-08-07 11:45:09 -05:00 · 2020-09-08 19:24:21 -04:00
parent 44033b43bb
commit 5473695fe1
6 changed files with 214 additions and 72 deletions
@@ -43,7 +43,7 @@
      <readonly/>
    </disk>
    <disk type="file" device="cdrom">
-      <driver name="qemu"/>
+      <driver name="qemu" type="raw"/>
      <source file="/VIRTINST-TESTSUITE/unattended.iso"/>
      <target dev="sdb" bus="sata"/>
      <readonly/>
@@ -0,0 +1,114 @@
+<domain type="test">
+  <name>win7</name>
+  <uuid>00000000-1111-2222-3333-444444444444</uuid>
+  <metadata>
+    <libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
+      <libosinfo:os id="http://microsoft.com/win/7"/>
+    </libosinfo:libosinfo>
+  </metadata>
+  <memory>65536</memory>
+  <currentMemory>65536</currentMemory>
+  <vcpu>2</vcpu>
+  <os>
+    <type arch="i686">hvm</type>
+    <boot dev="cdrom"/>
+  </os>
+  <features>
+    <pae/>
+    <hyperv>
+      <relaxed state="on"/>
+      <vapic state="on"/>
+      <spinlocks state="on" retries="8191"/>
+    </hyperv>
+  </features>
+  <clock offset="localtime"/>
+  <on_reboot>destroy</on_reboot>
+  <pm>
+    <suspend-to-mem enabled="no"/>
+    <suspend-to-disk enabled="no"/>
+  </pm>
+  <devices>
+    <emulator>/usr/bin/test-hv</emulator>
+    <disk type="file" device="cdrom">
+      <source file="/dev/default-pool/testvol1.img"/>
+      <target dev="hda" bus="ide"/>
+      <readonly/>
+    </disk>
+    <disk type="file" device="cdrom">
+      <source file="/VIRTINST-TESTSUITE/unattended.iso"/>
+      <target dev="hdb" bus="ide"/>
+      <readonly/>
+    </disk>
+    <controller type="usb" model="ich9-ehci1"/>
+    <controller type="usb" model="ich9-uhci1">
+      <master startport="0"/>
+    </controller>
+    <controller type="usb" model="ich9-uhci2">
+      <master startport="2"/>
+    </controller>
+    <controller type="usb" model="ich9-uhci3">
+      <master startport="4"/>
+    </controller>
+    <interface type="user">
+      <mac address="00:11:22:33:44:55"/>
+      <model type="e1000"/>
+    </interface>
+    <console type="pty"/>
+  </devices>
+</domain>
+<domain type="test">
+  <name>win7</name>
+  <uuid>00000000-1111-2222-3333-444444444444</uuid>
+  <metadata>
+    <libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
+      <libosinfo:os id="http://microsoft.com/win/7"/>
+    </libosinfo:libosinfo>
+  </metadata>
+  <memory>65536</memory>
+  <currentMemory>65536</currentMemory>
+  <vcpu>2</vcpu>
+  <os>
+    <type arch="i686">hvm</type>
+    <boot dev="cdrom"/>
+  </os>
+  <features>
+    <pae/>
+    <hyperv>
+      <relaxed state="on"/>
+      <vapic state="on"/>
+      <spinlocks state="on" retries="8191"/>
+    </hyperv>
+  </features>
+  <clock offset="localtime"/>
+  <pm>
+    <suspend-to-mem enabled="no"/>
+    <suspend-to-disk enabled="no"/>
+  </pm>
+  <devices>
+    <emulator>/usr/bin/test-hv</emulator>
+    <disk type="file" device="cdrom">
+      <source file="/dev/default-pool/testvol1.img"/>
+      <target dev="hda" bus="ide"/>
+      <readonly/>
+    </disk>
+    <disk type="file" device="cdrom">
+      <target dev="hdb" bus="ide"/>
+      <readonly/>
+    </disk>
+    <controller type="usb" model="ich9-ehci1"/>
+    <controller type="usb" model="ich9-uhci1">
+      <master startport="0"/>
+    </controller>
+    <controller type="usb" model="ich9-uhci2">
+      <master startport="2"/>
+    </controller>
+    <controller type="usb" model="ich9-uhci3">
+      <master startport="4"/>
+    </controller>
+    <interface type="user">
+      <mac address="00:11:22:33:44:55"/>
+      <model type="e1000"/>
+    </interface>
+    <console type="pty"/>
+  </devices>
+</domain>
@@ -920,15 +920,16 @@ c.add_compare("--location %(TREEDIR)s --unattended", "osinfo-unattended-treeapis
 c.add_compare("--cdrom %(ISO-WIN7)s --unattended profile=desktop,admin-password-file=%(ADMIN-PASSWORD-FILE)s", "osinfo-win7-unattended", prerun_check=no_osinfo_unattended_win_drivers_cb)  # unattended install for win7
 c.add_compare("--os-variant fedora26 --unattended profile=jeos,admin-password-file=%(ADMIN-PASSWORD-FILE)s --location %(ISO-F26-NETINST)s", "osinfo-netinst-unattended")  # triggering the special netinst checking code
 c.add_compare("--os-variant silverblue29 --location http://example.com", "network-install-resources")  # triggering network-install resources override
+c.add_compare("--connect %(URI-TEST-REMOTE)s --os-variant win7 --cdrom %(EXISTIMG1)s --unattended", "unattended-remote-cdrom")
 c.add_valid("--pxe --os-variant fedora26 --unattended", grep="Using unattended profile 'desktop'")  # filling in default 'desktop' profile
 c.add_invalid("--os-variant fedora26 --unattended profile=jeos --location http://example.foo", grep="admin-password")  # will trigger admin-password required error
 c.add_invalid("--os-variant fedora26 --unattended profile=jeos --location http://example.foo", grep="admin-password")  # will trigger admin-password required error
+c.add_invalid("--os-variant fedora26 --unattended profile=jeos --location http://example.foo", grep="admin-password")  # will trigger admin-password required error
 c.add_invalid("--os-variant debian9 --unattended profile=desktop,admin-password-file=%(ADMIN-PASSWORD-FILE)s --location http://example.foo", grep="user-password")  # will trigger user-password required error
 c.add_invalid("--os-variant debian9 --unattended profile=FRIBBER,admin-password-file=%(ADMIN-PASSWORD-FILE)s --location http://example.foo", grep="Available profiles")  # will trigger unknown profile error
 c.add_invalid("--os-variant fedora29 --unattended profile=desktop,admin-password-file=%(ADMIN-PASSWORD-FILE)s --cdrom %(ISO-F29-LIVE)s", grep="media does not support")  # live media doesn't support installscript
 c.add_invalid("--os-variant msdos --unattended profile=desktop --location http://example.com")  # msdos doesn't support unattended install
 c.add_invalid("--os-variant winxp --unattended profile=desktop --cdrom %(ISO-WIN7)s")  # winxp doesn't support expected injection method 'cdrom'
-c.add_invalid("--connect %(URI-TEST-REMOTE)s --os-variant win7 --cdrom %(EXISTIMG1)s --unattended")  # --unattended method=cdrom rejected for remote connections
 c.add_invalid("--install fedora29 --unattended user-login=root", grep="as user-login")  # will trigger an invalid user-login error


@@ -972,6 +973,7 @@ c.add_compare("--arch s390x --machine s390-ccw-virtio --connect " + utils.URIs.k

 # qemu:///session tests
 c.add_compare("--connect " + utils.URIs.kvm_session + " --disk size=8 --os-variant fedora21 --cdrom %(EXISTIMG1)s", "kvm-session-defaults", prerun_check=has_old_osinfo)
+c.add_valid("--connect " + utils.URIs.kvm_session + " --install fedora21", prerun_check=has_old_osinfo)  # hits some get_search_paths and media_upload code paths

 # misc KVM config tests
 c.add_compare("--disk none --location %(ISO-NO-OS)s,kernel=frib.img,initrd=/frob.img", "location-manual-kernel", prerun_check=missing_isoinfo)  # --location with an unknown ISO but manually specified kernel paths
@@ -10,6 +10,7 @@ import os

 from . import cloudinit
 from . import unattended
+from . import volumeupload
 from .installertreemedia import InstallerTreeMedia
 from .installerinject import perform_cdrom_injections
 from ..domain import DomainOs
@@ -17,6 +18,7 @@ from ..devices import DeviceDisk
 from ..osdict import OSDB
 from ..logger import log
 from .. import progress
+from .. import xmlutil


 def _make_testsuite_path(path):
@@ -59,6 +61,7 @@ class Installer(object):
        self._install_cdrom_device_added = False
        self._unattended_install_cdrom_device = None
        self._tmpfiles = []
+        self._tmpvols = []
        self._defaults_are_set = False
        self._unattended_data = None
        self._cloudinit_data = None
@@ -266,11 +269,50 @@ class Installer(object):
            guest.currentMemory = ram


-    ##########################
-    # Internal API overrides #
-    ##########################
+    ################
+    # Internal API #
+    ################
+
+    def _should_upload_media(self, guest):
+        """
+        Return True if we should upload media to the connection scratchdir.
+        This doesn't consider if there is any media to upload, just whether
+        we _should_ upload if there _is_ media.
+        """
+        scratchdir = InstallerTreeMedia.make_scratchdir(guest)
+        system_scratchdir = InstallerTreeMedia.get_system_scratchdir(guest)
+
+        if self.conn.is_remote():
+            return True
+        if self.conn.is_unprivileged():
+            return False
+        if scratchdir == system_scratchdir:
+            return False  # pragma: no cover
+        return True
+
+    def _upload_media(self, guest, meter, paths):
+        system_scratchdir = InstallerTreeMedia.get_system_scratchdir(guest)
+
+        if (not self._should_upload_media(guest) and
+            not xmlutil.in_testsuite()):
+            # We have access to system scratchdir, don't jump through hoops
+            log.debug("Have access to preferred scratchdir so"
+                        " nothing to upload")  # pragma: no cover
+            return paths  # pragma: no cover
+
+        if not guest.conn.support_remote_url_install():
+            # Needed for the test_urls suite
+            log.debug("Media upload not supported")  # pragma: no cover
+            return paths  # pragma: no cover
+
+        newpaths, tmpvols = volumeupload.upload_paths(
+                guest.conn, system_scratchdir, meter, paths)
+        self._tmpvols += tmpvols
+        return newpaths

    def _prepare_unattended_data(self, guest, meter, scripts):
+        scratchdir = InstallerTreeMedia.make_scratchdir(guest)
+
        injections = []
        for script in scripts:
            expected_filename = script.get_expected_filename()
@@ -283,14 +325,14 @@ class Installer(object):

        drivers_location = guest.osinfo.get_pre_installable_drivers_location(
                guest.os.arch)
-        drivers = unattended.download_drivers(drivers_location,
-                InstallerTreeMedia.make_scratchdir(guest), meter)
+        drivers = unattended.download_drivers(
+                drivers_location, scratchdir, meter)
        injections.extend(drivers)
        self._tmpfiles.extend([driverpair[0] for driverpair in drivers])

-        iso = perform_cdrom_injections(injections,
-                InstallerTreeMedia.make_scratchdir(guest))
+        iso = perform_cdrom_injections(injections, scratchdir)
        self._tmpfiles.append(iso)
+        iso = self._upload_media(guest, meter, [iso])[0]
        self._add_unattended_install_cdrom_device(guest, iso)

    def _prepare_unattended_scripts(self, guest, meter):
@@ -303,9 +345,6 @@ class Installer(object):
            os_tree = self._treemedia.get_os_tree(guest, meter)
            injection_method = "initrd"
        else:
-            if self.conn.is_remote():
-                raise RuntimeError("Unattended method=cdrom installs are "
-                        "not yet supported for remote connections.")
            if not guest.osinfo.is_windows():
                log.warning("Attempting unattended method=cdrom injection "
                        "for a non-windows OS. If this doesn't work, try "
@@ -318,6 +357,25 @@ class Installer(object):
                guest, self._unattended_data, url,
                os_media, os_tree, injection_method)

+    def _prepare_treemedia(self, guest, meter, unattended_scripts):
+        kernel, initrd, kernel_args = self._treemedia.prepare(guest, meter,
+                unattended_scripts)
+
+        paths = [kernel, initrd]
+        kernel, initrd = self._upload_media(guest, meter, paths)
+        self._treemedia_bootconfig = (kernel, initrd, kernel_args)
+
+    def _prepare_cloudinit(self, guest, meter):
+        scratchdir = InstallerTreeMedia.make_scratchdir(guest)
+        filepairs = cloudinit.create_files(scratchdir, self._cloudinit_data)
+        for filepair in filepairs:
+            self._tmpfiles.append(filepair[0])
+
+        iso = perform_cdrom_injections(filepairs, scratchdir, cloudinit=True)
+        self._tmpfiles.append(iso)
+        iso = self._upload_media(guest, meter, [iso])[0]
+        self._add_unattended_install_cdrom_device(guest, iso)
+
    def _prepare(self, guest, meter):
        if self._is_reinstall:
            self._pre_reinstall_xml = guest.get_xml()
@@ -327,22 +385,27 @@ class Installer(object):
            unattended_scripts = self._prepare_unattended_scripts(guest, meter)

        if self._treemedia:
-            self._treemedia_bootconfig = self._treemedia.prepare(guest, meter,
-                    unattended_scripts)
+            self._prepare_treemedia(guest, meter, unattended_scripts)

        elif unattended_scripts:
            self._prepare_unattended_data(guest, meter, unattended_scripts)

        elif self._cloudinit_data:
-            self._install_cloudinit(guest)
+            self._prepare_cloudinit(guest, meter)

    def _cleanup(self, guest):
        if self._treemedia:
            self._treemedia.cleanup(guest)

+        for vol in self._tmpvols:
+            log.debug("Removing volume '%s'", vol.name())
+            vol.delete(0)
+        self._tmpvols = []
+
        for f in self._tmpfiles:
            log.debug("Removing %s", str(f))
            os.unlink(f)
+        self._tmpfiles = []

    def _get_postinstall_bootdev(self, guest):
        if self.cdrom and self._no_install:
@@ -419,9 +482,10 @@ class Installer(object):
        for to perform this install.
        """
        search_paths = []
-        if (self._treemedia or
-            self._cloudinit_data or
-            self._unattended_data):
+        if ((self._treemedia or
+             self._cloudinit_data or
+             self._unattended_data) and
+             not self._should_upload_media(guest)):
            search_paths.append(InstallerTreeMedia.make_scratchdir(guest))
        if self._cdrom_path():
            search_paths.append(self._cdrom_path())
@@ -479,20 +543,6 @@ class Installer(object):
    def set_cloudinit_data(self, cloudinit_data):
        self._cloudinit_data = cloudinit_data

-    def _install_cloudinit(self, guest):
-        filepairs = cloudinit.create_files(
-                InstallerTreeMedia.make_scratchdir(guest),
-                self._cloudinit_data)
-        for filepair in filepairs:
-            self._tmpfiles.append(filepair[0])
-
-        iso = perform_cdrom_injections(
-                filepairs,
-                InstallerTreeMedia.make_scratchdir(guest),
-                cloudinit=True)
-        self._tmpfiles.append(iso)
-        self._add_unattended_install_cdrom_device(guest, iso)
-
    def has_cloudinit(self):
        return bool(self._cloudinit_data)

@@ -9,7 +9,6 @@ import os
 from . import urldetect
 from . import urlfetcher
 from .installerinject import perform_initrd_injections
-from .kernelupload import upload_kernel_initrd
 from .. import progress
 from ..devices import DeviceDisk
 from ..logger import log
@@ -127,7 +126,6 @@ class InstallerTreeMedia(object):
        self._cached_data = None

        self._tmpfiles = []
-        self._tmpvols = []

        if self._install_kernel or self._install_initrd:
            self._media_type = MEDIA_KERNEL
@@ -204,6 +202,8 @@ class InstallerTreeMedia(object):
        return self._cached_data

    def _prepare_kernel_url(self, guest, cache, fetcher):
+        ignore = guest
+
        def _check_kernel_pairs():
            for kpath, ipath in cache.kernel_pairs:
                if fetcher.hasFile(kpath) and fetcher.hasFile(ipath):
@@ -221,12 +221,6 @@ class InstallerTreeMedia(object):
                                  self._initrd_injections,
                                  fetcher.scratchdir)

-        system_scratchdir = InstallerTreeMedia.get_system_scratchdir(guest)
-        kernel, initrd, tmpvols = upload_kernel_initrd(
-                guest.conn, fetcher.scratchdir, system_scratchdir,
-                fetcher.meter, kernel, initrd)
-        self._tmpvols += tmpvols
-
        return kernel, initrd


@@ -294,11 +288,6 @@ class InstallerTreeMedia(object):
            log.debug("Removing %s", str(f))
            os.unlink(f)

-        for vol in self._tmpvols:
-            log.debug("Removing volume '%s'", vol.name())
-            vol.delete(0)
-
-        self._tmpvols = []
        self._tmpfiles = []

    def set_initrd_injections(self, initrd_injections):
@@ -7,7 +7,6 @@
 import os

 from .. import progress
-from .. import xmlutil
 from ..devices import DeviceDisk
 from ..logger import log
 from ..storage import StoragePool, StorageVolume
@@ -129,36 +128,24 @@ def _upload_file(conn, meter, destpool, src):
    return vol


-def upload_kernel_initrd(conn, scratchdir, system_scratchdir,
-                         meter, kernel, initrd):
+def upload_paths(conn, system_scratchdir, meter, pathlist):
    """
-    Upload kernel/initrd media to remote connection if necessary
+    Upload passed paths to the connection scratchdir
    """
-    tmpvols = []
-
-    if (not conn.is_remote() and
-        (conn.is_unprivileged() or scratchdir == system_scratchdir) and
-        not xmlutil.in_testsuite()):
-        # We have access to system scratchdir, don't jump through hoops
-        log.debug("Have access to preferred scratchdir so"
-                    " nothing to upload")  # pragma: no cover
-        return kernel, initrd, tmpvols  # pragma: no cover
-
-    if not conn.support_remote_url_install():
-        # Needed for the test_urls suite
-        log.debug("Media upload not supported")  # pragma: no cover
-        return kernel, initrd, tmpvols  # pragma: no cover
-
    # Build pool
    log.debug("Uploading kernel/initrd media")
    pool = _build_pool(conn, meter, system_scratchdir)

-    kvol = _upload_file(conn, meter, pool, kernel)
-    newkernel = kvol.path()
-    tmpvols.append(kvol)
+    tmpvols = []
+    newpaths = []
+    try:
+        for path in pathlist:
+            vol = _upload_file(conn, meter, pool, path)
+            newpaths.append(vol.path())
+            tmpvols.append(vol)
+    except Exception:  # pragma: no cover
+        for vol in tmpvols:
+            vol.delete(0)
+        raise

-    ivol = _upload_file(conn, meter, pool, initrd)
-    newinitrd = ivol.path()
-    tmpvols.append(ivol)
-
-    return newkernel, newinitrd, tmpvols
+    return newpaths, tmpvols