Implement boot ordering as a separate class

This is required because in current versions of libvirt, it is not
possible to specify a boot order when attaching a device; therefore we
can only parse the entire domain XML after all devices have been created
and then assign boot ordering according to the Vagrantfile
specification. This allows us to specify exact boot order for hd, cdrom
and network.

README.md

@@ -173,7 +173,7 @@ end
 * `machine` - Sets machine type. Equivalent to qemu `-machine`. Use `qemu-system-x86_64 -machine help` to get a list of supported machines.
 * `machine_arch` - Sets machine architecture. This helps libvirt to determine the correct emulator type. Possible values depend on your version of qemu. For possible values, see which emulator executable `qemu-system-*` your system provides. Common examples are `aarch64`, `alpha`, `arm`, `cris`, `i386`, `lm32`, `m68k`, `microblaze`, `microblazeel`, `mips`, `mips64`, `mips64el`, `mipsel`, `moxie`, `or32`, `ppc`, `ppc64`, `ppcemb`, `s390x`, `sh4`, `sh4eb`, `sparc`, `sparc64`, `tricore`, `unicore32`, `x86_64`, `xtensa`, `xtensaeb`.
 * `machine_virtual_size` - Sets the disk size in GB for the machine overriding the default specified in the box. Allows boxes to defined with a minimal size disk by default and to be grown to a larger size at creation time. Will ignore sizes smaller than the size specified by the box metadata. Note that currently there is no support for automatically resizing the filesystem to take advantage of the larger disk.
-* `boot` - Change the boot order and enables the boot menu. Possible options are "hd" or "network". Defaults to "hd" with boot menu disabled. When "network" is set first, *all* NICs will be tried before the first disk is tried.
+* `boot` - Change the boot order and enables the boot menu. Possible options are "hd", "network", "cdrom". Defaults to "hd" with boot menu disabled. When "network" is set without "hd", only all NICs will be tried; see below for more detail.
 * `nic_adapter_count` - Defaults to '8'. Only use case for increasing this count is for VMs that virtualize switches such as Cumulus Linux. Max value for Cumulus Linux VMs is 33.
 
 
@@ -198,7 +198,8 @@ Vagrant.configure("2") do |config|
 
 The following example shows part of a Vagrantfile that enables the VM to
 boot from a network interface first and a hard disk second. This could be
-used to run VMs that are meant to be a PXE booted machines.
+used to run VMs that are meant to be a PXE booted machines. Be aware that 
+if `hd` is not specified as a boot option, it will never be tried.
 
 ```ruby
 Vagrant.configure("2") do |config|
@@ -446,6 +447,17 @@ Vagrant.configure("2") do |config|
     end
   end
 
+And an example for a PXE booted VM with no box but a blank disk which will boot from this HD if the NICs fail to PXE boot::
+
+Vagrant.configure("2") do |config|
+  config.vm.define :pxeclient do |pxeclient|
+    pxeclient.vm.provider :libvirt do |domain|
+      domain.storage :file, :size => '100G', :type => 'qcow2'
+      domain.boot 'network'
+      domain.boot 'hd'
+    end
+  end
+
 ## SSH Access To VM
 
 vagrant-libvirt supports vagrant's [standard ssh settings](https://docs.vagrantup.com/v2/vagrantfile/ssh_settings.html).

lib/vagrant-libvirt/action.rb

@@ -27,6 +27,7 @@ module VagrantPlugins
                 b2.use CreateDomain
                 b2.use CreateNetworks
                 b2.use CreateNetworkInterfaces
+                b2.use SetBootOrder
                 b2.use StartDomain
               else
                 b2.use HandleStoragePool
@@ -43,6 +44,7 @@ module VagrantPlugins
                 b2.use ShareFolders
                 b2.use CreateNetworks
                 b2.use CreateNetworkInterfaces
+                b2.use SetBootOrder
 
                 b2.use StartDomain
                 b2.use WaitTillUp
@@ -78,6 +80,7 @@ module VagrantPlugins
               if !env[:machine].box
                 # With no box, we just care about network creation and starting it
                 b3.use CreateNetworks
+                b3.use SetBootOrder
                 b3.use StartDomain
               else
                 # VM is not running or suspended.
@@ -86,6 +89,7 @@ module VagrantPlugins
 
                 # Ensure networks are created and active
                 b3.use CreateNetworks
+                b3.use SetBootOrder
 
                 b3.use PrepareNFSValidIds
                 b3.use SyncedFolderCleanup
@@ -167,7 +171,7 @@ module VagrantPlugins
             if !env[:result]
               # Try to remove stale volumes anyway
               b2.use SetNameOfDomain
-	      if env[:machine].box
+              if env[:machine].box
                 b2.use RemoveStaleVolume
               end
               if !env[:result]
@@ -336,6 +340,7 @@ module VagrantPlugins
       autoload :ReadMacAddresses, action_root.join('read_mac_addresses')
       autoload :ResumeDomain, action_root.join('resume_domain')
       autoload :SetNameOfDomain, action_root.join('set_name_of_domain')
+      autoload :SetBootOrder, action_root.join('set_boot_order')
 
       # I don't think we need it anymore
       autoload :ShareFolders, action_root.join('share_folders')

lib/vagrant-libvirt/action/set_boot_order.rb

@@ -0,0 +1,66 @@
+require "log4r"
+require 'nokogiri'
+
+module VagrantPlugins
+  module ProviderLibvirt
+    module Action
+      class SetBootOrder
+        def initialize(app, env)
+          @app    = app
+          @logger = Log4r::Logger.new("vagrant_libvirt::action::set_boot_order")
+          config = env[:machine].provider_config
+          @boot_order = config.boot_order
+        end
+
+        def call(env)
+         # Get domain first
+          begin
+            domain = env[:machine].provider.driver.connection.client.lookup_domain_by_uuid(
+              env[:machine].id.to_s)
+          rescue => e
+            raise Errors::NoDomainError,
+              :error_message => e.message
+          end
+
+          # Only execute specific boot ordering if this is defined in the Vagrant file
+          if @boot_order.count >= 1
+          
+            # If a domain is initially defined with no box or disk or with an explicit boot order, libvirt adds <boot dev="foo">
+            # This conflicts with an explicit boot_order configuration, so we need to remove it from the domain xml and feed it back.
+            # Also see https://bugzilla.redhat.com/show_bug.cgi?id=1248514 as to why we have to do this after all devices have been defined.
+            xml = Nokogiri::XML(domain.xml_desc)
+            xml.search("/domain/os/boot").each do |node|
+              node.remove
+            end
+
+            # Parse the XML and find each defined drive and network interfacee
+            hd = xml.search("/domain/devices/disk[@device='disk']")
+            cdrom = xml.search("/domain/devices/disk[@device='cdrom']")
+            network = xml.search("/domain/devices/interface[@type='network']")
+
+            # Generate an array per device group and a flattened array from all of those
+            devices = {"hd" => hd, "cdrom" => cdrom, "network" => network}
+            final_boot_order = @boot_order.flat_map {|category| devices[category] }
+
+            # Loop over the entire defined boot order array and create boot order entries in the domain XML
+            final_boot_order.each_with_index do |node, index|
+              boot = "<boot order='#{index+1}'/>"
+              node.add_child(boot)
+              if node.name == 'disk'
+                @logger.debug "Setting #{node['device']} to boot index #{index+1}"
+              elsif node.name == 'interface'
+                @logger.debug "Setting #{node.name} to boot index #{index+1}"
+              end
+            end
+
+            # Finally redefine the domain XML through libvirt to apply the boot ordering
+            env[:machine].provider.driver.connection.client.define_domain_xml(xml.to_s)
+          end
+
+          @app.call(env)
+
+        end
+      end
+    end
+  end
+end

lib/vagrant-libvirt/templates/domain.xml.erb

@@ -31,12 +31,7 @@
       <loader readonly='yes' type='rom'><%= @loader %></loader>
     <% end %>
     <% if @boot_order.count >= 1 %>
-      <% @boot_order.each do |b| %>
-        <boot dev='<%= b %>'/>
-      <% end %> 
       <bootmenu enable='yes'/>
-    <% else %>
-      <boot dev='hd' />
     <% end %>
     <kernel><%= @kernel %></kernel>
     <initrd><%= @initrd %></initrd>