Allow to use many disks in vagrant box for libvirt provider

Adds support for a new multi disk box format and handling to upload the
multiple disks to the storage pool.

New format is:
{
  'disks': [
    {
      'name': 'disk1.img',
      'virtual_size': 10,
      'format': 'qcow2'
    },
    {
      'name': 'disk2.img',
      'virtual_size': 15,
      'format': 'qcow2'
    },
    {
      'name': 'disk3.img',
    }
  ],
  'provider': 'libvirt',
  'format': 'qcow2'
}

It is expected to remove format from being set at the top level when
using the new format, with the assuming that qcow2 should be the default
format, and other formats should be permitted to be specified as needed.

Includes tests for handling the box images and creation of domain
volumes. Additionally includes an integration test to ensure a box with
2 disks will work as expected.

Partially fixes: #602

spec/unit/action/create_domain_spec.rb

@@ -31,6 +31,12 @@ describe VagrantPlugins::ProviderLibvirt::Action::CreateDomain do
 
       env[:domain_name] = "vagrant-test_default"
 
+      env[:box_volumes] = []
+      env[:box_volumes].push({
+        :path=>"/test/box.img",
+        :name=>"test_vagrant_box_image_1.1.1_0.img", 
+        :virtual_size=>5
+      })
       # should be ignored for system session and used for user session
       allow(Process).to receive(:uid).and_return(9999)
       allow(Process).to receive(:gid).and_return(9999)

spec/unit/action/create_domain_volume_spec.rb

@@ -0,0 +1,102 @@
+require 'spec_helper'
+require 'support/sharedcontext'
+require 'support/libvirt_context'
+
+require 'vagrant-libvirt/action/destroy_domain'
+
+describe VagrantPlugins::ProviderLibvirt::Action::CreateDomainVolume do
+  subject { described_class.new(app, env) }
+
+  include_context 'unit'
+  include_context 'libvirt'
+
+  let(:libvirt_domain) { double('libvirt_domain') }
+  let(:libvirt_client) { double('libvirt_client') }
+  let(:volumes) { double('volumes') }
+  let(:all) { double('all') }
+  let(:box_volume) { double('box_volume') }
+
+  def read_test_file(name)
+    File.read(File.join(File.dirname(__FILE__), File.basename(__FILE__, '.rb'), name))
+  end
+
+  describe '#call' do
+    before do
+      allow_any_instance_of(VagrantPlugins::ProviderLibvirt::Driver)
+        .to receive(:connection).and_return(connection)
+      allow(connection).to receive(:client).and_return(libvirt_client)
+      allow(connection).to receive(:volumes).and_return(volumes)
+      allow(volumes).to receive(:all).and_return(all)
+      allow(all).to receive(:first).and_return(box_volume)
+      allow(box_volume).to receive(:id).and_return(nil)
+      env[:domain_name] = 'test'
+    end
+
+    context 'when one disk' do
+      before do
+        allow(box_volume).to receive(:path).and_return('/test/path_0.img')
+        env[:box_volumes] = [
+          {
+            :name=>"test_vagrant_box_image_1.1.1_0.img",
+            :virtual_size=>5
+          }
+        ]
+      end
+
+      it 'should create one disk in storage' do
+        expected_xml = read_test_file('one_disk_in_storage.xml')
+        expect(ui).to receive(:info).with('Creating image (snapshot of base box volume).')
+        expect(logger).to receive(:debug).with('Using pool default for base box snapshot')
+        expect(volumes).to receive(:create).with(
+          :xml => expected_xml,
+          :pool_name => "default"
+        )
+        expect(subject.call(env)).to be_nil
+      end
+    end
+
+    context 'when three disks' do
+      before do
+        allow(box_volume).to receive(:path).and_return(
+          '/test/path_0.img',
+          '/test/path_1.img',
+          '/test/path_2.img',
+        )
+        env[:box_volumes] = [
+          {
+            :name=>"test_vagrant_box_image_1.1.1_0.img",
+            :virtual_size=>5
+          },
+          {
+            :name=>"test_vagrant_box_image_1.1.1_1.img",
+            :virtual_size=>10
+          },
+          {
+            :name=>"test_vagrant_box_image_1.1.1_2.img",
+            :virtual_size=>20
+          }
+        ]
+      end
+
+      it 'should create three disks in storage' do
+        expect(ui).to receive(:info).with('Creating image (snapshot of base box volume).')
+        expect(logger).to receive(:debug).with('Using pool default for base box snapshot')
+        expect(volumes).to receive(:create).with(
+          :xml => read_test_file('three_disks_in_storage_disk_0.xml'),
+          :pool_name => "default"
+        )
+        expect(logger).to receive(:debug).with('Using pool default for base box snapshot')
+        expect(volumes).to receive(:create).with(
+          :xml => read_test_file('three_disks_in_storage_disk_1.xml'),
+          :pool_name => "default"
+        )
+        expect(logger).to receive(:debug).with('Using pool default for base box snapshot')
+        expect(volumes).to receive(:create).with(
+          :xml => read_test_file('three_disks_in_storage_disk_2.xml'),
+          :pool_name => "default"
+        )
+        expect(subject.call(env)).to be_nil
+      end
+    end
+  end
+end

spec/unit/action/create_domain_volume_spec/one_disk_in_storage.xml

@@ -0,0 +1,21 @@
+<volume>
+  <name>test.img</name>
+  <capacity unit="G">5</capacity>
+  <target>
+    <format type="qcow2"></format>
+    <permissions>
+      <owner>0</owner>
+      <group>0</group>
+      <label>virt_image_t</label>
+    </permissions>
+  </target>
+  <backingStore>
+    <path>/test/path_0.img</path>
+    <format type="qcow2"></format>
+    <permissions>
+      <owner>0</owner>
+      <group>0</group>
+      <label>virt_image_t</label>
+    </permissions>
+  </backingStore>
+</volume>

spec/unit/action/create_domain_volume_spec/three_disks_in_storage_disk_0.xml

@@ -0,0 +1,21 @@
+<volume>
+  <name>test.img</name>
+  <capacity unit="G">5</capacity>
+  <target>
+    <format type="qcow2"></format>
+    <permissions>
+      <owner>0</owner>
+      <group>0</group>
+      <label>virt_image_t</label>
+    </permissions>
+  </target>
+  <backingStore>
+    <path>/test/path_0.img</path>
+    <format type="qcow2"></format>
+    <permissions>
+      <owner>0</owner>
+      <group>0</group>
+      <label>virt_image_t</label>
+    </permissions>
+  </backingStore>
+</volume>

spec/unit/action/create_domain_volume_spec/three_disks_in_storage_disk_1.xml

@@ -0,0 +1,21 @@
+<volume>
+  <name>test_1.img</name>
+  <capacity unit="G">10</capacity>
+  <target>
+    <format type="qcow2"></format>
+    <permissions>
+      <owner>0</owner>
+      <group>0</group>
+      <label>virt_image_t</label>
+    </permissions>
+  </target>
+  <backingStore>
+    <path>/test/path_1.img</path>
+    <format type="qcow2"></format>
+    <permissions>
+      <owner>0</owner>
+      <group>0</group>
+      <label>virt_image_t</label>
+    </permissions>
+  </backingStore>
+</volume>

spec/unit/action/create_domain_volume_spec/three_disks_in_storage_disk_2.xml

@@ -0,0 +1,21 @@
+<volume>
+  <name>test_2.img</name>
+  <capacity unit="G">20</capacity>
+  <target>
+    <format type="qcow2"></format>
+    <permissions>
+      <owner>0</owner>
+      <group>0</group>
+      <label>virt_image_t</label>
+    </permissions>
+  </target>
+  <backingStore>
+    <path>/test/path_2.img</path>
+    <format type="qcow2"></format>
+    <permissions>
+      <owner>0</owner>
+      <group>0</group>
+      <label>virt_image_t</label>
+    </permissions>
+  </backingStore>
+</volume>

spec/unit/action/handle_box_image_spec.rb

@@ -0,0 +1,296 @@
+require 'spec_helper'
+require 'support/sharedcontext'
+require 'support/libvirt_context'
+
+require 'vagrant-libvirt/action/destroy_domain'
+
+describe VagrantPlugins::ProviderLibvirt::Action::HandleBoxImage do
+  subject { described_class.new(app, env) }
+
+  include_context 'unit'
+  include_context 'libvirt'
+
+  let(:libvirt_client) { double('libvirt_client') }
+  let(:volumes) { double('volumes') }
+  let(:all) { double('all') }
+  let(:box_volume) { double('box_volume') }
+  let(:fog_volume) { double('fog_volume') }
+
+  describe '#call' do
+    before do
+      allow_any_instance_of(VagrantPlugins::ProviderLibvirt::Driver)
+        .to receive(:connection).and_return(connection)
+      allow(connection).to receive(:client).and_return(libvirt_client)
+      allow(connection).to receive(:volumes).and_return(volumes)
+      allow(volumes).to receive(:all).and_return(all)
+      allow(env[:ui]).to receive(:clear_line)
+
+    end
+
+    context 'when one disk in metadata.json' do
+      before do
+        allow(all).to receive(:first).and_return(box_volume)
+        allow(box_volume).to receive(:id).and_return(1)
+        allow(env[:machine]).to receive_message_chain("box.name") { 'test' }
+        allow(env[:machine]).to receive_message_chain("box.version") { '1.1.1' }
+        allow(env[:machine]).to receive_message_chain("box.metadata") { Hash[
+          'virtual_size'=> 5,
+          'format' => 'qcow2'
+          ]
+        }
+        allow(env[:machine]).to receive_message_chain("box.directory.join") do |arg|
+          '/test/'.concat(arg.to_s)
+        end
+      end
+
+      it 'should have one disk in machine env' do
+        expect(subject.call(env)).to be_nil
+        expect(env[:box_volume_number]).to eq(1)
+        expect(env[:box_volumes]).to eq(
+          [
+            {
+              :path=>"/test/box.img",
+              :name=>"test_vagrant_box_image_1.1.1_0.img", 
+              :virtual_size=>5,
+              :format=>"qcow2"
+            }
+          ]
+        )
+      end
+
+      context 'when disk image not in storage pool' do
+        before do
+          allow(File).to receive(:exist?).and_return(true)
+          allow(File).to receive(:size).and_return(5*1024*1024*1024)
+          allow(all).to receive(:first).and_return(nil)
+          allow(subject).to receive(:upload_image).and_return(true)
+          allow(volumes).to receive(:create).and_return(fog_volume)
+        end
+
+        it 'should upload disk' do
+          expect(ui).to receive(:info).with('Uploading base box image as volume into Libvirt storage...')
+          expect(logger).to receive(:info).with('Creating volume test_vagrant_box_image_1.1.1_0.img in storage pool default.')
+          expect(volumes).to receive(:create).with(
+            hash_including(
+              :name => "test_vagrant_box_image_1.1.1_0.img",
+              :allocation => "5120M",
+              :capacity => "5G",
+            )
+          )
+          expect(subject).to receive(:upload_image)
+          expect(subject.call(env)).to be_nil
+        end
+      end
+
+      context 'when disk image already in storage pool' do
+        before do
+          allow(all).to receive(:first).and_return(box_volume)
+          allow(box_volume).to receive(:id).and_return(1)
+        end
+
+        it 'should skip disk upload' do
+          expect(volumes).not_to receive(:create)
+          expect(subject).not_to receive(:upload_image)
+          expect(subject.call(env)).to be_nil
+        end
+      end
+    end
+
+    context 'when three disks in metadata.json' do
+      before do
+        allow(all).to receive(:first).and_return(box_volume)
+        allow(box_volume).to receive(:id).and_return(1)
+        allow(env[:machine]).to receive_message_chain("box.name") { 'test' }
+        allow(env[:machine]).to receive_message_chain("box.version") { '1.1.1' }
+        allow(env[:machine]).to receive_message_chain("box.metadata") { Hash[
+          'disks' => [
+            {
+              'name'=>'send_box_name.img',
+              'virtual_size'=> 5,
+            },
+            {
+              'path' => 'disk.qcow2',
+              'virtual_size'=> 10
+            },
+            {
+              'virtual_size'=> 20,
+            },
+          ],
+          'format' => 'qcow2'
+          ]
+        }
+        allow(env[:machine]).to receive_message_chain("box.directory.join") do |arg|
+          '/test/'.concat(arg.to_s)
+        end
+      end
+
+      it 'should have three disks in machine env' do
+        expect(subject.call(env)).to be_nil
+        expect(env[:box_volume_number]).to eq(3)
+        expect(env[:box_volumes]).to eq(
+          [
+            {
+              :path=>"/test/box.img",
+              :name=>"send_box_name.img",
+              :virtual_size=>5,
+              :format=>"qcow2"
+            },
+            {
+              :path=>"/test/disk.qcow2",
+              :name=>"test_vagrant_box_image_1.1.1_1.img", 
+              :virtual_size=>10,
+              :format=>"qcow2"
+            },
+            {
+              :path=>"/test/box_2.img",
+              :name=>"test_vagrant_box_image_1.1.1_2.img", 
+              :virtual_size=>20,
+              :format=>"qcow2"
+            }
+          ]
+        )
+      end
+
+      context 'when none of the disks in storage pool' do
+        before do
+          allow(File).to receive(:exist?).and_return(true)
+          allow(File).to receive(:size).and_return(5*1024*1024*1024, 10*1024*1024*1024, 20*1024*1024*1024)
+          allow(all).to receive(:first).and_return(nil)
+          allow(subject).to receive(:upload_image).and_return(true)
+          allow(volumes).to receive(:create).and_return(fog_volume)
+        end
+
+        it 'should upload all 3 disks' do
+          expect(ui).to receive(:info).with('Uploading base box image as volume into Libvirt storage...')
+          expect(logger).to receive(:info).with('Creating volume send_box_name.img in storage pool default.')
+          expect(volumes).to receive(:create).with(
+            hash_including(
+              :name => "send_box_name.img",
+              :allocation => "5120M",
+              :capacity => "5G",
+            )
+          )
+          expect(subject).to receive(:upload_image)
+          expect(ui).to receive(:info).with('Uploading base box image as volume into Libvirt storage...')
+          expect(logger).to receive(:info).with('Creating volume test_vagrant_box_image_1.1.1_1.img in storage pool default.')
+          expect(volumes).to receive(:create).with(
+            hash_including(
+              :name => "test_vagrant_box_image_1.1.1_1.img",
+              :allocation => "10240M",
+              :capacity => "10G",
+            )
+          )
+          expect(subject).to receive(:upload_image)
+          expect(ui).to receive(:info).with('Uploading base box image as volume into Libvirt storage...')
+          expect(logger).to receive(:info).with('Creating volume test_vagrant_box_image_1.1.1_2.img in storage pool default.')
+          expect(volumes).to receive(:create).with(
+            hash_including(
+              :name => "test_vagrant_box_image_1.1.1_2.img",
+              :allocation => "20480M",
+              :capacity => "20G",
+            )
+          )
+          expect(subject).to receive(:upload_image)
+
+          expect(subject.call(env)).to be_nil
+        end
+      end
+
+      context 'when only disk 0 in storage pool' do
+        before do
+          allow(File).to receive(:exist?).and_return(true)
+          allow(File).to receive(:size).and_return(10*1024*1024*1024, 20*1024*1024*1024)
+          allow(all).to receive(:first).and_return(box_volume, nil, nil)
+          allow(box_volume).to receive(:id).and_return(1)
+          allow(subject).to receive(:upload_image).and_return(true)
+          allow(volumes).to receive(:create).and_return(fog_volume)
+        end
+
+        it 'upload disks 1 and 2 only' do
+          expect(ui).to receive(:info).with('Uploading base box image as volume into Libvirt storage...')
+          expect(logger).to receive(:info).with('Creating volume test_vagrant_box_image_1.1.1_1.img in storage pool default.')
+          expect(volumes).to receive(:create).with(hash_including(:name => "test_vagrant_box_image_1.1.1_1.img"))
+          expect(subject).to receive(:upload_image)
+          expect(ui).to receive(:info).with('Uploading base box image as volume into Libvirt storage...')
+          expect(logger).to receive(:info).with('Creating volume test_vagrant_box_image_1.1.1_2.img in storage pool default.')
+          expect(volumes).to receive(:create).with(hash_including(:name => "test_vagrant_box_image_1.1.1_2.img"))
+          expect(subject).to receive(:upload_image)
+
+          expect(subject.call(env)).to be_nil
+        end
+      end
+
+      context 'when has all disks on storage pool' do
+        before do
+          allow(all).to receive(:first).and_return(box_volume)
+          allow(box_volume).to receive(:id).and_return(1)
+        end
+
+        it 'should skip disk upload' do
+          expect(ui).not_to receive(:info).with('Uploading base box image as volume into Libvirt storage...')
+          expect(volumes).not_to receive(:create)
+          expect(subject).not_to receive(:upload_image)
+          expect(subject.call(env)).to be_nil
+        end
+      end
+    end
+
+    context 'when wrong box format in metadata.json' do
+      before do
+        allow(all).to receive(:first).and_return(box_volume)
+        allow(box_volume).to receive(:id).and_return(1)
+        allow(env[:machine]).to receive_message_chain("box.name") { 'test' }
+        allow(env[:machine]).to receive_message_chain("box.version") { '1.1.1' }
+        allow(env[:machine]).to receive_message_chain("box.metadata") { Hash[
+          'virtual_size'=> 5,
+          'format' => 'wrongFormat'
+          ]
+        }
+        allow(env[:machine]).to receive_message_chain("box.directory.join") do |arg|
+          '/test/'.concat(arg.to_s)
+        end
+      end
+
+      it 'should raise WrongBoxFormatSet exception' do
+        expect{ subject.call(env) }.to raise_error(VagrantPlugins::ProviderLibvirt::Errors::WrongBoxFormatSet)
+      end
+
+    end
+
+    context 'when one of a multi disk definition has wrong disk format in metadata.json' do
+      before do
+        allow(all).to receive(:first).and_return(box_volume)
+        allow(box_volume).to receive(:id).and_return(1)
+        allow(env[:machine]).to receive_message_chain("box.name") { 'test' }
+        allow(env[:machine]).to receive_message_chain("box.version") { '1.1.1' }
+        allow(env[:machine]).to receive_message_chain("box.metadata") {
+          Hash[
+            'disks' => [
+              {
+                'name'=>'send_box_name.img',
+                'virtual_size'=> 5,
+                'format'=> 'wrongFormat'
+              },
+              {
+                'path' => 'disk.qcow2',
+                'virtual_size'=> 10
+              },
+              {
+                'virtual_size'=> 20,
+              },
+            ],
+            'format' => 'qcow2',
+          ]
+        }
+        allow(env[:machine]).to receive_message_chain("box.directory.join") do |arg|
+          '/test/'.concat(arg.to_s)
+        end
+      end
+
+      it 'should raise WrongDiskFormatSet exception' do
+        expect { subject.call(env) }.to raise_error(VagrantPlugins::ProviderLibvirt::Errors::WrongDiskFormatSet)
+      end
+    end
+
+  end
+end

spec/unit/templates/domain_all_settings.xml

@@ -47,6 +47,11 @@
       <source file='/var/lib/libvirt/images/test.qcow2'/>
       <target dev='vda' bus='ide'/>
     </disk>
+    <disk type='file' device='disk'>
+      <driver name='qemu' type='qcow2' cache='unsafe' io='threads' copy_on_read='on' discard='unmap' detect_zeroes='on'/>
+      <source file='/var/lib/libvirt/images/test2.qcow2'/>
+      <target dev='vdb' bus='ide'/>
+    </disk>
     <disk type='file' device='disk'>
       <driver name='qemu' type='qcow2' cache='default'/>
       <source file='/var/lib/libvirt/images/test-disk1.qcow2'/>

spec/unit/templates/domain_spec.rb

@@ -9,6 +9,13 @@ describe 'templates/domain' do
   class DomainTemplateHelper < VagrantPlugins::ProviderLibvirt::Config
     include VagrantPlugins::ProviderLibvirt::Util::ErbTemplate
 
+    attr_accessor :domain_volumes
+
+    def initialize
+      super
+      @domain_volumes = []
+    end
+
     def finalize!
       super
     end
@@ -44,11 +51,22 @@ describe 'templates/domain' do
       domain.boot('cdrom')
       domain.boot('hd')
       domain.emulator_path = '/usr/bin/kvm-spice'
-      domain.instance_variable_set('@domain_volume_path', '/var/lib/libvirt/images/test.qcow2')
       domain.instance_variable_set('@domain_volume_cache', 'deprecated')
       domain.disk_bus = 'ide'
       domain.disk_device = 'vda'
       domain.disk_driver(:cache => 'unsafe', :io => 'threads', :copy_on_read => 'on', :discard => 'unmap', :detect_zeroes => 'on')
+      domain.domain_volumes.push({
+        :dev => 1.vdev.to_s,
+        :cache => 'unsafe',
+        :bus => domain.disk_bus,
+        :path => '/var/lib/libvirt/images/test.qcow2'
+      })
+      domain.domain_volumes.push({
+        :dev => 2.vdev.to_s,
+        :cache => 'unsafe',
+        :bus => domain.disk_bus,
+        :path => '/var/lib/libvirt/images/test2.qcow2'
+      })
       domain.storage(:file, path: 'test-disk1.qcow2')
       domain.storage(:file, path: 'test-disk2.qcow2', io: 'threads', copy_on_read: 'on', discard: 'unmap', detect_zeroes: 'on')
       domain.disks.each do |disk|