From 56282b965cde3bd175e1c8d29f142d6f665574e5 Mon Sep 17 00:00:00 2001 From: Darragh Bailey Date: Fri, 8 Oct 2021 11:05:10 +0100 Subject: [PATCH] Provide support for packaging into v2 format (#1360) Support packaging multi disk machines however default to v1 format unless v2 format is explicitly enabled. Output a warning to alert users of the possible change in behaviour in the future. Allows selecting the format to use via the environment, where use of v1 format with a multi disk machine will ignore the other disks. --- lib/vagrant-libvirt/action/package_domain.rb | 132 +++++++++++--- spec/unit/action/package_domain_spec.rb | 179 ++++++++++++++++++- 2 files changed, 281 insertions(+), 30 deletions(-) diff --git a/lib/vagrant-libvirt/action/package_domain.rb b/lib/vagrant-libvirt/action/package_domain.rb index f72e705..09b1bd6 100644 --- a/lib/vagrant-libvirt/action/package_domain.rb +++ b/lib/vagrant-libvirt/action/package_domain.rb @@ -31,45 +31,75 @@ module VagrantPlugins env[:machine].id ) domain = env[:machine].provider.driver.connection.servers.get(env[:machine].id.to_s) - root_disk = domain.volumes.select do |x| - !x.nil? && x.name == libvirt_domain.name + '.img' + + volumes = domain.volumes.select { |x| !x.nil? } + root_disk = volumes.select do |x| + x.name == libvirt_domain.name + '.img' end.first raise Errors::NoDomainVolume if root_disk.nil? - package_directory = env["package.directory"] - domain_img = package_directory + '/box.img' - env[:ui].info("Downloading #{root_disk.name} to #{domain_img}") - ret = download_image(domain_img, env[:machine].provider_config.storage_pool_name, - root_disk.name, env) do |progress,image_size| - rewriting(env[:ui]) do |ui| - ui.clear_line - ui.report_progress(progress, image_size, false) - end - end - # Clear the line one last time since the progress meter doesn't - # disappear immediately. - rewriting(env[:ui]) {|ui| ui.clear_line} + package_func = method(:package_v1) - # Prep domain disk - backing = `qemu-img info "#{domain_img}" | grep 'backing file:' | cut -d ':' -f2`.chomp - if backing - env[:ui].info('Image has backing image, copying image and rebasing ...') - `qemu-img rebase -p -b "" #{domain_img}` + box_format = ENV.fetch('VAGRANT_LIBVIRT_BOX_FORMAT_VERSION', nil) + + case box_format + when nil + if volumes.length() > 1 + msg = "Detected more than one volume for machine, in the future this will switch to using the v2 " + msg += "box format v2 automatically." + msg += "\nIf you want to include the additional disks attached when packaging please set the " + msg += "env variable VAGRANT_LIBVIRT_BOX_FORMAT_VERSION=v2 to use the new format. If you want " + msg += "to ensure that your box uses the old format for single disk only, please set the " + msg += "environment variable explicitly to 'v1'" + env[:ui].warn(msg) + end + when 'v2' + package_func = method(:package_v2) + when 'v1' + else + env[:ui].warn("Unrecognized value for 'VAGRANT_LIBVIRT_BOX_FORMAT_VERSION', defaulting to v1") end - # remove hw association with interface - # working for centos with lvs default disks - `virt-sysprep --no-logfile --operations #{@operations} -a #{domain_img} #{@options}` - `virt-sparsify --in-place #{domain_img}` + + metadata = package_func.call(env, volumes) # metadata / Vagrantfile - info = JSON.parse(`qemu-img info --output=json #{domain_img}`) - img_size = (Float(info['virtual-size'])/(1024**3)).ceil - File.write(package_directory + '/metadata.json', metadata_content(img_size)) + package_directory = env["package.directory"] + File.write(package_directory + '/metadata.json', metadata) File.write(package_directory + '/Vagrantfile', vagrantfile_content(env)) @app.call(env) end + def package_v1(env, volumes) + domain_img = download_volume(env, volumes.first, 'box.img') + + sysprep_domain(domain_img) + sparsify_volume(domain_img) + + info = JSON.parse(`qemu-img info --output=json #{domain_img}`) + img_size = (Float(info['virtual-size'])/(1024**3)).ceil + + return metadata_content_v1(img_size) + end + + def package_v2(env, volumes) + disks = [] + volumes.each_with_index do |vol, idx| + disk = {:path => "box_#{idx+1}.img"} + volume_img = download_volume(env, vol, disk[:path]) + + if idx == 0 + sysprep_domain(volume_img) + end + + sparsify_volume(volume_img) + + disks.push(disk) + end + + return metadata_content_v2(disks) + end + def vagrantfile_content(env) include_vagrantfile = "" @@ -93,7 +123,7 @@ module VagrantPlugins EOF end - def metadata_content(filesize) + def metadata_content_v1(filesize) <<-EOF.unindent { "provider": "libvirt", @@ -103,8 +133,54 @@ module VagrantPlugins EOF end + def metadata_content_v2(disks) + data = { + "provider": "libvirt", + "format": "qcow2", + "disks": disks.each do |disk| + {'path': disk[:path]} + end + } + JSON.pretty_generate(data) + end + protected + def sparsify_volume(volume_img) + `virt-sparsify --in-place #{volume_img}` + end + + def sysprep_domain(domain_img) + # remove hw association with interface + # working for centos with lvs default disks + `virt-sysprep --no-logfile --operations #{@operations} -a #{domain_img} #{@options}` + end + + def download_volume(env, volume, disk_path) + package_directory = env["package.directory"] + volume_img = package_directory + '/' + disk_path + env[:ui].info("Downloading #{volume.name} to #{volume_img}") + download_image(volume_img, env[:machine].provider_config.storage_pool_name, + volume.name, env) do |progress,image_size| + rewriting(env[:ui]) do |ui| + ui.clear_line + ui.report_progress(progress, image_size, false) + end + end + # Clear the line one last time since the progress meter doesn't + # disappear immediately. + rewriting(env[:ui]) {|ui| ui.clear_line} + + # Prep domain disk + backing = `qemu-img info "#{volume_img}" | grep 'backing file:' | cut -d ':' -f2`.chomp + if backing + env[:ui].info('Image has backing image, copying image and rebasing ...') + `qemu-img rebase -p -b "" #{volume_img}` + end + + return volume_img + end + # Fog libvirt currently doesn't support downloading images from storage # pool volumes. Use ruby-libvirt client instead. def download_image(image_file, pool_name, volume_name, env) diff --git a/spec/unit/action/package_domain_spec.rb b/spec/unit/action/package_domain_spec.rb index 1a8d571..2f00cee 100644 --- a/spec/unit/action/package_domain_spec.rb +++ b/spec/unit/action/package_domain_spec.rb @@ -16,6 +16,8 @@ describe VagrantPlugins::ProviderLibvirt::Action::PackageDomain do let(:libvirt_domain) { double('libvirt_domain') } let(:servers) { double('servers') } let(:volumes) { double('volumes') } + let(:metadata_file) { double('file') } + let(:vagrantfile_file) { double('file') } describe '#call' do before do @@ -54,10 +56,19 @@ describe VagrantPlugins::ProviderLibvirt::Action::PackageDomain do expect(subject).to receive(:`).with(/qemu-img info --output=json .*\/box.img/).and_return( { 'virtual-size': 5*1024*1024*1024 }.to_json ) + expect(File).to receive(:write).with( + /.*\/metadata.json/, + <<-EOF.unindent + { + "provider": "libvirt", + "format": "qcow2", + "virtual_size": 5 + } + EOF + ) + expect(File).to receive(:write).with(/.*\/Vagrantfile/, /.*/) expect(subject.call(env)).to be_nil - expect(File.exist?(File.join(temp_dir, 'metadata.json'))).to eq(true) - expect(File.exist?(File.join(temp_dir, 'Vagrantfile'))).to eq(true) end end @@ -85,6 +96,170 @@ describe VagrantPlugins::ProviderLibvirt::Action::PackageDomain do expect(subject.call(env)).to be_nil end end + + context 'when detecting the format' do + let(:root_disk) { double('libvirt_domain_disk') } + let(:disk2) { double('libvirt_additional_disk') } + let(:fake_env) { Hash.new } + + before do + allow(root_disk).to receive(:name).and_return('default_domain.img') + allow(disk2).to receive(:name).and_return('disk2.img') + allow(libvirt_domain).to receive(:name).and_return('default_domain') + end + + context 'with two disks' do + before do + allow(domain).to receive(:volumes).and_return([root_disk, disk2]) + end + + it 'should emit a warning' do + expect(ui).to receive(:info).with('Packaging domain...') + expect(ui).to receive(:warn).with(/Detected more than one volume for machine.*\n.*/) + expect(subject).to receive(:package_v1) + + expect(subject.call(env)).to be_nil + end + end + + context 'with format set to v1' do + before do + allow(domain).to receive(:volumes).and_return([root_disk]) + stub_const("ENV", fake_env) + fake_env['VAGRANT_LIBVIRT_BOX_FORMAT_VERSION'] = "v1" + end + + it 'should call v1 packaging' do + expect(ui).to receive(:info).with('Packaging domain...') + expect(subject).to receive(:package_v1) + + expect(subject.call(env)).to be_nil + end + end + + context 'with format set to v2' do + before do + allow(domain).to receive(:volumes).and_return([root_disk]) + stub_const("ENV", fake_env) + fake_env['VAGRANT_LIBVIRT_BOX_FORMAT_VERSION'] = "v2" + end + + it 'should call v1 packaging' do + expect(ui).to receive(:info).with('Packaging domain...') + expect(subject).to receive(:package_v2) + + expect(subject.call(env)).to be_nil + end + end + + context 'with invalid format' do + before do + allow(domain).to receive(:volumes).and_return([root_disk]) + stub_const("ENV", fake_env) + fake_env['VAGRANT_LIBVIRT_BOX_FORMAT_VERSION'] = "bad format" + end + + it 'should emit a warning and default to v1' do + expect(ui).to receive(:info).with('Packaging domain...') + expect(ui).to receive(:warn).with(/Unrecognized value for.*defaulting to v1/) + expect(subject).to receive(:package_v1) + + expect(subject.call(env)).to be_nil + end + end + end + + context 'with v2 format' do + let(:disk1) { double('libvirt_domain_disk') } + let(:disk2) { double('libvirt_additional_disk') } + let(:fake_env) { Hash.new } + + before do + allow(disk1).to receive(:name).and_return('default_domain.img') + allow(disk2).to receive(:name).and_return('disk2.img') + allow(libvirt_domain).to receive(:name).and_return('default_domain') + allow(subject).to receive(:download_image).and_return(true).twice() + + stub_const("ENV", fake_env) + fake_env['VAGRANT_LIBVIRT_BOX_FORMAT_VERSION'] = "v2" + end + + context 'with 2 disks' do + before do + allow(domain).to receive(:volumes).and_return([disk1, disk2]) + end + + it 'should succeed' do + expect(ui).to receive(:info).with('Packaging domain...') + expect(ui).to receive(:info).with(/Downloading default_domain.img to .*\/box_1.img/) + expect(ui).to receive(:info).with('Image has backing image, copying image and rebasing ...') + expect(subject).to receive(:`).with(/qemu-img info .*\/box_1.img | grep 'backing file:' | cut -d ':' -f2/).and_return("some image") + expect(subject).to receive(:`).with(/qemu-img rebase -p -b "" .*\/box_1.img/) + expect(subject).to receive(:`).with(/virt-sysprep --no-logfile --operations .* -a .*\/box_1.img .*/) + expect(subject).to receive(:`).with(/virt-sparsify --in-place .*\/box_1.img/) + expect(ui).to receive(:info).with(/Downloading disk2.img to .*\/box_2.img/) + expect(ui).to receive(:info).with('Image has backing image, copying image and rebasing ...') + expect(subject).to receive(:`).with(/qemu-img info .*\/box_2.img | grep 'backing file:' | cut -d ':' -f2/).and_return("some image") + expect(subject).to receive(:`).with(/qemu-img rebase -p -b "" .*\/box_2.img/) + expect(subject).to receive(:`).with(/virt-sparsify --in-place .*\/box_2.img/) + + expect(File).to receive(:write).with( + /.*\/metadata.json/, + <<-EOF.unindent.rstrip() + { + "provider": "libvirt", + "format": "qcow2", + "disks": [ + { + "path": "box_1.img" + }, + { + "path": "box_2.img" + } + ] + } + EOF + ) + expect(File).to receive(:write).with(/.*\/Vagrantfile/, /.*/) + + expect(subject.call(env)).to be_nil + end + end + + context 'with 1 disk' do + before do + allow(domain).to receive(:volumes).and_return([disk1]) + end + + it 'should succeed' do + expect(ui).to receive(:info).with('Packaging domain...') + expect(ui).to receive(:info).with(/Downloading default_domain.img to .*\/box_1.img/) + expect(ui).to receive(:info).with('Image has backing image, copying image and rebasing ...') + expect(subject).to receive(:`).with(/qemu-img info .*\/box_1.img | grep 'backing file:' | cut -d ':' -f2/).and_return("some image") + expect(subject).to receive(:`).with(/qemu-img rebase -p -b "" .*\/box_1.img/) + expect(subject).to receive(:`).with(/virt-sysprep --no-logfile --operations .* -a .*\/box_1.img .*/) + expect(subject).to receive(:`).with(/virt-sparsify --in-place .*\/box_1.img/) + + expect(File).to receive(:write).with( + /.*\/metadata.json/, + <<-EOF.unindent.rstrip() + { + "provider": "libvirt", + "format": "qcow2", + "disks": [ + { + "path": "box_1.img" + } + ] + } + EOF + ) + expect(File).to receive(:write).with(/.*\/Vagrantfile/, /.*/) + + expect(subject.call(env)).to be_nil + end + end + end end describe '#vagrantfile_content' do