On Fri, Jun 27, 2025 at 01:02:22PM -0700, Pierrick Bouvier wrote:
> This test allows to document and exercise device passthrough, using a
> nested virtual machine setup. Two disks are generated and passed to the
> VM, and their content is compared to original images.
>
> Guest and nested guests commands are executed through two scripts, and
> init used in both system is configured to trigger a kernel panic in case
> any command fails. This is more reliable and readable than executing all
> commands through prompt injection and trying to guess what failed.
>
> Initially, this test was supposed to test smmuv3 nested emulation
> (combining both stages of translation), but I could not find any setup
> (kernel + vmm) able to do the passthrough correctly, despite several
> tries.
>
> Signed-off-by: Pierrick Bouvier <pierrick.bouv...@linaro.org>
> ---
> tests/functional/meson.build | 2 +
> .../test_aarch64_device_passthrough.py | 142 ++++++++++++++++++
> 2 files changed, 144 insertions(+)
> create mode 100755 tests/functional/test_aarch64_device_passthrough.py
>
> diff --git a/tests/functional/meson.build b/tests/functional/meson.build
> index 3021928a9d4..6cc78abb123 100644
> --- a/tests/functional/meson.build
> +++ b/tests/functional/meson.build
> @@ -13,6 +13,7 @@ endif
> test_timeouts = {
> 'aarch64_aspeed_ast2700' : 600,
> 'aarch64_aspeed_ast2700fc' : 600,
> + 'aarch64_device_passthrough' : 720,
> 'aarch64_imx8mp_evk' : 240,
> 'aarch64_raspi4' : 480,
> 'aarch64_reverse_debug' : 180,
> @@ -84,6 +85,7 @@ tests_aarch64_system_quick = [
> tests_aarch64_system_thorough = [
> 'aarch64_aspeed_ast2700',
> 'aarch64_aspeed_ast2700fc',
> + 'aarch64_device_passthrough',
> 'aarch64_imx8mp_evk',
> 'aarch64_raspi3',
> 'aarch64_raspi4',
> diff --git a/tests/functional/test_aarch64_device_passthrough.py
> b/tests/functional/test_aarch64_device_passthrough.py
> new file mode 100755
> index 00000000000..1f3f158a9ff
> --- /dev/null
> +++ b/tests/functional/test_aarch64_device_passthrough.py
> @@ -0,0 +1,142 @@
> +#!/usr/bin/env python3
> +#
> +# Boots a nested guest and compare content of a device (passthrough) to a
> +# reference image. Both vfio group and iommufd passthrough methods are
> tested.
> +#
> +# Copyright (c) 2025 Linaro Ltd.
> +#
> +# Author: Pierrick Bouvier <pierrick.bouv...@linaro.org>
> +#
> +# SPDX-License-Identifier: GPL-2.0-or-later
> +
> +import os
> +
> +from qemu_test import QemuSystemTest, Asset
> +from qemu_test import exec_command, wait_for_console_pattern
> +from qemu_test import exec_command_and_wait_for_pattern
> +from random import randbytes
> +
> +guest_script = '''
> +#!/usr/bin/env bash
> +
> +set -euo pipefail
> +set -x
> +
> +# find disks from nvme serial
> +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
> +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
> +pci_vfio=$(basename $(readlink -f /sys/block/$dev_vfio/../../../))
> +pci_iommufd=$(basename $(readlink -f /sys/block/$dev_iommufd/../../../))
> +
> +# bind disks to vfio
> +for p in "$pci_vfio" "$pci_iommufd"; do
> + if [ "$(cat /sys/bus/pci/devices/$p/driver_override)" == vfio-pci ]; then
> + continue
> + fi
> + echo $p > /sys/bus/pci/drivers/nvme/unbind
> + echo vfio-pci > /sys/bus/pci/devices/$p/driver_override
> + echo $p > /sys/bus/pci/drivers/vfio-pci/bind
> +done
> +
> +# boot nested guest and execute /host/nested_guest.sh
> +# one disk is passed through vfio group, the other, through iommufd
> +qemu-system-aarch64 \
> +-M virt \
> +-display none \
> +-serial stdio \
> +-cpu host \
> +-enable-kvm \
> +-m 1G \
> +-kernel /host/Image.gz \
> +-drive format=raw,file=/host/guest.ext4,if=virtio \
> +-append "root=/dev/vda init=/init -- bash /host/nested_guest.sh" \
> +-virtfs local,path=/host,mount_tag=host,security_model=mapped,readonly=off \
> +-device vfio-pci,host=$pci_vfio \
> +-object iommufd,id=iommufd0 \
> +-device vfio-pci,host=$pci_iommufd,iommufd=iommufd0
> +'''
> +
> +nested_guest_script = '''
> +#!/usr/bin/env bash
> +
> +set -euo pipefail
> +set -x
> +
> +image_vfio=/host/disk_vfio
> +image_iommufd=/host/disk_iommufd
> +
> +dev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ')
> +dev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ')
> +
> +# compare if devices are identical to original images
> +diff $image_vfio /dev/$dev_vfio
> +diff $image_iommufd /dev/$dev_iommufd
> +
> +echo device_passthrough_test_ok
> +'''
> +
> +class Aarch64DevicePassthrough(QemuSystemTest):
> +
> + # https://github.com/pbo-linaro/qemu-linux-stack
> + #
> + # Linux kernel is compiled with defconfig +
> + # IOMMUFD + VFIO_DEVICE_CDEV + ARM_SMMU_V3_IOMMUFD
> + # https://docs.kernel.org/driver-api/vfio.html#vfio-device-cde
> + ASSET_DEVICE_PASSTHROUGH_STACK = Asset(
> + ('https://fileserver.linaro.org/s/fx5DXxBYme8dw2G/'
> + 'download/device_passthrough.tar.xz'),
> + '812750b664d61c2986f2b149939ae28cafbd60d53e9c7e4b16e97143845e196d')
> +
> + # This tests the device passthrough implementation, by booting a VM
> + # supporting it with two nvme disks attached, and launching a nested VM
> + # reading their content.
> + def test_aarch64_device_passthrough(self):
> + self.set_machine('virt')
> + self.require_accelerator('tcg')
> +
> + self.vm.set_console()
> +
> + stack_path_tar_gz = self.ASSET_DEVICE_PASSTHROUGH_STACK.fetch()
> + self.archive_extract(stack_path_tar_gz, format="tar")
> +
> + stack = self.scratch_file('out')
> + kernel = os.path.join(stack, 'Image.gz')
> + rootfs_host = os.path.join(stack, 'host.ext4')
> + disk_vfio = os.path.join(stack, 'disk_vfio')
> + disk_iommufd = os.path.join(stack, 'disk_iommufd')
> + guest_cmd = os.path.join(stack, 'guest.sh')
> + nested_guest_cmd = os.path.join(stack, 'nested_guest.sh')
Don't incrementally create paths like this - use the
'scratch_file' method for all components
ie
kernel = self.scratch_file('out', 'Image.gz')
rootfs_host = self.scratch_file('out', 'host.ext4')
...etc...
> + # we generate two random disks
> + with open(disk_vfio, "wb") as d: d.write(randbytes(512))
> + with open(disk_iommufd, "wb") as d: d.write(randbytes(1024))
> + with open(guest_cmd, 'w') as s: s.write(guest_script)
> + with open(nested_guest_cmd, 'w') as s: s.write(nested_guest_script)
> +
> + self.vm.add_args('-cpu', 'max')
> + self.vm.add_args('-m', '2G')
> + self.vm.add_args('-M', 'virt,'
> + 'virtualization=on,'
> + 'gic-version=max,'
> + 'iommu=smmuv3')
> + self.vm.add_args('-kernel', kernel)
> + self.vm.add_args('-drive', f'format=raw,file={rootfs_host}')
> + self.vm.add_args('-drive',
> + f'file={disk_vfio},if=none,id=vfio,format=raw')
> + self.vm.add_args('-device', 'nvme,serial=vfio,drive=vfio')
> + self.vm.add_args('-drive',
> +
> f'file={disk_iommufd},if=none,id=iommufd,format=raw')
> + self.vm.add_args('-device', 'nvme,serial=iommufd,drive=iommufd')
> + self.vm.add_args('-virtfs',
> + f'local,path={stack}/,mount_tag=host,'
> + 'security_model=mapped,readonly=off')
> + # boot and execute guest script
> + # init will trigger a kernel panic if script fails
> + self.vm.add_args('-append',
> + 'root=/dev/vda init=/init -- bash /host/guest.sh')
> +
> + self.vm.launch()
> + wait_for_console_pattern(self, 'device_passthrough_test_ok',
> + failure_message='Kernel panic')
> +
> +if __name__ == '__main__':
> + QemuSystemTest.main()
> --
> 2.47.2
>
With regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
