Nitro Enclaves are a confidential compute technology which allows a parent instance to carve out resources from itself and spawn a confidential sibling VM next to itself. Similar to other confidential compute solutions, this sibling is controlled by an underlying vmm, but still has a higher level vmm (QEMU) to implement some of its I/O functionality and lifecycle.
Add an accelerator to drive this interface. In combination with follow-on patches to enhance the Nitro Enclaves machine model, this will allow users to run a Nitro Enclave using QEMU. Signed-off-by: Alexander Graf <[email protected]> --- MAINTAINERS | 6 + accel/Kconfig | 3 + accel/meson.build | 1 + accel/nitro/meson.build | 3 + accel/nitro/nitro-accel.c | 333 ++++++++++++++++++++++++++++++++++ accel/nitro/trace-events | 6 + accel/nitro/trace.h | 2 + accel/stubs/meson.build | 1 + accel/stubs/nitro-stub.c | 11 ++ include/system/hw_accel.h | 1 + include/system/nitro-accel.h | 25 +++ meson.build | 11 ++ meson_options.txt | 2 + qemu-options.hx | 8 +- scripts/meson-buildoptions.sh | 3 + 15 files changed, 412 insertions(+), 4 deletions(-) create mode 100644 accel/nitro/meson.build create mode 100644 accel/nitro/nitro-accel.c create mode 100644 accel/nitro/trace-events create mode 100644 accel/nitro/trace.h create mode 100644 accel/stubs/nitro-stub.c create mode 100644 include/system/nitro-accel.h diff --git a/MAINTAINERS b/MAINTAINERS index d3aa6d6732..3d002143ae 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -583,6 +583,12 @@ F: include/system/mshv.h F: include/hw/hyperv/hvgdk*.h F: include/hw/hyperv/hvhdk*.h +Nitro Enclaves (native) +M: Alexander Graf <[email protected]> +S: Maintained +F: accel/nitro/ +F: include/system/nitro-accel.h + X86 MSHV CPUs M: Magnus Kulke <[email protected]> R: Wei Liu <[email protected]> diff --git a/accel/Kconfig b/accel/Kconfig index a60f114923..6d052875ee 100644 --- a/accel/Kconfig +++ b/accel/Kconfig @@ -16,6 +16,9 @@ config KVM config MSHV bool +config NITRO + bool + config XEN bool select FSDEV_9P if VIRTFS diff --git a/accel/meson.build b/accel/meson.build index 289b7420ff..7da12b9741 100644 --- a/accel/meson.build +++ b/accel/meson.build @@ -12,6 +12,7 @@ if have_system subdir('xen') subdir('stubs') subdir('mshv') + subdir('nitro') endif # qtest diff --git a/accel/nitro/meson.build b/accel/nitro/meson.build new file mode 100644 index 0000000000..e01c1bab96 --- /dev/null +++ b/accel/nitro/meson.build @@ -0,0 +1,3 @@ +nitro_ss = ss.source_set() +nitro_ss.add(files('nitro-accel.c')) +system_ss.add_all(when: 'CONFIG_NITRO', if_true: nitro_ss) diff --git a/accel/nitro/nitro-accel.c b/accel/nitro/nitro-accel.c new file mode 100644 index 0000000000..bea76fcd0b --- /dev/null +++ b/accel/nitro/nitro-accel.c @@ -0,0 +1,333 @@ +/* + * Nitro Enclaves accelerator + * + * Copyright © 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Authors: + * Alexander Graf <[email protected]> + * + * Nitro Enclaves are a confidential compute technology which + * allows a parent instance to carve out resources from itself + * and spawn a confidential sibling VM next to itself. Similar + * to other confidential compute solutions, this sibling is + * controlled by an underlying vmm, but still has a higher level + * vmm (QEMU) to implement some of its I/O functionality and + * lifecycle. + * + * This accelerator drives /dev/nitro_enclaves to spawn a Nitro + * Enclave. It works in tandem with the nitro_enclaves machine + * which ensures the correct backend devices are available and + * that the initial seed (an EIF file) is loaded at the correct + * offset in memory. + * + * The accel starts the enclave on first vCPU 0 main loop entry, + * to ensure that all device setup is finished and that we have + * a working vCPU loop. + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" +#include "qemu/error-report.h" +#include "qapi/error.h" +#include "qapi/visitor.h" +#include "qemu/module.h" +#include "qemu/rcu.h" +#include "qemu/accel.h" +#include "qemu/guest-random.h" +#include "qemu/main-loop.h" +#include "accel/accel-ops.h" +#include "accel/accel-cpu-ops.h" +#include "system/cpus.h" +#include "hw/core/cpu.h" +#include "hw/core/boards.h" +#include "hw/core/sysbus.h" +#include "system/ramblock.h" +#include "system/nitro-accel.h" +#include "trace.h" + +#include <sys/ioctl.h> +#include "standard-headers/linux/nitro_enclaves.h" + +bool nitro_allowed; + +typedef struct NitroAccelState { + AccelState parent_obj; + + int ne_fd; + int enclave_fd; + uint64_t slot_uid; + uint64_t enclave_cid; + bool debug_mode; +} NitroAccelState; + +static int nitro_init_machine(AccelState *as, MachineState *ms) +{ + NitroAccelState *s = NITRO_ACCEL(as); + uint64_t slot_uid = 0; + int ret; + + s->ne_fd = open("/dev/nitro_enclaves", O_RDWR | O_CLOEXEC); + if (s->ne_fd < 0) { + error_report("nitro: failed to open /dev/nitro_enclaves: %s", + strerror(errno)); + return -errno; + } + + ret = ioctl(s->ne_fd, NE_CREATE_VM, &slot_uid); + if (ret < 0) { + error_report("nitro: NE_CREATE_VM failed: %s", strerror(errno)); + close(s->ne_fd); + return -errno; + } + s->enclave_fd = ret; + s->slot_uid = slot_uid; + + return 0; +} + +static int nitro_donate_ram_block(RAMBlock *rb, void *opaque) +{ + NitroAccelState *s = opaque; + struct ne_user_memory_region region = { + .flags = 0, + .memory_size = rb->used_length, + .userspace_addr = (uint64_t)(uintptr_t)rb->host, + }; + + if (!rb->used_length) { + return 0; + } + + if (ioctl(s->enclave_fd, NE_SET_USER_MEMORY_REGION, ®ion) < 0) { + error_report("nitro: NE_SET_USER_MEMORY_REGION failed for %s " + "(%" PRIu64 " bytes): %s", rb->idstr, rb->used_length, + strerror(errno)); + return -errno; + } + return 0; +} + +/* + * Start the Enclave. This gets called when the first vCPU 0 enters its main + * loop. At this point memory is set up and the EIF is loaded. This function + * donates memory, adds vCPUs, and starts the enclave. + */ +static void nitro_do_start(NitroAccelState *s) +{ + MachineState *ms = MACHINE(qdev_get_machine()); + int nr_cpus = ms->smp.cpus; + int i, ret; + struct ne_enclave_start_info start_info = { + .flags = s->debug_mode ? NE_ENCLAVE_DEBUG_MODE : 0, + .enclave_cid = s->enclave_cid, + }; + + ret = qemu_ram_foreach_block(nitro_donate_ram_block, s); + if (ret < 0) { + error_report("nitro: failed to donate memory"); + exit(1); + } + + for (i = 0; i < nr_cpus; i++) { + uint32_t cpu_id = 0; + if (ioctl(s->enclave_fd, NE_ADD_VCPU, &cpu_id) < 0) { + error_report("nitro: NE_ADD_VCPU failed: %s", strerror(errno)); + exit(1); + } + } + + ret = ioctl(s->enclave_fd, NE_START_ENCLAVE, &start_info); + if (ret < 0) { + switch (errno) { + case NE_ERR_NO_MEM_REGIONS_ADDED: + error_report("nitro: no memory regions added"); + break; + case NE_ERR_NO_VCPUS_ADDED: + error_report("nitro: no vCPUs added"); + break; + case NE_ERR_ENCLAVE_MEM_MIN_SIZE: + error_report("nitro: memory is below the minimum " + "required size. Try increasing -m"); + break; + case NE_ERR_FULL_CORES_NOT_USED: + error_report("nitro: requires full CPU cores. " + "Try increasing -smp to a multiple of threads " + "per core on this host (e.g. -smp 2)"); + break; + case NE_ERR_NOT_IN_INIT_STATE: + error_report("nitro: not in init state"); + break; + case NE_ERR_INVALID_FLAG_VALUE: + error_report("nitro: invalid flag value for NE_START_ENCLAVE"); + break; + case NE_ERR_INVALID_ENCLAVE_CID: + error_report("nitro: invalid enclave CID"); + break; + default: + error_report("nitro: NE_START_ENCLAVE failed: %s (errno %d)", + strerror(errno), errno); + break; + } + exit(1); + } + + s->enclave_cid = start_info.enclave_cid; + trace_nitro_enclave_started(s->enclave_cid); + + /* + * Push enclave CID to all devices that need it. + * Each device handles its own connection (console, heartbeat). + */ + { + BusState *sysbus = sysbus_get_default(); + BusChild *kid; + + QTAILQ_FOREACH(kid, &sysbus->children, sibling) { + DeviceState *dev = kid->child; + if (object_property_find(OBJECT(dev), "enclave-cid")) { + object_property_set_uint(OBJECT(dev), "enclave-cid", + s->enclave_cid, NULL); + } + } + } +} + +/* + * vCPU dummy thread function. The real vCPUs run inside the enclave. + * + * Based on dummy_cpu_thread_fn() from accel/dummy-cpus.c. + */ +static void *nitro_vcpu_thread_fn(void *arg) +{ + CPUState *cpu = arg; + NitroAccelState *s = NITRO_ACCEL(current_accel()); + sigset_t waitset; + + rcu_register_thread(); + + bql_lock(); + qemu_thread_get_self(cpu->thread); + cpu->thread_id = qemu_get_thread_id(); + current_cpu = cpu; + + sigemptyset(&waitset); + sigaddset(&waitset, SIG_IPI); + + cpu_thread_signal_created(cpu); + qemu_guest_random_seed_thread_part2(cpu->random_seed); + + /* vCPU 0 starts the enclave on first entry */ + if (cpu->cpu_index == 0) { + nitro_do_start(s); + } + + do { + qemu_process_cpu_events(cpu); + bql_unlock(); + { + int sig; + while (sigwait(&waitset, &sig) == -1 && + (errno == EAGAIN || errno == EINTR)) { + /* retry */ + } + } + bql_lock(); + } while (!cpu->unplug); + + bql_unlock(); + rcu_unregister_thread(); + return NULL; +} + +static void nitro_start_vcpu_thread(CPUState *cpu) +{ + char thread_name[VCPU_THREAD_NAME_SIZE]; + + snprintf(thread_name, VCPU_THREAD_NAME_SIZE, "CPU %d/Nitro", + cpu->cpu_index); + qemu_thread_create(cpu->thread, thread_name, nitro_vcpu_thread_fn, + cpu, QEMU_THREAD_JOINABLE); +} + +/* QOM properties */ + +static bool nitro_get_debug_mode(Object *obj, Error **errp) +{ + return NITRO_ACCEL(obj)->debug_mode; +} + +static void nitro_set_debug_mode(Object *obj, bool value, Error **errp) +{ + NITRO_ACCEL(obj)->debug_mode = value; +} + +static void nitro_get_enclave_cid(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + uint64_t val = NITRO_ACCEL(obj)->enclave_cid; + visit_type_uint64(v, name, &val, errp); +} + +static void nitro_set_enclave_cid(Object *obj, Visitor *v, + const char *name, void *opaque, + Error **errp) +{ + uint64_t val; + if (visit_type_uint64(v, name, &val, errp)) { + NITRO_ACCEL(obj)->enclave_cid = val; + } +} + +static void nitro_accel_class_init(ObjectClass *oc, const void *data) +{ + AccelClass *ac = ACCEL_CLASS(oc); + ac->name = "Nitro"; + ac->init_machine = nitro_init_machine; + ac->allowed = &nitro_allowed; + + object_class_property_add_bool(oc, "debug-mode", + nitro_get_debug_mode, + nitro_set_debug_mode); + object_class_property_set_description(oc, "debug-mode", + "Start enclave in debug mode (enables console output)"); + + object_class_property_add(oc, "enclave-cid", "uint64", + nitro_get_enclave_cid, + nitro_set_enclave_cid, + NULL, NULL); + object_class_property_set_description(oc, "enclave-cid", + "Enclave CID (0 = auto-assigned by Nitro)"); +} + +static const TypeInfo nitro_accel_type = { + .name = TYPE_NITRO_ACCEL, + .parent = TYPE_ACCEL, + .instance_size = sizeof(NitroAccelState), + .class_init = nitro_accel_class_init, +}; +module_obj(TYPE_NITRO_ACCEL); + +static void nitro_accel_ops_class_init(ObjectClass *oc, const void *data) +{ + AccelOpsClass *ops = ACCEL_OPS_CLASS(oc); + ops->create_vcpu_thread = nitro_start_vcpu_thread; + ops->handle_interrupt = generic_handle_interrupt; +} + +static const TypeInfo nitro_accel_ops_type = { + .name = ACCEL_OPS_NAME("nitro"), + .parent = TYPE_ACCEL_OPS, + .class_init = nitro_accel_ops_class_init, + .abstract = true, +}; +module_obj(ACCEL_OPS_NAME("nitro")); + +static void nitro_type_init(void) +{ + type_register_static(&nitro_accel_type); + type_register_static(&nitro_accel_ops_type); +} + +type_init(nitro_type_init); diff --git a/accel/nitro/trace-events b/accel/nitro/trace-events new file mode 100644 index 0000000000..9673eb5aa2 --- /dev/null +++ b/accel/nitro/trace-events @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# See docs/devel/tracing.rst for syntax documentation. + +# nitro-accel.c +nitro_enclave_started(uint64_t cid) "nitro: enclave started, CID=%"PRIu64 diff --git a/accel/nitro/trace.h b/accel/nitro/trace.h new file mode 100644 index 0000000000..8c5564725d --- /dev/null +++ b/accel/nitro/trace.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#include "trace/trace-accel_nitro.h" diff --git a/accel/stubs/meson.build b/accel/stubs/meson.build index 48eccd1b86..5de4a279ff 100644 --- a/accel/stubs/meson.build +++ b/accel/stubs/meson.build @@ -3,6 +3,7 @@ system_stubs_ss.add(when: 'CONFIG_XEN', if_false: files('xen-stub.c')) system_stubs_ss.add(when: 'CONFIG_KVM', if_false: files('kvm-stub.c')) system_stubs_ss.add(when: 'CONFIG_TCG', if_false: files('tcg-stub.c')) system_stubs_ss.add(when: 'CONFIG_HVF', if_false: files('hvf-stub.c')) +system_stubs_ss.add(when: 'CONFIG_NITRO', if_false: files('nitro-stub.c')) system_stubs_ss.add(when: 'CONFIG_NVMM', if_false: files('nvmm-stub.c')) system_stubs_ss.add(when: 'CONFIG_WHPX', if_false: files('whpx-stub.c')) system_stubs_ss.add(when: 'CONFIG_MSHV', if_false: files('mshv-stub.c')) diff --git a/accel/stubs/nitro-stub.c b/accel/stubs/nitro-stub.c new file mode 100644 index 0000000000..186c8444f8 --- /dev/null +++ b/accel/stubs/nitro-stub.c @@ -0,0 +1,11 @@ +/* + * Nitro accel stubs for QEMU + * + * Copyright © 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" + +bool nitro_allowed; diff --git a/include/system/hw_accel.h b/include/system/hw_accel.h index 628a50e066..f0c10b6d80 100644 --- a/include/system/hw_accel.h +++ b/include/system/hw_accel.h @@ -17,6 +17,7 @@ #include "system/mshv.h" #include "system/whpx.h" #include "system/nvmm.h" +#include "system/nitro-accel.h" /** * cpu_synchronize_state: diff --git a/include/system/nitro-accel.h b/include/system/nitro-accel.h new file mode 100644 index 0000000000..a93aa6fb00 --- /dev/null +++ b/include/system/nitro-accel.h @@ -0,0 +1,25 @@ +/* + * Nitro Enclaves accelerator - public interface + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#ifndef SYSTEM_NITRO_ACCEL_H +#define SYSTEM_NITRO_ACCEL_H + +#include "qemu/accel.h" + +extern bool nitro_allowed; + +static inline bool nitro_enabled(void) +{ + return nitro_allowed; +} + +#define TYPE_NITRO_ACCEL ACCEL_CLASS_NAME("nitro") + +typedef struct NitroAccelState NitroAccelState; +DECLARE_INSTANCE_CHECKER(NitroAccelState, NITRO_ACCEL, + TYPE_NITRO_ACCEL) + +#endif /* SYSTEM_NITRO_ACCEL_H */ diff --git a/meson.build b/meson.build index 4af32c3e1f..bdeee65db2 100644 --- a/meson.build +++ b/meson.build @@ -302,11 +302,13 @@ accelerator_targets += { 'CONFIG_XEN': xen_targets } if cpu == 'aarch64' accelerator_targets += { 'CONFIG_HVF': ['aarch64-softmmu'], + 'CONFIG_NITRO': ['aarch64-softmmu'], 'CONFIG_WHPX': ['aarch64-softmmu'] } elif cpu == 'x86_64' accelerator_targets += { 'CONFIG_HVF': ['x86_64-softmmu'], + 'CONFIG_NITRO': ['x86_64-softmmu'], 'CONFIG_NVMM': ['i386-softmmu', 'x86_64-softmmu'], 'CONFIG_WHPX': ['i386-softmmu', 'x86_64-softmmu'], 'CONFIG_MSHV': ['x86_64-softmmu'], @@ -881,6 +883,11 @@ if get_option('hvf').allowed() endif endif +nitro = not_found +if get_option('nitro').allowed() and host_os == 'linux' + accelerators += 'CONFIG_NITRO' +endif + nvmm = not_found if host_os == 'netbsd' nvmm = cc.find_library('nvmm', required: get_option('nvmm')) @@ -922,6 +929,9 @@ endif if 'CONFIG_HVF' not in accelerators and get_option('hvf').enabled() error('HVF not available on this platform') endif +if 'CONFIG_NITRO' not in accelerators and get_option('nitro').enabled() + error('NITRO not available on this platform') +endif if 'CONFIG_NVMM' not in accelerators and get_option('nvmm').enabled() error('NVMM not available on this platform') endif @@ -3593,6 +3603,7 @@ if have_system 'accel/hvf', 'accel/kvm', 'accel/mshv', + 'accel/nitro', 'audio', 'backends', 'backends/tpm', diff --git a/meson_options.txt b/meson_options.txt index 2836156257..31d5916cfc 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -79,6 +79,8 @@ option('whpx', type: 'feature', value: 'auto', description: 'WHPX acceleration support') option('hvf', type: 'feature', value: 'auto', description: 'HVF acceleration support') +option('nitro', type: 'feature', value: 'auto', + description: 'Nitro acceleration support') option('nvmm', type: 'feature', value: 'auto', description: 'NVMM acceleration support') option('xen', type: 'feature', value: 'auto', diff --git a/qemu-options.hx b/qemu-options.hx index 33fcfe7ce6..9b6fb247f7 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -28,7 +28,7 @@ DEF("machine", HAS_ARG, QEMU_OPTION_machine, \ "-machine [type=]name[,prop[=value][,...]]\n" " selects emulated machine ('-machine help' for list)\n" " property accel=accel1[:accel2[:...]] selects accelerator\n" - " supported accelerators are kvm, xen, hvf, nvmm, whpx, mshv or tcg (default: tcg)\n" + " supported accelerators are kvm, xen, hvf, nitro, nvmm, whpx, mshv or tcg (default: tcg)\n" " vmport=on|off|auto controls emulation of vmport (default: auto)\n" " dump-guest-core=on|off include guest memory in a core dump (default=on)\n" " mem-merge=on|off controls memory merge support (default: on)\n" @@ -67,7 +67,7 @@ SRST ``accel=accels1[:accels2[:...]]`` This is used to enable an accelerator. Depending on the target - architecture, kvm, xen, hvf, nvmm, whpx, mshv or tcg can be + architecture, kvm, xen, hvf, nitro, nvmm, whpx, mshv or tcg can be available. By default, tcg is used. If there is more than one accelerator specified, the next one is used if the previous one fails to initialize. @@ -228,7 +228,7 @@ ERST DEF("accel", HAS_ARG, QEMU_OPTION_accel, "-accel [accel=]accelerator[,prop[=value][,...]]\n" - " select accelerator (kvm, xen, hvf, nvmm, whpx, mshv or tcg; use 'help' for a list)\n" + " select accelerator (kvm, xen, hvf, nitro, nvmm, whpx, mshv or tcg; use 'help' for a list)\n" " igd-passthru=on|off (enable Xen integrated Intel graphics passthrough, default=off)\n" " kernel-irqchip=on|off|split controls accelerated irqchip support (default=on)\n" " kvm-shadow-mem=size of KVM shadow MMU in bytes\n" @@ -243,7 +243,7 @@ DEF("accel", HAS_ARG, QEMU_OPTION_accel, SRST ``-accel name[,prop=value[,...]]`` This is used to enable an accelerator. Depending on the target - architecture, kvm, xen, hvf, nvmm, whpx, mshv or tcg can be available. + architecture, kvm, xen, hvf, nitro, nvmm, whpx, mshv or tcg can be available. By default, tcg is used. If there is more than one accelerator specified, the next one is used if the previous one fails to initialize. diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh index e8edc5252a..ca5b113119 100644 --- a/scripts/meson-buildoptions.sh +++ b/scripts/meson-buildoptions.sh @@ -158,6 +158,7 @@ meson_options_help() { printf "%s\n" ' multiprocess Out of process device emulation support' printf "%s\n" ' netmap netmap network backend support' printf "%s\n" ' nettle nettle cryptography support' + printf "%s\n" ' nitro Nitro acceleration support' printf "%s\n" ' numa libnuma support' printf "%s\n" ' nvmm NVMM acceleration support' printf "%s\n" ' opengl OpenGL support' @@ -418,6 +419,8 @@ _meson_option_parse() { --disable-netmap) printf "%s" -Dnetmap=disabled ;; --enable-nettle) printf "%s" -Dnettle=enabled ;; --disable-nettle) printf "%s" -Dnettle=disabled ;; + --enable-nitro) printf "%s" -Dnitro=enabled ;; + --disable-nitro) printf "%s" -Dnitro=disabled ;; --enable-numa) printf "%s" -Dnuma=enabled ;; --disable-numa) printf "%s" -Dnuma=disabled ;; --enable-nvmm) printf "%s" -Dnvmm=enabled ;; -- 2.47.1 Amazon Web Services Development Center Germany GmbH Tamara-Danz-Str. 13 10243 Berlin Geschaeftsfuehrung: Christof Hellmis, Andreas Stieger Eingetragen am Amtsgericht Charlottenburg unter HRB 257764 B Sitz: Berlin Ust-ID: DE 365 538 597
