x2APIC guests with more than 128 vCPUs need destination IDs beyond the
8-bit range provided by the standard MSI address and IO-APIC RTE fields.
The Intel spec allows bits 11:5 of the MSI address and bits 55:49 of the
IO-APIC RTE to carry the high 7 bits of the destination ID when the
platform advertises support, expanding the range to 15 bits total.
Add MSI_ADDR_EXT_DEST_ID_MASK for the extended MSI address bits, and
IO_APIC_REDIR_{DEST,EXT_DEST}_MASK with a VIOAPIC_RTE_DEST() helper to
extract the combined 15-bit destination from an IO-APIC RTE. Extend the
IO-APIC RTE save/restore struct with an ext_dest_id field, and add
XEN_DOMCTL_VMSI_X86_EXT_DEST_ID_MASK plus
XEN_DOMCTL_VMSI_X86_FULL_DEST() to the domctl MSI gflags interface so
callers can pass and extract the full destination ID through the
existing pt_irq_bind path.Signed-off-by: Julian Vetter <[email protected]> --- Changes in V3: - Addressed comments from Jan - Replaced reserved field by generically named reserved2 - Added proper constants for the shift of the upper bits of the MSI/RTE IDs - Added reference for the MSI_ADDR_EXT_DEST_ID_MASK --- xen/arch/x86/hvm/vioapic.c | 2 +- xen/arch/x86/include/asm/hvm/vioapic.h | 13 +++++++++++++ xen/arch/x86/include/asm/msi.h | 10 +++++++++- xen/include/public/arch-x86/hvm/save.h | 4 +++- xen/include/public/domctl.h | 18 ++++++++++++------ 5 files changed, 38 insertions(+), 9 deletions(-) diff --git a/xen/arch/x86/hvm/vioapic.c b/xen/arch/x86/hvm/vioapic.c index b9acdd8af6..d7a4105a57 100644 --- a/xen/arch/x86/hvm/vioapic.c +++ b/xen/arch/x86/hvm/vioapic.c @@ -616,7 +616,7 @@ static int cf_check ioapic_check(const struct domain *d, hvm_domain_context_t *h */ if ( e->fields.reserve || e->fields.reserved[0] || e->fields.reserved[1] || - e->fields.reserved[2] || e->fields.reserved[3] ) + e->fields.reserved[2] || e->fields.reserved2 ) return -EINVAL; } diff --git a/xen/arch/x86/include/asm/hvm/vioapic.h b/xen/arch/x86/include/asm/hvm/vioapic.h index 68af6dce79..5d7da5a092 100644 --- a/xen/arch/x86/include/asm/hvm/vioapic.h +++ b/xen/arch/x86/include/asm/hvm/vioapic.h @@ -32,6 +32,19 @@ #define VIOAPIC_EDGE_TRIG 0 #define VIOAPIC_LEVEL_TRIG 1 +/* + * Extract the destination ID from a 64-bit IO-APIC RTE, including the + * extended bits (55:49) used when XEN_HVM_CPUID_EXT_DEST_ID is advertised. + */ +#define IO_APIC_REDIR_DEST_MASK (0xffULL << 56) +#define IO_APIC_REDIR_EXT_DEST_MASK (0x7fULL << 49) + +#define VIOAPIC_RTE_DEST_ID_UPPER_BITS 8 +#define VIOAPIC_RTE_DEST(rte) \ + (MASK_EXTR((rte), IO_APIC_REDIR_DEST_MASK) | \ + (MASK_EXTR((rte), IO_APIC_REDIR_EXT_DEST_MASK) << \ + VIOAPIC_RTE_DEST_ID_UPPER_BITS)) + #define VIOAPIC_DEFAULT_BASE_ADDRESS 0xfec00000U #define VIOAPIC_MEM_LENGTH 0x100 diff --git a/xen/arch/x86/include/asm/msi.h b/xen/arch/x86/include/asm/msi.h index 00059d4a3a..8d87d0c10d 100644 --- a/xen/arch/x86/include/asm/msi.h +++ b/xen/arch/x86/include/asm/msi.h @@ -51,9 +51,17 @@ #define MSI_ADDR_REDIRECTION_MASK (1 << MSI_ADDR_REDIRECTION_SHIFT) #define MSI_ADDR_DEST_ID_SHIFT 12 -#define MSI_ADDR_DEST_ID_MASK 0x00ff000 +#define MSI_ADDR_DEST_ID_UPPER_BITS 8 +#define MSI_ADDR_DEST_ID_MASK 0x00ff000 #define MSI_ADDR_DEST_ID(dest) (((dest) << MSI_ADDR_DEST_ID_SHIFT) & MSI_ADDR_DEST_ID_MASK) +/* + * Per the Intel x2APIC specification, in physical destination mode bits 11:5 + * of the MSI address carry APIC ID bits [14:8] (the "Extended Destination ID"), + * extending the addressable range from 8 to 15 bits. + */ +#define MSI_ADDR_EXT_DEST_ID_MASK 0x0000fe0 + /* MAX fixed pages reserved for mapping MSIX tables. */ #define FIX_MSIX_MAX_PAGES 512 diff --git a/xen/include/public/arch-x86/hvm/save.h b/xen/include/public/arch-x86/hvm/save.h index 9c4bfc7ebd..483097d940 100644 --- a/xen/include/public/arch-x86/hvm/save.h +++ b/xen/include/public/arch-x86/hvm/save.h @@ -359,7 +359,9 @@ union vioapic_redir_entry uint8_t trig_mode:1; uint8_t mask:1; uint8_t reserve:7; - uint8_t reserved[4]; + uint8_t reserved[3]; + uint8_t reserved2:1; + uint8_t ext_dest_id:7; uint8_t dest_id; } fields; }; diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h index 8f6708c0a7..6d425e34ac 100644 --- a/xen/include/public/domctl.h +++ b/xen/include/public/domctl.h @@ -615,12 +615,14 @@ struct xen_domctl_bind_pt_irq { struct { uint8_t gvec; uint32_t gflags; -#define XEN_DOMCTL_VMSI_X86_DEST_ID_MASK 0x0000ff -#define XEN_DOMCTL_VMSI_X86_RH_MASK 0x000100 -#define XEN_DOMCTL_VMSI_X86_DM_MASK 0x000200 -#define XEN_DOMCTL_VMSI_X86_DELIV_MASK 0x007000 -#define XEN_DOMCTL_VMSI_X86_TRIG_MASK 0x008000 -#define XEN_DOMCTL_VMSI_X86_UNMASKED 0x010000 +#define XEN_DOMCTL_VMSI_X86_DEST_ID_MASK 0x0000ff +#define XEN_DOMCTL_VMSI_X86_DEST_ID_BITS 8 +#define XEN_DOMCTL_VMSI_X86_RH_MASK 0x000100 +#define XEN_DOMCTL_VMSI_X86_DM_MASK 0x000200 +#define XEN_DOMCTL_VMSI_X86_DELIV_MASK 0x007000 +#define XEN_DOMCTL_VMSI_X86_TRIG_MASK 0x008000 +#define XEN_DOMCTL_VMSI_X86_UNMASKED 0x010000 +#define XEN_DOMCTL_VMSI_X86_EXT_DEST_ID_MASK 0xfe0000 uint64_aligned_t gtable; } msi; @@ -630,6 +632,10 @@ struct xen_domctl_bind_pt_irq { } u; }; +#define XEN_DOMCTL_VMSI_X86_FULL_DEST(gflags) \ + (MASK_EXTR((gflags), XEN_DOMCTL_VMSI_X86_DEST_ID_MASK) | \ + (MASK_EXTR((gflags), XEN_DOMCTL_VMSI_X86_EXT_DEST_ID_MASK) << \ + XEN_DOMCTL_VMSI_X86_DEST_ID_BITS)) /* Bind machine I/O address range -> HVM address range. */ /* XEN_DOMCTL_memory_mapping */ -- 2.51.0 -- Julian Vetter | Vates Hypervisor & Kernel Developer XCP-ng & Xen Orchestra - Vates solutions web: https://vates.tech
