From: Manish Honap <[email protected]> A CXL Type-2 guest maps the device HDM memory to use its coherent CXL.mem. The HDM memory is a host physical range with no struct page, so the guest and KVM need a write-back mapping of it.
Register the range as a region under VFIO_REGION_TYPE_PCI_VENDOR_TYPE with the CXL vendor id rather than a bespoke region type, per open because vfio-pci-core tears down all dynamic regions on close and on a failed open. This patch serves read and write on the region fd; mmap follows. Own the resolved host physical range exclusively (IORESOURCE_EXCLUSIVE) so nothing, /dev/mem included, can map a conflicting cacheable alias that would fault the host once the range is mapped write-back. Release it in the provider .release op rather than through a devm action on the PCI device, which could run after vfio-cxl has unloaded. vfio-cxl now holds the range, so its reset op runs CXL Reset with pci_cxl_reset_owned(). Host access to the range is safe only while the committed decoder is in place. Track that in hdm_valid, set at bind once the decoder is verified committed. Low power loses the decoder, so close the gate in power_down and restore the decoder with cxl_restore_state_after_pci_reset() in power_up, reopening the gate on success. Device memory can stay unavailable for seconds after the decoder is back: the device clears Memory_Active in DVSEC CXL Range 1 Size Low until the memory is usable (CXL r4.0 sec 8.1.3.8.2). At bind, wait up to the Memory_Active_Timeout the device reports for Memory_Active, since a reset just before the bind also clears it. A device whose memory then reads active brings it up by itself, and power_up waits for Memory_Active, up to the same timeout, before it reopens the gate. A device whose memory stays inactive relies on its driver and is not waited for. CXL.mem decode follows the committed decoder and Mem_Enable, not Memory Space Enable (CXL r4.0 sec 8.2.4.20.2, 8.2.4.20.7 and 8.1.3.2), so the gate does not check the Command register. Read and write copy through a page-sized kernel buffer, so memory_lock is never held across a user copy. Assisted-by: LLM Signed-off-by: Manish Honap <[email protected]> --- drivers/vfio/pci/cxl/vfio_cxl_core.c | 285 ++++++++++++++++++++++++++- include/uapi/linux/vfio.h | 4 + 2 files changed, 279 insertions(+), 10 deletions(-) diff --git a/drivers/vfio/pci/cxl/vfio_cxl_core.c b/drivers/vfio/pci/cxl/vfio_cxl_core.c index 052be27bc46f..e6e633902ecd 100644 --- a/drivers/vfio/pci/cxl/vfio_cxl_core.c +++ b/drivers/vfio/pci/cxl/vfio_cxl_core.c @@ -5,11 +5,15 @@ * Copyright (c) 2026 NVIDIA Corporation & Affiliates */ +#include <linux/bitfield.h> #include <linux/cleanup.h> +#include <linux/delay.h> +#include <linux/io.h> #include <linux/module.h> #include <linux/pci.h> #include <linux/range.h> #include <linux/slab.h> +#include <linux/uaccess.h> #include <linux/vfio_pci_core.h> #include <cxl/cxl.h> #include <cxl/hdm.h> @@ -20,13 +24,114 @@ * @cxlds: CXL device state; kept first for devm_cxl_dev_state_create() * @cxlmd: memory device joined to the CXL topology at bind * @hpa_range: host physical range of the HDM region + * @hpa_res: exclusive reservation of @hpa_range, held for the bind * @hdm: copy of the HDM state cached at PCI enumeration, taken at bind + * @hdm_valid: true when host CPU access to the HDM range is safe; under memory_lock + * @await_mem_active: the device reported its memory active at bind, so wait + * for Memory_Active before host access reopens after a reset or low power */ struct vfio_cxl_state { struct cxl_dev_state cxlds; struct cxl_memdev *cxlmd; struct range hpa_range; + struct resource *hpa_res; struct cxl_hdm_info *hdm; + bool hdm_valid; + bool await_mem_active; +}; + +/* + * Copy one chunk between the bounce buffer and the HDM range under + * memory_lock. The user copy stays outside the lock: a user buffer that is + * itself an mmap of this device faults through a handler that takes + * memory_lock for read, and a writer queued in between would deadlock the + * two readers. + */ +static int vfio_cxl_mem_copy(struct vfio_pci_core_device *vdev, void *bounce, + u64 pos, size_t len, bool iswrite) +{ + struct vfio_cxl_state *cxl = vdev->cxl; + void *mem; + + guard(rwsem_read)(&vdev->memory_lock); + + /* + * Only touch the HDM range with the decoder in a known-good state, or + * a host CPU access aborts as a fatal SError. CXL.mem decode follows + * the committed decoder and Mem_Enable, not Memory Space Enable (CXL + * r4.0 sec 8.2.4.20.2, 8.1.3.2). + */ + if (!cxl->hdm_valid || !cxl->cxlds.media_ready) + return -EIO; + + mem = memremap(cxl->hpa_range.start + pos, len, MEMREMAP_WB); + if (!mem) + return -ENOMEM; + if (iswrite) + memcpy(mem, bounce, len); + else + memcpy(bounce, mem, len); + memunmap(mem); + + return 0; +} + +static ssize_t vfio_cxl_mem_rw(struct vfio_pci_core_device *vdev, + char __user *buf, size_t count, loff_t *ppos, + bool iswrite) +{ + struct vfio_cxl_state *cxl = vdev->cxl; + u64 pos = *ppos & VFIO_PCI_OFFSET_MASK; + ssize_t done = 0; + void *bounce; + int ret = 0; + + if (pos >= range_len(&cxl->hpa_range)) + return -EINVAL; + count = min_t(u64, count, range_len(&cxl->hpa_range) - pos); + + bounce = kmalloc(min_t(size_t, count, PAGE_SIZE), GFP_KERNEL); + if (!bounce) + return -ENOMEM; + + while (count) { + size_t len = min_t(size_t, count, PAGE_SIZE); + + if (iswrite && copy_from_user(bounce, buf + done, len)) { + ret = -EFAULT; + break; + } + ret = vfio_cxl_mem_copy(vdev, bounce, pos + done, len, iswrite); + if (ret) + break; + if (!iswrite && copy_to_user(buf + done, bounce, len)) { + ret = -EFAULT; + break; + } + done += len; + count -= len; + } + kfree(bounce); + + if (!done) + return ret; + + *ppos += done; + return done; +} + +/* + * The CXL regions carry no per-region state (region->data is the shared, + * devm-managed vfio_cxl_state), so releasing a region is a no-op. + */ +static void vfio_cxl_region_release(struct vfio_pci_core_device *vdev, + struct vfio_pci_region *region) +{ +} + +static const struct vfio_pci_regops vfio_cxl_mem_regops = { + .rw = vfio_cxl_mem_rw, + .release = vfio_cxl_region_release, }; /* @@ -94,12 +199,75 @@ vfio_cxl_create_memdev(struct pci_dev *pdev, const struct cxl_hdm_info *hdm, return cxl; } +/* CXL r4.0 sec 8.1.3.8.2: Range 1 Size Low Memory_Active_Timeout encodings */ +static const unsigned int vfio_cxl_mem_active_timeout_s[] = { 1, 4, 16, 64, 256 }; + +/* + * Poll Memory_Active in DVSEC CXL Range 1 Size Low (CXL r4.0 sec 8.1.3.8.2) + * for up to the Memory_Active_Timeout the device reports, which is stored in + * @timeout. Return 0 once it is set, -ETIMEDOUT, or -ENODEV if config space + * does not respond. + */ +static int vfio_cxl_poll_mem_active(struct pci_dev *pdev, u16 dvsec, + unsigned int *timeout) +{ + int pos = dvsec + PCI_DVSEC_CXL_RANGE_SIZE_LOW(0); + unsigned long deadline; + u32 size_lo; + + if (pci_read_config_dword(pdev, pos, &size_lo) || + PCI_POSSIBLE_ERROR(size_lo)) + return -ENODEV; + + *timeout = FIELD_GET(PCI_DVSEC_CXL_MEM_ACTIVE_TIMEOUT, size_lo); + *timeout = vfio_cxl_mem_active_timeout_s[min_t(unsigned int, *timeout, + ARRAY_SIZE(vfio_cxl_mem_active_timeout_s) - 1)]; + deadline = jiffies + secs_to_jiffies(*timeout); + + while (!(size_lo & PCI_DVSEC_CXL_MEM_ACTIVE)) { + if (time_after(jiffies, deadline)) + return -ETIMEDOUT; + msleep(20); + if (pci_read_config_dword(pdev, pos, &size_lo) || + PCI_POSSIBLE_ERROR(size_lo)) + return -ENODEV; + } + + return 0; +} + +/* + * A reset or a return from low power leaves device memory unavailable until + * the device sets Memory_Active. That can take seconds after the reset + * completes and the decoder is restored, and a host access to the HDM range + * before then can abort. Wait for it on a device that brings its memory up + * by itself; a device that relies on its driver for that was found at bind + * and is not waited for. + */ +static int vfio_cxl_await_mem_active(struct vfio_pci_core_device *vdev) +{ + struct vfio_cxl_state *cxl = vdev->cxl; + unsigned int timeout = 0; + int ret; + + if (!cxl->await_mem_active) + return 0; + + ret = vfio_cxl_poll_mem_active(vdev->pdev, cxl->hdm->dvsec, &timeout); + if (ret == -ETIMEDOUT) + pci_err(vdev->pdev, "vfio-cxl: device memory not active after %us\n", + timeout); + return ret; +} + static int vfio_cxl_init_device(struct vfio_pci_core_device *vdev) { struct pci_dev *pdev = vdev->pdev; struct cxl_hdm_info *hdm __free(kfree) = NULL; struct vfio_cxl_state *cxl; + unsigned int timeout = 0; u64 hdm_size; + int ret; /* * pdev->hdm is cached at PCI enumeration, before any driver binds, so a @@ -150,9 +318,44 @@ static int vfio_cxl_init_device(struct vfio_pci_core_device *vdev) devres_release_group(&pdev->dev, NULL); return PTR_ERR(cxl); } + + /* + * Claim the range IORESOURCE_EXCLUSIVE so no conflicting cacheable + * alias can fault the host once it is mapped write-back. Release it + * explicitly in vfio_cxl_release_device() rather than through a devm + * action on pdev->dev: the provider module ref is dropped as soon as + * that .release op returns, so a pdev-devres callback could run after + * vfio-cxl has unloaded and jump into freed module text. + */ + cxl->hpa_res = request_mem_region_exclusive(cxl->hpa_range.start, + range_len(&cxl->hpa_range), + "vfio-cxl-hdm"); + if (!cxl->hpa_res) { + devres_release_group(&pdev->dev, NULL); + return -EBUSY; + } devres_close_group(&pdev->dev, NULL); cxl->hdm = no_free_ptr(hdm); + /* + * Wait for Memory_Active after a reset only on a device that brings its + * memory up by itself; see vfio_cxl_await_mem_active(). Memory_Active + * also reads clear for seconds after a reset, for example one the host + * ran just before this bind, so give the device its Memory_Active_Timeout + * before deciding. A device that relies on its driver times out here + * once. + */ + ret = vfio_cxl_poll_mem_active(pdev, cxl->hdm->dvsec, &timeout); + cxl->await_mem_active = !ret; + if (ret == -ETIMEDOUT) + pci_info(pdev, "vfio-cxl: device memory not active after %us, left to its driver\n", + timeout); + /* + * The decoder decodes a non-empty range, so it is committed, and host + * access to the HDM range is safe. Reset, bus reset and low power + * close this gate until the decoder is restored. + */ + cxl->hdm_valid = true; vdev->cxl = cxl; return 0; @@ -160,13 +363,40 @@ static int vfio_cxl_init_device(struct vfio_pci_core_device *vdev) static void vfio_cxl_release_device(struct vfio_pci_core_device *vdev) { - kfree(vdev->cxl->hdm); + struct vfio_cxl_state *cxl = vdev->cxl; + + /* + * Drop the exclusive HDM range claim before vfio_pci_core_cxl_release() + * drops the provider module ref; see vfio_cxl_init_device(). + */ + release_mem_region(cxl->hpa_range.start, range_len(&cxl->hpa_range)); + kfree(cxl->hdm); vdev->cxl = NULL; } +static int vfio_cxl_add_region(struct vfio_pci_core_device *vdev, u32 subtype, + const struct vfio_pci_regops *ops, size_t size, + u32 flags) +{ + u32 type = VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_CXL; + + return vfio_pci_core_register_dev_region(vdev, type, subtype, ops, + size, flags, vdev->cxl); +} + static int vfio_cxl_open_device(struct vfio_pci_core_device *vdev) { - return 0; + struct vfio_cxl_state *cxl = vdev->cxl; + + /* + * vfio-pci-core frees all dynamic regions on close and on a failed + * open, so register them here per open rather than at bind. + */ + return vfio_cxl_add_region(vdev, VFIO_REGION_SUBTYPE_CXL_MEM, + &vfio_cxl_mem_regops, + range_len(&cxl->hpa_range), + VFIO_REGION_INFO_FLAG_READ | + VFIO_REGION_INFO_FLAG_WRITE); } static void vfio_cxl_close_device(struct vfio_pci_core_device *vdev) @@ -183,21 +413,54 @@ static int vfio_cxl_reset_done(struct vfio_pci_core_device *vdev) } /* - * Until vfio-cxl serves the device memory, reset through the PCI core's - * reset methods, which run CXL Reset ahead of FLR. Restore the PCI state - * here rather than through pci_reset_function_locked(): its reset_prepare - * and reset_done callbacks take memory_lock, which the caller already holds. - * The caller has saved the state, or on close loaded the open-time copy. + * vfio-cxl holds the HDM range, so the PCI core's cxl_reset method would + * fail to request it. Run the same CXL Reset with that claim. */ static int vfio_cxl_reset(struct vfio_pci_core_device *vdev) { + /* + * The reset restores PCI state from the saved copy. The caller provides + * it: the try-reset helper saves the current state, and the close path + * keeps the open-time state that vfio_pci_core_disable() loaded. + */ + return pci_cxl_reset_owned(vdev->pdev, vdev->cxl->hpa_res); +} + +/* Low power loses the committed decoder: close host access until restored. */ +static void vfio_cxl_power_down(struct vfio_pci_core_device *vdev) +{ + lockdep_assert_held_write(&vdev->memory_lock); + + vdev->cxl->hdm_valid = false; +} + +static void vfio_cxl_power_up(struct vfio_pci_core_device *vdev) +{ + struct vfio_cxl_state *cxl = vdev->cxl; struct pci_dev *pdev = vdev->pdev; int ret; - ret = __pci_reset_function_locked(pdev); - pci_restore_state(pdev); + lockdep_assert_held_write(&vdev->memory_lock); - return ret; + /* + * Restore the decoder that low power lost. The restore needs the device + * lock; if it is contended, keep host access closed until a reset + * restores the decoder. The restore refreshes the decoders in cxl->hdm + * from the cache; global and DVSEC control stay as taken at bind, when + * pci_cxl_reset_capable() required HDM decode and Mem_Enable on. + */ + if (!pci_dev_trylock(pdev)) + return; + + ret = pci_save_state(pdev); + if (!ret) + ret = cxl_restore_state_after_pci_reset(pdev, cxl->hdm); + pci_dev_unlock(pdev); + + if (ret) + pci_warn(pdev, "vfio-cxl: HDM decoder restore failed: %d\n", ret); + else if (!vfio_cxl_await_mem_active(vdev)) + cxl->hdm_valid = true; } static const struct vfio_cxl_ops vfio_cxl_ops = { @@ -208,6 +471,8 @@ static const struct vfio_cxl_ops vfio_cxl_ops = { .reset_prepare = vfio_cxl_reset_prepare, .reset_done = vfio_cxl_reset_done, .reset = vfio_cxl_reset, + .power_down = vfio_cxl_power_down, + .power_up = vfio_cxl_power_up, .owner = THIS_MODULE, }; diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index e41437fa17ad..883b51521e6a 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -370,6 +370,10 @@ struct vfio_region_info_cap_type { */ #define VFIO_REGION_SUBTYPE_IBM_NVLINK2_ATSD (1) +/* CXL Type-2 device (0x1e98) sub-types for VFIO_REGION_TYPE_PCI_VENDOR_TYPE */ +/* CXL.mem HDM region of a Type-2 device, read and write */ +#define VFIO_REGION_SUBTYPE_CXL_MEM (1) + /* sub-types for VFIO_REGION_TYPE_GFX */ #define VFIO_REGION_SUBTYPE_GFX_EDID (1) -- 2.53.0
