On Wed, 2025-11-26 at 08:32 -0800, Matthew Brost wrote:
> On Tue, Nov 11, 2025 at 05:44:06PM +0100, Thomas Hellström wrote:
> > Support migration over interconnect when migrating from
> > device-private pages with the same dev_pagemap owner.
> >
> > Since we now also collect device-private pages to migrate,
> > also abort migration if the range to migrate is already
> > fully populated with pages from the desired pagemap.
> >
> > Finally return -EBUSY from drm_pagemap_populate_mm()
> > if the migration can't be completed without first migrating all
> > pages in the range to system. It is expected that the caller
> > will perform that before retrying the call to
> > drm_pagemap_populate_mm().
> >
> > Assume for now that the drm_pagemap implementation is *not*
> > capable of migrating data within the pagemap itself. This
> > restriction will be configurable in upcoming patches.
> >
> > Signed-off-by: Thomas Hellström <[email protected]>
> > ---
> > drivers/gpu/drm/drm_pagemap.c | 177 +++++++++++++++++++++++++-----
> > ----
> > drivers/gpu/drm/xe/xe_svm.c | 20 ++--
> > 2 files changed, 143 insertions(+), 54 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/drm_pagemap.c
> > b/drivers/gpu/drm/drm_pagemap.c
> > index 1477a2057a15..e87676313ff9 100644
> > --- a/drivers/gpu/drm/drm_pagemap.c
> > +++ b/drivers/gpu/drm/drm_pagemap.c
> > @@ -210,6 +210,7 @@ static void drm_pagemap_get_devmem_page(struct
> > page *page,
> > /**
> > * drm_pagemap_migrate_map_pages() - Map migration pages for GPU
> > SVM migration
> > * @dev: The device for which the pages are being mapped
> > + * @local_dpagemap: The drm_pagemap pointer of the local
> > drm_pagemap.
> > * @pagemap_addr: Array to store DMA information corresponding to
> > mapped pages
> > * @migrate_pfn: Array of migrate page frame numbers to map
> > * @npages: Number of pages to map
> > @@ -223,12 +224,14 @@ static void
> > drm_pagemap_get_devmem_page(struct page *page,
> > * Returns: 0 on success, -EFAULT if an error occurs during
> > mapping.
> > */
> > static int drm_pagemap_migrate_map_pages(struct device *dev,
> > + struct drm_pagemap
> > *local_dpagemap,
> > struct drm_pagemap_addr
> > *pagemap_addr,
> > unsigned long
> > *migrate_pfn,
> > unsigned long npages,
> > enum dma_data_direction
> > dir)
> > {
> > unsigned long i;
> > + unsigned long num_peer_pages = 0;
> >
> > for (i = 0; i < npages;) {
> > struct page *page =
> > migrate_pfn_to_page(migrate_pfn[i]);
> > @@ -239,31 +242,48 @@ static int
> > drm_pagemap_migrate_map_pages(struct device *dev,
> > if (!page)
> > goto next;
> >
> > - if (WARN_ON_ONCE(is_zone_device_page(page)))
> > - return -EFAULT;
> > -
> > folio = page_folio(page);
> > order = folio_order(folio);
> >
> > - dma_addr = dma_map_page(dev, page, 0,
> > page_size(page), dir);
> > - if (dma_mapping_error(dev, dma_addr))
> > - return -EFAULT;
> > + if (is_zone_device_page(page)) {
> > + struct drm_pagemap_zdd *zdd = page-
> > >zone_device_data;
> > + struct drm_pagemap *dpagemap = zdd-
> > >dpagemap;
> > + struct drm_pagemap_addr addr;
> > +
> > + if (dpagemap == local_dpagemap)
> > + goto next;
> >
> > - pagemap_addr[i] =
> > - drm_pagemap_addr_encode(dma_addr,
> > -
> > DRM_INTERCONNECT_SYSTEM,
> > - order, dir);
> > + num_peer_pages += NR_PAGES(order);
> > + addr = dpagemap->ops->device_map(dpagemap,
> > dev, page, order, dir);
> > + if (dma_mapping_error(dev, addr.addr))
> > + return -EFAULT;
> > + } else {
> > + dma_addr = dma_map_page(dev, page, 0,
> > page_size(page), dir);
> > + if (dma_mapping_error(dev, dma_addr))
> > + return -EFAULT;
> > +
> > + pagemap_addr[i] =
> > + drm_pagemap_addr_encode(dma_addr,
> > + DRM_INTERC
> > ONNECT_SYSTEM,
> > + order,
> > dir);
> > + }
> >
> > next:
> > i += NR_PAGES(order);
> > }
> >
> > + if (num_peer_pages)
> > + drm_dbg(local_dpagemap->drm, "Migrating %lu peer
> > pages over interconnect.\n",
> > + num_peer_pages);
> > +
> > return 0;
> > }
> >
> > /**
> > * drm_pagemap_migrate_unmap_pages() - Unmap pages previously
> > mapped for GPU SVM migration
> > * @dev: The device for which the pages were mapped
> > + * @migrate_pfn: Array of migrate pfns set up for the mapped
> > pages. Used to
> > + * determine the drm_pagemap of a peer device private page.
> > * @pagemap_addr: Array of DMA information corresponding to mapped
> > pages
> > * @npages: Number of pages to unmap
> > * @dir: Direction of data transfer (e.g., DMA_BIDIRECTIONAL)
> > @@ -274,16 +294,27 @@ static int
> > drm_pagemap_migrate_map_pages(struct device *dev,
> > */
> > static void drm_pagemap_migrate_unmap_pages(struct device *dev,
> > struct
> > drm_pagemap_addr *pagemap_addr,
> > + unsigned long
> > *migrate_pfn,
> > unsigned long npages,
> > enum
> > dma_data_direction dir)
> > {
> > unsigned long i;
> >
> > for (i = 0; i < npages;) {
> > - if (!pagemap_addr[i].addr ||
> > dma_mapping_error(dev, pagemap_addr[i].addr))
> > + struct page *page =
> > migrate_pfn_to_page(migrate_pfn[i]);
> > +
> > + if (!page || !pagemap_addr[i].addr ||
> > dma_mapping_error(dev, pagemap_addr[i].addr))
> > goto next;
> >
> > - dma_unmap_page(dev, pagemap_addr[i].addr,
> > PAGE_SIZE << pagemap_addr[i].order, dir);
> > + if (is_zone_device_page(page)) {
> > + struct drm_pagemap_zdd *zdd = page-
> > >zone_device_data;
> > + struct drm_pagemap *dpagemap = zdd-
> > >dpagemap;
> > +
> > + dpagemap->ops->device_unmap(dpagemap, dev,
> > pagemap_addr[i]);
> > + } else {
> > + dma_unmap_page(dev, pagemap_addr[i].addr,
> > + PAGE_SIZE <<
> > pagemap_addr[i].order, dir);
> > + }
> >
> > next:
> > i += NR_PAGES(pagemap_addr[i].order);
> > @@ -308,6 +339,7 @@ npages_in_range(unsigned long start, unsigned
> > long end)
> > * @timeslice_ms: The time requested for the migrated pagemap
> > pages to
> > * be present in @mm before being allowed to be migrated back.
> > * @pgmap_owner: Not used currently, since only system memory is
> > considered.
> > + * @mflags: Flags governing the migration.
> > *
> > * This function migrates the specified virtual address range to
> > device memory.
> > * It performs the necessary setup and invokes the driver-specific
> > operations for
> > @@ -333,13 +365,18 @@ int drm_pagemap_migrate_to_devmem(struct
> > drm_pagemap_devmem *devmem_allocation,
> > .start = start,
> > .end = end,
> > .pgmap_owner = pgmap_owner,
> > - .flags = MIGRATE_VMA_SELECT_SYSTEM,
> > + .flags = MIGRATE_VMA_SELECT_SYSTEM |
> > + MIGRATE_VMA_SELECT_DEVICE_PRIVATE |
> > + MIGRATE_VMA_SELECT_DEVICE_COHERENT,
> > };
> > unsigned long i, npages = npages_in_range(start, end);
> > + unsigned long own_pages = 0, migrated_pages = 0;
> > struct vm_area_struct *vas;
> > struct drm_pagemap_zdd *zdd = NULL;
> > struct page **pages;
> > struct drm_pagemap_addr *pagemap_addr;
> > + struct drm_pagemap *dpagemap = devmem_allocation-
> > >dpagemap;
> > + struct dev_pagemap *pagemap = dpagemap->pagemap;
> > void *buf;
> > int err;
> >
> > @@ -374,11 +411,13 @@ int drm_pagemap_migrate_to_devmem(struct
> > drm_pagemap_devmem *devmem_allocation,
> > pagemap_addr = buf + (2 * sizeof(*migrate.src) * npages);
> > pages = buf + (2 * sizeof(*migrate.src) +
> > sizeof(*pagemap_addr)) * npages;
> >
> > - zdd = drm_pagemap_zdd_alloc(devmem_allocation->dpagemap,
> > pgmap_owner);
> > + zdd = drm_pagemap_zdd_alloc(dpagemap, pgmap_owner);
> > if (!zdd) {
> > err = -ENOMEM;
> > - goto err_free;
> > + kvfree(buf);
> > + goto err_out;
> > }
> > + zdd->devmem_allocation = devmem_allocation; /* Owns
> > ref */
> >
> > migrate.vma = vas;
> > migrate.src = buf;
> > @@ -389,54 +428,108 @@ int drm_pagemap_migrate_to_devmem(struct
> > drm_pagemap_devmem *devmem_allocation,
> > goto err_free;
> >
> > if (!migrate.cpages) {
> > - err = -EFAULT;
> > + /* No pages to migrate. Raced or unknown device
> > pages. */
> > + err = -EBUSY;
> > goto err_free;
> > }
> >
> > if (migrate.cpages != npages) {
> > + /*
> > + * Some pages to migrate. But we want to migrate
> > all or
> > + * nothing. Raced or unknown device pages.
> > + */
> > err = -EBUSY;
> > - goto err_finalize;
> > + goto err_aborted_migration;
> > + }
> > +
> > + /* Count device-private pages to migrate */
> > + for (i = 0; i < npages; ++i) {
> > + struct page *src_page =
> > migrate_pfn_to_page(migrate.src[i]);
> > +
> > + if (src_page && is_zone_device_page(src_page)) {
> > + if (page_pgmap(src_page) == pagemap)
> > + own_pages++;
> > + }
> > + }
>
> I understand what this is doing—aborting the migration if the pages
> are
> in the correct location. Conceptually, I believe this is correct, but
> implementation-wise it is likely not. The pages collected here are
> gathered via migrate_vma_setup. This step issues an MMU notifier and
> installs migration PTEs, which are expensive operations. For example,
> if
> another GPU already has the correct mappings and the pages are in the
> correct location, migrate_vma_setup will result in the range being
> invalidated and the GPU<->GPU mapping being removed. Installing
> migration PTEs is also CPU-intensive.
>
> I think the step to check own_pages should be built on top of
> hmm_range_fault without HMM_PFN_REQ_FAULT set, which is fast and will
> not issue an MMU notifier.
I agree fully and what you say above is also what we outlined in a
previous discussion. I was planning to do the hmm_range_fault() check
in drm_gpusvm, as a follow-up, though, before calling into drm_pagemap
to migrate.
Reason is that in drm_pagemap, the current implementation doesn't
register any mmu notifier callbacks.
But I realize now that this is pretty much required for any efficiency
with multiple gpus that would otherwise invalidate just checking that
the pages are in place....
/Thomas
>
> Matt
>
> > +
> > + drm_dbg(dpagemap->drm, "Total pages %lu; Own pages:
> > %lu.\n",
> > + npages, own_pages);
> > + if (own_pages == npages) {
> > + err = 0;
> > + drm_dbg(dpagemap->drm, "Migration wasn't
> > necessary.\n");
> > + goto err_aborted_migration;
> > + } else if (own_pages) {
> > + err = -EBUSY;
> > + drm_dbg(dpagemap->drm, "Migration aborted due to
> > fragmentation.\n");
> > + goto err_aborted_migration;
> > }
> >
> > err = ops->populate_devmem_pfn(devmem_allocation, npages,
> > migrate.dst);
> > if (err)
> > goto err_finalize;
> >
> > - err = drm_pagemap_migrate_map_pages(devmem_allocation-
> > >dev, pagemap_addr,
> > + err = drm_pagemap_migrate_map_pages(devmem_allocation-
> > >dev,
> > + devmem_allocation-
> > >dpagemap, pagemap_addr,
> > migrate.src, npages,
> > DMA_TO_DEVICE);
> >
> > - if (err)
> > + if (err) {
> > + drm_pagemap_migrate_unmap_pages(devmem_allocation-
> > >dev, pagemap_addr,
> > + migrate.src,
> > npages, DMA_TO_DEVICE);
> > +
> > goto err_finalize;
> > + }
> >
> > + own_pages = 0;
> > for (i = 0; i < npages; ++i) {
> > struct page *page = pfn_to_page(migrate.dst[i]);
> > + struct page *src_page =
> > migrate_pfn_to_page(migrate.src[i]);
> >
> > + if (unlikely(src_page &&
> > is_zone_device_page(src_page) &&
> > + page_pgmap(src_page) == pagemap)) {
> > + migrate.dst[i] = 0;
> > + pages[i] = NULL;
> > + own_pages++;
> > + continue;
> > + }
> > pages[i] = page;
> > migrate.dst[i] = migrate_pfn(migrate.dst[i]);
> > drm_pagemap_get_devmem_page(page, zdd);
> > }
> > + drm_WARN_ON(dpagemap->drm, !!own_pages);
> >
> > err = ops->copy_to_devmem(pages, pagemap_addr, npages);
> > + drm_pagemap_migrate_unmap_pages(devmem_allocation->dev,
> > pagemap_addr,
> > + migrate.src, npages,
> > DMA_TO_DEVICE);
> > if (err)
> > goto err_finalize;
> >
> > /* Upon success bind devmem allocation to range and zdd */
> > devmem_allocation->timeslice_expiration = get_jiffies_64()
> > +
> > msecs_to_jiffies(timeslice_ms);
> > - zdd->devmem_allocation = devmem_allocation; /* Owns
> > ref */
> >
> > err_finalize:
> > if (err)
> > drm_pagemap_migration_unlock_put_pages(npages,
> > migrate.dst);
> > +err_aborted_migration:
> > migrate_vma_pages(&migrate);
> > +
> > + for (i = 0; i < npages; ++i)
> > + if (migrate.src[i] & MIGRATE_PFN_MIGRATE)
> > + migrated_pages++;
> > +
> > + if (!err && migrated_pages < npages - own_pages) {
> > + drm_dbg(dpagemap->drm, "Raced while finalizing
> > migration.\n");
> > + err = -EBUSY;
> > + }
> > +
> > migrate_vma_finalize(&migrate);
> > - drm_pagemap_migrate_unmap_pages(devmem_allocation->dev,
> > pagemap_addr, npages,
> > - DMA_TO_DEVICE);
> > err_free:
> > - if (zdd)
> > - drm_pagemap_zdd_put(zdd);
> > + drm_pagemap_zdd_put(zdd);
> > kvfree(buf);
> > + return err;
> > +
> > err_out:
> > + devmem_allocation->ops->devmem_release(devmem_allocation);
> > return err;
> > }
> > EXPORT_SYMBOL_GPL(drm_pagemap_migrate_to_devmem);
> > @@ -747,7 +840,8 @@ int drm_pagemap_evict_to_ram(struct
> > drm_pagemap_devmem *devmem_allocation)
> > if (err || !mpages)
> > goto err_finalize;
> >
> > - err = drm_pagemap_migrate_map_pages(devmem_allocation-
> > >dev, pagemap_addr,
> > + err = drm_pagemap_migrate_map_pages(devmem_allocation-
> > >dev,
> > + devmem_allocation-
> > >dpagemap, pagemap_addr,
> > dst, npages,
> > DMA_FROM_DEVICE);
> > if (err)
> > goto err_finalize;
> > @@ -764,7 +858,7 @@ int drm_pagemap_evict_to_ram(struct
> > drm_pagemap_devmem *devmem_allocation)
> > drm_pagemap_migration_unlock_put_pages(npages,
> > dst);
> > migrate_device_pages(src, dst, npages);
> > migrate_device_finalize(src, dst, npages);
> > - drm_pagemap_migrate_unmap_pages(devmem_allocation->dev,
> > pagemap_addr, npages,
> > + drm_pagemap_migrate_unmap_pages(devmem_allocation->dev,
> > pagemap_addr, dst, npages,
> > DMA_FROM_DEVICE);
> > err_free:
> > kvfree(buf);
> > @@ -820,12 +914,10 @@ static int
> > __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas,
> > void *buf;
> > int i, err = 0;
> >
> > - if (page) {
> > - zdd = page->zone_device_data;
> > - if (time_before64(get_jiffies_64(),
> > - zdd->devmem_allocation-
> > >timeslice_expiration))
> > - return 0;
> > - }
> > + zdd = page->zone_device_data;
> > + if (time_before64(get_jiffies_64(),
> > + zdd->devmem_allocation-
> > >timeslice_expiration))
> > + return 0;
> >
> > start = ALIGN_DOWN(fault_addr, size);
> > end = ALIGN(fault_addr + 1, size);
> > @@ -861,19 +953,6 @@ static int __drm_pagemap_migrate_to_ram(struct
> > vm_area_struct *vas,
> > if (!migrate.cpages)
> > goto err_free;
> >
> > - if (!page) {
> > - for (i = 0; i < npages; ++i) {
> > - if (!(migrate.src[i] &
> > MIGRATE_PFN_MIGRATE))
> > - continue;
> > -
> > - page =
> > migrate_pfn_to_page(migrate.src[i]);
> > - break;
> > - }
> > -
> > - if (!page)
> > - goto err_finalize;
> > - }
> > - zdd = page->zone_device_data;
> > ops = zdd->devmem_allocation->ops;
> > dev = zdd->devmem_allocation->dev;
> >
> > @@ -883,7 +962,7 @@ static int __drm_pagemap_migrate_to_ram(struct
> > vm_area_struct *vas,
> > if (err)
> > goto err_finalize;
> >
> > - err = drm_pagemap_migrate_map_pages(dev, pagemap_addr,
> > migrate.dst, npages,
> > + err = drm_pagemap_migrate_map_pages(dev, zdd->dpagemap,
> > pagemap_addr, migrate.dst, npages,
> > DMA_FROM_DEVICE);
> > if (err)
> > goto err_finalize;
> > @@ -901,8 +980,8 @@ static int __drm_pagemap_migrate_to_ram(struct
> > vm_area_struct *vas,
> > migrate_vma_pages(&migrate);
> > migrate_vma_finalize(&migrate);
> > if (dev)
> > - drm_pagemap_migrate_unmap_pages(dev, pagemap_addr,
> > npages,
> > - DMA_FROM_DEVICE);
> > + drm_pagemap_migrate_unmap_pages(dev, pagemap_addr,
> > migrate.dst,
> > + npages,
> > DMA_FROM_DEVICE);
> > err_free:
> > kvfree(buf);
> > err_out:
> > @@ -938,10 +1017,12 @@ static vm_fault_t
> > drm_pagemap_migrate_to_ram(struct vm_fault *vmf)
> > struct drm_pagemap_zdd *zdd = vmf->page->zone_device_data;
> > int err;
> >
> > + drm_pagemap_zdd_get(zdd);
> > err = __drm_pagemap_migrate_to_ram(vmf->vma,
> > zdd-
> > >device_private_page_owner,
> > vmf->page, vmf-
> > >address,
> > zdd->devmem_allocation-
> > >size);
> > + drm_pagemap_zdd_put(zdd);
> >
> > return err ? VM_FAULT_SIGBUS : 0;
> > }
> > diff --git a/drivers/gpu/drm/xe/xe_svm.c
> > b/drivers/gpu/drm/xe/xe_svm.c
> > index 0b39905c9312..56bb3896b89a 100644
> > --- a/drivers/gpu/drm/xe/xe_svm.c
> > +++ b/drivers/gpu/drm/xe/xe_svm.c
> > @@ -1028,11 +1028,10 @@ static int
> > xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap,
> >
> > /* Ensure the device has a pm ref while there are
> > device pages active. */
> > xe_pm_runtime_get_noresume(xe);
> > + /* Consumes the devmem allocation. */
> > err = drm_pagemap_migrate_to_devmem(&bo-
> > >devmem_allocation, mm,
> > start, end,
> > timeslice_ms,
> > xpagemap-
> > >pagemap.owner);
> > - if (err)
> > - xe_svm_devmem_release(&bo-
> > >devmem_allocation);
> > xe_bo_unlock(bo);
> > xe_bo_put(bo);
> > }
> > @@ -1546,6 +1545,7 @@ int xe_svm_alloc_vram(struct xe_svm_range
> > *range, const struct drm_gpusvm_ctx *c
> > struct drm_pagemap *dpagemap)
> > {
> > struct xe_device *xe = range_to_vm(&range->base)->xe;
> > + int err, retries = 1;
> >
> > xe_assert(range_to_vm(&range->base)->xe, range-
> > >base.pages.flags.migrate_devmem);
> > range_debug(range, "ALLOCATE VRAM");
> > @@ -1554,10 +1554,18 @@ int xe_svm_alloc_vram(struct xe_svm_range
> > *range, const struct drm_gpusvm_ctx *c
> > drm_dbg(&xe->drm, "Request migration to device
> > memory on \"%s\".\n",
> > dpagemap->drm->unique);
> >
> > - return drm_pagemap_populate_mm(dpagemap,
> > xe_svm_range_start(range),
> > - xe_svm_range_end(range),
> > - range->base.gpusvm->mm,
> > - ctx->timeslice_ms);
> > + do {
> > + err = drm_pagemap_populate_mm(dpagemap,
> > xe_svm_range_start(range),
> > +
> > xe_svm_range_end(range),
> > + range->base.gpusvm-
> > >mm,
> > + ctx->timeslice_ms);
> > +
> > + if (err == -EBUSY && retries)
> > + drm_gpusvm_range_evict(range->base.gpusvm,
> > &range->base);
> > +
> > + } while (err == -EBUSY && retries--);
> > +
> > + return err;
> > }
> >
> > static struct drm_pagemap_addr
> > --
> > 2.51.1
> >
