From: Honglei Huang <[email protected]>
The drm_gpusvm framework has been refactored [1] so that:
- drm_gpusvm_range carries only MM/VA-range state.
- drm_gpusvm_pages owns physical/DMA state and embeds drm_device.
- drm_gpusvm itself is now a pure MM-level object.
Adapt amdgpu SVM to the new shape, mirroring the xe conversion:
- Embed struct drm_gpusvm_pages in amdgpu_svm_range; amdgpu is
1 svm : 1 drm_device, so a single instance is enough.
- amdgpu_svm_range_alloc() seeds pages.notifier_seq;
amdgpu_svm_range_free() calls drm_gpusvm_free_pages() before
kmem_cache_free().
- Replace drm_gpusvm_range_{get,unmap}_pages() and
drm_gpusvm_range_pages_valid() with the drm_gpusvm_pages
helpers, passing &adev->drm and &svm_range->pages explicitly.
- Drop adev_to_drm(adev) from drm_gpusvm_init().
- Move unmapped/partial_unmap reads from base->pages.flags to
base->flags.
- Add a small driver-local amdgpu_svm_range_unmap_pages() inline
helper since the call site count justifies it.
No functional change intended.
tests:
Tested on gfx943 (MI300X) and gfx906 (MI60) with XNACK on:
- KFD test: 95%+ passed.
- ROCR test: all passed.
- HIP catch test: gfx943 (MI300X): 96% passed.
gfx906 (MI60): 99% passed.
Depends on:
[1] [RFC 0/5] drm/gpusvm: split MM and device state across
gpusvm/range/pages
Base amdgpu SVM series this sits on top of:
[2]
https://lore.kernel.org/amd-gfx/[email protected]/
[3]
https://lore.kernel.org/amd-gfx/[email protected]/
Signed-off-by: Honglei Huang <[email protected]>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c | 5 +++-
drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c | 30 +++++++++++--------
drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h | 18 ++++++++++-
drivers/gpu/drm/amd/amdgpu/amdgpu_userptr.c | 4 +--
4 files changed, 41 insertions(+), 16 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c
index 626c5790e4d..57668f1e06b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm.c
@@ -96,11 +96,14 @@ static struct drm_gpusvm_range
*amdgpu_svm_range_alloc(struct drm_gpusvm *gpusvm
INIT_LIST_HEAD(&range->work_node);
range->pending_start_page = ULONG_MAX;
+ range->pages.notifier_seq = LONG_MAX;
return &range->base;
}
static void amdgpu_svm_range_free(struct drm_gpusvm_range *range)
{
+ drm_gpusvm_free_pages(range->gpusvm,
&(to_amdgpu_svm_range(range)->pages),
+ drm_gpusvm_range_size(range) >> PAGE_SHIFT);
kmem_cache_free(amdgpu_svm_range_cache, to_amdgpu_svm_range(range));
}
@@ -411,7 +414,7 @@ static int amdgpu_svm_init_with_ops(struct amdgpu_svm *svm,
}
ret = drm_gpusvm_init(&svm->gpusvm, "AMDGPU SVM",
- adev_to_drm(adev), current->mm,
0,
+ current->mm, 0,
adev->vm_manager.max_pfn <<
AMDGPU_GPU_PAGE_SHIFT,
AMDGPU_SVM_DEFAULT_SVM_NOTIFIER_SIZE * SZ_1M,
&amdgpu_gpusvm_ops,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
index ef913a2363e..27756256105 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.c
@@ -43,10 +43,10 @@ amdgpu_svm_range_pages_valid(struct amdgpu_svm *svm,
lockdep_assert_held(&svm->gpusvm.notifier_lock);
- if (base->pages.flags.unmapped || base->pages.flags.partial_unmap)
+ if (base->flags.unmapped || base->flags.partial_unmap)
return false;
- return drm_gpusvm_range_pages_valid(&svm->gpusvm, base);
+ return drm_gpusvm_pages_valid(&svm->gpusvm, &range->pages);
}
bool amdgpu_svm_range_is_valid(struct amdgpu_svm *svm,
@@ -267,11 +267,11 @@ amdgpu_svm_range_update_gpu_range(struct amdgpu_svm *svm,
unsigned long dma_idx = 0;
int ret;
- if (!base->pages.dma_addr || !npages)
+ if (!range->pages.dma_addr || !npages)
return -EINVAL;
while (mapped_pages < npages) {
- const struct drm_pagemap_addr *entry =
&base->pages.dma_addr[dma_idx++];
+ const struct drm_pagemap_addr *entry =
&range->pages.dma_addr[dma_idx++];
unsigned long seg_pages = min_t(unsigned long, 1UL <<
entry->order,
npages - mapped_pages);
uint64_t pte_flags;
@@ -332,10 +332,15 @@ int amdgpu_svm_range_get_pages(struct amdgpu_svm *svm,
struct drm_gpusvm_range *range,
struct drm_gpusvm_ctx *ctx)
{
+ struct amdgpu_svm_range *svm_range = to_amdgpu_svm_range(range);
int ret;
retry:
- ret = drm_gpusvm_range_get_pages(&svm->gpusvm, range, ctx);
+ ret = drm_gpusvm_get_pages(&svm->gpusvm, &svm_range->pages,
+ adev_to_drm(svm->adev), svm->gpusvm.mm,
+ &range->notifier->notifier,
+ drm_gpusvm_range_start(range),
+ drm_gpusvm_range_end(range), ctx);
/*
* HMM returns -EPERM when write access is requested for a read-only
* VMA. Retry as read-only so the eventual GPU mapping follows the CPU
@@ -360,7 +365,9 @@ int amdgpu_svm_range_get_pages(struct amdgpu_svm *svm,
void amdgpu_svm_range_evict(struct amdgpu_svm *svm,
struct drm_gpusvm_range *range)
{
- if (!range->pages.flags.has_devmem_pages)
+ struct amdgpu_svm_range *svm_range = to_amdgpu_svm_range(range);
+
+ if (!svm_range->pages.flags.has_devmem_pages)
return;
drm_gpusvm_range_evict(&svm->gpusvm, range);
@@ -496,8 +503,8 @@ void amdgpu_svm_range_remove(struct amdgpu_svm *svm,
amdgpu_svm_assert_locked(svm);
- if (!base->pages.flags.unmapped && !base->pages.flags.partial_unmap)
- drm_gpusvm_range_unmap_pages(&svm->gpusvm, base, ctx);
+ if (!base->flags.unmapped && !base->flags.partial_unmap)
+ amdgpu_svm_range_unmap_pages(svm, base, ctx);
amdgpu_svm_range_invalidate_gpu_mapping(range);
drm_gpusvm_range_remove(&svm->gpusvm, base);
@@ -515,7 +522,7 @@ amdgpu_svm_range_notifier_event_begin(struct amdgpu_svm
*svm,
AMDGPU_SVM_RANGE_DEBUG(svm_range, "NOTIFIER");
- if (range->pages.flags.unmapped || !svm_range->gpu_mapped)
+ if (range->flags.unmapped || !svm_range->gpu_mapped)
return false;
AMDGPU_SVM_RANGE_DEBUG(svm_range, "NOTIFIER - EXECUTE");
@@ -580,10 +587,9 @@ amdgpu_svm_range_notifier_event_end(struct amdgpu_svm *svm,
amdgpu_svm_assert_in_notifier(svm);
- drm_gpusvm_range_unmap_pages(&svm->gpusvm, range, &ctx);
+ amdgpu_svm_range_unmap_pages(svm, range, &ctx);
if (mmu_range->event == MMU_NOTIFY_UNMAP)
- amdgpu_svm_gc_add_range(svm, to_amdgpu_svm_range(range),
- mmu_range);
+ amdgpu_svm_gc_add_range(svm, to_amdgpu_svm_range(range),
mmu_range);
}
int
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h
index 2d677f0965b..90ad1c55438 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_svm_range.h
@@ -53,6 +53,11 @@ enum amdgpu_svm_range_queue_state {
struct amdgpu_svm_range {
struct drm_gpusvm_range base;
+ /*
+ * Per-device DMA mapping state; single instance since amdgpu svm
+ * is 1 svm : 1 drm_device.
+ */
+ struct drm_gpusvm_pages pages;
struct list_head work_node;
bool gpu_mapped;
u8 queue_state;
@@ -75,13 +80,24 @@ amdgpu_svm_range_invalidate_gpu_mapping(struct
amdgpu_svm_range *range)
WRITE_ONCE(range->gpu_mapped, false);
}
+static inline void
+amdgpu_svm_range_unmap_pages(struct amdgpu_svm *svm,
+ struct drm_gpusvm_range *range,
+ struct drm_gpusvm_ctx *ctx)
+{
+ struct amdgpu_svm_range *svm_range = to_amdgpu_svm_range(range);
+
+ drm_gpusvm_unmap_pages(&svm->gpusvm, &svm_range->pages,
+ drm_gpusvm_range_size(range) >> PAGE_SHIFT, ctx);
+}
+
#define AMDGPU_SVM_RANGE_DEBUG(r__, op__)
\
AMDGPU_SVM_TRACE("%s: pasid=%u, gpusvm=%p, mapped=%d, " \
"seqno=%lu, range: [0x%lx-0x%lx]-" \
"0x%lx\n", \
(op__), to_amdgpu_svm((r__)->base.gpusvm)->vm->pasid, \
(r__)->base.gpusvm, READ_ONCE((r__)->gpu_mapped), \
- (r__)->base.pages.notifier_seq, \
+ (r__)->pages.notifier_seq, \
drm_gpusvm_range_start(&(r__)->base) >> PAGE_SHIFT, \
drm_gpusvm_range_end(&(r__)->base) >> PAGE_SHIFT, \
(drm_gpusvm_range_end(&(r__)->base) - \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userptr.c
b/drivers/gpu/drm/amd/amdgpu/amdgpu_userptr.c
index 4582fb9453b..77ef6f41ab9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userptr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userptr.c
@@ -58,7 +58,7 @@ svm_restore_notifier_event_end(struct amdgpu_svm *svm,
amdgpu_svm_assert_in_notifier(svm);
- drm_gpusvm_range_unmap_pages(&svm->gpusvm, range, &ctx);
+ amdgpu_svm_range_unmap_pages(svm, range, &ctx);
if (mmu_range->event == MMU_NOTIFY_UNMAP) {
AMDGPU_SVM_RANGE_DEBUG(svm_range, "GARBAGE COLLECTOR ADD");
drm_gpusvm_range_set_unmapped(&svm_range->base, mmu_range);
@@ -623,7 +623,7 @@ void amdgpu_svm_restore_gc_work_func(struct work_struct *w)
spin_unlock(&svm->work_lock);
range_start_page = drm_gpusvm_range_start(&op_ctx.range->base)
>> PAGE_SHIFT;
range_last_page = (drm_gpusvm_range_end(&op_ctx.range->base) >>
PAGE_SHIFT) - 1;
- partial = op_ctx.range->base.pages.flags.partial_unmap;
+ partial = op_ctx.range->base.flags.partial_unmap;
ret = 0;
WARN_ON(!UNMAP_WORK(op_ctx.pending_ops));
--
2.34.1
