Add per-device tracking of BOs whose backing TT pages were allocated below the pool's beneficial order, as reported by ttm_tt::beneficial_order_failed. These BOs are future candidates for a defragmentation pass that reallocates their pages at the beneficial order.
Add a mem.defrag sub-struct (lock, list, atomic count) to struct xe_device and a defrag_link to struct xe_bo, initialised at BO create and device init. Membership is updated from xe_bo_move() via xe_bo_defrag_update(): a BO is added when it is a populated, non-pinned ttm_bo_type_device object flagged beneficial_order_failed, and removed otherwise. BOs are also removed on pin, on purge, and at destroy. Only ttm_bo_type_device BOs are tracked. Cc: Carlos Santa <[email protected]> Cc: Ryan Neph <[email protected]> Cc: Christian Koenig <[email protected]> Cc: Huang Rui <[email protected]> Cc: Matthew Auld <[email protected]> Cc: Maarten Lankhorst <[email protected]> Cc: Maxime Ripard <[email protected]> Cc: Thomas Zimmermann <[email protected]> Cc: David Airlie <[email protected]> Cc: Simona Vetter <[email protected]> Cc: [email protected] Cc: [email protected] Cc: Thomas Hellström <[email protected]> Assisted-by: GitHub_Copilot:claude-opus-4.8 Signed-off-by: Matthew Brost <[email protected]> --- drivers/gpu/drm/xe/xe_bo.c | 77 ++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_bo.h | 2 + drivers/gpu/drm/xe/xe_bo_types.h | 6 +++ drivers/gpu/drm/xe/xe_device.c | 2 + drivers/gpu/drm/xe/xe_device_types.h | 19 +++++++ 5 files changed, 106 insertions(+) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 4c80bac67622..4acacc50a28f 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -947,6 +947,72 @@ static int xe_ttm_bo_purge(struct ttm_buffer_object *ttm_bo, struct ttm_operatio return 0; } +/** + * xe_bo_defrag_init() - Initialize the device defrag BO tracking + * @xe: The xe device + * + * Initialize the list, lock and count used to track BOs whose backing TT + * pages were allocated at a sub-optimal order. + */ +void xe_bo_defrag_init(struct xe_device *xe) +{ + spin_lock_init(&xe->mem.defrag.lock); + INIT_LIST_HEAD(&xe->mem.defrag.list); + atomic_set(&xe->mem.defrag.count, 0); +} + +static void xe_bo_defrag_add(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + + scoped_guard(spinlock, &xe->mem.defrag.lock) { + if (list_empty(&bo->defrag_link)) { + list_add_tail(&bo->defrag_link, &xe->mem.defrag.list); + atomic_inc(&xe->mem.defrag.count); + } + } +} + +/** + * xe_bo_defrag_remove() - Remove a BO from the device defrag list + * @bo: The buffer object + * + * Remove @bo from the defrag list if present. Safe to call on a BO that is + * not currently on the list. + */ +void xe_bo_defrag_remove(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + + guard(spinlock)(&xe->mem.defrag.lock); + + if (!list_empty(&bo->defrag_link)) { + list_del_init(&bo->defrag_link); + atomic_dec(&xe->mem.defrag.count); + } +} + +/** + * xe_bo_defrag_update() - Update defrag list membership for a BO + * @bo: The buffer object + * + * Add @bo to the device defrag list when it has a populated, non-pinned TT of + * type ttm_bo_type_device whose pages were allocated at a sub-optimal order + * (tt->beneficial_order_failed). Otherwise ensure it is removed from the list. + */ +static void xe_bo_defrag_update(struct xe_bo *bo) +{ + struct ttm_buffer_object *ttm_bo = &bo->ttm; + struct ttm_tt *tt = ttm_bo->ttm; + + if (ttm_bo->type == ttm_bo_type_device && tt && + ttm_tt_is_populated(tt) && tt->beneficial_order_failed && + !xe_bo_is_pinned(bo)) + xe_bo_defrag_add(bo); + else + xe_bo_defrag_remove(bo); +} + static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, struct ttm_operation_ctx *ctx, struct ttm_resource *new_mem, @@ -975,6 +1041,8 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, if (ret) return ret; + xe_bo_defrag_remove(bo); + /* Free the unused eviction destination resource */ ttm_resource_free(ttm_bo, &new_mem); return 0; @@ -1172,6 +1240,8 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, xe_tt_unmap_sg(xe, ttm_bo->ttm); } + xe_bo_defrag_update(bo); + return ret; } @@ -1850,6 +1920,8 @@ static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo) list_del(&bo->vram_userfault_link); mutex_unlock(&xe->mem_access.vram_userfault.lock); + xe_bo_defrag_remove(bo); + kfree(bo); } @@ -2355,6 +2427,7 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, bo->ttm.base.funcs = &xe_gem_object_funcs; bo->ttm.priority = XE_BO_PRIORITY_NORMAL; INIT_LIST_HEAD(&bo->pinned_link); + INIT_LIST_HEAD(&bo->defrag_link); #ifdef CONFIG_PROC_FS INIT_LIST_HEAD(&bo->client_link); #endif @@ -2961,6 +3034,8 @@ int xe_bo_pin_external(struct xe_bo *bo, bool in_place, struct drm_exec *exec) if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm)) xe_ttm_tt_account_subtract(xe, bo->ttm.ttm); + xe_bo_defrag_remove(bo); + /* * FIXME: If we always use the reserve / unreserve functions for locking * we do not need this. @@ -3019,6 +3094,8 @@ int xe_bo_pin(struct xe_bo *bo, struct drm_exec *exec) if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm)) xe_ttm_tt_account_subtract(xe, bo->ttm.ttm); + xe_bo_defrag_remove(bo); + /* * FIXME: If we always use the reserve / unreserve functions for locking * we do not need this. diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index 6340317f7d2e..6b7c2e99a1d2 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -224,6 +224,8 @@ int xe_bo_pin_external(struct xe_bo *bo, bool in_place, struct drm_exec *exec); int xe_bo_pin(struct xe_bo *bo, struct drm_exec *exec); void xe_bo_unpin_external(struct xe_bo *bo); void xe_bo_unpin(struct xe_bo *bo); +void xe_bo_defrag_init(struct xe_device *xe); +void xe_bo_defrag_remove(struct xe_bo *bo); int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict, struct drm_exec *exec); diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h index fcc63ae3f455..9eea9ab380e5 100644 --- a/drivers/gpu/drm/xe/xe_bo_types.h +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -54,6 +54,12 @@ struct xe_bo { struct ttm_bo_kmap_obj kmap; /** @pinned_link: link to present / evicted list of pinned BO */ struct list_head pinned_link; + /** + * @defrag_link: link into @xe_device.mem.defrag.list for BOs whose + * backing TT pages were allocated at a sub-optimal order. Protected by + * @xe_device.mem.defrag.lock. + */ + struct list_head defrag_link; #ifdef CONFIG_PROC_FS /** * @client: @xe_drm_client which created the bo diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index b6e49309a99f..c8c90e9b1bc1 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -577,6 +577,8 @@ int xe_device_init_early(struct xe_device *xe) if (err) return err; + xe_bo_defrag_init(xe); + xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq", WQ_MEM_RECLAIM); xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0); diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 4e2f115f14e2..fe67b0ad938c 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -281,6 +281,25 @@ struct xe_device { struct xe_shrinker *shrinker; /** @mem.stolen_mgr: stolen memory manager. */ struct xe_ttm_stolen_mgr *stolen_mgr; + /** + * @mem.defrag: Tracking of BOs whose backing TT pages were + * allocated at a sub-optimal (smaller than beneficial) order. + * + * Such BOs are candidates for a future defragmentation pass that + * tries to reallocate their pages at the device's beneficial + * order. Only ttm_bo_type_device BOs are tracked. + */ + struct { + /** @mem.defrag.lock: Protects @mem.defrag.list. */ + spinlock_t lock; + /** @mem.defrag.list: List of struct xe_bo defrag_link. */ + struct list_head list; + /** + * @mem.defrag.count: Number of BOs currently on + * @mem.defrag.list. + */ + atomic_t count; + } defrag; } mem; /** @sriov: device level virtualization data */ -- 2.34.1
