Extract the scheduling queue from "execlists" into the per-engine scheduling structs, for reuse by other backends.
Signed-off-by: Chris Wilson <[email protected]> Reviewed-by: Tvrtko Ursulin <[email protected]> --- .../gpu/drm/i915/gem/i915_gem_context_types.h | 2 +- drivers/gpu/drm/i915/gem/i915_gem_wait.c | 1 + drivers/gpu/drm/i915/gt/intel_engine_cs.c | 7 +- drivers/gpu/drm/i915/gt/intel_engine_pm.c | 3 +- drivers/gpu/drm/i915/gt/intel_engine_types.h | 14 --- .../drm/i915/gt/intel_execlists_submission.c | 54 ++++------- .../gpu/drm/i915/gt/intel_ring_submission.c | 9 +- drivers/gpu/drm/i915/gt/mock_engine.c | 7 +- .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 32 ++----- drivers/gpu/drm/i915/i915_drv.h | 1 - drivers/gpu/drm/i915/i915_request.h | 2 +- drivers/gpu/drm/i915/i915_scheduler.c | 90 ++++++++++++++----- drivers/gpu/drm/i915/i915_scheduler.h | 17 ++++ drivers/gpu/drm/i915/i915_scheduler_types.h | 34 +++++++ .../gpu/drm/i915/selftests/i915_scheduler.c | 13 ++- 15 files changed, 159 insertions(+), 127 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context_types.h b/drivers/gpu/drm/i915/gem/i915_gem_context_types.h index 085f6a3735e8..d5bc75508048 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_context_types.h @@ -19,7 +19,7 @@ #include "gt/intel_context_types.h" -#include "i915_scheduler.h" +#include "i915_scheduler_types.h" #include "i915_sw_fence.h" struct pid; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_wait.c b/drivers/gpu/drm/i915/gem/i915_gem_wait.c index d79bf16083bd..4d1897c347b9 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_wait.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_wait.c @@ -13,6 +13,7 @@ #include "dma_resv_utils.h" #include "i915_gem_ioctls.h" #include "i915_gem_object.h" +#include "i915_scheduler.h" static long i915_gem_object_wait_fence(struct dma_fence *fence, diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index 9fbb2e924522..2ce65189d5e2 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -581,7 +581,6 @@ void intel_engine_init_execlists(struct intel_engine_cs *engine) memset(execlists->inflight, 0, sizeof(execlists->inflight)); execlists->queue_priority_hint = INT_MIN; - execlists->queue = RB_ROOT_CACHED; } static void cleanup_status_page(struct intel_engine_cs *engine) @@ -918,7 +917,7 @@ int intel_engines_init(struct intel_gt *gt) */ void intel_engine_cleanup_common(struct intel_engine_cs *engine) { - GEM_BUG_ON(!list_empty(&engine->sched.requests)); + i915_sched_fini(intel_engine_get_scheduler(engine)); tasklet_kill(&engine->execlists.tasklet); /* flush the callback */ intel_breadcrumbs_free(engine->breadcrumbs); @@ -1232,6 +1231,8 @@ void __intel_engine_flush_submission(struct intel_engine_cs *engine, bool sync) */ bool intel_engine_is_idle(struct intel_engine_cs *engine) { + struct i915_sched *se = intel_engine_get_scheduler(engine); + /* More white lies, if wedged, hw state is inconsistent */ if (intel_gt_is_wedged(engine->gt)) return true; @@ -1244,7 +1245,7 @@ bool intel_engine_is_idle(struct intel_engine_cs *engine) intel_engine_flush_submission(engine); /* ELSP is empty, but there are ready requests? E.g. after reset */ - if (!RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)) + if (!i915_sched_is_idle(se)) return false; /* Ring stopped? */ diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c index 6372d7826bc9..3510c9236334 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c @@ -4,6 +4,7 @@ */ #include "i915_drv.h" +#include "i915_scheduler.h" #include "intel_breadcrumbs.h" #include "intel_context.h" @@ -276,7 +277,7 @@ static int __engine_park(struct intel_wakeref *wf) if (engine->park) engine->park(engine); - engine->execlists.no_priolist = false; + i915_sched_park(intel_engine_get_scheduler(engine)); /* While gt calls i915_vma_parked(), we have to break the lock cycle */ intel_gt_pm_put_async(engine->gt); diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index 5a28113c9a98..9ea38b01121a 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -153,11 +153,6 @@ struct intel_engine_execlists { */ struct timer_list preempt; - /** - * @default_priolist: priority list for I915_PRIORITY_NORMAL - */ - struct i915_priolist default_priolist; - /** * @ccid: identifier for contexts submitted to this engine */ @@ -192,11 +187,6 @@ struct intel_engine_execlists { */ u32 reset_ccid; - /** - * @no_priolist: priority lists disabled - */ - bool no_priolist; - /** * @submit_reg: gen-specific execlist submission register * set to the ExecList Submission Port (elsp) register pre-Gen11 and to @@ -252,10 +242,6 @@ struct intel_engine_execlists { */ int queue_priority_hint; - /** - * @queue: queue of requests, in priority lists - */ - struct rb_root_cached queue; struct rb_root_cached virtual; /** diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c index 1b65df24cacd..12e20a8eb5d5 100644 --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c @@ -273,11 +273,11 @@ static int effective_prio(const struct i915_request *rq) return prio; } -static int queue_prio(const struct intel_engine_execlists *execlists) +static int queue_prio(const struct i915_sched *se) { struct rb_node *rb; - rb = rb_first_cached(&execlists->queue); + rb = rb_first_cached(&se->queue); if (!rb) return INT_MIN; @@ -341,7 +341,7 @@ static bool need_preempt(const struct intel_engine_cs *engine, * context, it's priority would not exceed ELSP[0] aka last_prio. */ return max(virtual_prio(&engine->execlists), - queue_prio(&engine->execlists)) > last_prio; + queue_prio(se)) > last_prio; } __maybe_unused static bool @@ -1034,13 +1034,13 @@ static bool needs_timeslice(const struct intel_engine_cs *engine, return false; /* If ELSP[1] is occupied, always check to see if worth slicing */ - if (!list_is_last_rcu(&rq->sched.link, &se->requests)) { + if (!i915_sched_is_last_request(se, rq)) { ENGINE_TRACE(engine, "timeslice required for second inflight context\n"); return true; } /* Otherwise, ELSP[0] is by itself, but may be waiting in the queue */ - if (!RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)) { + if (!i915_sched_is_idle(se)) { ENGINE_TRACE(engine, "timeslice required for queue\n"); return true; } @@ -1286,7 +1286,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) GEM_BUG_ON(rq->engine != &ve->base); GEM_BUG_ON(rq->context != &ve->context); - if (unlikely(rq_prio(rq) < queue_prio(execlists))) { + if (unlikely(rq_prio(rq) < queue_prio(se))) { spin_unlock(&ve->base.sched.lock); break; } @@ -1352,7 +1352,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) break; } - while ((rb = rb_first_cached(&execlists->queue))) { + while ((rb = rb_first_cached(&se->queue))) { struct i915_priolist *p = to_priolist(rb); struct i915_request *rq, *rn; @@ -1431,7 +1431,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) } } - rb_erase_cached(&p->node, &execlists->queue); + rb_erase_cached(&p->node, &se->queue); i915_priolist_free(p); } done: @@ -1453,7 +1453,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * request triggering preemption on the next dequeue (or subsequent * interrupt for secondary ports). */ - execlists->queue_priority_hint = queue_prio(execlists); + execlists->queue_priority_hint = queue_prio(se); spin_unlock(&se->lock); /* @@ -2667,7 +2667,6 @@ static void execlists_reset_cancel(struct intel_engine_cs *engine) { struct intel_engine_execlists * const execlists = &engine->execlists; struct i915_sched *se = intel_engine_get_scheduler(engine); - struct i915_request *rq, *rn; struct rb_node *rb; unsigned long flags; @@ -2692,34 +2691,13 @@ static void execlists_reset_cancel(struct intel_engine_cs *engine) rcu_read_lock(); spin_lock_irqsave(&se->lock, flags); - /* Mark all executing requests as skipped. */ - list_for_each_entry(rq, &se->requests, sched.link) - i915_request_put(i915_request_mark_eio(rq)); - intel_engine_signal_breadcrumbs(engine); - - /* Flush the queued requests to the timeline list (for retiring). */ - while ((rb = rb_first_cached(&execlists->queue))) { - struct i915_priolist *p = to_priolist(rb); - - priolist_for_each_request_consume(rq, rn, p) { - if (i915_request_mark_eio(rq)) { - __i915_request_submit(rq); - i915_request_put(rq); - } - } - - rb_erase_cached(&p->node, &execlists->queue); - i915_priolist_free(p); - } - - /* On-hold requests will be flushed to timeline upon their release */ - list_for_each_entry(rq, &se->hold, sched.link) - i915_request_put(i915_request_mark_eio(rq)); + __i915_sched_cancel_queue(se); /* Cancel all attached virtual engines */ while ((rb = rb_first_cached(&execlists->virtual))) { struct virtual_engine *ve = rb_entry(rb, typeof(*ve), nodes[engine->id].rb); + struct i915_request *rq; rb_erase_cached(rb, &execlists->virtual); RB_CLEAR_NODE(rb); @@ -2739,16 +2717,16 @@ static void execlists_reset_cancel(struct intel_engine_cs *engine) spin_unlock(&ve->base.sched.lock); } - /* Remaining _unready_ requests will be nop'ed when submitted */ - execlists->queue_priority_hint = INT_MIN; - execlists->queue = RB_ROOT_CACHED; + se->queue = RB_ROOT_CACHED; GEM_BUG_ON(__tasklet_is_enabled(&execlists->tasklet)); execlists->tasklet.callback = nop_submission_tasklet; spin_unlock_irqrestore(&se->lock, flags); rcu_read_unlock(); + + intel_engine_signal_breadcrumbs(engine); } static void execlists_reset_finish(struct intel_engine_cs *engine) @@ -2984,7 +2962,7 @@ int intel_execlists_submission_setup(struct intel_engine_cs *engine) static struct list_head *virtual_queue(struct virtual_engine *ve) { - return &ve->base.execlists.default_priolist.requests; + return &ve->base.sched.default_priolist.requests; } static void rcu_virtual_context_destroy(struct work_struct *wrk) @@ -3585,7 +3563,7 @@ void intel_execlists_show_requests(struct intel_engine_cs *engine, last = NULL; count = 0; - for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) { + for (rb = rb_first_cached(&se->queue); rb; rb = rb_next(rb)) { struct i915_priolist *p = rb_entry(rb, typeof(*p), node); priolist_for_each_request(rq, p) { diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c index 68dcb8a1eb8c..4a7d3420cc9d 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c @@ -396,19 +396,14 @@ static void reset_finish(struct intel_engine_cs *engine) static void reset_cancel(struct intel_engine_cs *engine) { struct i915_sched *se = intel_engine_get_scheduler(engine); - struct i915_request *request; unsigned long flags; spin_lock_irqsave(&se->lock, flags); - /* Mark all submitted requests as skipped. */ - list_for_each_entry(request, &se->requests, sched.link) - i915_request_put(i915_request_mark_eio(request)); - intel_engine_signal_breadcrumbs(engine); - - /* Remaining _unready_ requests will be nop'ed when submitted */ + __i915_sched_cancel_queue(se); spin_unlock_irqrestore(&se->lock, flags); + intel_engine_signal_breadcrumbs(engine); } static void i9xx_submit_request(struct i915_request *request) diff --git a/drivers/gpu/drm/i915/gt/mock_engine.c b/drivers/gpu/drm/i915/gt/mock_engine.c index 8b1c2727d25c..2081deed94b7 100644 --- a/drivers/gpu/drm/i915/gt/mock_engine.c +++ b/drivers/gpu/drm/i915/gt/mock_engine.c @@ -238,10 +238,7 @@ static void mock_reset_cancel(struct intel_engine_cs *engine) spin_lock_irqsave(&se->lock, flags); - /* Mark all submitted requests as skipped. */ - list_for_each_entry(rq, &se->requests, sched.link) - i915_request_put(i915_request_mark_eio(rq)); - intel_engine_signal_breadcrumbs(engine); + __i915_sched_cancel_queue(se); /* Cancel and submit all pending requests. */ list_for_each_entry(rq, &mock->hw_queue, mock.link) { @@ -252,6 +249,8 @@ static void mock_reset_cancel(struct intel_engine_cs *engine) } INIT_LIST_HEAD(&mock->hw_queue); + intel_engine_signal_breadcrumbs(engine); + spin_unlock_irqrestore(&se->lock, flags); } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 4ecbb26391b7..7b1780d1652f 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -205,7 +205,7 @@ static void __guc_dequeue(struct intel_engine_cs *engine) * event. */ port = first; - while ((rb = rb_first_cached(&execlists->queue))) { + while ((rb = rb_first_cached(&se->queue))) { struct i915_priolist *p = to_priolist(rb); struct i915_request *rq, *rn; @@ -225,7 +225,7 @@ static void __guc_dequeue(struct intel_engine_cs *engine) last = rq; } - rb_erase_cached(&p->node, &execlists->queue); + rb_erase_cached(&p->node, &se->queue); i915_priolist_free(p); } done: @@ -342,8 +342,6 @@ static void guc_reset_cancel(struct intel_engine_cs *engine) { struct intel_engine_execlists * const execlists = &engine->execlists; struct i915_sched *se = intel_engine_get_scheduler(engine); - struct i915_request *rq, *rn; - struct rb_node *rb; unsigned long flags; ENGINE_TRACE(engine, "\n"); @@ -364,33 +362,13 @@ static void guc_reset_cancel(struct intel_engine_cs *engine) */ spin_lock_irqsave(&se->lock, flags); - /* Mark all executing requests as skipped. */ - list_for_each_entry(rq, &se->requests, sched.link) { - i915_request_set_error_once(rq, -EIO); - i915_request_mark_complete(rq); - } - - /* Flush the queued requests to the timeline list (for retiring). */ - while ((rb = rb_first_cached(&execlists->queue))) { - struct i915_priolist *p = to_priolist(rb); - - priolist_for_each_request_consume(rq, rn, p) { - list_del_init(&rq->sched.link); - __i915_request_submit(rq); - dma_fence_set_error(&rq->fence, -EIO); - i915_request_mark_complete(rq); - } - - rb_erase_cached(&p->node, &execlists->queue); - i915_priolist_free(p); - } - - /* Remaining _unready_ requests will be nop'ed when submitted */ + __i915_sched_cancel_queue(se); execlists->queue_priority_hint = INT_MIN; - execlists->queue = RB_ROOT_CACHED; + se->queue = RB_ROOT_CACHED; spin_unlock_irqrestore(&se->lock, flags); + intel_engine_signal_breadcrumbs(engine); } static void guc_reset_finish(struct intel_engine_cs *engine) diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index faf7eafdad45..cee180ca7f5a 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -99,7 +99,6 @@ #include "i915_gpu_error.h" #include "i915_perf_types.h" #include "i915_request.h" -#include "i915_scheduler.h" #include "gt/intel_timeline.h" #include "i915_vma.h" #include "i915_irq.h" diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h index 3a5d6bdcd8dd..c41582b96b46 100644 --- a/drivers/gpu/drm/i915/i915_request.h +++ b/drivers/gpu/drm/i915/i915_request.h @@ -35,7 +35,7 @@ #include "gt/intel_timeline_types.h" #include "i915_gem.h" -#include "i915_scheduler.h" +#include "i915_scheduler_types.h" #include "i915_selftest.h" #include "i915_sw_fence.h" diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c index aec99142f712..4d281e990a86 100644 --- a/drivers/gpu/drm/i915/i915_scheduler.c +++ b/drivers/gpu/drm/i915/i915_scheduler.c @@ -129,10 +129,24 @@ void i915_sched_init(struct i915_sched *se, INIT_LIST_HEAD(&se->requests); INIT_LIST_HEAD(&se->hold); + se->queue = RB_ROOT_CACHED; init_ipi(&se->ipi); } +void i915_sched_park(struct i915_sched *se) +{ + GEM_BUG_ON(!i915_sched_is_idle(se)); + se->no_priolist = false; +} + +void i915_sched_fini(struct i915_sched *se) +{ + GEM_BUG_ON(!list_empty(&se->requests)); + + i915_sched_park(se); +} + static void __ipi_add(struct i915_request *rq) { #define STUB ((struct i915_request *)1) @@ -181,7 +195,7 @@ static inline struct i915_priolist *to_priolist(struct rb_node *rb) return rb_entry(rb, struct i915_priolist, node); } -static void assert_priolists(struct intel_engine_execlists * const execlists) +static void assert_priolists(struct i915_sched * const se) { struct rb_node *rb; long last_prio; @@ -189,11 +203,11 @@ static void assert_priolists(struct intel_engine_execlists * const execlists) if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) return; - GEM_BUG_ON(rb_first_cached(&execlists->queue) != - rb_first(&execlists->queue.rb_root)); + GEM_BUG_ON(rb_first_cached(&se->queue) != + rb_first(&se->queue.rb_root)); last_prio = INT_MAX; - for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) { + for (rb = rb_first_cached(&se->queue); rb; rb = rb_next(rb)) { const struct i915_priolist *p = to_priolist(rb); GEM_BUG_ON(p->priority > last_prio); @@ -202,24 +216,22 @@ static void assert_priolists(struct intel_engine_execlists * const execlists) } static struct list_head * -lookup_priolist(struct intel_engine_cs *engine, int prio) +lookup_priolist(struct i915_sched *se, int prio) { - struct intel_engine_execlists * const execlists = &engine->execlists; - struct i915_sched *se = intel_engine_get_scheduler(engine); struct i915_priolist *p; struct rb_node **parent, *rb; bool first = true; lockdep_assert_held(&se->lock); - assert_priolists(execlists); + assert_priolists(se); - if (unlikely(execlists->no_priolist)) + if (unlikely(se->no_priolist)) prio = I915_PRIORITY_NORMAL; find_priolist: /* most positive priority is scheduled first, equal priorities fifo */ rb = NULL; - parent = &execlists->queue.rb_root.rb_node; + parent = &se->queue.rb_root.rb_node; while (*parent) { rb = *parent; p = to_priolist(rb); @@ -234,7 +246,7 @@ lookup_priolist(struct intel_engine_cs *engine, int prio) } if (prio == I915_PRIORITY_NORMAL) { - p = &execlists->default_priolist; + p = &se->default_priolist; } else { p = kmem_cache_alloc(global.slab_priorities, GFP_ATOMIC); /* Convert an allocation failure to a priority bump */ @@ -249,7 +261,7 @@ lookup_priolist(struct intel_engine_cs *engine, int prio) * requests, so if userspace lied about their * dependencies that reordering may be visible. */ - execlists->no_priolist = true; + se->no_priolist = true; goto find_priolist; } } @@ -258,7 +270,7 @@ lookup_priolist(struct intel_engine_cs *engine, int prio) INIT_LIST_HEAD(&p->requests); rb_link_node(&p->node, rb, parent); - rb_insert_color_cached(&p->node, &execlists->queue, first); + rb_insert_color_cached(&p->node, &se->queue, first); return &p->requests; } @@ -351,13 +363,14 @@ static void ipi_priority(struct i915_request *rq, int prio) static void __i915_request_set_priority(struct i915_request *rq, int prio) { struct intel_engine_cs *engine = rq->engine; + struct i915_sched *se = intel_engine_get_scheduler(engine); struct list_head *pos = &rq->sched.signalers_list; struct list_head *plist; SCHED_TRACE(&engine->sched, "PI for " RQ_FMT ", prio:%d\n", RQ_ARG(rq), prio); - plist = lookup_priolist(engine, prio); + plist = lookup_priolist(se, prio); /* * Recursively bump all dependent priorities to match the new request. @@ -505,7 +518,7 @@ void __i915_sched_defer_request(struct intel_engine_cs *engine, * to those that are waiting upon it. So we traverse its chain of * waiters and move any that are earlier than the request to after it. */ - pl = lookup_priolist(engine, rq_prio(rq)); + pl = lookup_priolist(se, rq_prio(rq)); do { struct i915_dependency *p; @@ -543,11 +556,10 @@ void __i915_sched_defer_request(struct intel_engine_cs *engine, } while (rq); } -static void queue_request(struct intel_engine_cs *engine, - struct i915_request *rq) +static void queue_request(struct i915_sched *se, struct i915_request *rq) { GEM_BUG_ON(!list_empty(&rq->sched.link)); - list_add_tail(&rq->sched.link, lookup_priolist(engine, rq_prio(rq))); + list_add_tail(&rq->sched.link, lookup_priolist(se, rq_prio(rq))); set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); } @@ -614,9 +626,9 @@ void i915_request_enqueue(struct i915_request *rq) list_add_tail(&rq->sched.link, &se->hold); i915_request_set_hold(rq); } else { - queue_request(engine, rq); + queue_request(se, rq); - GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)); + GEM_BUG_ON(i915_sched_is_idle(se)); kick = submit_queue(engine, rq); } @@ -648,9 +660,9 @@ __i915_sched_rewind_requests(struct intel_engine_cs *engine) GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID); if (rq_prio(rq) != prio) { prio = rq_prio(rq); - pl = lookup_priolist(engine, prio); + pl = lookup_priolist(se, prio); } - GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)); + GEM_BUG_ON(i915_sched_is_idle(se)); list_move(&rq->sched.link, pl); set_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); @@ -785,7 +797,7 @@ void __i915_sched_resume_request(struct intel_engine_cs *engine, i915_request_clear_hold(rq); list_del_init(&rq->sched.link); - queue_request(engine, rq); + queue_request(se, rq); /* Also release any children on this engine that are ready */ for_each_waiter(p, rq) { @@ -827,6 +839,38 @@ void i915_sched_resume_request(struct intel_engine_cs *engine, spin_unlock_irq(&se->lock); } +void __i915_sched_cancel_queue(struct i915_sched *se) +{ + struct i915_request *rq, *rn; + struct rb_node *rb; + + lockdep_assert_held(&se->lock); + + /* Mark all executing requests as skipped. */ + list_for_each_entry(rq, &se->requests, sched.link) + i915_request_put(i915_request_mark_eio(rq)); + + /* Flush the queued requests to the timeline list (for retiring). */ + while ((rb = rb_first_cached(&se->queue))) { + struct i915_priolist *p = to_priolist(rb); + + priolist_for_each_request_consume(rq, rn, p) { + i915_request_put(i915_request_mark_eio(rq)); + __i915_request_submit(rq); + } + + rb_erase_cached(&p->node, &se->queue); + i915_priolist_free(p); + } + GEM_BUG_ON(!i915_sched_is_idle(se)); + + /* On-hold requests will be flushed to timeline upon their release */ + list_for_each_entry(rq, &se->hold, sched.link) + i915_request_put(i915_request_mark_eio(rq)); + + /* Remaining _unready_ requests will be nop'ed when submitted */ +} + void i915_sched_node_init(struct i915_sched_node *node) { spin_lock_init(&node->lock); diff --git a/drivers/gpu/drm/i915/i915_scheduler.h b/drivers/gpu/drm/i915/i915_scheduler.h index ebd93ae303b4..6b80df7feb78 100644 --- a/drivers/gpu/drm/i915/i915_scheduler.h +++ b/drivers/gpu/drm/i915/i915_scheduler.h @@ -12,6 +12,7 @@ #include <linux/kernel.h> #include "i915_scheduler_types.h" +#include "i915_request.h" struct drm_printer; struct intel_engine_cs; @@ -48,6 +49,8 @@ void i915_sched_init(struct i915_sched *se, const char *name, unsigned long mask, unsigned int subclass); +void i915_sched_park(struct i915_sched *se); +void i915_sched_fini(struct i915_sched *se); void i915_request_set_priority(struct i915_request *request, int prio); @@ -68,6 +71,8 @@ bool i915_sched_suspend_request(struct intel_engine_cs *engine, void i915_sched_resume_request(struct intel_engine_cs *engine, struct i915_request *rq); +void __i915_sched_cancel_queue(struct i915_sched *se); + void __i915_priolist_free(struct i915_priolist *p); static inline void i915_priolist_free(struct i915_priolist *p) { @@ -75,6 +80,18 @@ static inline void i915_priolist_free(struct i915_priolist *p) __i915_priolist_free(p); } +static inline bool i915_sched_is_idle(const struct i915_sched *se) +{ + return RB_EMPTY_ROOT(&se->queue.rb_root); +} + +static inline bool +i915_sched_is_last_request(const struct i915_sched *se, + const struct i915_request *rq) +{ + return list_is_last_rcu(&rq->sched.link, &se->requests); +} + void i915_request_show_with_schedule(struct drm_printer *m, const struct i915_request *rq, const char *prefix, diff --git a/drivers/gpu/drm/i915/i915_scheduler_types.h b/drivers/gpu/drm/i915/i915_scheduler_types.h index b7ee122d4f28..2b34d9882835 100644 --- a/drivers/gpu/drm/i915/i915_scheduler_types.h +++ b/drivers/gpu/drm/i915/i915_scheduler_types.h @@ -30,12 +30,46 @@ struct i915_sched { struct list_head requests; /* active request, on HW */ struct list_head hold; /* ready requests, but on hold */ + /** + * @queue: queue of requests, in priority lists + * + * During request construction, we build a list of fence dependencies + * that must be completed before the fence is executed. Then when the + * request is committed, it waits for all of those fences before it is + * submitted to the scheduler. + * + * The scheduler only sees requests that are ready to be executed. + * However, the number that we may execute at any one time may be + * limited, and so we store them in the @queue. This queue is sorted + * in execution order, such that when the backend may submit more + * requests to the HW, it can fill the HW submission ports from the + * head of the queue. It also allows the backends to inspect the head + * of the queue against the currently active requests to see if + * we need to preempt the current execution in order to run higher + * priority requests from the queue. + * + * In the simplest cases where the HW can consume everything, the + * @queue is only used to transfer requests from the scheduler + * frontend to the back. + */ + struct rb_root_cached queue; + /* Inter-engine scheduling delegate */ struct i915_sched_ipi { struct i915_request *list; struct work_struct work; } ipi; + /** + * @default_priolist: priority list for I915_PRIORITY_NORMAL + */ + struct i915_priolist default_priolist; + + /** + * @no_priolist: priority lists disabled + */ + bool no_priolist; + /* Pretty device names for debug messages */ struct { struct device *dev; diff --git a/drivers/gpu/drm/i915/selftests/i915_scheduler.c b/drivers/gpu/drm/i915/selftests/i915_scheduler.c index 88ebe92bed2d..956a9af6f7e5 100644 --- a/drivers/gpu/drm/i915/selftests/i915_scheduler.c +++ b/drivers/gpu/drm/i915/selftests/i915_scheduler.c @@ -77,8 +77,7 @@ static int all_engines(struct drm_i915_private *i915, return 0; } -static bool check_context_order(struct i915_sched *se, - struct intel_engine_cs *engine) +static bool check_context_order(struct i915_sched *se) { u64 last_seqno, last_context; unsigned long count; @@ -93,7 +92,7 @@ static bool check_context_order(struct i915_sched *se, last_context = 0; last_seqno = 0; last_prio = 0; - for (rb = rb_first_cached(&engine->execlists.queue); rb; rb = rb_next(rb)) { + for (rb = rb_first_cached(&se->queue); rb; rb = rb_next(rb)) { struct i915_priolist *p = rb_entry(rb, typeof(*p), node); struct i915_request *rq; @@ -175,7 +174,7 @@ static int __single_chain(struct intel_engine_cs *engine, unsigned long length, intel_engine_flush_submission(engine); execlists_active_lock_bh(&engine->execlists); - if (fn(rq, count, count - 1) && !check_context_order(se, engine)) + if (fn(rq, count, count - 1) && !check_context_order(se)) err = -EINVAL; execlists_active_unlock_bh(&engine->execlists); @@ -260,7 +259,7 @@ static int __wide_chain(struct intel_engine_cs *engine, unsigned long width, intel_engine_flush_submission(engine); execlists_active_lock_bh(&engine->execlists); - if (fn(rq[i - 1], i, count) && !check_context_order(se, engine)) + if (fn(rq[i - 1], i, count) && !check_context_order(se)) err = -EINVAL; execlists_active_unlock_bh(&engine->execlists); @@ -349,7 +348,7 @@ static int __inv_chain(struct intel_engine_cs *engine, unsigned long width, intel_engine_flush_submission(engine); execlists_active_lock_bh(&engine->execlists); - if (fn(rq[i - 1], i, count) && !check_context_order(se, engine)) + if (fn(rq[i - 1], i, count) && !check_context_order(se)) err = -EINVAL; execlists_active_unlock_bh(&engine->execlists); @@ -455,7 +454,7 @@ static int __sparse_chain(struct intel_engine_cs *engine, unsigned long width, intel_engine_flush_submission(engine); execlists_active_lock_bh(&engine->execlists); - if (fn(rq[i - 1], i, count) && !check_context_order(se, engine)) + if (fn(rq[i - 1], i, count) && !check_context_order(se)) err = -EINVAL; execlists_active_unlock_bh(&engine->execlists); -- 2.20.1 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
