To further enchance port processing, keep track of reserved ports. This way we can iterate only the used subset of port space. Note that we lift the responsibility of execlists_submit_request() to inspect hw availability and always do dequeuing. This is to ensure that only the irq handler will be responsible for keeping track of available ports.
v2: rebase, comment fix, READ_ONCE only outside of irq handler (Chris) v3: rebase to new naming Cc: Chris Wilson <[email protected]> Cc: Michał Winiarski <[email protected]> Signed-off-by: Mika Kuoppala <[email protected]> --- drivers/gpu/drm/i915/i915_guc_submission.c | 52 ++++++++------ drivers/gpu/drm/i915/i915_irq.c | 2 +- drivers/gpu/drm/i915/intel_engine_cs.c | 7 +- drivers/gpu/drm/i915/intel_lrc.c | 112 ++++++++++++++++++----------- drivers/gpu/drm/i915/intel_ringbuffer.h | 45 +++++++++--- 5 files changed, 141 insertions(+), 77 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c index c6cd05a5347c..d6071396da32 100644 --- a/drivers/gpu/drm/i915/i915_guc_submission.c +++ b/drivers/gpu/drm/i915/i915_guc_submission.c @@ -487,7 +487,7 @@ static void guc_ring_doorbell(struct i915_guc_client *client) * @engine: engine associated with the commands * * The only error here arises if the doorbell hardware isn't functioning - * as expected, which really shouln't happen. + * as expected, which really shouldn't happen. */ static void i915_guc_submit(struct intel_engine_cs *engine) { @@ -495,17 +495,19 @@ static void i915_guc_submit(struct intel_engine_cs *engine) struct intel_guc *guc = &dev_priv->guc; struct i915_guc_client *client = guc->execbuf_client; struct intel_engine_execlists * const execlists = &engine->execlists; - struct execlist_port *port = execlists->port; const unsigned int engine_id = engine->id; unsigned int n; - for (n = 0; n < ARRAY_SIZE(execlists->port); n++) { + for (n = 0; n < execlists_active_ports(execlists); n++) { + struct execlist_port *port; struct drm_i915_gem_request *rq; unsigned int count; - rq = port_unpack(&port[n], &count); + port = execlists_port(execlists, n); + rq = port_unpack(port, &count); + if (rq && count == 0) { - port_set(&port[n], port_pack(rq, ++count)); + port_set(port, port_pack(rq, ++count)); if (i915_vma_is_map_and_fenceable(rq->ring->vma)) POSTING_READ_FW(GUC_STATUS); @@ -559,26 +561,29 @@ static void port_assign(struct execlist_port *port, static void i915_guc_dequeue(struct intel_engine_cs *engine) { + struct intel_engine_execlists * const execlists = &engine->execlists; - struct execlist_port *port = execlists->port; + struct execlist_port *port; struct drm_i915_gem_request *last = NULL; - const struct execlist_port * const last_port = execlists_port_tail(execlists); bool submit = false; struct rb_node *rb; - if (port_isset(port)) - port++; - spin_lock_irq(&engine->timeline->lock); rb = execlists->first; GEM_BUG_ON(rb_first(&execlists->queue) != rb); - while (rb) { + + if (unlikely(!rb)) + goto done; + + port = execlists_request_port(execlists); + + do { struct i915_priolist *p = rb_entry(rb, typeof(*p), node); struct drm_i915_gem_request *rq, *rn; list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) { if (last && rq->ctx != last->ctx) { - if (port == last_port) { + if (!execlists_inactive_ports(execlists)) { __list_del_many(&p->requests, &rq->priotree.link); goto done; @@ -587,7 +592,8 @@ static void i915_guc_dequeue(struct intel_engine_cs *engine) if (submit) port_assign(port, last); - port = execlists_port_next(execlists, port); + port = execlists_request_port(execlists); + GEM_BUG_ON(port_isset(port)); } INIT_LIST_HEAD(&rq->priotree.link); @@ -604,7 +610,7 @@ static void i915_guc_dequeue(struct intel_engine_cs *engine) INIT_LIST_HEAD(&p->requests); if (p->priority != I915_PRIORITY_NORMAL) kmem_cache_free(engine->i915->priorities, p); - } + } while (rb); done: execlists->first = rb; if (submit) { @@ -618,21 +624,21 @@ static void i915_guc_irq_handler(unsigned long data) { struct intel_engine_cs * const engine = (struct intel_engine_cs *)data; struct intel_engine_execlists * const execlists = &engine->execlists; - struct execlist_port *port = execlists_port_head(execlists); - const struct execlist_port * const last_port = execlists_port_tail(execlists); - struct drm_i915_gem_request *rq; - rq = port_request(port); - while (rq && i915_gem_request_completed(rq)) { + while (execlists_active_ports(execlists)) { + struct execlist_port *port = execlists_port_head(execlists); + struct drm_i915_gem_request *rq = port_request(port); + + if (!i915_gem_request_completed(rq)) + break; + trace_i915_gem_request_out(rq); i915_gem_request_put(rq); - port = execlists_port_complete(execlists, port); - - rq = port_request(port); + execlists_release_port(execlists, port); } - if (!port_isset(last_port)) + if (execlists_inactive_ports(execlists)) i915_guc_dequeue(engine); } diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index b9d1f379c5a0..424e960fa115 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -1382,7 +1382,7 @@ gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift) bool tasklet = false; if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift)) { - if (port_count(execlists_port_head(execlists))) { + if (READ_ONCE(execlists->port_count)) { __set_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted); tasklet = true; } diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c index 3f857786e2ed..65e9c3a66e70 100644 --- a/drivers/gpu/drm/i915/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/intel_engine_cs.c @@ -409,6 +409,9 @@ static void intel_engine_init_execlist(struct intel_engine_cs *engine) BUILD_BUG_ON_NOT_POWER_OF_2(execlists_num_ports(execlists)); GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS); + execlists->port_head = 0; + execlists->port_count = 0; + execlists->queue = RB_ROOT; execlists->first = NULL; } @@ -1504,8 +1507,8 @@ bool intel_engine_is_idle(struct intel_engine_cs *engine) if (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted)) return false; - /* Both ports drained, no more ELSP submission? */ - if (port_request(execlists_port_head(&engine->execlists))) + /* All ports drained, no more ELSP submission? */ + if (execlists_active_ports(&engine->execlists)) return false; /* ELSP is empty, but there are ready requests? */ diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c index cb7fb3c651ce..b056bf5ddd53 100644 --- a/drivers/gpu/drm/i915/intel_lrc.c +++ b/drivers/gpu/drm/i915/intel_lrc.c @@ -392,6 +392,12 @@ static u64 execlists_update_context(struct drm_i915_gem_request *rq) return ce->lrc_desc; } +static inline void elsp_write(u64 desc, u32 __iomem *elsp) +{ + writel(upper_32_bits(desc), elsp); + writel(lower_32_bits(desc), elsp); +} + static void execlists_submit_ports(struct intel_engine_cs *engine) { struct intel_engine_execlists * const execlists = &engine->execlists; @@ -399,28 +405,29 @@ static void execlists_submit_ports(struct intel_engine_cs *engine) engine->i915->regs + i915_mmio_reg_offset(RING_ELSP(engine)); unsigned int n; - for (n = execlists_num_ports(execlists); n--; ) { - struct execlist_port *port; + for (n = 0; n < execlists_inactive_ports(execlists); n++) + elsp_write(0, elsp); + + for (n = execlists_active_ports(execlists); n--; ) { struct drm_i915_gem_request *rq; + struct execlist_port *port; unsigned int count; u64 desc; port = execlists_port(execlists, n); rq = port_unpack(port, &count); - if (rq) { - GEM_BUG_ON(count > !n); - if (!count++) - execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN); - port_set(port, port_pack(rq, count)); - desc = execlists_update_context(rq); - GEM_DEBUG_EXEC(port->context_id = upper_32_bits(desc)); - } else { - GEM_BUG_ON(!n); - desc = 0; - } - writel(upper_32_bits(desc), elsp); - writel(lower_32_bits(desc), elsp); + GEM_BUG_ON(!rq); + GEM_BUG_ON(count > !n); + + if (!count++) + execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN); + + port_set(port, port_pack(rq, count)); + desc = execlists_update_context(rq); + GEM_DEBUG_EXEC(port->context_id = upper_32_bits(desc)); + + elsp_write(desc, elsp); } } @@ -455,15 +462,23 @@ static void port_assign(struct execlist_port *port, static void execlists_dequeue(struct intel_engine_cs *engine) { - struct drm_i915_gem_request *last; struct intel_engine_execlists * const execlists = &engine->execlists; - struct execlist_port *port = execlists_port_head(execlists); - const struct execlist_port * const last_port = execlists_port_tail(execlists); + struct execlist_port *port; + struct drm_i915_gem_request *last; struct rb_node *rb; bool submit = false; - last = port_request(port); - if (last) + spin_lock_irq(&engine->timeline->lock); + rb = execlists->first; + GEM_BUG_ON(rb_first(&execlists->queue) != rb); + + if (unlikely(!rb)) + goto done; + + if (execlists_active_ports(execlists)) { + port = execlists_port_tail(execlists); + last = port_request(port); + /* WaIdleLiteRestore:bdw,skl * Apply the wa NOOPs to prevent ring:HEAD == req:TAIL * as we resubmit the request. See gen8_emit_breadcrumb() @@ -471,6 +486,11 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * request. */ last->tail = last->wa_tail; + } else { + /* Allocate first port to coalesce into */ + port = execlists_request_port(execlists); + last = NULL; + } /* Hardware submission is through 2 ports. Conceptually each port * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is @@ -493,10 +513,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * and context switches) submission. */ - spin_lock_irq(&engine->timeline->lock); - rb = execlists->first; - GEM_BUG_ON(rb_first(&execlists->queue) != rb); - while (rb) { + do { struct i915_priolist *p = rb_entry(rb, typeof(*p), node); struct drm_i915_gem_request *rq, *rn; @@ -514,11 +531,11 @@ static void execlists_dequeue(struct intel_engine_cs *engine) */ if (last && !can_merge_ctx(rq->ctx, last->ctx)) { /* - * If we are on the second port and cannot + * If we are on the last port and cannot * combine this request with the last, then we * are done. */ - if (port == last_port) { + if (!execlists_inactive_ports(execlists)) { __list_del_many(&p->requests, &rq->priotree.link); goto done; @@ -543,8 +560,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) if (submit) port_assign(port, last); - port = execlists_port_next(execlists, port); - + port = execlists_request_port(execlists); GEM_BUG_ON(port_isset(port)); } @@ -562,7 +578,8 @@ static void execlists_dequeue(struct intel_engine_cs *engine) INIT_LIST_HEAD(&p->requests); if (p->priority != I915_PRIORITY_NORMAL) kmem_cache_free(engine->i915->priorities, p); - } + } while (rb); + done: execlists->first = rb; if (submit) @@ -576,18 +593,26 @@ static void execlists_dequeue(struct intel_engine_cs *engine) static void execlists_cancel_port_requests(struct intel_engine_execlists *execlists) { - struct execlist_port *port = execlists->port; - unsigned int num_ports = ARRAY_SIZE(execlists->port); + while (execlists_active_ports(execlists)) { + struct execlist_port *port; + struct drm_i915_gem_request *rq; - while (num_ports-- && port_isset(port)) { - struct drm_i915_gem_request *rq = port_request(port); + port = execlists_port_head(execlists); + GEM_BUG_ON(!port_isset(port)); + + rq = port_request(port); + GEM_BUG_ON(!rq); execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT); i915_gem_request_put(rq); - memset(port, 0, sizeof(*port)); - port++; + execlists_release_port(execlists, port); } + + GEM_BUG_ON(execlists->port_count); + + execlists->port_count = 0; + execlists->port_head = 0; } static void execlists_cancel_requests(struct intel_engine_cs *engine) @@ -649,10 +674,12 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine) static bool execlists_elsp_ready(struct intel_engine_execlists * const execlists) { - struct execlist_port * const port0 = execlists_port_head(execlists); - struct execlist_port * const port1 = execlists_port_next(execlists, port0); + const unsigned int active = execlists_active_ports(execlists); + + if (!active) + return true; - return port_count(port0) + port_count(port1) < 2; + return port_count(execlists_port_tail(execlists)) + active < 2; } /* @@ -663,7 +690,6 @@ static void intel_lrc_irq_handler(unsigned long data) { struct intel_engine_cs * const engine = (struct intel_engine_cs *)data; struct intel_engine_execlists * const execlists = &engine->execlists; - struct execlist_port *port = execlists_port_head(execlists); struct drm_i915_private *dev_priv = engine->i915; /* We can skip acquiring intel_runtime_pm_get() here as it was taken @@ -720,6 +746,7 @@ static void intel_lrc_irq_handler(unsigned long data) } while (head != tail) { + struct execlist_port *port; struct drm_i915_gem_request *rq; unsigned int status; unsigned int count; @@ -748,6 +775,7 @@ static void intel_lrc_irq_handler(unsigned long data) if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK)) continue; + port = execlists_port_head(execlists); /* Check the context/desc id for this event matches */ GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id); @@ -761,13 +789,13 @@ static void intel_lrc_irq_handler(unsigned long data) trace_i915_gem_request_out(rq); i915_gem_request_put(rq); - port = execlists_port_complete(execlists, port); + execlists_release_port(execlists, port); } else { port_set(port, port_pack(rq, count)); } /* After the final element, the hw should be idle */ - GEM_BUG_ON(port_count(port) == 0 && + GEM_BUG_ON(execlists_active_ports(execlists) == 0 && !(status & GEN8_CTX_STATUS_ACTIVE_IDLE)); } @@ -792,7 +820,7 @@ static void insert_request(struct intel_engine_cs *engine, struct i915_priolist *p = lookup_priolist(engine, pt, prio); list_add_tail(&pt->link, &ptr_mask_bits(p, 1)->requests); - if (ptr_unmask_bits(p, 1) && execlists_elsp_ready(execlists)) + if (ptr_unmask_bits(p, 1)) tasklet_hi_schedule(&execlists->irq_tasklet); } diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index 2e795b44a942..c09d1c93fd15 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -249,6 +249,11 @@ struct intel_engine_execlists { unsigned int port_head; /** + * @port_count: reserved ports + */ + unsigned int port_count; + + /** * @queue: queue of requests, in priority lists */ struct rb_root queue; @@ -529,6 +534,18 @@ execlists_num_ports(const struct intel_engine_execlists * const execlists) return execlists->port_mask + 1; } +static inline unsigned int +execlists_active_ports(const struct intel_engine_execlists * const execlists) +{ + return execlists->port_count; +} + +static inline unsigned int +execlists_inactive_ports(const struct intel_engine_execlists * const execlists) +{ + return execlists_num_ports(execlists) - execlists_active_ports(execlists); +} + #define __port_n(start, n, mask) (((start) + (n)) & (mask)) #define port_n(e, n) __port_n((e)->port_head, n, (e)->port_mask) @@ -543,33 +560,43 @@ execlists_port(struct intel_engine_execlists * const execlists, static inline struct execlist_port * execlists_port_head(struct intel_engine_execlists * const execlists) { + GEM_BUG_ON(!execlists->port_count); + return execlists_port(execlists, 0); } static inline struct execlist_port * execlists_port_tail(struct intel_engine_execlists * const execlists) { - return execlists_port(execlists, -1); + GEM_BUG_ON(!execlists->port_count); + + return execlists_port(execlists, execlists->port_count - 1); } static inline struct execlist_port * -execlists_port_next(struct intel_engine_execlists * const execlists, - const struct execlist_port * const port) +execlists_request_port(struct intel_engine_execlists * const execlists) { - const unsigned int n = port_index(port, execlists); + GEM_BUG_ON(execlists->port_count == execlists->port_mask + 1); + + execlists->port_count++; - return execlists_port(execlists, n + 1); + GEM_BUG_ON(port_isset(execlists_port_tail(execlists))); + + return execlists_port_tail(execlists); } -static inline struct execlist_port * -execlists_port_complete(struct intel_engine_execlists * const execlists, - struct execlist_port * const port) +static inline void +execlists_release_port(struct intel_engine_execlists * const execlists, + struct execlist_port * const port) { + GEM_BUG_ON(port_index(port, execlists) != execlists->port_head); + GEM_BUG_ON(!port_isset(port)); + GEM_BUG_ON(!execlists->port_count); memset(port, 0, sizeof(struct execlist_port)); execlists->port_head = port_n(execlists, 1); - return execlists_port_head(execlists); + execlists->port_count--; } static inline unsigned int -- 2.11.0 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
