Add UAPI and infostructure to support sending submission to the LPAC
ring.
Signed-off-by: Anna Maniscalco <[email protected]>
---
drivers/gpu/drm/msm/adreno/a6xx_gpu.c | 86 ++++++++++++++++++++++-------------
drivers/gpu/drm/msm/adreno/a6xx_gpu.h | 2 +-
drivers/gpu/drm/msm/msm_gem_submit.c | 6 ++-
drivers/gpu/drm/msm/msm_gpu.c | 8 ++--
drivers/gpu/drm/msm/msm_gpu.h | 4 +-
drivers/gpu/drm/msm/msm_submitqueue.c | 11 ++++-
include/uapi/drm/msm_drm.h | 2 +
7 files changed, 78 insertions(+), 41 deletions(-)
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
index 9f69aada9b7b..4417a9d04d7c 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -204,8 +204,10 @@ void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer
*ring)
void
a6xx_flush_yield(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
{
+ bool is_lpac = ring == gpu->lpac_rb;
+
/* If preemption is enabled */
- if (gpu->nr_rings > 1) {
+ if (gpu->nr_rings > 1 && !is_lpac) {
/* Yield the floor on command completion */
OUT_PKT7(ring, CP_CONTEXT_SWITCH_YIELD, 4);
@@ -244,6 +246,7 @@ static void a6xx_set_pagetable(struct a6xx_gpu *a6xx_gpu,
struct drm_gpuvm *vm = msm_context_vm(submit->dev, ctx);
struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
phys_addr_t ttbr;
+ bool is_lpac = ring == a6xx_gpu->base.base.lpac_rb;
u32 asid;
u64 memptr = rbmemptr(ring, ttbr0);
@@ -261,25 +264,27 @@ static void a6xx_set_pagetable(struct a6xx_gpu *a6xx_gpu,
OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
OUT_RING(ring, submit->seqno - 1);
- OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
- OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS |
CP_SET_THREAD_BOTH);
+ if (!is_lpac) {
+ OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
+ OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS |
CP_SET_THREAD_BOTH);
- /* Reset state used to synchronize BR and BV */
- OUT_PKT7(ring, CP_RESET_CONTEXT_STATE, 1);
- OUT_RING(ring,
- CP_RESET_CONTEXT_STATE_0_CLEAR_ON_CHIP_TS |
- CP_RESET_CONTEXT_STATE_0_CLEAR_RESOURCE_TABLE |
- CP_RESET_CONTEXT_STATE_0_CLEAR_BV_BR_COUNTER |
- CP_RESET_CONTEXT_STATE_0_RESET_GLOBAL_LOCAL_TS);
+ /* Reset state used to synchronize BR and BV */
+ OUT_PKT7(ring, CP_RESET_CONTEXT_STATE, 1);
+ OUT_RING(ring,
+ CP_RESET_CONTEXT_STATE_0_CLEAR_ON_CHIP_TS |
+ CP_RESET_CONTEXT_STATE_0_CLEAR_RESOURCE_TABLE |
+ CP_RESET_CONTEXT_STATE_0_CLEAR_BV_BR_COUNTER |
+
CP_RESET_CONTEXT_STATE_0_RESET_GLOBAL_LOCAL_TS);
- OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
- OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS |
CP_SET_THREAD_BOTH);
+ OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
+ OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS |
CP_SET_THREAD_BOTH);
- OUT_PKT7(ring, CP_EVENT_WRITE, 1);
- OUT_RING(ring, LRZ_FLUSH_INVALIDATE);
+ OUT_PKT7(ring, CP_EVENT_WRITE, 1);
+ OUT_RING(ring, LRZ_FLUSH_INVALIDATE);
- OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
- OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS |
CP_SET_THREAD_BR);
+ OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
+ OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS |
CP_SET_THREAD_BR);
+ }
}
if (!sysprof) {
@@ -493,7 +498,10 @@ static void a7xx_submit(struct msm_gpu *gpu, struct
msm_gem_submit *submit)
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
struct msm_ringbuffer *ring = submit->ring;
- u32 rbbm_perfctr_cp0, cp_always_on_context;
+ u32 rbbm_perfctr_cp0, cp_always_on_context,
+ cp_always_on_counter;
+ bool is_lpac = ring == gpu->lpac_rb;
+ u64 timestamp_iova;
unsigned int i, ibs = 0;
adreno_check_and_reenable_stall(adreno_gpu);
@@ -511,19 +519,24 @@ static void a7xx_submit(struct msm_gpu *gpu, struct
msm_gem_submit *submit)
* If preemption is enabled, then set the pseudo register for the save
* sequence
*/
- if (gpu->nr_rings > 1)
+ if (gpu->nr_rings > 1 && !is_lpac)
a6xx_emit_set_pseudo_reg(ring, a6xx_gpu, submit->queue);
if (adreno_is_a8xx(adreno_gpu)) {
rbbm_perfctr_cp0 = REG_A8XX_RBBM_PERFCTR_CP(0);
cp_always_on_context = REG_A8XX_CP_ALWAYS_ON_CONTEXT;
+ cp_always_on_counter = REG_A8XX_CP_ALWAYS_ON_COUNTER;
} else {
rbbm_perfctr_cp0 = REG_A7XX_RBBM_PERFCTR_CP(0);
cp_always_on_context = REG_A6XX_CP_ALWAYS_ON_CONTEXT;
+ cp_always_on_counter = REG_A6XX_CP_ALWAYS_ON_COUNTER;
}
get_stats_counter(ring, rbbm_perfctr_cp0, rbmemptr_stats(ring, index,
cpcycles_start));
- get_stats_counter(ring, cp_always_on_context, rbmemptr_stats(ring,
index, alwayson_start));
+ if (is_lpac)
+ get_stats_counter(ring, cp_always_on_counter,
rbmemptr_stats(ring, index, alwayson_start));
+ else
+ get_stats_counter(ring, cp_always_on_context,
rbmemptr_stats(ring, index, alwayson_start));
OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
OUT_RING(ring, CP_SET_THREAD_BOTH);
@@ -582,17 +595,19 @@ static void a7xx_submit(struct msm_gpu *gpu, struct
msm_gem_submit *submit)
OUT_RING(ring, submit->seqno);
}
- OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
- OUT_RING(ring, CP_SET_THREAD_BR);
+ if (!is_lpac) {
+ OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
+ OUT_RING(ring, CP_SET_THREAD_BR);
- OUT_PKT7(ring, CP_EVENT_WRITE, 1);
- OUT_RING(ring, CCU_INVALIDATE_DEPTH);
+ OUT_PKT7(ring, CP_EVENT_WRITE, 1);
+ OUT_RING(ring, CCU_INVALIDATE_DEPTH);
- OUT_PKT7(ring, CP_EVENT_WRITE, 1);
- OUT_RING(ring, CCU_INVALIDATE_COLOR);
+ OUT_PKT7(ring, CP_EVENT_WRITE, 1);
+ OUT_RING(ring, CCU_INVALIDATE_COLOR);
- OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
- OUT_RING(ring, CP_SET_THREAD_BV);
+ OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
+ OUT_RING(ring, CP_SET_THREAD_BV);
+ }
/*
* Make sure the timestamp is committed once BV pipe is
@@ -638,10 +653,12 @@ static void a7xx_submit(struct msm_gpu *gpu, struct
msm_gem_submit *submit)
a6xx_flush_yield(gpu, ring);
/* Check to see if we need to start preemption */
- if (adreno_is_a8xx(adreno_gpu))
- a8xx_preempt_trigger(gpu);
- else
- a6xx_preempt_trigger(gpu);
+ if (!is_lpac) {
+ if (adreno_is_a8xx(adreno_gpu))
+ a8xx_preempt_trigger(gpu);
+ else
+ a6xx_preempt_trigger(gpu);
+ }
}
static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state)
@@ -1219,6 +1236,7 @@ int a6xx_zap_shader_init(struct msm_gpu *gpu)
A6XX_RBBM_INT_0_MASK_PM4CPINTERRUPT | \
A6XX_RBBM_INT_0_MASK_CP_RB_DONE_TS | \
A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
+ A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS_LPAC | \
A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \
A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \
A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
@@ -2030,6 +2048,9 @@ static irqreturn_t a6xx_irq(struct msm_gpu *gpu)
a6xx_preempt_trigger(gpu);
}
+ if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS_LPAC)
+ msm_gpu_retire(gpu);
+
if (status & A6XX_RBBM_INT_0_MASK_CP_SW)
a6xx_preempt_irq(gpu);
@@ -2500,6 +2521,9 @@ static bool a6xx_progress(struct msm_gpu *gpu, struct
msm_ringbuffer *ring)
if (to_adreno_gpu(gpu)->info->quirks & ADRENO_QUIRK_IFPC)
return true;
+ if (ring == gpu->lpac_rb)
+ return true;
+
cp_state = (struct msm_cp_state) {
.ib1_base = gpu_read64(gpu, REG_A6XX_CP_IB1_BASE),
.ib2_base = gpu_read64(gpu, REG_A6XX_CP_IB2_BASE),
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
index b50c57f427b4..d85f2536551c 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
@@ -75,7 +75,7 @@ struct a6xx_gpu {
struct drm_gem_object *preempt_smmu_bo[MSM_GPU_MAX_RINGS];
void *preempt_smmu[MSM_GPU_MAX_RINGS];
uint64_t preempt_smmu_iova[MSM_GPU_MAX_RINGS];
- uint32_t last_seqno[MSM_GPU_MAX_RINGS];
+ uint32_t last_seqno[MSM_GPU_MAX_RINGS + 1];
atomic_t preempt_state;
spinlock_t eval_lock;
diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c
b/drivers/gpu/drm/msm/msm_gem_submit.c
index 26ea8a28be47..45e7f8b72dc9 100644
--- a/drivers/gpu/drm/msm/msm_gem_submit.c
+++ b/drivers/gpu/drm/msm/msm_gem_submit.c
@@ -71,7 +71,8 @@ static struct msm_gem_submit *submit_create(struct drm_device
*dev,
submit->cmd = (void *)&submit->bos[nr_bos];
submit->queue = queue;
submit->pid = get_pid(task_pid(current));
- submit->ring = gpu->rb[queue->ring_nr];
+ submit->ring = queue->ring_nr == gpu->nr_rings ?
+ gpu->lpac_rb : gpu->rb[queue->ring_nr];
submit->fault_dumped = false;
/* Get a unique identifier for the submission for logging purposes */
@@ -599,7 +600,8 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
goto out_post_unlock;
}
- ring = gpu->rb[queue->ring_nr];
+ ring = queue->ring_nr == gpu->nr_rings ?
+ gpu->lpac_rb : gpu->rb[queue->ring_nr];
if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 932e2a7c24b3..38fdf8d95cc0 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -537,8 +537,8 @@ static void recover_worker(struct kthread_work *work)
* needs to happen after msm_rd_dump_submit() to ensure that the
* bo's referenced by the offending submit are still around.
*/
- for (i = 0; i < gpu->nr_rings; i++) {
- struct msm_ringbuffer *ring = gpu->rb[i];
+ for (i = 0; i < gpu->nr_rings + !!gpu->lpac_rb; i++) {
+ struct msm_ringbuffer *ring = i < gpu->nr_rings ? gpu->rb[i] :
gpu->lpac_rb;
uint32_t fence = ring->memptrs->fence;
@@ -561,8 +561,8 @@ static void recover_worker(struct kthread_work *work)
* Replay all remaining submits starting with highest priority
* ring
*/
- for (i = 0; i < gpu->nr_rings; i++) {
- struct msm_ringbuffer *ring = gpu->rb[i];
+ for (i = 0; i < gpu->nr_rings + !!gpu->lpac_rb; i++) {
+ struct msm_ringbuffer *ring = i < gpu->nr_rings ? gpu->rb[i] :
gpu->lpac_rb;
unsigned long flags;
spin_lock_irqsave(&ring->submit_lock, flags);
diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h
index 9a213fb65b4f..263dafeb9652 100644
--- a/drivers/gpu/drm/msm/msm_gpu.h
+++ b/drivers/gpu/drm/msm/msm_gpu.h
@@ -490,9 +490,11 @@ struct msm_context {
* create at most one &drm_sched_entity per-process per-priority-
* level.
*/
- struct drm_sched_entity *entities[NR_SCHED_PRIORITIES *
MSM_GPU_MAX_RINGS];
+ struct drm_sched_entity *entities[NR_SCHED_PRIORITIES *
(MSM_GPU_MAX_RINGS + 1)];
+
/**
* @ctx_mem:
+ *
* Total amount of memory of GEM buffers with handles attached for
* this context.
*/
diff --git a/drivers/gpu/drm/msm/msm_submitqueue.c
b/drivers/gpu/drm/msm/msm_submitqueue.c
index 1a5a77b28016..042b4f90fb73 100644
--- a/drivers/gpu/drm/msm/msm_submitqueue.c
+++ b/drivers/gpu/drm/msm/msm_submitqueue.c
@@ -175,6 +175,7 @@ int msm_submitqueue_create(struct drm_device *drm, struct
msm_context *ctx,
struct msm_drm_private *priv = drm->dev_private;
struct msm_gpu_submitqueue *queue;
enum drm_sched_priority sched_prio;
+ struct msm_ringbuffer *ring;
unsigned ring_nr;
int ret;
@@ -211,6 +212,13 @@ int msm_submitqueue_create(struct drm_device *drm, struct
msm_context *ctx,
queue = kzalloc_obj(*queue);
}
+ if (flags & MSM_SUBMITQUEUE_LPAC) {
+ ring_nr = priv->gpu->nr_rings;
+ ring = priv->gpu->lpac_rb;
+ } else {
+ ring = priv->gpu->rb[ring_nr];
+ }
+
if (!queue)
return -ENOMEM;
@@ -227,8 +235,7 @@ int msm_submitqueue_create(struct drm_device *drm, struct
msm_context *ctx,
} else {
queue->ring_nr = ring_nr;
- queue->entity = get_sched_entity(ctx, priv->gpu->rb[ring_nr],
- ring_nr, sched_prio);
+ queue->entity = get_sched_entity(ctx, ring, ring_nr,
sched_prio);
}
if (IS_ERR(queue->entity)) {
diff --git a/include/uapi/drm/msm_drm.h b/include/uapi/drm/msm_drm.h
index 7f2e594be4eb..058228a56213 100644
--- a/include/uapi/drm/msm_drm.h
+++ b/include/uapi/drm/msm_drm.h
@@ -465,10 +465,12 @@ struct drm_msm_gem_madvise {
#define MSM_SUBMITQUEUE_ALLOW_PREEMPT 0x00000001
#define MSM_SUBMITQUEUE_VM_BIND 0x00000002 /* virtual queue for
VM_BIND ops */
+#define MSM_SUBMITQUEUE_LPAC 0x00000004
#define MSM_SUBMITQUEUE_FLAGS ( \
MSM_SUBMITQUEUE_ALLOW_PREEMPT | \
MSM_SUBMITQUEUE_VM_BIND | \
+ MSM_SUBMITQUEUE_LPAC | \
0)
/*
--
2.54.0
