From: "Siluvery, Arun" <[email protected]> Instead of full GPU reset, where possible a single ring can be reset individually. This patch adds functions to save ring's current state and it will be restored with the same state after reset. The state comprises of a set of ring specific registers. The actual hang detection and recovery changes are in subsequent patches.
Signed-off-by: Siluvery, Arun <[email protected]> --- drivers/gpu/drm/i915/i915_drv.h | 9 +- drivers/gpu/drm/i915/i915_reg.h | 7 + drivers/gpu/drm/i915/intel_ringbuffer.c | 595 ++++++++++++++++++++++++++++++++ drivers/gpu/drm/i915/intel_ringbuffer.h | 49 +++ drivers/gpu/drm/i915/intel_uncore.c | 25 ++ 5 files changed, 681 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index b98a7c8..b0a244d 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -2423,6 +2423,7 @@ extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e, */ void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv); void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv); +void gen6_gt_force_wake_restore(struct drm_device *dev); int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u8 mbox, u32 *val); int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val); @@ -2456,13 +2457,13 @@ int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val); #define I915_READ16(reg) dev_priv->uncore.funcs.mmio_readw(dev_priv, (reg), true) #define I915_WRITE16(reg, val) dev_priv->uncore.funcs.mmio_writew(dev_priv, (reg), (val), true) -#define I915_READ16_NOTRACE(reg) dev_priv->uncore.funcs.mmio_readw(dev_priv, (reg), false) -#define I915_WRITE16_NOTRACE(reg, val) dev_priv->uncore.funcs.mmio_writew(dev_priv, (reg), (val), false) +#define I915_READ16_NOTRACE(reg) dev_priv->uncore.funcs.mmio_readw(dev_priv, (reg), false) +#define I915_WRITE16_NOTRACE(reg, val) dev_priv->uncore.funcs.mmio_writew(dev_priv, (reg), (val), false) #define I915_READ(reg) dev_priv->uncore.funcs.mmio_readl(dev_priv, (reg), true) #define I915_WRITE(reg, val) dev_priv->uncore.funcs.mmio_writel(dev_priv, (reg), (val), true) -#define I915_READ_NOTRACE(reg) dev_priv->uncore.funcs.mmio_readl(dev_priv, (reg), false) -#define I915_WRITE_NOTRACE(reg, val) dev_priv->uncore.funcs.mmio_writel(dev_priv, (reg), (val), false) +#define I915_READ_NOTRACE(reg) dev_priv->uncore.funcs.mmio_readl(dev_priv, (reg), false) +#define I915_WRITE_NOTRACE(reg, val) dev_priv->uncore.funcs.mmio_writel(dev_priv, (reg), (val), false) #define I915_WRITE64(reg, val) dev_priv->uncore.funcs.mmio_writeq(dev_priv, (reg), (val), true) #define I915_READ64(reg) dev_priv->uncore.funcs.mmio_readq(dev_priv, (reg), true) diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 849e595..02e6ec6 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -33,6 +33,7 @@ #define _MASKED_BIT_ENABLE(a) (((a) << 16) | (a)) #define _MASKED_BIT_DISABLE(a) ((a) << 16) +#define _MASKED_BIT_ENABLE_ALL(a) (0xFFFF0000 | (a)) /* PCI config space */ @@ -104,6 +105,7 @@ #define GEN6_GRDOM_RENDER (1 << 1) #define GEN6_GRDOM_MEDIA (1 << 2) #define GEN6_GRDOM_BLT (1 << 3) +#define GEN6_GRDOM_VEBOX (1 << 4) #define RING_PP_DIR_BASE(ring) ((ring)->mmio_base+0x228) #define RING_PP_DIR_BASE_READ(ring) ((ring)->mmio_base+0x518) @@ -642,6 +644,8 @@ #define RING_SYNC_0(base) ((base)+0x40) #define RING_SYNC_1(base) ((base)+0x44) #define RING_SYNC_2(base) ((base)+0x48) +#define RING_MI_MODE(base) ((base)+0x9c) +#define RING_UHPTR(base) ((base)+0x134) #define GEN6_RVSYNC (RING_SYNC_0(RENDER_RING_BASE)) #define GEN6_RBSYNC (RING_SYNC_1(RENDER_RING_BASE)) #define GEN6_RVESYNC (RING_SYNC_2(RENDER_RING_BASE)) @@ -789,6 +793,8 @@ #define MI_MODE 0x0209c # define VS_TIMER_DISPATCH (1 << 6) +# define MODE_STOP (1 << 8) +# define MODE_IDLE (1 << 9) # define MI_FLUSH_ENABLE (1 << 12) # define ASYNC_FLIP_PERF_DISABLE (1 << 14) @@ -799,6 +805,7 @@ #define GFX_MODE 0x02520 #define GFX_MODE_GEN7 0x0229c #define RING_MODE_GEN7(ring) ((ring)->mmio_base+0x29c) +#define RING_EXCC_GEN7(ring) ((ring)->mmio_base+0x028) #define GFX_RUN_LIST_ENABLE (1<<15) #define GFX_TLB_INVALIDATE_ALWAYS (1<<13) #define GFX_SURFACE_FAULT_ENABLE (1<<12) diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index b620337..cce29d0 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -46,6 +46,7 @@ void __intel_ring_advance(struct intel_ring_buffer *ring) struct drm_i915_private *dev_priv = ring->dev->dev_private; ring->tail &= ring->size - 1; + if (dev_priv->gpu_error.stop_rings & intel_ring_flag(ring)) return; ring->write_tail(ring, ring->tail); @@ -428,6 +429,21 @@ static void ring_setup_phys_status_page(struct intel_ring_buffer *ring) I915_WRITE(HWS_PGA, addr); } +void intel_ring_resample(struct intel_ring_buffer *ring) +{ + struct drm_device *dev = ring->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + + if (!drm_core_check_feature(ring->dev, DRIVER_MODESET)) + i915_kernel_lost_context(ring->dev); + else { + ring->head = I915_READ_HEAD(ring); + ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; + ring->space = ring_space(ring); + ring->last_retired_head = -1; + } +} + static int init_ring_common(struct intel_ring_buffer *ring) { struct drm_device *dev = ring->dev; @@ -508,6 +524,9 @@ static int init_ring_common(struct intel_ring_buffer *ring) memset(&ring->hangcheck, 0, sizeof(ring->hangcheck)); + if (ring->invalidate_tlb) + ring->invalidate_tlb(ring); + out: gen6_gt_force_wake_put(dev_priv); @@ -1318,6 +1337,40 @@ static int init_phys_status_page(struct intel_ring_buffer *ring) return 0; } +/* gen6_ring_invalidate_tlb + * GFX soft resets do not invalidate TLBs, it is up to + * GFX driver to explicitly invalidate TLBs post reset. + */ +static int gen6_ring_invalidate_tlb(struct intel_ring_buffer *ring) +{ + struct drm_device *dev = ring->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + u32 reg; + int ret; + + if ((INTEL_INFO(dev)->gen < 6) || (!ring->stop) || (!ring->start)) + return -EINVAL; + + /* stop the ring before sync_flush */ + ret = ring->stop(ring); + if ((ret) && (ret != -EALREADY)) + DRM_ERROR("%s: unable to stop the ring\n", ring->name); + + /* Invalidate TLB */ + reg = RING_INSTPM(ring->mmio_base); + I915_WRITE(reg, _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | + INSTPM_SYNC_FLUSH)); + if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0, 1000)) + DRM_ERROR("%s: wait for SyncFlush to complete timed out\n", + ring->name); + + /* only start if stop was sucessfull */ + if (!ret) + ring->start(ring); + + return 0; +} + static int intel_init_ring_buffer(struct drm_device *dev, struct intel_ring_buffer *ring) { @@ -1801,6 +1854,516 @@ gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, return 0; } +static int +gen6_ring_stop(struct intel_ring_buffer *ring) +{ + struct drm_device *dev = ring->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + + /* check if ring is already stopped */ + if (I915_READ_MODE(ring) & MODE_STOP) + return -EALREADY; + + /* Request the ring to go idle */ + I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(MODE_STOP)); + + /* Wait for idle */ + if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) { + DRM_ERROR("%s :timed out trying to stop ring", ring->name); + return -ETIMEDOUT; + } + + return 0; +} + +static int +gen6_ring_start(struct intel_ring_buffer *ring) +{ + struct drm_device *dev = ring->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + uint32_t mode; + + /* Clear the MI_MODE stop bit */ + I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(MODE_STOP)); + mode = I915_READ_MODE(ring); /* Barrier read */ + + return 0; +} + +int intel_ring_disable(struct intel_ring_buffer *ring) +{ + if (ring && ring->disable) + return ring->disable(ring); + else { + DRM_ERROR("ring disable not supported\n"); + return -EINVAL; + } +} + +static int +gen6_ring_disable(struct intel_ring_buffer *ring) +{ + struct drm_device *dev = ring->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + uint32_t ring_ctl; + uint32_t mi_mode; + uint32_t retries = 10000; + + /* Request the ring to go idle */ + I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(MODE_STOP)); + + /* Wait for idle */ + do { + mi_mode = I915_READ_MODE(ring); + } while (retries-- && !(mi_mode & MODE_IDLE)); + + if (retries == 0) { + DRM_ERROR("timed out trying to disable ring %d\n", ring->id); + return -ETIMEDOUT; + } + + /* Disable the ring */ + ring_ctl = I915_READ_CTL(ring); + ring_ctl &= (RING_NR_PAGES | RING_REPORT_MASK); + I915_WRITE_CTL(ring, ring_ctl); + ring_ctl = I915_READ_CTL(ring); /* Barrier read */ + + return ((ring_ctl & RING_VALID) == 0) ? 0 : -EIO; +} + +int intel_ring_enable(struct intel_ring_buffer *ring) +{ + if (ring && ring->enable) + return ring->enable(ring); + else { + DRM_ERROR("ring enable not supported\n"); + return -EINVAL; + } +} + +static int +gen6_ring_enable(struct intel_ring_buffer *ring) +{ + struct drm_device *dev = ring->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + uint32_t ring_ctl; + uint32_t mode; + + /* Clear the MI_MODE stop bit */ + I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(MODE_STOP)); + mode = I915_READ_MODE(ring); /* Barrier read */ + + /* Enable the ring */ + ring_ctl = I915_READ_CTL(ring); + ring_ctl &= (RING_NR_PAGES | RING_REPORT_MASK); + I915_WRITE_CTL(ring, ring_ctl | RING_VALID); + ring_ctl = I915_READ_CTL(ring); /* Barrier read */ + + return ((ring_ctl & RING_VALID) == 0) ? -EIO : 0; +} + + +/* raw read/write, no need for forcewake etc. */ +#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__)) +#define __raw_i915_write32(dev_priv__, reg__, val__) writel(val__, (dev_priv__)->regs + (reg__)) + +int intel_ring_reset(struct intel_ring_buffer *ring) +{ + if (ring && ring->reset) { + int ret = ring->reset(ring); + /* invalidate TLB, if we got reset due to TLB giving + * stale addr (after soft reset) for HW status page, + * resulting in seqno not getting updated. + */ + if (ring->invalidate_tlb) + ring->invalidate_tlb(ring); + return ret; + } else { + DRM_ERROR("ring reset not supported\n"); + return -EINVAL; + } +} + +static int +gen6_ring_reset(struct intel_ring_buffer *ring) +{ + struct drm_device *dev = ring->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + int ret = 0; + char *reset_event[2]; + unsigned long irqflags; + reset_event[1] = NULL; + + /* Hold uncore.lock across reset to prevent any register access + * with forcewake not set correctly + */ + spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + + switch (ring->id) { + case RCS: + /* GEN6_GDRST is not in the gt power well, no need to check + * for fifo space for the write or forcewake the chip for + * the read + */ + __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_RENDER); + + /* Spin waiting for the device to ack the reset request */ + ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) + & GEN6_GRDOM_RENDER) == 0, 500); + DRM_DEBUG("RCS Reset\n"); + break; + + + case BCS: + __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_BLT); + + /* Spin waiting for the device to ack the reset request */ + ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) + & GEN6_GRDOM_BLT) == 0, 500); + DRM_DEBUG("BCS Reset\n"); + break; + + + case VCS: + __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_MEDIA); + + /* Spin waiting for the device to ack the reset request */ + ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) + & GEN6_GRDOM_MEDIA) == 0, 500); + DRM_DEBUG("VCS Reset\n"); + break; + + case VECS: + __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_VEBOX); + + /* Spin waiting for the device to ack the reset request */ + ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) + & GEN6_GRDOM_VEBOX) == 0, 500); + DRM_DEBUG("VECS Reset\n"); + break; + + default: + DRM_ERROR("Unexpected ring ID\n"); + break; + } + + /* Request power management to restore the power state based + * on the current reference count(s)*/ + gen6_gt_force_wake_restore(dev); + + spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); + + /* Do uevent outside of spinlock as uevent can sleep */ + reset_event[0] = kasprintf(GFP_KERNEL, "RESET RING=%d", ring->id); + kobject_uevent_env(&dev->primary->kdev->kobj, + KOBJ_CHANGE, reset_event); + kfree(reset_event[0]); + + return ret; +} + +int intel_ring_save(struct intel_ring_buffer *ring, u32 flags) +{ + if (ring && ring->save) + return ring->save(ring, ring->saved_state, + I915_RING_CONTEXT_SIZE, flags); + else { + DRM_ERROR("ring save not supported\n"); + return -EINVAL; + } +} + +static int +gen6_ring_save(struct intel_ring_buffer *ring, uint32_t *data, uint32_t max, + u32 flags) +{ + struct drm_device *dev = ring->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + uint32_t idx = 0; + uint32_t gen = INTEL_INFO(dev)->gen; + uint32_t head; + uint32_t tail; + uint32_t head_addr; + uint32_t tail_addr; + int clamp_to_tail = 0; + + /* Ring save only added for gen >= 7 */ + WARN_ON(gen < 7); + + /* Save common registers */ + if (max < COMMON_RING_CTX_SIZE) + return -EINVAL; + + head = I915_READ_HEAD(ring); + tail = I915_READ_TAIL(ring); + + head_addr = head & HEAD_ADDR; + tail_addr = tail & TAIL_ADDR; + + if (flags & FORCE_ADVANCE) { + /* The head must always chase the tail. + * If the tail is beyond the head then do not allow + * the head to overtake it. If the tail is less than + * the head then the tail has already wrapped and + * there is no problem in advancing the head or even + * wrapping the head back to 0 as worst case it will + * become equal to tail */ + if (head_addr <= tail_addr) + clamp_to_tail = 1; + + /* Force head to next QWORD boundary */ + head_addr &= ~0x7; + head_addr += 8; + + if (clamp_to_tail && (head_addr > tail_addr)) { + head_addr = tail_addr; + } else if (head_addr >= ring->size) { + /* Wrap head back to start if it exceeds ring size*/ + head_addr = 0; + } + + /* Update the register */ + head &= ~HEAD_ADDR; + head |= (head_addr & HEAD_ADDR); + + DRM_DEBUG("Forced head to 0x%08x\n", head); + } else if (head & 0x7) { + /* Ensure head pointer is pointing to a QWORD boundary */ + DRM_DEBUG("Rounding up head 0x%08x\n", head); + head += 0x7; + head &= ~0x7; + } + + /* Saved with enable = 0 */ + data[idx++] = I915_READ_CTL(ring) & (RING_NR_PAGES | RING_REPORT_MASK); + + data[idx++] = (flags & RESET_HEAD_TAIL) ? 0 : tail; + + + if (flags & RESET_HEAD_TAIL) { + /* Save head as 0 so head is reset on restore */ + data[idx++] = 0; + } else { + /* Head will already have advanced to next instruction location + * even if the current instruction caused a hang, so we just + * save the current value as the value to restart at */ + data[idx++] = head; + } + + data[idx++] = I915_READ_START(ring); + + /* Workaround for reading DCLV registers for gen < 8 */ + data[idx++] = (gen < 8) ? + I915_READ(RING_PP_DIR_DCLV(&dev_priv->ring[VCS])) + : I915_READ(RING_PP_DIR_DCLV(ring)); + data[idx++] = (gen < 8) ? + I915_READ(RING_PP_DIR_BASE(&dev_priv->ring[VCS])) + : I915_READ(RING_PP_DIR_BASE(ring)); + + switch (ring->id) { + case RCS: + if (max < (COMMON_RING_CTX_SIZE + RCS_RING_CTX_SIZE)) + return -EINVAL; + + data[idx++] = I915_READ(RENDER_HWS_PGA_GEN7); + data[idx++] = I915_READ(RING_UHPTR(ring->mmio_base)); + data[idx++] = I915_READ(RING_INSTPM(ring->mmio_base)); + data[idx++] = I915_READ(RING_IMR(ring->mmio_base)); + data[idx++] = I915_READ(CACHE_MODE_1); + data[idx++] = I915_READ(RING_MI_MODE(ring->mmio_base)); + data[idx++] = I915_READ(FW_BLC2); + data[idx++] = I915_READ(_3D_CHICKEN3); + data[idx++] = I915_READ(GAM_ECOCHK); + data[idx++] = I915_READ(RING_MODE_GEN7(ring)); + data[idx++] = I915_READ(GEN6_RBSYNC); + data[idx++] = I915_READ(GEN7_FF_THREAD_MODE); + data[idx++] = I915_READ(MI_ARB_STATE); + break; + + case VCS: + if (max < (COMMON_RING_CTX_SIZE + VCS_RING_CTX_SIZE)) + return -EINVAL; + + data[idx++] = I915_READ(BSD_HWS_PGA_GEN7); + data[idx++] = I915_READ(RING_MI_MODE(ring->mmio_base)); + data[idx++] = I915_READ(RING_IMR(ring->mmio_base)); + data[idx++] = I915_READ(RING_UHPTR(ring->mmio_base)); + data[idx++] = I915_READ(RING_INSTPM(ring->mmio_base)); + data[idx++] = I915_READ(RING_EXCC_GEN7(ring)); + data[idx++] = I915_READ(GAC_ECO_BITS); + data[idx++] = I915_READ(RING_MODE_GEN7(ring)); + data[idx++] = I915_READ(GEN6_VRSYNC); + data[idx++] = I915_READ(RING_MAX_IDLE(ring->mmio_base)); + break; + + case BCS: + if (max < (COMMON_RING_CTX_SIZE + BCS_RING_CTX_SIZE)) + return -EINVAL; + + data[idx++] = I915_READ(BLT_HWS_PGA_GEN7); + data[idx++] = I915_READ(RING_MI_MODE(ring->mmio_base)); + data[idx++] = I915_READ(RING_IMR(ring->mmio_base)); + data[idx++] = I915_READ(RING_UHPTR(ring->mmio_base)); + data[idx++] = I915_READ(RING_INSTPM(ring->mmio_base)); + data[idx++] = I915_READ(RING_EXCC_GEN7(ring)); + data[idx++] = I915_READ(GAB_CTL); + data[idx++] = I915_READ(RING_MODE_GEN7(ring)); + data[idx++] = I915_READ(GEN6_BRSYNC); + data[idx++] = I915_READ(GEN6_BVSYNC); + data[idx++] = I915_READ(RING_MAX_IDLE(ring->mmio_base)); + break; + + case VECS: + if (max < (COMMON_RING_CTX_SIZE + VECS_RING_CTX_SIZE)) + return -EINVAL; + + data[idx++] = I915_READ(VEBOX_HWS_PGA_GEN7); + data[idx++] = I915_READ(RING_MI_MODE(ring->mmio_base)); + data[idx++] = I915_READ(RING_IMR(ring->mmio_base)); + data[idx++] = I915_READ(RING_UHPTR(ring->mmio_base)); + data[idx++] = I915_READ(RING_INSTPM(ring->mmio_base)); + data[idx++] = I915_READ(RING_EXCC_GEN7(ring)); + data[idx++] = I915_READ(RING_MODE_GEN7(ring)); + data[idx++] = I915_READ(GEN6_VEVSYNC); + break; + } + + return 0; +} + +int intel_ring_restore(struct intel_ring_buffer *ring) +{ + if (ring && ring->restore) + return ring->restore(ring, ring->saved_state, + I915_RING_CONTEXT_SIZE); + else { + DRM_ERROR("ring restore not supported\n"); + return -EINVAL; + } +} + + +static int +gen6_ring_restore(struct intel_ring_buffer *ring, uint32_t *data, + uint32_t max) +{ + struct drm_device *dev = ring->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + uint32_t idx = 0; + uint32_t x; + + /* NOTE: Registers are restored in reverse order from when + * they were saved. */ + switch (ring->id) { + case RCS: + if (max < (COMMON_RING_CTX_SIZE + RCS_RING_CTX_SIZE)) + return -EINVAL; + + idx = COMMON_RING_CTX_SIZE + RCS_RING_CTX_SIZE - 1; + + I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(GEN7_FF_THREAD_MODE, data[idx--]); + I915_WRITE(GEN6_RBSYNC, data[idx--]); + I915_WRITE(RING_MODE_GEN7(ring), + _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(GAM_ECOCHK, data[idx--]); + I915_WRITE(_3D_CHICKEN3, _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(FW_BLC2, _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(RING_MI_MODE(ring->mmio_base), + _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(CACHE_MODE_1, _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(RING_IMR(ring->mmio_base), data[idx--]); + I915_WRITE(RING_INSTPM(ring->mmio_base), + _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(RING_UHPTR(ring->mmio_base), data[idx--]); + I915_WRITE(RENDER_HWS_PGA_GEN7, data[idx--]); + break; + + case VCS: + if (max < (COMMON_RING_CTX_SIZE + VCS_RING_CTX_SIZE)) + return -EINVAL; + + idx = COMMON_RING_CTX_SIZE + VCS_RING_CTX_SIZE - 1; + + I915_WRITE(RING_MAX_IDLE(ring->mmio_base), data[idx--]); + I915_WRITE(GEN6_VRSYNC, data[idx--]); + I915_WRITE(RING_MODE_GEN7(ring), + _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(GAC_ECO_BITS, data[idx--]); + I915_WRITE(RING_EXCC_GEN7(ring), + _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(RING_INSTPM(ring->mmio_base), + _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(RING_UHPTR(ring->mmio_base), data[idx--]); + I915_WRITE(RING_IMR(ring->mmio_base), data[idx--]); + I915_WRITE(RING_MI_MODE(ring->mmio_base), + _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(BSD_HWS_PGA_GEN7, data[idx--]); + break; + + case BCS: + if (max < (COMMON_RING_CTX_SIZE + BCS_RING_CTX_SIZE)) + return -EINVAL; + + idx = COMMON_RING_CTX_SIZE + BCS_RING_CTX_SIZE - 1; + + I915_WRITE(RING_MAX_IDLE(ring->mmio_base), data[idx--]); + I915_WRITE(GEN6_BVSYNC, data[idx--]); + I915_WRITE(GEN6_BRSYNC, data[idx--]); + I915_WRITE(RING_MODE_GEN7(ring), + _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(GAB_CTL, data[idx--]); + I915_WRITE(RING_EXCC_GEN7(ring), + _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(RING_INSTPM(ring->mmio_base), + _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(RING_UHPTR(ring->mmio_base), data[idx--]); + I915_WRITE(RING_IMR(ring->mmio_base), data[idx--]); + I915_WRITE(RING_MI_MODE(ring->mmio_base), + _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(BLT_HWS_PGA_GEN7, data[idx--]); + break; + + case VECS: + if (max < (COMMON_RING_CTX_SIZE + VECS_RING_CTX_SIZE)) + return -EINVAL; + + idx = COMMON_RING_CTX_SIZE + VECS_RING_CTX_SIZE - 1; + + I915_WRITE(GEN6_VEVSYNC, data[idx--]); + I915_WRITE(RING_MODE_GEN7(ring), + _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(RING_EXCC_GEN7(ring), + _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(RING_INSTPM(ring->mmio_base), + _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(RING_UHPTR(ring->mmio_base), data[idx--]); + I915_WRITE(RING_IMR(ring->mmio_base), data[idx--]); + I915_WRITE(RING_MI_MODE(ring->mmio_base), + _MASKED_BIT_ENABLE_ALL(data[idx--])); + I915_WRITE(VEBOX_HWS_PGA_GEN7, data[idx--]); + break; + } + + /* Restore common registers */ + if (max < COMMON_RING_CTX_SIZE) + return -EINVAL; + + idx = COMMON_RING_CTX_SIZE - 1; + + I915_WRITE(RING_PP_DIR_BASE(ring), data[idx--]); + I915_WRITE(RING_PP_DIR_DCLV(ring), data[idx--]); + + /* Write ring base address before head/tail as it clears head to 0 */ + I915_WRITE_START(ring, data[idx--]); + x = I915_READ_START(ring); + I915_WRITE_HEAD(ring, data[idx--]); + I915_WRITE_TAIL(ring, data[idx--]); + I915_WRITE_CTL(ring, data[idx--]); + + return 0; +} + /* Blitter support (SandyBridge+) */ static int gen6_ring_flush(struct intel_ring_buffer *ring, @@ -1869,6 +2432,14 @@ int intel_init_render_ring_buffer(struct drm_device *dev) ring->get_seqno = gen6_ring_get_seqno; ring->set_seqno = ring_set_seqno; ring->sync_to = gen6_ring_sync; + ring->enable = gen6_ring_enable; + ring->disable = gen6_ring_disable; + ring->start = gen6_ring_start; + ring->stop = gen6_ring_stop; + ring->reset = gen6_ring_reset; + ring->save = gen6_ring_save; + ring->restore = gen6_ring_restore; + ring->invalidate_tlb = gen6_ring_invalidate_tlb; ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_INVALID; ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_RV; ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_RB; @@ -2045,6 +2616,14 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev) gen6_ring_dispatch_execbuffer; } ring->sync_to = gen6_ring_sync; + ring->enable = gen6_ring_enable; + ring->disable = gen6_ring_disable; + ring->start = gen6_ring_start; + ring->stop = gen6_ring_stop; + ring->reset = gen6_ring_reset; + ring->save = gen6_ring_save; + ring->restore = gen6_ring_restore; + ring->invalidate_tlb = gen6_ring_invalidate_tlb; ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VR; ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_INVALID; ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VB; @@ -2102,6 +2681,14 @@ int intel_init_blt_ring_buffer(struct drm_device *dev) ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; } ring->sync_to = gen6_ring_sync; + ring->enable = gen6_ring_enable; + ring->disable = gen6_ring_disable; + ring->start = gen6_ring_start; + ring->stop = gen6_ring_stop; + ring->reset = gen6_ring_reset; + ring->save = gen6_ring_save; + ring->restore = gen6_ring_restore; + ring->invalidate_tlb = gen6_ring_invalidate_tlb; ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_BR; ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_BV; ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_INVALID; @@ -2143,6 +2730,14 @@ int intel_init_vebox_ring_buffer(struct drm_device *dev) ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer; } ring->sync_to = gen6_ring_sync; + ring->enable = gen6_ring_enable; + ring->disable = gen6_ring_disable; + ring->start = gen6_ring_start; + ring->stop = gen6_ring_stop; + ring->reset = gen6_ring_reset; + ring->save = gen6_ring_save; + ring->restore = gen6_ring_restore; + ring->invalidate_tlb = gen6_ring_invalidate_tlb; ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VER; ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_VEV; ring->semaphore_register[BCS] = MI_SEMAPHORE_SYNC_VEB; diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index 71a73f4..cd96ad9 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -18,6 +18,25 @@ struct intel_hw_status_page { struct drm_i915_gem_object *obj; }; +/* These values must match the requirements of the ring save/restore functions + * which may need to change for different versions of the chip */ +#define COMMON_RING_CTX_SIZE 6 + +#define RCS_RING_CTX_SIZE 13 +#define VCS_RING_CTX_SIZE 10 +#define BCS_RING_CTX_SIZE 11 +#define VECS_RING_CTX_SIZE 8 + +#define MAX_CTX(a, b) (((a) > (b)) ? (a) : (b)) + +/* Largest of individual rings + common*/ +#define I915_RING_CONTEXT_SIZE (COMMON_RING_CTX_SIZE + \ + MAX_CTX(MAX_CTX(RCS_RING_CTX_SIZE, \ + VCS_RING_CTX_SIZE), \ + MAX_CTX(BCS_RING_CTX_SIZE, \ + VECS_RING_CTX_SIZE))) + + #define I915_READ_TAIL(ring) I915_READ(RING_TAIL((ring)->mmio_base)) #define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL((ring)->mmio_base), val) @@ -33,6 +52,13 @@ struct intel_hw_status_page { #define I915_READ_IMR(ring) I915_READ(RING_IMR((ring)->mmio_base)) #define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val) +#define I915_READ_MODE(ring) I915_READ(RING_MI_MODE((ring)->mmio_base)) +#define I915_WRITE_MODE(ring, val) \ + I915_WRITE(RING_MI_MODE((ring)->mmio_base), val) + +#define RESET_HEAD_TAIL 0x1 +#define FORCE_ADVANCE 0x2 + enum intel_ring_hangcheck_action { HANGCHECK_IDLE = 0, HANGCHECK_WAIT, @@ -115,6 +141,18 @@ struct intel_ring_buffer { struct intel_ring_buffer *to, u32 seqno); + int (*enable)(struct intel_ring_buffer *ring); + int (*disable)(struct intel_ring_buffer *ring); + int (*start)(struct intel_ring_buffer *ring); + int (*stop)(struct intel_ring_buffer *ring); + int (*reset)(struct intel_ring_buffer *ring); + int (*save)(struct intel_ring_buffer *ring, + uint32_t *data, uint32_t max, + u32 flags); + int (*restore)(struct intel_ring_buffer *ring, + uint32_t *data, uint32_t max); + int (*invalidate_tlb)(struct intel_ring_buffer *ring); + /* our mbox written by others */ u32 semaphore_register[I915_NUM_RINGS]; /* mboxes this ring signals to */ @@ -155,6 +193,9 @@ struct intel_ring_buffer { struct i915_hw_context *default_context; struct i915_hw_context *last_context; + /* Area large enough to store all the register + * data associated with this ring */ + u32 saved_state[I915_RING_CONTEXT_SIZE]; struct intel_ring_hangcheck hangcheck; struct { @@ -229,6 +270,7 @@ intel_write_status_page(struct intel_ring_buffer *ring, #define I915_GEM_HWS_INDEX 0x20 #define I915_GEM_HWS_SCRATCH_INDEX 0x30 #define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT) +#define I915_GEM_PGFLIP_INDEX 0x38 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring); @@ -278,4 +320,11 @@ static inline void i915_trace_irq_get(struct intel_ring_buffer *ring, u32 seqno) /* DRI warts */ int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size); +void intel_ring_resample(struct intel_ring_buffer *ring); +int intel_ring_disable(struct intel_ring_buffer *ring); +int intel_ring_enable(struct intel_ring_buffer *ring); +int intel_ring_reset(struct intel_ring_buffer *ring); +int intel_ring_save(struct intel_ring_buffer *ring, u32 flags); +int intel_ring_restore(struct intel_ring_buffer *ring); + #endif /* _INTEL_RINGBUFFER_H_ */ diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index f9883ce..5349215 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -312,6 +312,31 @@ void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv) spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } +void gen6_gt_force_wake_restore(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + /* Restore the current expected force wake state with the + * hardware. This may be required following a reset. + * + * WARNING: Caller *MUST* hold uncore.lock whilst calling this. + * + * uncore.lock isn't taken in this function to allow the caller the + * flexibility to do other work immediately before/after + * whilst holding the lock */ + + intel_uncore_forcewake_reset(dev); + + /* If reset with a user forcewake, try to restore, otherwise turn it off */ + if (dev_priv->uncore.forcewake_count) + dev_priv->uncore.funcs.force_wake_get(dev_priv); + else + dev_priv->uncore.funcs.force_wake_put(dev_priv); + + /* Restore fifo count */ + dev_priv->uncore.fifo_count = __raw_i915_read32(dev_priv, GT_FIFO_FREE_ENTRIES); +} + /* We give fast paths for the really cool registers */ #define NEEDS_FORCE_WAKE(dev_priv, reg) \ ((reg) < 0x40000 && (reg) != FORCEWAKE) -- 1.8.4 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
