Program RCID and MCID for RISC-V IOMMU groups through the device context TA fields. The resctrl group assignment is per device group, so reject BARE mode where only the per-IOMMU iommu_qosid global default is available.
Validate every group member, firmware ID, device context, field value, and QoS ID capability before changing hardware. Then update all members through the checked IOMMU group helper so a validation failure leaves the group unchanged. Serialize DC.ta changes with context setup under qosid_lock. Change only the RCID and MCID fields with ordinary accesses so fixed DDT mappings are not subject to atomic LR/SC operations, invalidate active device contexts after an update, and clear the IDs when a device is released. Signed-off-by: Zhanpeng Zhang <[email protected]> --- arch/riscv/include/asm/qos.h | 16 +++ drivers/iommu/riscv/iommu-bits.h | 15 +++ drivers/iommu/riscv/iommu.c | 200 ++++++++++++++++++++++++++++++- drivers/iommu/riscv/iommu.h | 3 + 4 files changed, 232 insertions(+), 2 deletions(-) diff --git a/arch/riscv/include/asm/qos.h b/arch/riscv/include/asm/qos.h index cf19e8438bb9..daa758d4efff 100644 --- a/arch/riscv/include/asm/qos.h +++ b/arch/riscv/include/asm/qos.h @@ -2,7 +2,23 @@ #ifndef _ASM_RISCV_QOS_H #define _ASM_RISCV_QOS_H +#include <linux/errno.h> #include <linux/percpu-defs.h> +#include <linux/types.h> + +struct iommu_group; + +#ifdef CONFIG_RISCV_IOMMU +int riscv_iommu_group_set_qosid(struct iommu_group *group, u32 rcid, + u32 mcid); +#else +static inline int riscv_iommu_group_set_qosid(struct iommu_group *group, + u32 rcid, u32 mcid) +{ + return -EOPNOTSUPP; +} + +#endif #ifdef CONFIG_RISCV_ISA_SSQOSID diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h index f2ef9bd3cde9..782de5c92727 100644 --- a/drivers/iommu/riscv/iommu-bits.h +++ b/drivers/iommu/riscv/iommu-bits.h @@ -63,6 +63,7 @@ #define RISCV_IOMMU_CAPABILITIES_PD8 BIT_ULL(38) #define RISCV_IOMMU_CAPABILITIES_PD17 BIT_ULL(39) #define RISCV_IOMMU_CAPABILITIES_PD20 BIT_ULL(40) +#define RISCV_IOMMU_CAPABILITIES_QOSID BIT_ULL(41) #define RISCV_IOMMU_CAPABILITIES_NL BIT_ULL(42) #define RISCV_IOMMU_CAPABILITIES_S BIT_ULL(43) @@ -274,6 +275,14 @@ enum riscv_iommu_hpmevent_id { #define RISCV_IOMMU_TR_RESPONSE_SZ BIT_ULL(9) #define RISCV_IOMMU_TR_RESPONSE_PPN RISCV_IOMMU_PPN_FIELD +/* 6.27 IOMMU QoS IDs for IOMMU-initiated requests (32bits) */ +#define RISCV_IOMMU_REG_IOMMU_QOSID 0x0270 +#define RISCV_IOMMU_IOMMU_QOSID_RCID GENMASK(11, 0) +#define RISCV_IOMMU_IOMMU_QOSID_MCID GENMASK(27, 16) + +#define RISCV_IOMMU_IOMMU_QOSID_RCID_SHIFT 0 +#define RISCV_IOMMU_IOMMU_QOSID_MCID_SHIFT 16 + /* 5.27 Interrupt cause to vector (64bits) */ #define RISCV_IOMMU_REG_ICVEC 0x02F8 #define RISCV_IOMMU_ICVEC_CIV GENMASK_ULL(3, 0) @@ -371,6 +380,12 @@ enum riscv_iommu_dc_iohgatp_modes { /* Translation attributes fields */ #define RISCV_IOMMU_DC_TA_PSCID GENMASK_ULL(31, 12) +/* + * QoS IDs for translated device requests and IOMMU accesses with a + * device context (when capabilities.QOSID == 1). + */ +#define RISCV_IOMMU_DC_TA_RCID GENMASK_ULL(51, 40) +#define RISCV_IOMMU_DC_TA_MCID GENMASK_ULL(63, 52) /* First-stage context fields */ #define RISCV_IOMMU_DC_FSC_PPN RISCV_IOMMU_ATP_PPN_FIELD diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index cec3ddd7ab10..deab646bb1ea 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -48,6 +48,8 @@ static DEFINE_IDA(riscv_iommu_pscids); #define RISCV_IOMMU_MAX_PSCID (BIT(20) - 1) +static const struct iommu_ops riscv_iommu_ops; + /* Device resource-managed allocations */ struct riscv_iommu_devres { void *addr; @@ -1091,6 +1093,28 @@ static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain, } #define RISCV_IOMMU_FSC_BARE 0 +#define RISCV_IOMMU_DC_TA_QOSID \ + (RISCV_IOMMU_DC_TA_RCID | RISCV_IOMMU_DC_TA_MCID) + +static u64 riscv_iommu_qosid_ta(u32 rcid, u32 mcid) +{ + return FIELD_PREP(RISCV_IOMMU_DC_TA_RCID, rcid) | + FIELD_PREP(RISCV_IOMMU_DC_TA_MCID, mcid); +} + +static void riscv_iommu_dc_update_qosid(struct riscv_iommu_device *iommu, + struct riscv_iommu_dc *dc, + u32 rcid, u32 mcid) +{ + u64 qos_ta = riscv_iommu_qosid_ta(rcid, mcid); + u64 ta; + + lockdep_assert_held(&iommu->qosid_lock); + ta = READ_ONCE(dc->ta); + ta = (ta & ~RISCV_IOMMU_DC_TA_QOSID) | qos_ta; + WRITE_ONCE(dc->ta, ta); +} + /* * This function sends IOTINVAL commands as required by the RISC-V * IOMMU specification (Section 6.3.1 and 6.3.2 in 1.0 spec version) @@ -1202,12 +1226,23 @@ static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu, * is stored as DC_TC_V bit (both sharing the same location at BIT(0)). */ for (i = 0; i < fwspec->num_ids; i++) { + u64 dc_ta; + u64 ta_mask = RISCV_IOMMU_PC_TA_PSCID; + dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]); tc = READ_ONCE(dc->tc); - tc |= ta & RISCV_IOMMU_DC_TC_V; + dc_ta = ta; + if (iommu->caps & RISCV_IOMMU_CAPABILITIES_QOSID) { + dc_ta |= READ_ONCE(dc->ta) & + (RISCV_IOMMU_DC_TA_RCID | + RISCV_IOMMU_DC_TA_MCID); + ta_mask |= RISCV_IOMMU_DC_TA_RCID | + RISCV_IOMMU_DC_TA_MCID; + } + tc |= dc_ta & RISCV_IOMMU_DC_TC_V; WRITE_ONCE(dc->fsc, fsc); - WRITE_ONCE(dc->ta, ta & RISCV_IOMMU_PC_TA_PSCID); + WRITE_ONCE(dc->ta, dc_ta & ta_mask); /* Update device context, write TC.V as the last step. */ dma_wmb(); WRITE_ONCE(dc->tc, tc); @@ -1474,13 +1509,174 @@ static struct iommu_device *riscv_iommu_probe_device(struct device *dev) return &iommu->iommu; } +static void riscv_iommu_qosid_invalidate_did(struct riscv_iommu_device *iommu, + unsigned int did) +{ + struct riscv_iommu_command cmd; + + riscv_iommu_cmd_iodir_inval_ddt(&cmd); + riscv_iommu_cmd_iodir_set_did(&cmd, did); + riscv_iommu_cmd_send(iommu, &cmd); +} + static void riscv_iommu_release_device(struct device *dev) { struct riscv_iommu_info *info = dev_iommu_priv_get(dev); + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct riscv_iommu_device *iommu = dev_to_iommu(dev); + bool sync_required = false; + unsigned int i; + + if (iommu->caps & RISCV_IOMMU_CAPABILITIES_QOSID) { + mutex_lock(&iommu->qosid_lock); + for (i = 0; fwspec && i < fwspec->num_ids; i++) { + struct riscv_iommu_dc *dc; + u64 tc; + + dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]); + if (!dc) + continue; + + tc = READ_ONCE(dc->tc); + riscv_iommu_dc_update_qosid(iommu, dc, 0, 0); + if (!(tc & RISCV_IOMMU_DC_TC_V)) + continue; + + dma_wmb(); + riscv_iommu_qosid_invalidate_did(iommu, fwspec->ids[i]); + riscv_iommu_iodir_iotinval(iommu, false, dc->iohgatp, + dc, NULL); + sync_required = true; + } + + if (sync_required) + riscv_iommu_cmd_sync(iommu, + RISCV_IOMMU_IOTINVAL_TIMEOUT); + mutex_unlock(&iommu->qosid_lock); + } kfree_rcu_mightsleep(info); } +struct riscv_iommu_qosid_hw_ctx { + u32 rcid; + u32 mcid; + bool has_devices; + bool has_qosid; + bool reset; +}; + +static int riscv_iommu_qosid_validate_dev(struct device *dev, void *data) +{ + struct riscv_iommu_qosid_hw_ctx *ctx = data; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct riscv_iommu_device *iommu; + unsigned int i; + + ctx->has_devices = true; + + if (!dev->iommu || !dev->iommu->iommu_dev || + dev->iommu->iommu_dev->ops != &riscv_iommu_ops) + return -EOPNOTSUPP; + + if (!fwspec || !fwspec->num_ids) + return -ENODEV; + + iommu = dev_to_iommu(dev); + + if (!(iommu->caps & RISCV_IOMMU_CAPABILITIES_QOSID)) + return ctx->reset ? 0 : -EOPNOTSUPP; + + ctx->has_qosid = true; + + /* + * IOMMU group QoS is a per-device assignment. BARE mode only has the + * per-IOMMU iommu_qosid register, which is a global default rather + * than a safe target for moving an individual group between resctrl + * groups. + */ + if (iommu->ddt_mode <= RISCV_IOMMU_DDTP_IOMMU_MODE_BARE) + return -EOPNOTSUPP; + + for (i = 0; i < fwspec->num_ids; i++) { + if (!riscv_iommu_get_dc(iommu, fwspec->ids[i])) + return -ENODEV; + } + + return 0; +} + +static void riscv_iommu_qosid_apply_dev(struct device *dev, void *data) +{ + struct riscv_iommu_qosid_hw_ctx *ctx = data; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct riscv_iommu_device *iommu; + bool sync_required = false; + unsigned int i; + + iommu = dev_to_iommu(dev); + if (!(iommu->caps & RISCV_IOMMU_CAPABILITIES_QOSID)) + return; + + mutex_lock(&iommu->qosid_lock); + for (i = 0; i < fwspec->num_ids; i++) { + struct riscv_iommu_dc *dc; + bool dc_is_valid; + u64 tc; + + dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]); + if (WARN_ON_ONCE(!dc)) + continue; + + tc = READ_ONCE(dc->tc); + dc_is_valid = tc & RISCV_IOMMU_DC_TC_V; + + riscv_iommu_dc_update_qosid(iommu, dc, ctx->rcid, ctx->mcid); + dev_dbg(dev, "set QoS ID DC.ta did=%u rcid=%u mcid=%u\n", + fwspec->ids[i], ctx->rcid, ctx->mcid); + + if (dc_is_valid) { + dma_wmb(); + riscv_iommu_qosid_invalidate_did(iommu, fwspec->ids[i]); + riscv_iommu_iodir_iotinval(iommu, false, dc->iohgatp, + dc, NULL); + sync_required = true; + } + } + + if (sync_required) + riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT); + mutex_unlock(&iommu->qosid_lock); +} + +int riscv_iommu_group_set_qosid(struct iommu_group *group, u32 rcid, u32 mcid) +{ + struct riscv_iommu_qosid_hw_ctx hw = { + .rcid = rcid, + .mcid = mcid, + .reset = !rcid && !mcid, + }; + int ret; + + if (rcid > FIELD_MAX(RISCV_IOMMU_DC_TA_RCID) || + mcid > FIELD_MAX(RISCV_IOMMU_DC_TA_MCID)) + return -ERANGE; + + ret = iommu_group_update_devices(group, &hw, + riscv_iommu_qosid_validate_dev, + riscv_iommu_qosid_apply_dev); + if (ret) + return ret; + if (!hw.has_devices) + return -ENODATA; + if (!hw.has_qosid && !hw.reset) + return -EOPNOTSUPP; + + pr_debug("set qosid: group=%d rcid=%u mcid=%u\n", + iommu_group_id(group), rcid, mcid); + return 0; +} + static const struct iommu_ops riscv_iommu_ops = { .of_xlate = riscv_iommu_of_xlate, .identity_domain = &riscv_iommu_identity_domain, diff --git a/drivers/iommu/riscv/iommu.h b/drivers/iommu/riscv/iommu.h index 46df79dd5495..2c57625637bf 100644 --- a/drivers/iommu/riscv/iommu.h +++ b/drivers/iommu/riscv/iommu.h @@ -19,6 +19,9 @@ struct riscv_iommu_device; +int riscv_iommu_group_set_qosid(struct iommu_group *group, u32 rcid, + u32 mcid); + struct riscv_iommu_queue { atomic_t prod; /* unbounded producer allocation index */ atomic_t head; /* unbounded shadow ring buffer consumer index */ -- 2.50.1 (Apple Git-155)
