Enable BIO_DMABUF_MAP backed requests. It creates a prp list for the
dmabuf when it's mapped, which is then used to initialise requests.
Suggested-by: Keith Busch <[email protected]>
Signed-off-by: Pavel Begunkov <[email protected]>
---
drivers/nvme/host/core.c | 12 ++
drivers/nvme/host/nvme.h | 2 +
drivers/nvme/host/pci.c | 259 +++++++++++++++++++++++++++++++++++++++
3 files changed, 273 insertions(+)
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 453c1f0b2dd0..ce66a1843bec 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -2676,6 +2676,17 @@ static int nvme_report_zones(struct gendisk *disk,
sector_t sector,
#define nvme_report_zones NULL
#endif /* CONFIG_BLK_DEV_ZONED */
+static int nvme_init_dma_buf_io_ctx(struct block_device *bdev,
+ struct dma_buf_io_ctx *ctx)
+{
+ struct nvme_ns *ns = bdev->bd_disk->private_data;
+ struct nvme_ctrl *ctrl = ns->ctrl;
+
+ if (!ctrl->ops->init_dma_buf_io_ctx)
+ return -EINVAL;
+ return ctrl->ops->init_dma_buf_io_ctx(ctrl, ctx);
+}
+
const struct block_device_operations nvme_bdev_ops = {
.owner = THIS_MODULE,
.ioctl = nvme_ioctl,
@@ -2686,6 +2697,7 @@ const struct block_device_operations nvme_bdev_ops = {
.get_unique_id = nvme_get_unique_id,
.report_zones = nvme_report_zones,
.pr_ops = &nvme_pr_ops,
+ .init_dma_buf_io_ctx = nvme_init_dma_buf_io_ctx,
};
static int nvme_wait_ready(struct nvme_ctrl *ctrl, u32 mask, u32 val,
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 824651cc898d..034c31af8fec 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -652,6 +652,8 @@ struct nvme_ctrl_ops {
int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size);
void (*print_device_info)(struct nvme_ctrl *ctrl);
bool (*supports_pci_p2pdma)(struct nvme_ctrl *ctrl);
+ int (*init_dma_buf_io_ctx)(struct nvme_ctrl *ctrl,
+ struct dma_buf_io_ctx *ctx);
unsigned long (*get_virt_boundary)(struct nvme_ctrl *ctrl, bool
is_admin);
};
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 69932d640b53..f1b67c191892 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -27,6 +27,8 @@
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/io-64-nonatomic-hi-lo.h>
#include <linux/sed-opal.h>
+#include <linux/dma-buf-io.h>
+#include <linux/dma-resv.h>
#include "trace.h"
#include "nvme.h"
@@ -393,6 +395,13 @@ struct nvme_queue {
struct completion delete_done;
};
+struct nvme_dmabuf_map {
+ struct dma_buf_io_map base;
+ struct sg_table *sgt;
+ unsigned nr_entries;
+ dma_addr_t dma_list[];
+};
+
/* bits for iod->flags */
enum nvme_iod_flags {
/* this command has been aborted by the timeout handler */
@@ -859,6 +868,138 @@ static void nvme_free_descriptors(struct request *req)
}
}
+static inline struct nvme_dmabuf_map *
+to_nvme_dmabuf_map(struct dma_buf_io_map *map)
+{
+ return container_of(map, struct nvme_dmabuf_map, base);
+}
+
+static void nvme_dmabuf_map_sync_for_cpu(struct nvme_dev *nvme_dev,
+ struct request *req)
+{
+ struct device *dev = nvme_dev->dev;
+ enum dma_data_direction dma_dir;
+ struct bio *bio = req->bio;
+ struct nvme_dmabuf_map *map = to_nvme_dmabuf_map(bio->bi_dmabuf_map);
+ dma_addr_t *dma_list = map->dma_list;
+ unsigned offset = bio->bi_iter.bi_offset;
+ unsigned map_idx = offset / NVME_CTRL_PAGE_SIZE;
+ int length = blk_rq_payload_bytes(req) +
+ (offset & (NVME_CTRL_PAGE_SIZE - 1));
+
+ dma_dir = rq_data_dir(req) == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
+
+ while (length > 0) {
+ dma_sync_single_for_cpu(dev, dma_list[map_idx++],
+ NVME_CTRL_PAGE_SIZE, dma_dir);
+ length -= NVME_CTRL_PAGE_SIZE;
+ }
+}
+
+static void nvme_dmabuf_map_sync_for_device(struct nvme_dev *nvme_dev,
+ struct request *req)
+{
+ struct device *dev = nvme_dev->dev;
+ enum dma_data_direction dma_dir;
+ struct bio *bio = req->bio;
+ struct nvme_dmabuf_map *map = to_nvme_dmabuf_map(bio->bi_dmabuf_map);
+ dma_addr_t *dma_list = map->dma_list;
+ unsigned offset = bio->bi_iter.bi_offset;
+ unsigned map_idx = offset / NVME_CTRL_PAGE_SIZE;
+ int length = blk_rq_payload_bytes(req) +
+ (offset & (NVME_CTRL_PAGE_SIZE - 1));
+
+ dma_dir = rq_data_dir(req) == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
+
+ while (length > 0) {
+ dma_sync_single_for_device(dev, dma_list[map_idx++],
+ NVME_CTRL_PAGE_SIZE, dma_dir);
+ length -= NVME_CTRL_PAGE_SIZE;
+ }
+}
+
+static void nvme_rq_clean_dmabuf_map(struct nvme_dev *dev,
+ struct request *req)
+{
+ struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+
+ nvme_dmabuf_map_sync_for_cpu(dev, req);
+
+ if (!(iod->flags & IOD_SINGLE_SEGMENT))
+ nvme_free_descriptors(req);
+}
+
+static blk_status_t nvme_rq_setup_dmabuf_map(struct request *req,
+ struct nvme_queue *nvmeq)
+{
+ struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
+ struct bio *bio = req->bio;
+ struct nvme_dmabuf_map *map = to_nvme_dmabuf_map(bio->bi_dmabuf_map);
+ unsigned bvec_done = bio->bi_iter.bi_offset;
+ unsigned map_idx = bvec_done / NVME_CTRL_PAGE_SIZE;
+ unsigned offset = bvec_done & (NVME_CTRL_PAGE_SIZE - 1);
+ int length = blk_rq_payload_bytes(req) - (NVME_CTRL_PAGE_SIZE - offset);
+ dma_addr_t *dma_list = map->dma_list;
+ u64 prp1_dma = dma_list[map_idx++] + offset;
+ u64 dma_addr, prp2_dma;
+ dma_addr_t prp_dma;
+ __le64 *prp_list;
+ unsigned i;
+
+ nvme_dmabuf_map_sync_for_device(nvmeq->dev, req);
+
+ if (length <= 0) {
+ prp2_dma = 0;
+ goto done;
+ }
+
+ if (length <= NVME_CTRL_PAGE_SIZE) {
+ prp2_dma = dma_list[map_idx];
+ goto done;
+ }
+
+ if (DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE) <=
+ NVME_SMALL_POOL_SIZE / sizeof(__le64))
+ iod->flags |= IOD_SMALL_DESCRIPTOR;
+
+ prp_list = dma_pool_alloc(nvme_dma_pool(nvmeq, iod), GFP_ATOMIC,
+ &prp_dma);
