From: Kenneth Lee <liguo...@hisilicon.com>

This patch add uacce support to the Hislicon QM driver, any accelerator
that use QM can share its queues to the user space.

Signed-off-by: Kenneth Lee <liguo...@hisilicon.com>
Signed-off-by: Zhou Wang <wangzh...@hisilicon.com>
Signed-off-by: Hao Fang <fangha...@huawei.com>
Signed-off-by: Zaibo Xu <xuza...@huawei.com>
---
 drivers/crypto/hisilicon/Kconfig        |   7 +
 drivers/crypto/hisilicon/qm.c           | 227 +++++++++++++++++++++---
 drivers/crypto/hisilicon/qm.h           |  16 +-
 drivers/crypto/hisilicon/zip/zip_main.c |  27 ++-
 4 files changed, 249 insertions(+), 28 deletions(-)

diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig
index ce9deefbf037..819e4995f361 100644
--- a/drivers/crypto/hisilicon/Kconfig
+++ b/drivers/crypto/hisilicon/Kconfig
@@ -16,6 +16,13 @@ config CRYPTO_DEV_HISI_QM
        tristate
        depends on ARM64 && PCI
 
+config CRYPTO_QM_UACCE
+       bool "enable UACCE support for all acceleartor with Hisi QM"
+       depends on CRYPTO_DEV_HISI_QM
+       select UACCE
+       help
+         Support UACCE interface in Hisi QM.
+
 config CRYPTO_DEV_HISI_ZIP
        tristate "Support for HISI ZIP Driver"
        depends on ARM64
diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c
index 5b810a6f4dd5..750d8c069d92 100644
--- a/drivers/crypto/hisilicon/qm.c
+++ b/drivers/crypto/hisilicon/qm.c
@@ -5,6 +5,7 @@
 #include <linux/io.h>
 #include <linux/irqreturn.h>
 #include <linux/log2.h>
+#include <linux/uacce.h>
 #include "qm.h"
 
 #define QM_DEF_Q_NUM                   128
@@ -435,17 +436,29 @@ int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long 
arg)
        qp->sqc_dma = qm->sqc_dma + qp_index * sizeof(struct sqc);
        qp->cqc_dma = qm->cqc_dma + qp_index * sizeof(struct cqc);
 
-       qp->qdma.size = qm->sqe_size * QM_Q_DEPTH +
-                       sizeof(struct cqe) * QM_Q_DEPTH,
-       qp->qdma.va = dma_alloc_coherent(dev, qp->qdma.size,
-                                        &qp->qdma.dma,
-                                        GFP_KERNEL | __GFP_ZERO);
-       dev_dbg(dev, "allocate qp dma buf(va=%p, dma=%pad, size=%lx)\n",
-               qp->qdma.va, &qp->qdma.dma, qp->qdma.size);
+       if (qm->uacce_mode) {
+               dev_dbg(dev, "User shared DMA Buffer used: (%lx/%x)\n",
+                       off, QM_DUS_PAGE_NR << PAGE_SHIFT);
+               if (off > (QM_DUS_PAGE_NR << PAGE_SHIFT))
+                       return -EINVAL;
+       } else {
+
+               /*
+                * todo: we are using dma api here. it should be updated to
+                * uacce api for user and kernel mode working at the same time
+                */
+               qp->qdma.size = qm->sqe_size * QM_Q_DEPTH +
+                               sizeof(struct cqe) * QM_Q_DEPTH,
+               qp->qdma.va = dma_alloc_coherent(dev, qp->qdma.size,
+                                                &qp->qdma.dma,
+                                                GFP_KERNEL | __GFP_ZERO);
+               dev_dbg(dev, "allocate qp dma buf(va=%p, dma=%pad, size=%lx)\n",
+                       qp->qdma.va, &qp->qdma.dma, qp->qdma.size);
+       }
 
        if (!qp->qdma.va) {
                dev_err(dev, "cannot get qm dma buffer\n");
-               return -ENOMEM;
+               return qm->uacce_mode ? -EINVAL : -ENOMEM;
        }
 
        QP_INIT_BUF(qp, sqe, qm->sqe_size * QM_Q_DEPTH);
@@ -491,7 +504,8 @@ void hisi_qm_release_qp(struct hisi_qp *qp)
        bitmap_clear(qm->qp_bitmap, qid, 1);
        write_unlock(&qm->qps_lock);
 
-       dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma);
+       if (!qm->uacce_mode)
+               dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma);
 
        kfree(qp);
 }
@@ -535,6 +549,149 @@ int hisi_qp_send(struct hisi_qp *qp, void *msg)
 }
 EXPORT_SYMBOL_GPL(hisi_qp_send);
 
+#ifdef CONFIG_CRYPTO_QM_UACCE
+static void qm_qp_event_notifier(struct hisi_qp *qp)
+{
+       uacce_wake_up(qp->uacce_q);
+}
+
+static int hisi_qm_get_queue(struct uacce *uacce, unsigned long arg,
+                            struct uacce_queue **q)
+{
+       struct qm_info *qm = uacce->priv;
+       struct hisi_qp *qp = NULL;
+       struct uacce_queue *wd_q;
+       u8 alg_type = 0; /* fix me here */
+       int ret;
+
+       qp = hisi_qm_create_qp(qm, alg_type);
+       if (IS_ERR(qp))
+               return PTR_ERR(qp);
+
+       wd_q = kzalloc(sizeof(struct uacce_queue), GFP_KERNEL);
+       if (!wd_q) {
+               ret = -ENOMEM;
+               goto err_with_qp;
+       }
+
+       wd_q->priv = qp;
+       wd_q->uacce = uacce;
+       *q = wd_q;
+       qp->uacce_q = wd_q;
+       qp->event_cb = qm_qp_event_notifier;
+       qp->pasid = arg;
+
+       return 0;
+
+err_with_qp:
+       hisi_qm_release_qp(qp);
+       return ret;
+}
+
+static void hisi_qm_put_queue(struct uacce_queue *q)
+{
+       struct hisi_qp *qp = q->priv;
+
+       /* need to stop hardware, but can not support in v1 */
+       hisi_qm_release_qp(qp);
+       kfree(q);
+}
+
+/* map sq/cq/doorbell to user space */
+static int hisi_qm_mmap(struct uacce_queue *q,
+                       struct vm_area_struct *vma)
+{
+       struct hisi_qp *qp = (struct hisi_qp *)q->priv;
+       struct qm_info *qm = qp->qm;
+       size_t sz = vma->vm_end - vma->vm_start;
+       u8 region;
+
+       region = vma->vm_pgoff;
+
+       switch (region) {
+       case 0:
+               if (sz > PAGE_SIZE)
+                       return -EINVAL;
+
+               vma->vm_flags |= VM_IO;
+               /*
+                * Warning: This is not safe as multiple queues use the same
+                * doorbell, v1 hardware interface problem. will fix it in v2
+                */
+               return remap_pfn_range(vma, vma->vm_start,
+                                      qm->phys_base >> PAGE_SHIFT,
+                                      sz, pgprot_noncached(vma->vm_page_prot));
+
+       default:
+               return -EINVAL;
+       }
+}
+
+static int hisi_qm_start_queue(struct uacce_queue *q)
+{
+       int ret;
+       struct qm_info *qm = q->uacce->priv;
+       struct hisi_qp *qp = (struct hisi_qp *)q->priv;
+
+       /* todo: we don't need to start qm here in SVA version */
+       qm->qdma.dma = q->qfrs[UACCE_QFRT_DKO]->iova;
+       qm->qdma.va = q->qfrs[UACCE_QFRT_DKO]->kaddr;
+
+       ret = hisi_qm_start(qm);
+       if (ret)
+               return ret;
+
+       qp->qdma.dma = q->qfrs[UACCE_QFRT_DUS]->iova;
+       qp->qdma.va = q->qfrs[UACCE_QFRT_DUS]->kaddr;
+       ret = hisi_qm_start_qp(qp, qp->pasid);
+       if (ret)
+               hisi_qm_stop(qm);
+
+       return 0;
+}
+
+static void hisi_qm_stop_queue(struct uacce_queue *q)
+{
+       struct qm_info *qm = q->uacce->priv;
+
+       /* todo: we don't need to stop qm in SVA version */
+       hisi_qm_stop(qm);
+}
+
+/*
+ * the device is set the UACCE_DEV_SVA, but it will be cut if SVA patch is not
+ * available
+ */
+static struct uacce_ops uacce_qm_ops = {
+       .owner = THIS_MODULE,
+       .flags = UACCE_DEV_SVA | UACCE_DEV_KMAP_DUS,
+       .api_ver = "hisi_qm_v1",
+       .qf_pg_start = {QM_DOORBELL_PAGE_NR,
+                       QM_DOORBELL_PAGE_NR + QM_DKO_PAGE_NR,
+                       QM_DOORBELL_PAGE_NR + QM_DKO_PAGE_NR + QM_DUS_PAGE_NR},
+
+       .get_queue = hisi_qm_get_queue,
+       .put_queue = hisi_qm_put_queue,
+       .start_queue = hisi_qm_start_queue,
+       .stop_queue = hisi_qm_stop_queue,
+       .mmap = hisi_qm_mmap,
+};
+
+static int qm_register_uacce(struct qm_info *qm)
+{
+       struct pci_dev *pdev = qm->pdev;
+       struct uacce *uacce = &qm->uacce;
+
+       uacce->name = dev_name(&pdev->dev);
+       uacce->dev = &pdev->dev;
+       uacce->is_vf = pdev->is_virtfn;
+       uacce->priv = qm;
+       uacce->ops = &uacce_qm_ops;
+
+       return uacce_register(uacce);
+}
+#endif
+
 static irqreturn_t qm_irq(int irq, void *data)
 {
        struct qm_info *qm = data;
@@ -635,21 +792,34 @@ int hisi_qm_init(struct qm_info *qm)
                }
        }
 
-       qm->qdma.size = max_t(size_t, sizeof(struct eqc),
-                             sizeof(struct aeqc)) +
-                       sizeof(struct eqe) * QM_Q_DEPTH +
-                       sizeof(struct sqc) * qm->qp_num +
-                       sizeof(struct cqc) * qm->qp_num;
-       qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size,
-                                        &qm->qdma.dma,
-                                        GFP_KERNEL | __GFP_ZERO);
-       dev_dbg(dev, "allocate qm dma buf(va=%p, dma=%pad, size=%lx)\n",
-               qm->qdma.va, &qm->qdma.dma, qm->qdma.size);
-       ret = qm->qdma.va ? 0 : -ENOMEM;
+       if (qm->uacce_mode) {
+#ifdef CONFIG_CRYPTO_QM_UACCE
+               ret = qm_register_uacce(qm);
+#else
+               dev_err(dev, "qm uacce feature is not enabled\n");
+               ret = -EINVAL;
+#endif
+
+       } else {
+               qm->qdma.size = max_t(size_t, sizeof(struct eqc),
+                                     sizeof(struct aeqc)) +
+                               sizeof(struct eqe) * QM_Q_DEPTH +
+                               sizeof(struct sqc) * qm->qp_num +
+                               sizeof(struct cqc) * qm->qp_num;
+               qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size,
+                                                &qm->qdma.dma,
+                                                GFP_KERNEL | __GFP_ZERO);
+               dev_dbg(dev, "allocate qm dma buf(va=%p, dma=%pad, size=%lx)\n",
+                       qm->qdma.va, &qm->qdma.dma, qm->qdma.size);
+               ret = qm->qdma.va ? 0 : -ENOMEM;
+       }
 
        if (ret)
                goto err_with_irq;
 
+       dev_dbg(dev, "init qm %s to %s mode\n", pdev->is_physfn ? "pf" : "vf",
+               qm->uacce_mode ? "uacce" : "crypto");
+
        return 0;
 
 err_with_irq:
@@ -669,7 +839,13 @@ void hisi_qm_uninit(struct qm_info *qm)
 {
        struct pci_dev *pdev = qm->pdev;
 
-       dma_free_coherent(&pdev->dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma);
+       if (qm->uacce_mode) {
+#ifdef CONFIG_CRYPTO_QM_UACCE
+               uacce_unregister(&qm->uacce);
+#endif
+       } else
+               dma_free_coherent(&pdev->dev, qm->qdma.size, qm->qdma.va,
+                                 qm->qdma.dma);
 
        devm_free_irq(&pdev->dev, pci_irq_vector(pdev, 0), qm);
        pci_free_irq_vectors(pdev);
@@ -690,7 +866,7 @@ int hisi_qm_start(struct qm_info *qm)
 } while (0)
 
        if (!qm->qdma.va)
-               return -EINVAL;
+               return qm->uacce_mode ? 0 : -EINVAL;
 
        if (qm->pdev->is_physfn)
                qm->ops->vft_config(qm, qm->qp_base, qm->qp_num);
@@ -705,6 +881,13 @@ int hisi_qm_start(struct qm_info *qm)
        QM_INIT_BUF(qm, eqc,
                    max_t(size_t, sizeof(struct eqc), sizeof(struct aeqc)));
 
+       if (qm->uacce_mode) {
+               dev_dbg(&qm->pdev->dev, "kernel-only buffer used 
(0x%lx/0x%x)\n",
+                       off, QM_DKO_PAGE_NR << PAGE_SHIFT);
+               if (off > (QM_DKO_PAGE_NR << PAGE_SHIFT))
+                       return -EINVAL;
+       }
+
        qm->eqc->base_l = lower_32_bits(qm->eqe_dma);
        qm->eqc->base_h = upper_32_bits(qm->eqe_dma);
        qm->eqc->dw3 = 2 << MB_EQC_EQE_SHIFT;
diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h
index 6d124d948738..81b0b8c1f0b0 100644
--- a/drivers/crypto/hisilicon/qm.h
+++ b/drivers/crypto/hisilicon/qm.h
@@ -9,6 +9,10 @@
 #include <linux/slab.h>
 #include "qm_usr_if.h"
 
+#ifdef CONFIG_CRYPTO_QM_UACCE
+#include <linux/uacce.h>
+#endif
+
 /* qm user domain */
 #define QM_ARUSER_M_CFG_1              0x100088
 #define QM_ARUSER_M_CFG_ENABLE         0x100090
@@ -146,6 +150,12 @@ struct qm_info {
        struct mutex mailbox_lock;
 
        struct hisi_acc_qm_hw_ops *ops;
+
+       bool uacce_mode;
+
+#ifdef CONFIG_CRYPTO_QM_UACCE
+       struct uacce uacce;
+#endif
 };
 #define QM_ADDR(qm, off) ((qm)->io_base + off)
 
@@ -186,6 +196,10 @@ struct hisi_qp {
 
        struct qm_info *qm;
 
+#ifdef CONFIG_CRYPTO_QM_UACCE
+       struct uacce_queue *uacce_q;
+#endif
+
        /* for crypto sync API */
        struct completion completion;
 
@@ -197,7 +211,7 @@ struct hisi_qp {
 
 /* QM external interface for accelerator driver.
  * To use qm:
- * 1. Set qm with pdev, and sqe_size set accordingly
+ * 1. Set qm with pdev, uacce_mode, and sqe_size set accordingly
  * 2. hisi_qm_init()
  * 3. config the accelerator hardware
  * 4. hisi_qm_start()
diff --git a/drivers/crypto/hisilicon/zip/zip_main.c 
b/drivers/crypto/hisilicon/zip/zip_main.c
index f4f3b6d89340..f5fcd0f4b836 100644
--- a/drivers/crypto/hisilicon/zip/zip_main.c
+++ b/drivers/crypto/hisilicon/zip/zip_main.c
@@ -5,6 +5,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/pci.h>
+#include <linux/uacce.h>
 #include "zip.h"
 #include "zip_crypto.h"
 
@@ -28,6 +29,9 @@
 LIST_HEAD(hisi_zip_list);
 DEFINE_MUTEX(hisi_zip_list_lock);
 
+static bool uacce_mode;
+module_param(uacce_mode, bool, 0);
+
 static const char hisi_zip_name[] = "hisi_zip";
 
 static const struct pci_device_id hisi_zip_dev_ids[] = {
@@ -96,6 +100,7 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct 
pci_device_id *id)
 
        qm = &hisi_zip->qm;
        qm->pdev = pdev;
+       qm->uacce_mode = uacce_mode;
        qm->qp_base = HZIP_PF_DEF_Q_BASE;
        qm->qp_num = HZIP_PF_DEF_Q_NUM;
        qm->sqe_size = HZIP_SQE_SIZE;
@@ -150,10 +155,21 @@ static int __init hisi_zip_init(void)
                return ret;
        }
 
-       ret = hisi_zip_register_to_crypto();
-       if (ret < 0) {
-               pci_unregister_driver(&hisi_zip_pci_driver);
-               return ret;
+       /* todo:
+        *
+        * Before JPB's SVA patch is enabled, SMMU/IOMMU cannot support PASID.
+        * When it is accepted in the mainline kernel, we can add a
+        * IOMMU_DOMAIN_DAUL mode to IOMMU, then the dma and iommu API can
+        * work together. We then can let crypto and uacce mode works at the
+        * same time.
+        */
+       if (!uacce_mode) {
+               pr_debug("hisi_zip: init crypto mode\n");
+               ret = hisi_zip_register_to_crypto();
+               if (ret < 0) {
+                       pci_unregister_driver(&hisi_zip_pci_driver);
+                       return ret;
+               }
        }
 
        return 0;
@@ -161,7 +177,8 @@ static int __init hisi_zip_init(void)
 
 static void __exit hisi_zip_exit(void)
 {
-       hisi_zip_unregister_from_crypto();
+       if (!uacce_mode)
+               hisi_zip_unregister_from_crypto();
        pci_unregister_driver(&hisi_zip_pci_driver);
 }
 
-- 
2.17.1

Reply via email to