Am Freitag, 6. März 2015, 02:58:26 schrieb James Hartley: Hi James,
>This adds support for the Imagination Technologies hash >accelerator which provides hardware acceleration for >SHA1 SHA224 SHA256 and MD5 hashes. > >Signed-off-by: James Hartley <[email protected]> >--- > drivers/crypto/Kconfig | 14 + > drivers/crypto/Makefile | 2 + > drivers/crypto/img-hash.c | 1037 >+++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1053 >insertions(+) > create mode 100644 drivers/crypto/img-hash.c > >diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig >index 2fb0fdf..c72223e 100644 >--- a/drivers/crypto/Kconfig >+++ b/drivers/crypto/Kconfig >@@ -436,4 +436,18 @@ config CRYPTO_DEV_QCE > hardware. To compile this driver as a module, choose M here. The > module will be called qcrypto. > >+config CRYPTO_DEV_IMGTEC_HASH >+ depends on MIPS || COMPILE_TEST >+ tristate "Imagination Technologies hardware hash accelerator" >+ select CRYPTO_ALG_API >+ select CRYPTO_MD5 >+ select CRYPTO_SHA1 >+ select CRYPTO_SHA224 >+ select CRYPTO_SHA256 >+ select CRYPTO_HASH >+ help >+ This driver interfaces with the Imagination Technologies >+ hardware hash accelerator. Supporting MD5/SHA1/SHA224/SHA256 >+ hashing algorithms. >+ > endif # CRYPTO_HW >diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile >index 3924f93..fb84be7 100644 >--- a/drivers/crypto/Makefile >+++ b/drivers/crypto/Makefile >@@ -6,6 +6,7 @@ obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ > obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/ > obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o > obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o >+obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o > obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o > obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o > obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o >@@ -25,3 +26,4 @@ obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o > obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/ > obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/ > obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/ >+obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/ >diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c >new file mode 100644 >index 0000000..94a3a6f >--- /dev/null >+++ b/drivers/crypto/img-hash.c >@@ -0,0 +1,1037 @@ >+/* >+ * Copyright (c) 2014 Imagination Technologies >+ * Authors: Will Thomas, James Hartley >+ * >+ * This program is free software; you can redistribute it and/or >modify + * it under the terms of the GNU General Public License >version 2 as published + * by the Free Software Foundation. >+ * >+ * Interface structure taken from omap-sham driver >+ */ >+ >+#include <linux/clk.h> >+#include <linux/dmaengine.h> >+#include <linux/interrupt.h> >+#include <linux/io.h> >+#include <linux/kernel.h> >+#include <linux/module.h> >+#include <linux/of_device.h> >+#include <linux/platform_device.h> >+#include <linux/scatterlist.h> >+ >+#include <crypto/internal/hash.h> >+#include <crypto/md5.h> >+#include <crypto/sha.h> >+ >+#define CR_RESET 0 >+#define CR_RESET_SET 1 >+#define CR_RESET_UNSET 0 >+ >+#define CR_MESSAGE_LENGTH_H 0x4 >+#define CR_MESSAGE_LENGTH_L 0x8 >+ >+#define CR_CONTROL 0xc >+#define CR_CONTROL_BYTE_ORDER_3210 0 >+#define CR_CONTROL_BYTE_ORDER_0123 1 >+#define CR_CONTROL_BYTE_ORDER_2310 2 >+#define CR_CONTROL_BYTE_ORDER_1032 3 >+#define CR_CONTROL_BYTE_ORDER_SHIFT 8 >+#define CR_CONTROL_ALGO_MD5 0 >+#define CR_CONTROL_ALGO_SHA1 1 >+#define CR_CONTROL_ALGO_SHA224 2 >+#define CR_CONTROL_ALGO_SHA256 3 >+ >+#define CR_INTSTAT 0x10 >+#define CR_INTENAB 0x14 >+#define CR_INTCLEAR 0x18 >+#define CR_INT_RESULTS_AVAILABLE BIT(0) >+#define CR_INT_NEW_RESULTS_SET BIT(1) >+#define CR_INT_RESULT_READ_ERR BIT(2) >+#define CR_INT_MESSAGE_WRITE_ERROR BIT(3) >+#define CR_INT_STATUS BIT(8) >+ >+#define CR_RESULT_QUEUE 0x1c >+#define CR_RSD0 0x40 >+#define CR_CORE_REV 0x50 >+#define CR_CORE_DES1 0x60 >+#define CR_CORE_DES2 0x70 >+ >+#define DRIVER_FLAGS_BUSY BIT(0) >+#define DRIVER_FLAGS_FINAL BIT(1) >+#define DRIVER_FLAGS_DMA_ACTIVE BIT(2) >+#define DRIVER_FLAGS_OUTPUT_READY BIT(3) >+#define DRIVER_FLAGS_INIT BIT(4) >+#define DRIVER_FLAGS_CPU BIT(5) >+#define DRIVER_FLAGS_DMA_READY BIT(6) >+#define DRIVER_FLAGS_ERROR BIT(7) >+#define DRIVER_FLAGS_SG BIT(8) >+#define DRIVER_FLAGS_SHA1 BIT(18) >+#define DRIVER_FLAGS_SHA224 BIT(19) >+#define DRIVER_FLAGS_SHA256 BIT(20) >+#define DRIVER_FLAGS_MD5 BIT(21) >+ >+#define IMG_HASH_QUEUE_LENGTH 20 >+#define IMG_HASH_DMA_THRESHOLD 64 >+ >+#ifdef __LITTLE_ENDIAN >+#define IMG_HASH_BYTE_ORDER (CR_CONTROL_BYTE_ORDER_3210) >+#else >+#define IMG_HASH_BYTE_ORDER (CR_CONTROL_BYTE_ORDER_0123) >+#endif >+ >+struct img_hash_dev; >+ >+struct img_hash_request_ctx { >+ struct img_hash_dev *hdev; >+ u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32)); >+ unsigned long flags; >+ size_t digsize; >+ >+ dma_addr_t dma_addr; >+ size_t dma_ct; >+ >+ /* sg root */ >+ struct scatterlist *sgfirst; >+ /* walk state */ >+ struct scatterlist *sg; >+ size_t nents; >+ size_t offset; >+ unsigned int total; >+ size_t sent; >+ >+ unsigned long op; >+ >+ size_t bufcnt; >+ u8 buffer[0] __aligned(sizeof(u32)); >+ struct ahash_request fallback_req; >+}; >+ >+struct img_hash_ctx { >+ struct img_hash_dev *hdev; >+ unsigned long flags; >+ struct crypto_ahash *fallback; >+}; >+ >+struct img_hash_dev { >+ struct list_head list; >+ struct device *dev; >+ struct clk *iclk; >+ struct clk *fclk; >+ int irq; >+ void __iomem *io_base; >+ >+ phys_addr_t bus_addr; >+ void __iomem *cpu_addr; >+ >+ spinlock_t lock; >+ int err; >+ struct tasklet_struct done_task; >+ struct tasklet_struct dma_task; >+ >+ unsigned long flags; >+ struct crypto_queue queue; >+ struct ahash_request *req; >+ >+ struct dma_chan *dma_lch; >+}; >+ >+struct img_hash_drv { >+ struct list_head dev_list; >+ spinlock_t lock; >+}; >+ >+static struct img_hash_drv img_hash = { >+ .dev_list = LIST_HEAD_INIT(img_hash.dev_list), >+ .lock = __SPIN_LOCK_UNLOCKED(img_hash.lock), >+}; >+ >+static inline u32 img_hash_read(struct img_hash_dev *hdev, u32 offset) >+{ >+ return readl_relaxed(hdev->io_base + offset); >+} >+ >+static inline void img_hash_write(struct img_hash_dev *hdev, >+ u32 offset, u32 value) >+{ >+ writel_relaxed(value, hdev->io_base + offset); >+} >+ >+static inline u32 img_hash_read_result_queue(struct img_hash_dev >*hdev) +{ >+ return be32_to_cpu(img_hash_read(hdev, CR_RESULT_QUEUE)); >+} >+ >+static void img_hash_start(struct img_hash_dev *hdev, bool dma) >+{ >+ struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); >+ u32 cr = IMG_HASH_BYTE_ORDER << CR_CONTROL_BYTE_ORDER_SHIFT; >+ >+ if (ctx->flags & DRIVER_FLAGS_MD5) >+ cr |= CR_CONTROL_ALGO_MD5; >+ else if (ctx->flags & DRIVER_FLAGS_SHA1) >+ cr |= CR_CONTROL_ALGO_SHA1; >+ else if (ctx->flags & DRIVER_FLAGS_SHA224) >+ cr |= CR_CONTROL_ALGO_SHA224; >+ else if (ctx->flags & DRIVER_FLAGS_SHA256) >+ cr |= CR_CONTROL_ALGO_SHA256; >+ dev_dbg(hdev->dev, "Starting hash process\n"); >+ img_hash_write(hdev, CR_CONTROL, cr); >+ >+ /* >+ * The hardware block requires two cycles between writing the control >+ * register and writing the first word of data in non DMA mode, to >+ * ensure the first data write is not grouped in burst with the >control + * register write a read is issued to 'flush' the bus. >+ */ >+ if (!dma) >+ img_hash_read(hdev, CR_CONTROL); >+} >+ >+static int img_hash_xmit_cpu(struct img_hash_dev *hdev, const u8 *buf, >+ size_t length, int final) >+{ >+ u32 count, len32; >+ const u32 *buffer = (const u32 *)buf; >+ >+ dev_dbg(hdev->dev, "xmit_cpu: length: %u bytes\n", length); >+ >+ if (final) >+ hdev->flags |= DRIVER_FLAGS_FINAL; >+ >+ len32 = DIV_ROUND_UP(length, sizeof(u32)); >+ >+ for (count = 0; count < len32; count++) >+ writel_relaxed(buffer[count], hdev->cpu_addr); >+ >+ return -EINPROGRESS; >+} >+ >+static void img_hash_dma_callback(void *data) >+{ >+ struct img_hash_dev *hdev = (struct img_hash_dev *)data; >+ struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); >+ >+ if (ctx->bufcnt) { >+ img_hash_xmit_cpu(hdev, ctx->buffer, ctx->bufcnt, 0); >+ ctx->bufcnt = 0; >+ } >+ if (ctx->sg) >+ tasklet_schedule(&hdev->dma_task); >+} >+ >+static int img_hash_xmit_dma(struct img_hash_dev *hdev, struct >scatterlist *sg) +{ >+ struct dma_async_tx_descriptor *desc; >+ struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); >+ >+ ctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, DMA_MEM_TO_DEV); >+ if (ctx->dma_ct == 0) { >+ dev_err(hdev->dev, "Invalid DMA sg\n"); >+ hdev->err = -EINVAL; >+ return -EINVAL; >+ } >+ >+ desc = dmaengine_prep_slave_sg(hdev->dma_lch, >+ sg, >+ ctx->dma_ct, >+ DMA_MEM_TO_DEV, >+ DMA_PREP_INTERRUPT | DMA_CTRL_ACK); >+ if (!desc) { >+ dev_err(hdev->dev, "Null DMA descriptor\n"); >+ hdev->err = -EINVAL; >+ dma_unmap_sg(hdev->dev, sg, 1, DMA_MEM_TO_DEV); >+ return -EINVAL; >+ } >+ desc->callback = img_hash_dma_callback; >+ desc->callback_param = hdev; >+ dmaengine_submit(desc); >+ dma_async_issue_pending(hdev->dma_lch); >+ >+ return 0; >+} >+ >+static int img_hash_write_via_cpu(struct img_hash_dev *hdev) >+{ >+ struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); >+ int ret = 0; >+ >+ ctx->bufcnt = sg_copy_to_buffer(hdev->req->src, sg_nents(ctx- >sg), >+ ctx->buffer, hdev->req->nbytes); >+ >+ ctx->total = hdev->req->nbytes; >+ ctx->bufcnt = 0; >+ >+ hdev->flags |= (DRIVER_FLAGS_CPU | DRIVER_FLAGS_FINAL); >+ >+ img_hash_start(hdev, false); >+ >+ if (ctx->total) >+ ret = img_hash_xmit_cpu(hdev, ctx->buffer, ctx->total, 1); >+ else >+ ret = 0; >+ >+ return ret; >+} >+ >+static int img_hash_finish(struct ahash_request *req) >+{ >+ struct img_hash_request_ctx *ctx = ahash_request_ctx(req); >+ >+ if (!req->result) >+ return -EINVAL; >+ >+ memcpy(req->result, ctx->digest, ctx->digsize); >+ >+ return 0; >+} >+ >+static void img_hash_copy_hash(struct ahash_request *req) >+{ >+ struct img_hash_request_ctx *ctx = ahash_request_ctx(req); >+ u32 *hash = (u32 *)ctx->digest; >+ int i; >+ >+ for (i = (ctx->digsize / sizeof(u32)) - 1; i >= 0; i--) >+ hash[i] = img_hash_read_result_queue(ctx->hdev); >+} >+ >+static void img_hash_finish_req(struct ahash_request *req, int err) >+{ >+ struct img_hash_request_ctx *ctx = ahash_request_ctx(req); >+ struct img_hash_dev *hdev = ctx->hdev; >+ >+ if (!err) { >+ img_hash_copy_hash(req); >+ if (DRIVER_FLAGS_FINAL & hdev->flags) >+ err = img_hash_finish(req); >+ } else { >+ dev_warn(hdev->dev, "Hash failed with error %d\n", err); >+ ctx->flags |= DRIVER_FLAGS_ERROR; >+ } >+ >+ hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | DRIVER_FLAGS_OUTPUT_READY | >+ DRIVER_FLAGS_CPU | DRIVER_FLAGS_BUSY | DRIVER_FLAGS_FINAL); + >+ if (req->base.complete) >+ req->base.complete(&req->base, err); >+} >+ >+static int img_hash_write_via_dma(struct img_hash_dev *hdev) >+{ >+ struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); >+ >+ img_hash_start(hdev, true); >+ >+ dev_dbg(hdev->dev, "xmit dma size: %d\n", ctx->total); >+ >+ if (!ctx->total) >+ hdev->flags |= DRIVER_FLAGS_FINAL; >+ >+ hdev->flags |= DRIVER_FLAGS_DMA_ACTIVE | DRIVER_FLAGS_FINAL; >+ >+ tasklet_schedule(&hdev->dma_task); >+ >+ return -EINPROGRESS; >+} >+ >+static int img_hash_dma_init(struct img_hash_dev *hdev) >+{ >+ struct dma_slave_config dma_conf; >+ int err = -EINVAL; >+ >+ hdev->dma_lch = dma_request_slave_channel(hdev->dev, "tx"); >+ if (!hdev->dma_lch) { >+ dev_err(hdev->dev, "Couldn't aquire a slave DMA channel. \n"); >+ return -EBUSY; >+ } >+ dma_conf.direction = DMA_MEM_TO_DEV; >+ dma_conf.dst_addr = hdev->bus_addr; >+ dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; >+ dma_conf.dst_maxburst = 16; >+ dma_conf.device_fc = false; >+ >+ err = dmaengine_slave_config(hdev->dma_lch, &dma_conf); >+ >+ if (err) { >+ dev_err(hdev->dev, "Couldn't configure DMA slave.\n"); >+ dma_release_channel(hdev->dma_lch); >+ return err; >+ } >+ return 0; >+} >+ >+static void img_hash_dma_task(unsigned long d) >+{ >+ struct img_hash_dev *hdev = (struct img_hash_dev *)d; >+ struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); >+ u8 *addr; >+ size_t nbytes, bleft, wsend, len, tbc; >+ struct scatterlist tsg; >+ >+ if (!ctx->sg) >+ return; >+ >+ addr = sg_virt(ctx->sg); >+ nbytes = ctx->sg->length - ctx->offset; >+ >+ /* The hash accelerator does not support a data valid mask. This >means + * that if each dma (i.e. per page) is not a multiple of 4 >bytes, the + * padding bytes in the last word written by that dma >would erroneously + * be included in the hash. To avoid this we round >down the transfer, + * and add the excess to the start of the next >dma. It does not matter + * that the final dma may not be a multiple >of 4 bytes as the hashing + * block is programmed to accept the >correct number of bytes. + */ >+ >+ bleft = nbytes % 4; >+ wsend = (nbytes / 4); >+ >+ if (wsend) { >+ sg_init_one(&tsg, addr + ctx->offset, wsend * 4); >+ if (img_hash_xmit_dma(hdev, &tsg)) { >+ dev_err(hdev->dev, "DMA failed, falling back to CPU"); >+ ctx->flags |= DRIVER_FLAGS_CPU; >+ hdev->err = 0; >+ img_hash_xmit_cpu(hdev, addr + ctx->offset, >+ wsend * 4, 0); >+ ctx->sent += wsend * 4; >+ wsend = 0; >+ } else { >+ ctx->sent += wsend * 4; >+ } >+ } >+ >+ if (bleft) { >+ ctx->bufcnt = sg_pcopy_to_buffer(ctx->sgfirst, ctx- >nents, >+ ctx->buffer, bleft, ctx->sent); >+ tbc = 0; >+ ctx->sg = sg_next(ctx->sg); >+ while (ctx->sg && (ctx->bufcnt < 4)) { >+ len = ctx->sg->length; >+ if (likely(len > (4 - ctx->bufcnt))) >+ len = 4 - ctx->bufcnt; >+ tbc = sg_pcopy_to_buffer(ctx->sgfirst, ctx- >nents, >+ ctx->buffer + ctx- >bufcnt, len, >+ ctx->sent + ctx->bufcnt); >+ ctx->bufcnt += tbc; >+ if (tbc >= ctx->sg->length) { >+ ctx->sg = sg_next(ctx->sg); >+ tbc = 0; >+ } >+ } >+ >+ ctx->sent += ctx->bufcnt; >+ ctx->offset = tbc; >+ >+ if (!wsend) >+ img_hash_dma_callback(hdev); >+ } else { >+ ctx->offset = 0; >+ ctx->sg = sg_next(ctx->sg); >+ } >+} >+ >+static int img_hash_write_via_dma_stop(struct img_hash_dev *hdev) >+{ >+ struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req); >+ >+ if (ctx->flags & DRIVER_FLAGS_SG) >+ dma_unmap_sg(hdev->dev, ctx->sg, ctx->dma_ct, DMA_TO_DEVICE); >+ >+ return 0; >+} >+ >+static int img_hash_process_data(struct img_hash_dev *hdev) >+{ >+ struct ahash_request *req = hdev->req; >+ struct img_hash_request_ctx *ctx = ahash_request_ctx(req); >+ int err = 0; >+ >+ ctx->bufcnt = 0; >+ >+ if (req->nbytes >= IMG_HASH_DMA_THRESHOLD) { >+ dev_dbg(hdev->dev, "process data request(%d bytes) using DMA\n", >+ req->nbytes); >+ err = img_hash_write_via_dma(hdev); >+ } else { >+ dev_dbg(hdev->dev, "process data request(%d bytes) using CPU\n", >+ req->nbytes); >+ err = img_hash_write_via_cpu(hdev); >+ } >+ return err; >+} >+ >+static int img_hash_hw_init(struct img_hash_dev *hdev) >+{ >+ unsigned long long nbits; >+ u32 u, l; >+ int ret; >+ >+ ret = clk_prepare_enable(hdev->iclk); >+ if (ret) >+ return ret; >+ >+ img_hash_write(hdev, CR_RESET, CR_RESET_SET); >+ img_hash_write(hdev, CR_RESET, CR_RESET_UNSET); >+ img_hash_write(hdev, CR_INTENAB, CR_INT_NEW_RESULTS_SET); >+ >+ nbits = (hdev->req->nbytes << 3); >+ u = nbits >> 32; >+ l = nbits; >+ img_hash_write(hdev, CR_MESSAGE_LENGTH_H, u); >+ img_hash_write(hdev, CR_MESSAGE_LENGTH_L, l); >+ >+ if (!(DRIVER_FLAGS_INIT & hdev->flags)) { >+ hdev->flags |= DRIVER_FLAGS_INIT; >+ hdev->err = 0; >+ } >+ dev_dbg(hdev->dev, "hw initialized, nbits: %llx\n", nbits); >+ return 0; >+} >+ >+static int img_hash_init(struct ahash_request *req) >+{ >+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); >+ struct img_hash_request_ctx *rctx = ahash_request_ctx(req); >+ struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); >+ >+ ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); >+ rctx->fallback_req.base.flags = req->base.flags >+ & CRYPTO_TFM_REQ_MAY_SLEEP; >+ >+ return crypto_ahash_init(&rctx->fallback_req); >+} >+ >+static int img_hash_handle_queue(struct img_hash_dev *hdev, >+ struct ahash_request *req) >+{ >+ struct crypto_async_request *async_req, *backlog; >+ struct img_hash_request_ctx *ctx; >+ unsigned long flags; >+ int err = 0, res = 0; >+ >+ spin_lock_irqsave(&hdev->lock, flags); >+ >+ if (req) >+ res = ahash_enqueue_request(&hdev->queue, req); >+ >+ if (DRIVER_FLAGS_BUSY & hdev->flags) { >+ spin_unlock_irqrestore(&hdev->lock, flags); >+ return res; >+ } >+ >+ backlog = crypto_get_backlog(&hdev->queue); >+ async_req = crypto_dequeue_request(&hdev->queue); >+ if (async_req) >+ hdev->flags |= DRIVER_FLAGS_BUSY; >+ >+ spin_unlock_irqrestore(&hdev->lock, flags); >+ >+ if (!async_req) >+ return res; >+ >+ if (backlog) >+ backlog->complete(backlog, -EINPROGRESS); >+ >+ req = ahash_request_cast(async_req); >+ hdev->req = req; >+ >+ ctx = ahash_request_ctx(req); >+ >+ dev_info(hdev->dev, "processing req, op: %lu, bytes: %d\n", >+ ctx->op, req->nbytes); >+ >+ err = img_hash_hw_init(hdev); >+ >+ if (!err) >+ err = img_hash_process_data(hdev); >+ >+ if (err != -EINPROGRESS) { >+ /* done_task will not finish so do it here */ >+ img_hash_finish_req(req, err); >+ } >+ return res; >+} >+ >+static int img_hash_update(struct ahash_request *req) >+{ >+ struct img_hash_request_ctx *rctx = ahash_request_ctx(req); >+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); >+ struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); >+ >+ ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); >+ rctx->fallback_req.base.flags = req->base.flags >+ & CRYPTO_TFM_REQ_MAY_SLEEP; >+ rctx->fallback_req.nbytes = req->nbytes; >+ rctx->fallback_req.src = req->src; >+ >+ return crypto_ahash_update(&rctx->fallback_req); >+} >+ >+static int img_hash_final(struct ahash_request *req) >+{ >+ struct img_hash_request_ctx *rctx = ahash_request_ctx(req); >+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); >+ struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); >+ >+ ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); >+ rctx->fallback_req.base.flags = req->base.flags >+ & CRYPTO_TFM_REQ_MAY_SLEEP; >+ rctx->fallback_req.result = req->result; >+ >+ return crypto_ahash_final(&rctx->fallback_req); >+} >+ >+static int img_hash_finup(struct ahash_request *req) >+{ >+ struct img_hash_request_ctx *rctx = ahash_request_ctx(req); >+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); >+ struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm); >+ >+ ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback); >+ rctx->fallback_req.base.flags = req->base.flags >+ & CRYPTO_TFM_REQ_MAY_SLEEP; >+ rctx->fallback_req.nbytes = req->nbytes; >+ rctx->fallback_req.src = req->src; >+ rctx->fallback_req.result = req->result; >+ >+ return crypto_ahash_finup(&rctx->fallback_req); >+} >+ >+static int img_hash_digest(struct ahash_request *req) >+{ >+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); >+ struct img_hash_ctx *tctx = crypto_ahash_ctx(tfm); >+ struct img_hash_request_ctx *ctx = ahash_request_ctx(req); >+ struct img_hash_dev *hdev = NULL; >+ struct img_hash_dev *tmp; >+ int err; >+ >+ spin_lock_bh(&img_hash.lock); >+ if (!tctx->hdev) { >+ list_for_each_entry(tmp, &img_hash.dev_list, list) { >+ hdev = tmp; >+ break; >+ } >+ tctx->hdev = hdev; >+ >+ } else { >+ hdev = tctx->hdev; >+ } >+ >+ spin_unlock_bh(&img_hash.lock); >+ ctx->hdev = hdev; >+ ctx->flags = 0; >+ ctx->digsize = crypto_ahash_digestsize(tfm); >+ >+ switch (ctx->digsize) { >+ case SHA1_DIGEST_SIZE: >+ ctx->flags |= DRIVER_FLAGS_SHA1; >+ break; >+ case SHA256_DIGEST_SIZE: >+ ctx->flags |= DRIVER_FLAGS_SHA256; >+ break; >+ case SHA224_DIGEST_SIZE: >+ ctx->flags |= DRIVER_FLAGS_SHA224; >+ break; >+ case MD5_DIGEST_SIZE: >+ ctx->flags |= DRIVER_FLAGS_MD5; >+ break; >+ default: >+ return -EINVAL; >+ } >+ >+ ctx->bufcnt = 0; >+ ctx->offset = 0; >+ ctx->sent = 0; >+ ctx->total = req->nbytes; >+ ctx->sg = req->src; >+ ctx->sgfirst = req->src; >+ ctx->nents = sg_nents(ctx->sg); >+ >+ err = img_hash_handle_queue(tctx->hdev, req); >+ >+ return err; >+} >+ >+static int img_hash_cra_init(struct crypto_tfm *tfm) >+{ >+ struct img_hash_ctx *ctx = crypto_tfm_ctx(tfm); >+ const char *alg_name = crypto_tfm_alg_name(tfm); >+ int err = -ENOMEM; >+ >+ ctx->fallback = crypto_alloc_ahash(alg_name, 0, >+ CRYPTO_ALG_NEED_FALLBACK); >+ if (IS_ERR(ctx->fallback)) { >+ pr_err("img_hash: Could not load fallback driver.\n"); >+ err = PTR_ERR(ctx->fallback); >+ goto err; >+ } >+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), >+ sizeof(struct img_hash_request_ctx) + >+ IMG_HASH_DMA_THRESHOLD); >+ >+ return 0; >+ >+err: >+ return err; >+} >+ >+static void img_hash_cra_exit(struct crypto_tfm *tfm) >+{ >+ struct img_hash_ctx *tctx = crypto_tfm_ctx(tfm); >+ >+ crypto_free_ahash(tctx->fallback); >+} >+ >+static irqreturn_t img_irq_handler(int irq, void *dev_id) >+{ >+ struct img_hash_dev *hdev = dev_id; >+ u32 reg; >+ >+ reg = img_hash_read(hdev, CR_INTSTAT); >+ img_hash_write(hdev, CR_INTCLEAR, reg); >+ >+ if (reg & CR_INT_NEW_RESULTS_SET) { >+ dev_dbg(hdev->dev, "IRQ CR_INT_NEW_RESULTS_SET\n"); >+ if (DRIVER_FLAGS_BUSY & hdev->flags) { >+ hdev->flags |= DRIVER_FLAGS_OUTPUT_READY; >+ if (!(DRIVER_FLAGS_CPU & hdev->flags)) >+ hdev->flags |= DRIVER_FLAGS_DMA_READY; >+ tasklet_schedule(&hdev->done_task); >+ } else { >+ dev_warn(hdev->dev, >+ "HASH interrupt when no active requests.\n"); >+ } >+ } else if (reg & CR_INT_RESULTS_AVAILABLE) { >+ dev_warn(hdev->dev, >+ "IRQ triggered before the hash had completed\n"); >+ } else if (reg & CR_INT_RESULT_READ_ERR) { >+ dev_warn(hdev->dev, >+ "Attempt to read from an empty result queue\n"); >+ } else if (reg & CR_INT_MESSAGE_WRITE_ERROR) { >+ dev_warn(hdev->dev, >+ "Data written before the hardware was configured\n"); >+ } >+ return IRQ_HANDLED; >+} >+ >+static struct ahash_alg img_algs[] = { >+ { >+ .init = img_hash_init, >+ .update = img_hash_update, >+ .final = img_hash_final, >+ .finup = img_hash_finup, >+ .digest = img_hash_digest, >+ .halg = { >+ .digestsize = MD5_DIGEST_SIZE, >+ .base = { >+ .cra_name = "md5", >+ .cra_driver_name = "img-md5", >+ .cra_priority = 301, Just curious: why do you use such odd priorities of 301 or 3000? IMHO, all you need is a priority of more than 100 to "beat" the generic C prios. Maybe you also need to beat the standard assembler implementations which are routinely at 200 for hashes. So, a prio of 300 should suffice, should it not? >+ .cra_flags = >+ CRYPTO_ALG_ASYNC | >+ CRYPTO_ALG_NEED_FALLBACK, >+ .cra_blocksize = MD5_HMAC_BLOCK_SIZE, >+ .cra_ctxsize = sizeof(struct img_hash_ctx), >+ .cra_init = img_hash_cra_init, >+ .cra_exit = img_hash_cra_exit, >+ .cra_module = THIS_MODULE, >+ } >+ } >+ }, >+ { >+ .init = img_hash_init, >+ .update = img_hash_update, >+ .final = img_hash_final, >+ .finup = img_hash_finup, >+ .digest = img_hash_digest, >+ .halg = { >+ .digestsize = SHA1_DIGEST_SIZE, >+ .base = { >+ .cra_name = "sha1", >+ .cra_driver_name = "img-sha1", >+ .cra_priority = 3000, >+ .cra_flags = >+ CRYPTO_ALG_ASYNC | >+ CRYPTO_ALG_NEED_FALLBACK, >+ .cra_blocksize = SHA1_BLOCK_SIZE, >+ .cra_ctxsize = sizeof(struct img_hash_ctx), >+ .cra_init = img_hash_cra_init, >+ .cra_exit = img_hash_cra_exit, >+ .cra_module = THIS_MODULE, >+ } >+ } >+ }, >+ { >+ .init = img_hash_init, >+ .update = img_hash_update, >+ .final = img_hash_final, >+ .finup = img_hash_finup, >+ .digest = img_hash_digest, >+ .halg = { >+ .digestsize = SHA224_DIGEST_SIZE, >+ .base = { >+ .cra_name = "sha224", >+ .cra_driver_name = "img-sha224", >+ .cra_priority = 3000, >+ .cra_flags = >+ CRYPTO_ALG_ASYNC | >+ CRYPTO_ALG_NEED_FALLBACK, >+ .cra_blocksize = SHA224_BLOCK_SIZE, >+ .cra_ctxsize = sizeof(struct img_hash_ctx), >+ .cra_init = img_hash_cra_init, >+ .cra_exit = img_hash_cra_exit, >+ .cra_module = THIS_MODULE, >+ } >+ } >+ }, >+ { >+ .init = img_hash_init, >+ .update = img_hash_update, >+ .final = img_hash_final, >+ .finup = img_hash_finup, >+ .digest = img_hash_digest, >+ .halg = { >+ .digestsize = SHA256_DIGEST_SIZE, >+ .base = { >+ .cra_name = "sha256", >+ .cra_driver_name = "img-sha256", >+ .cra_priority = 301, >+ .cra_flags = >+ CRYPTO_ALG_ASYNC | >+ CRYPTO_ALG_NEED_FALLBACK, >+ .cra_blocksize = SHA256_BLOCK_SIZE, >+ .cra_ctxsize = sizeof(struct img_hash_ctx), >+ .cra_init = img_hash_cra_init, >+ .cra_exit = img_hash_cra_exit, >+ .cra_module = THIS_MODULE, >+ } >+ } >+ } >+}; >+ >+static int img_register_algs(struct img_hash_dev *hdev) >+{ >+ int i, err; >+ >+ for (i = 0; i < ARRAY_SIZE(img_algs); i++) { >+ err = crypto_register_ahash(&img_algs[i]); >+ if (err) >+ goto err_reg; >+ } >+ return 0; >+ >+err_reg: >+ for (; i--; ) >+ crypto_unregister_ahash(&img_algs[i]); >+ >+ return err; >+} >+ >+static int img_unregister_algs(struct img_hash_dev *hdev) >+{ >+ int i; >+ >+ for (i = 0; i < ARRAY_SIZE(img_algs); i++) >+ crypto_unregister_ahash(&img_algs[i]); >+ return 0; >+} >+ >+static void img_hash_done_task(unsigned long data) >+{ >+ struct img_hash_dev *hdev = (struct img_hash_dev *)data; >+ int err = 0; >+ >+ if (hdev->err == -EINVAL) { >+ err = hdev->err; >+ goto finish; >+ } >+ >+ if (!(DRIVER_FLAGS_BUSY & hdev->flags)) { >+ img_hash_handle_queue(hdev, NULL); >+ return; >+ } >+ >+ if (DRIVER_FLAGS_CPU & hdev->flags) { >+ if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) { >+ hdev->flags &= ~DRIVER_FLAGS_OUTPUT_READY; >+ goto finish; >+ } >+ } else if (DRIVER_FLAGS_DMA_READY & hdev->flags) { >+ if (DRIVER_FLAGS_DMA_ACTIVE & hdev->flags) { >+ hdev->flags &= ~DRIVER_FLAGS_DMA_ACTIVE; >+ img_hash_write_via_dma_stop(hdev); >+ if (hdev->err) { >+ err = hdev->err; >+ goto finish; >+ } >+ } >+ if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) { >+ hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | >+ DRIVER_FLAGS_OUTPUT_READY); >+ goto finish; >+ } >+ } >+ return; >+ >+finish: >+ img_hash_finish_req(hdev->req, err); >+} >+ >+static const struct of_device_id img_hash_match[] = { >+ { .compatible = "img,hash-accelerator" }, >+ {} >+}; >+MODULE_DEVICE_TABLE(of, img_hash_match) >+ >+static int img_hash_probe(struct platform_device *pdev) >+{ >+ struct img_hash_dev *hdev; >+ struct device *dev = &pdev->dev; >+ struct resource *hash_res; >+ int err; >+ >+ hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL); >+ if (hdev == NULL) { >+ err = -ENOMEM; >+ goto sha_dev_err; >+ } >+ spin_lock_init(&hdev->lock); >+ >+ hdev->dev = dev; >+ >+ platform_set_drvdata(pdev, hdev); >+ >+ INIT_LIST_HEAD(&hdev->list); >+ >+ tasklet_init(&hdev->done_task, img_hash_done_task, (unsigned >long)hdev); + tasklet_init(&hdev->dma_task, img_hash_dma_task, >(unsigned long)hdev); + >+ crypto_init_queue(&hdev->queue, IMG_HASH_QUEUE_LENGTH); >+ >+ /* Register bank */ >+ hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); >+ >+ hdev->io_base = devm_ioremap_resource(dev, hash_res); >+ if (IS_ERR(hdev->io_base)) { >+ dev_err(dev, "can't ioremap\n"); >+ err = PTR_ERR(hdev->io_base); >+ goto hash_io_err; >+ } >+ >+ /* Write port (DMA or CPU) */ >+ hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 1); >+ if (!hash_res) { >+ dev_err(dev, "no write port resource info\n"); >+ err = -ENODEV; >+ goto res_err; >+ } >+ hdev->bus_addr = hash_res->start; >+ hdev->cpu_addr = devm_ioremap_resource(dev, hash_res); >+ if (IS_ERR(hdev->cpu_addr)) { >+ dev_err(dev, "can't ioremap write port\n"); >+ err = PTR_ERR(hdev->cpu_addr); >+ goto res_err; >+ } >+ >+ hdev->irq = platform_get_irq(pdev, 0); >+ if (hdev->irq < 0) { >+ dev_err(dev, "no IRQ resource info\n"); >+ err = hdev->irq; >+ goto res_err; >+ } >+ >+ err = devm_request_irq(dev, hdev->irq, img_irq_handler, 0, >+ dev_name(dev), hdev); >+ if (err) { >+ dev_err(dev, "unable to request %s irq\n", dev_name(dev)); >+ goto res_err; >+ } >+ dev_dbg(dev, "using IRQ channel %d\n", hdev->irq); >+ >+ hdev->iclk = devm_clk_get(&pdev->dev, "hash_clk"); >+ if (IS_ERR(hdev->iclk)) { >+ dev_err(dev, "clock initialization failed.\n"); >+ err = PTR_ERR(hdev->iclk); >+ goto res_err; >+ } >+ >+ hdev->fclk = devm_clk_get(&pdev->dev, "hash_reg_clk"); >+ if (IS_ERR(hdev->fclk)) { >+ dev_err(dev, "clock initialization failed.\n"); >+ err = PTR_ERR(hdev->fclk); >+ goto res_err; >+ } >+ >+ err = img_hash_dma_init(hdev); >+ if (err) >+ goto err_dma; >+ >+ dev_dbg(dev, "using %s for DMA transfers\n", >+ dma_chan_name(hdev->dma_lch)); >+ >+ spin_lock(&img_hash.lock); >+ list_add_tail(&hdev->list, &img_hash.dev_list); >+ spin_unlock(&img_hash.lock); >+ >+ err = img_register_algs(hdev); >+ if (err) >+ goto err_algs; >+ dev_info(dev, "data bus: 0x%x;\tsize:0x%x\n", hdev->bus_addr, 0x4); >+ dev_info(dev, "Img MD5/SHA1/SHA224/SHA256 Hardware accelerator >initialized\n"); + >+ return 0; >+ >+err_algs: >+ spin_lock(&img_hash.lock); >+ list_del(&hdev->list); >+ spin_unlock(&img_hash.lock); >+ dma_release_channel(hdev->dma_lch); >+err_dma: >+hash_io_err: >+ devm_clk_put(dev, hdev->iclk); >+ devm_clk_put(dev, hdev->fclk); >+res_err: >+ tasklet_kill(&hdev->done_task); >+ tasklet_kill(&hdev->dma_task); >+sha_dev_err: >+ dev_err(dev, "initialization failed.\n"); >+ return err; >+} >+ >+static int img_hash_remove(struct platform_device *pdev) >+{ >+ static struct img_hash_dev *hdev; >+ >+ hdev = platform_get_drvdata(pdev); >+ spin_lock(&img_hash.lock); >+ list_del(&hdev->list); >+ spin_unlock(&img_hash.lock); >+ >+ img_unregister_algs(hdev); >+ >+ tasklet_kill(&hdev->done_task); >+ tasklet_kill(&hdev->dma_task); >+ >+ dma_release_channel(hdev->dma_lch); >+ >+ clk_disable_unprepare(hdev->iclk); >+ clk_disable_unprepare(hdev->fclk); >+ return 0; >+} >+ >+static struct platform_driver img_hash_driver = { >+ .probe = img_hash_probe, >+ .remove = img_hash_remove, >+ .driver = { >+ .name = "img-hash-accelerator", >+ .of_match_table = of_match_ptr(img_hash_match), >+ } >+}; >+module_platform_driver(img_hash_driver); >+ >+MODULE_LICENSE("GPL v2"); >+MODULE_DESCRIPTION("Imgtec SHA1/224/256 & MD5 hw accelerator driver"); >+MODULE_AUTHOR("Will Thomas."); >+MODULE_AUTHOR("James Hartley."); Ciao Stephan -- To unsubscribe from this list: send the line "unsubscribe linux-crypto" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
