From: Alex Ousherovitch <[email protected]> Register ChaCha20-Poly1305 AEAD and ChaCha20 skcipher algorithms using the CMH CCP core (core ID 0x18). Also registers the Poly1305 ahash for standalone use.
Co-developed-by: Saravanakrishnan Krishnamoorthy <[email protected]> Signed-off-by: Saravanakrishnan Krishnamoorthy <[email protected]> Signed-off-by: Alex Ousherovitch <[email protected]> Reviewed-by: Joel Wittenauer <[email protected]> Reviewed-by: Thi Nguyen <[email protected]> --- drivers/crypto/cmh/Makefile | 5 +- drivers/crypto/cmh/cmh_ccp.c | 364 +++++++++++++++++ drivers/crypto/cmh/cmh_ccp_aead.c | 583 +++++++++++++++++++++++++++ drivers/crypto/cmh/cmh_ccp_poly.c | 528 ++++++++++++++++++++++++ drivers/crypto/cmh/cmh_main.c | 25 ++ drivers/crypto/cmh/include/cmh_ccp.h | 24 ++ 6 files changed, 1528 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_ccp.c create mode 100644 drivers/crypto/cmh/cmh_ccp_aead.c create mode 100644 drivers/crypto/cmh/cmh_ccp_poly.c create mode 100644 drivers/crypto/cmh/include/cmh_ccp.h diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index 1f36cd9c0b98..4ebd0e1d10bc 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -25,7 +25,10 @@ cmh-y := \ cmh_aes_cmac.o \ cmh_sm4_skcipher.o \ cmh_sm4_aead.o \ - cmh_sm4_cmac.o + cmh_sm4_cmac.o \ + cmh_ccp.o \ + cmh_ccp_aead.o \ + cmh_ccp_poly.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_ccp.c b/drivers/crypto/cmh/cmh_ccp.c new file mode 100644 index 000000000000..deb1db9200f8 --- /dev/null +++ b/drivers/crypto/cmh/cmh_ccp.c @@ -0,0 +1,364 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API ChaCha20 (skcipher) Driver + * + * Registers the "chacha20" skcipher algorithm with the Linux crypto + * subsystem, backed by the CMH CCP core. + * + * VCQ sequence: + * [SYS_CMD_WRITE] + CCP_CMD_CHACHA20_INIT + CCP_CMD_FINAL + CCP_CMD_FLUSH + * + * The CCP core expects a 16-byte counter+nonce (ctrnonce): + * bytes [0..3] = 32-bit LE counter + * bytes [4..15] = 12-byte nonce + * + * The Linux chacha20 skcipher interface passes a 16-byte IV in the + * same format, so we forward it directly. + * + * ChaCha20 is a stream cipher -- arbitrary plaintext lengths are + * supported (no block-alignment requirement). + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/crypto.h> +#include <crypto/internal/skcipher.h> +#include <crypto/scatterwalk.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/unaligned.h> + +#include "cmh_ccp.h" +#include "cmh_vcq.h" +#include "cmh_ccp_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +/* Per-transform context */ + +struct cmh_ccp_tfm_ctx { + struct cmh_key_ctx key; +}; + +/* Per-request context (lives in skcipher_request::__ctx) */ + +/* + * Maximum payload commands: + * [SYS_CMD_WRITE] + CCP_CMD_CHACHA20_INIT + CCP_CMD_FINAL + FLUSH = 4 + */ +#define CMH_CCP_MAX_PAYLOAD 4 +#define CMH_CCP_MAX_PACKED (CMH_CCP_MAX_PAYLOAD * 2) + +struct cmh_ccp_reqctx { + dma_addr_t in_dma; + dma_addr_t out_dma; + dma_addr_t iv_dma; + dma_addr_t key_dma; + u8 *in_buf; + u8 *out_buf; + u8 *iv_buf; + u32 cryptlen; + u32 keylen; + struct vcq_cmd packed[CMH_CCP_MAX_PACKED]; +}; + +/* VCQ Builders -- ChaCha20-specific */ + +static void vcq_add_ccp_chacha_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref, + u64 ctrnonce_dma, u32 keylen, u32 op) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_CHACHA20_INIT); + slot->hwc.ccp.cmd_chacha.key = key_ref; + slot->hwc.ccp.cmd_chacha.ctrnonce = ctrnonce_dma; + slot->hwc.ccp.cmd_chacha.keylen = keylen; + slot->hwc.ccp.cmd_chacha.ctrnoncelen = CCP_CTRNONCE_SIZE; + slot->hwc.ccp.cmd_chacha.ctrlen = CCP_CHACHA_CTR_LEN; + slot->hwc.ccp.cmd_chacha.op = op; +} + +static void vcq_add_ccp_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma, + u64 output_dma, u32 iolen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_FINAL); + slot->hwc.ccp.cmd_final.input = input_dma; + slot->hwc.ccp.cmd_final.output = output_dma; + slot->hwc.ccp.cmd_final.tag = 0; + slot->hwc.ccp.cmd_final.iolen = iolen; + slot->hwc.ccp.cmd_final.taglen = 0; +} + +/* skcipher Operations */ +static int cmh_ccp_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_ccp_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + /* ChaCha20 requires 32-byte key per RFC 8439 */ + if (keylen != 32) + return -EINVAL; + + return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_CCP); +} + +static int cmh_ccp_init_tfm(struct crypto_skcipher *tfm) +{ + struct cmh_ccp_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + + memset(tctx, 0, sizeof(*tctx)); + crypto_skcipher_set_reqsize(tfm, sizeof(struct cmh_ccp_reqctx)); + return 0; +} + +static void cmh_ccp_exit_tfm(struct crypto_skcipher *tfm) +{ + struct cmh_ccp_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + + cmh_key_destroy(&tctx->key); +} + +/* DMA unmap helper */ +static void cmh_ccp_unmap_dma(struct cmh_ccp_reqctx *rctx) +{ + cmh_dma_unmap_single(rctx->iv_dma, CCP_CTRNONCE_SIZE, DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen, DMA_FROM_DEVICE); + cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen, DMA_TO_DEVICE); +} + +static void cmh_ccp_free_bufs(struct cmh_ccp_reqctx *rctx) +{ + kfree(rctx->iv_buf); + rctx->iv_buf = NULL; + kfree_sensitive(rctx->out_buf); + rctx->out_buf = NULL; + kfree_sensitive(rctx->in_buf); + rctx->in_buf = NULL; +} + +static void cmh_ccp_complete(void *data, int error) +{ + struct skcipher_request *req = data; + struct cmh_ccp_reqctx *rctx = skcipher_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + cmh_ccp_unmap_dma(rctx); + + if (!error) { + u32 counter, nblocks; + + scatterwalk_map_and_copy(rctx->out_buf, req->dst, + 0, rctx->cryptlen, 1); + + /* + * Update the 32-bit LE block counter at IV[0..3]. + * ChaCha20 processes 64-byte blocks; the nonce at + * IV[4..15] is unchanged. + */ + counter = get_unaligned_le32(req->iv); + nblocks = DIV_ROUND_UP(rctx->cryptlen, 64); + put_unaligned_le32(counter + nblocks, req->iv); + } + + cmh_ccp_free_bufs(rctx); + cmh_complete(&req->base, error); +} + +/* + * Core encrypt/decrypt -- builds a VCQ transaction and submits async. + * + * ChaCha20 is a stream cipher: encrypt and decrypt use the same + * underlying XOR operation. + */ +static int cmh_ccp_crypt(struct skcipher_request *req, u32 ccp_op) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct cmh_ccp_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + struct cmh_ccp_reqctx *rctx = skcipher_request_ctx(req); + struct vcq_cmd cmds[CMH_CCP_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + if (tctx->key.mode == CMH_KEY_NONE) + return -ENOKEY; + + if (!req->cryptlen) + return 0; + + /* Limit linearisation buffers to avoid large allocations. */ + if (req->cryptlen > SZ_1M) + return -EINVAL; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + memset(rctx, 0, sizeof(*rctx)); + rctx->cryptlen = req->cryptlen; + + /* Linearise input from scatterlist */ + rctx->in_buf = kmalloc(req->cryptlen, gfp); + if (!rctx->in_buf) + return -ENOMEM; + + scatterwalk_map_and_copy(rctx->in_buf, req->src, 0, req->cryptlen, 0); + + rctx->in_dma = cmh_dma_map_single(rctx->in_buf, req->cryptlen, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->in_dma)) { + ret = -ENOMEM; + goto out_free_in; + } + + rctx->out_buf = kmalloc(req->cryptlen, gfp); + if (!rctx->out_buf) { + ret = -ENOMEM; + goto out_unmap_in; + } + + rctx->out_dma = cmh_dma_map_single(rctx->out_buf, req->cryptlen, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->out_dma)) { + ret = -ENOMEM; + goto out_free_out; + } + + rctx->iv_buf = kmemdup(req->iv, CCP_CTRNONCE_SIZE, gfp); + if (!rctx->iv_buf) { + ret = -ENOMEM; + goto out_unmap_out; + } + + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, CCP_CTRNONCE_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->iv_dma)) { + ret = -ENOMEM; + goto out_free_iv; + } + + /* Resolve key reference */ + idx = 0; + + rctx->key_dma = tctx->key.raw.dma; + rctx->keylen = tctx->key.raw.len; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_CCP); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + vcq_add_ccp_chacha_init(&cmds[idx++], core_id, key_ref, + (u64)rctx->iv_dma, keylen, ccp_op); + + vcq_add_ccp_final(&cmds[idx++], core_id, (u64)rctx->in_dma, + (u64)rctx->out_dma, req->cryptlen); + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_CCP_MAX_PACKED, target_mbx, + cmh_ccp_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + /* -EBUSY = backlogged; ownership transferred to callback. */ + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + cmh_dma_unmap_single(rctx->iv_dma, CCP_CTRNONCE_SIZE, DMA_TO_DEVICE); +out_free_iv: + kfree(rctx->iv_buf); +out_unmap_out: + cmh_dma_unmap_single(rctx->out_dma, req->cryptlen, DMA_FROM_DEVICE); +out_free_out: + kfree_sensitive(rctx->out_buf); +out_unmap_in: + cmh_dma_unmap_single(rctx->in_dma, req->cryptlen, DMA_TO_DEVICE); +out_free_in: + kfree_sensitive(rctx->in_buf); + return ret; +} + +static int cmh_ccp_encrypt(struct skcipher_request *req) +{ + return cmh_ccp_crypt(req, CCP_OP_ENCRYPT); +} + +static int cmh_ccp_decrypt(struct skcipher_request *req) +{ + return cmh_ccp_crypt(req, CCP_OP_DECRYPT); +} + +/* Registration */ + +static struct skcipher_alg cmh_chacha20_alg = { + .setkey = cmh_ccp_setkey, + .encrypt = cmh_ccp_encrypt, + .decrypt = cmh_ccp_decrypt, + .init = cmh_ccp_init_tfm, + .exit = cmh_ccp_exit_tfm, + .min_keysize = 32, + .max_keysize = 32, + .ivsize = CCP_CTRNONCE_SIZE, + .base = { + .cra_name = "chacha20", + .cra_driver_name = "cri-cmh-chacha20", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, /* stream cipher */ + .cra_ctxsize = sizeof(struct cmh_ccp_tfm_ctx), + .cra_module = THIS_MODULE, + }, +}; + +/** + * cmh_ccp_register() - Register ChaCha20 skcipher algorithm with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_ccp_register(void) +{ + int ret; + + ret = crypto_register_skcipher(&cmh_chacha20_alg); + if (ret) + dev_err(cmh_dev(), "cmh_ccp: failed to register chacha20 (rc=%d)\n", ret); + else + dev_dbg(cmh_dev(), "cmh_ccp: registered chacha20\n"); + + return ret; +} + +/** + * cmh_ccp_unregister() - Unregister ChaCha20 skcipher algorithm from the crypto framework + */ +void cmh_ccp_unregister(void) +{ + crypto_unregister_skcipher(&cmh_chacha20_alg); + dev_dbg(cmh_dev(), "cmh_ccp: unregistered chacha20\n"); +} diff --git a/drivers/crypto/cmh/cmh_ccp_aead.c b/drivers/crypto/cmh/cmh_ccp_aead.c new file mode 100644 index 000000000000..20b6f9d1746a --- /dev/null +++ b/drivers/crypto/cmh/cmh_ccp_aead.c @@ -0,0 +1,583 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API ChaCha20-Poly1305 AEAD Driver (RFC 7539) + * + * Registers "rfc7539(chacha20,poly1305)" as an AEAD algorithm with the + * Linux crypto subsystem, backed by the CMH CCP core. + * + * VCQ sequence: + * [SYS_CMD_WRITE] + CCP_CMD_AEAD_INIT + [CCP_CMD_AAD_FINAL] + * + CCP_CMD_FINAL + CCP_CMD_FLUSH + * + * The RFC 7539 AEAD interface passes a 12-byte nonce via req->iv. + * The CCP core expects a 16-byte ctrnonce (4-byte LE counter + 12-byte + * nonce). We prepend a zero counter (per RFC 7539 S2.8: counter 0 + * generates the Poly1305 key, counter 1 starts encryption -- the + * CMH eSW handles this internally from the initial counter value of 0). + * + * Tag is always 16 bytes (Poly1305 authenticator). + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/crypto.h> +#include <crypto/chacha.h> +#include <crypto/internal/aead.h> +#include <crypto/scatterwalk.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include <linux/string.h> + +#include "cmh_ccp.h" +#include "cmh_vcq.h" +#include "cmh_ccp_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +#define CCP_AEAD_IV_SIZE 12U /* RFC 7539 nonce */ +#define CCP_ESP_IV_SIZE 8U /* RFC 7539 ESP nonce (4-byte salt at setkey) */ +#define CCP_ESP_SALT_SIZE 4U +#define CCP_AEAD_TAG_SIZE 16U /* Poly1305 tag */ + +struct cmh_ccp_aead_tfm_ctx { + struct cmh_key_ctx key; + u32 authsize; + u8 salt[CCP_ESP_SALT_SIZE]; /* ESP salt (unused for rfc7539) */ +}; + +/* Per-request context (lives in aead_request::__ctx) */ + +/* + * Maximum payload commands: + * [SYS_CMD_WRITE] + CCP_CMD_AEAD_INIT + CCP_CMD_AAD_FINAL + * + CCP_CMD_FINAL + FLUSH = 5 + */ +#define CMH_CCP_AEAD_MAX_PAYLOAD 5 +#define CMH_CCP_AEAD_MAX_PACKED (CMH_CCP_AEAD_MAX_PAYLOAD * 2) + +struct cmh_ccp_aead_reqctx { + dma_addr_t in_dma; + dma_addr_t out_dma; + dma_addr_t iv_dma; + dma_addr_t key_dma; + dma_addr_t aad_dma; + dma_addr_t tag_dma; + u8 *in_buf; + u8 *out_buf; + u8 *iv_buf; + u8 *aad_buf; + u8 *tag_buf; + u32 cryptlen; + u32 assoclen; + u32 authsize; + u32 keylen; + bool encrypting; + struct vcq_cmd packed[CMH_CCP_AEAD_MAX_PACKED]; +}; + +/* VCQ Builders -- CCP AEAD-specific */ + +static void vcq_add_ccp_aead_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref, + u64 ctrnonce_dma, u32 keylen, u32 op) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_AEAD_INIT); + slot->hwc.ccp.cmd_aead.key = key_ref; + slot->hwc.ccp.cmd_aead.ctrnonce = ctrnonce_dma; + slot->hwc.ccp.cmd_aead.keylen = keylen; + slot->hwc.ccp.cmd_aead.ctrnoncelen = CCP_CTRNONCE_SIZE; + slot->hwc.ccp.cmd_aead.op = op; +} + +static void vcq_add_ccp_aad_final(struct vcq_cmd *slot, u32 core_id, u64 aad_dma, + u32 aadlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_AAD_FINAL); + slot->hwc.ccp.cmd_aad_final.aad = aad_dma; + slot->hwc.ccp.cmd_aad_final.aadlen = aadlen; +} + +static void vcq_add_ccp_aead_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma, + u64 output_dma, u64 tag_dma, + u32 iolen, u32 taglen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_FINAL); + slot->hwc.ccp.cmd_final.input = input_dma; + slot->hwc.ccp.cmd_final.output = output_dma; + slot->hwc.ccp.cmd_final.tag = tag_dma; + slot->hwc.ccp.cmd_final.iolen = iolen; + slot->hwc.ccp.cmd_final.taglen = taglen; +} + +/* setkey */ +static int cmh_ccp_aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + /* RFC 7539 AEAD requires 32-byte key */ + if (keylen != CHACHA_KEY_SIZE) + return -EINVAL; + + return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_CCP); +} + +static int cmh_ccp_aead_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + + /* Poly1305 tag is always 16 bytes */ + if (authsize != CCP_AEAD_TAG_SIZE) + return -EINVAL; + + tctx->authsize = authsize; + return 0; +} + +static int cmh_ccp_aead_init_tfm(struct crypto_aead *tfm) +{ + struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + + memset(tctx, 0, sizeof(*tctx)); + tctx->authsize = CCP_AEAD_TAG_SIZE; + crypto_aead_set_reqsize(tfm, sizeof(struct cmh_ccp_aead_reqctx)); + return 0; +} + +static void cmh_ccp_aead_exit_tfm(struct crypto_aead *tfm) +{ + struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + + cmh_key_destroy(&tctx->key); +} + +/* DMA unmap helper */ +static void cmh_ccp_aead_unmap_dma(struct cmh_ccp_aead_reqctx *rctx) +{ + cmh_dma_unmap_single(rctx->iv_dma, CCP_CTRNONCE_SIZE, DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->tag_dma, rctx->authsize, + rctx->encrypting ? DMA_FROM_DEVICE : + DMA_TO_DEVICE); + if (rctx->cryptlen > 0) { + cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen, + DMA_FROM_DEVICE); + cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen, + DMA_TO_DEVICE); + } + if (rctx->assoclen > 0) + cmh_dma_unmap_single(rctx->aad_dma, rctx->assoclen, + DMA_TO_DEVICE); +} + +static void cmh_ccp_aead_free_bufs(struct cmh_ccp_aead_reqctx *rctx) +{ + kfree(rctx->iv_buf); + rctx->iv_buf = NULL; + kfree(rctx->tag_buf); + rctx->tag_buf = NULL; + kfree_sensitive(rctx->out_buf); + rctx->out_buf = NULL; + kfree_sensitive(rctx->in_buf); + rctx->in_buf = NULL; + kfree(rctx->aad_buf); + rctx->aad_buf = NULL; +} + +static void cmh_ccp_aead_complete(void *data, int error) +{ + struct aead_request *req = data; + struct cmh_ccp_aead_reqctx *rctx = aead_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + cmh_ccp_aead_unmap_dma(rctx); + + /* + * Map HW error on decrypt to -EBADMSG. The eSW CCP core uses a + * single error code (-EIO) for both authentication failures and + * other core errors (e.g. DMA timeout), so we cannot distinguish + * them from the MBX_STATUS alone. In practice the only error + * during a well-formed AEAD decrypt is auth-tag mismatch; a DMA + * timeout would indicate a fatal HW problem where -EBADMSG vs + * -EIO is moot. The kernel crypto API requires -EBADMSG for + * AEAD authentication failures. + */ + if (error == -EIO && !rctx->encrypting) + error = -EBADMSG; + + if (!error) { + if (rctx->cryptlen > 0) + scatterwalk_map_and_copy(rctx->out_buf, req->dst, + req->assoclen, + rctx->cryptlen, 1); + if (rctx->encrypting) + scatterwalk_map_and_copy(rctx->tag_buf, req->dst, + req->assoclen + + rctx->cryptlen, + rctx->authsize, 1); + } + + cmh_ccp_aead_free_bufs(rctx); + cmh_complete(&req->base, error); +} + +/* + * Core AEAD encrypt/decrypt -- async path. + * + * Encrypt: plaintext -> ciphertext + 16-byte tag + * Decrypt: ciphertext + tag -> plaintext (tag verified by CMH eSW) + * + * VCQ: [SYS_CMD_WRITE] + AEAD_INIT + [AAD_FINAL] + FINAL + FLUSH + */ +static int cmh_ccp_aead_crypt(struct aead_request *req, u32 ccp_op) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + struct cmh_ccp_aead_reqctx *rctx = aead_request_ctx(req); + struct vcq_cmd cmds[CMH_CCP_AEAD_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen, authsize, cryptlen; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + if (tctx->key.mode == CMH_KEY_NONE) + return -ENOKEY; + + authsize = tctx->authsize; + + if (ccp_op == CCP_OP_ENCRYPT) { + cryptlen = req->cryptlen; + } else { + if (req->cryptlen < authsize) + return -EINVAL; + cryptlen = req->cryptlen - authsize; + } + + /* + * HW uses a proprietary LLI scatter-gather format that is + * incompatible with struct scatterlist, so the payload is + * linearised into contiguous buffers for DMA. Cap total + * size to prevent excessive memory consumption. + */ + if ((u64)cryptlen + req->assoclen > SZ_1M) + return -EINVAL; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + memset(rctx, 0, sizeof(*rctx)); + rctx->cryptlen = cryptlen; + rctx->assoclen = req->assoclen; + rctx->authsize = authsize; + rctx->encrypting = (ccp_op == CCP_OP_ENCRYPT); + + /* + * rfc7539esp: the last ivsize (8) bytes of the AAD region are the + * IV/nonce, not actual associated data. Subtract them so HW only + * authenticates the real AAD. + */ + if (crypto_aead_ivsize(tfm) == CCP_ESP_IV_SIZE) { + if (rctx->assoclen < CCP_ESP_IV_SIZE) + return -EINVAL; + rctx->assoclen -= CCP_ESP_IV_SIZE; + } + + /* Linearise AAD */ + if (rctx->assoclen > 0) { + rctx->aad_buf = kmalloc(rctx->assoclen, gfp); + if (!rctx->aad_buf) + return -ENOMEM; + scatterwalk_map_and_copy(rctx->aad_buf, req->src, + 0, rctx->assoclen, 0); + rctx->aad_dma = cmh_dma_map_single(rctx->aad_buf, + rctx->assoclen, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->aad_dma)) { + ret = -ENOMEM; + goto out_free_aad; + } + } + + /* Linearise input */ + if (cryptlen > 0) { + rctx->in_buf = kmalloc(cryptlen, gfp); + if (!rctx->in_buf) { + ret = -ENOMEM; + goto out_unmap_aad; + } + scatterwalk_map_and_copy(rctx->in_buf, req->src, + req->assoclen, cryptlen, 0); + rctx->in_dma = cmh_dma_map_single(rctx->in_buf, cryptlen, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->in_dma)) { + ret = -ENOMEM; + goto out_free_in; + } + } + + /* Allocate output buffer */ + if (cryptlen > 0) { + rctx->out_buf = kmalloc(cryptlen, gfp); + if (!rctx->out_buf) { + ret = -ENOMEM; + goto out_unmap_in; + } + rctx->out_dma = cmh_dma_map_single(rctx->out_buf, cryptlen, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->out_dma)) { + ret = -ENOMEM; + goto out_free_out; + } + } + + /* Tag buffer */ + rctx->tag_buf = kmalloc(authsize, gfp); + if (!rctx->tag_buf) { + ret = -ENOMEM; + goto out_unmap_out; + } + + if (!rctx->encrypting) { + scatterwalk_map_and_copy(rctx->tag_buf, req->src, + req->assoclen + cryptlen, + authsize, 0); + } else { + memset(rctx->tag_buf, 0, authsize); + } + + rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, authsize, + rctx->encrypting ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->tag_dma)) { + ret = -ENOMEM; + goto out_free_tag; + } + + /* Build 16-byte ctrnonce: 4-byte zero counter + 12-byte nonce. + * rfc7539: counter(4) | req->iv(12) + * rfc7539esp: counter(4) | salt(4) | req->iv(8) + */ + rctx->iv_buf = kzalloc(CCP_CTRNONCE_SIZE, gfp); + if (!rctx->iv_buf) { + ret = -ENOMEM; + goto out_unmap_tag; + } + if (crypto_aead_ivsize(tfm) == CCP_ESP_IV_SIZE) { + memcpy(rctx->iv_buf + CCP_CHACHA_CTR_LEN, + tctx->salt, CCP_ESP_SALT_SIZE); + memcpy(rctx->iv_buf + CCP_CHACHA_CTR_LEN + CCP_ESP_SALT_SIZE, + req->iv, CCP_ESP_IV_SIZE); + } else { + memcpy(rctx->iv_buf + CCP_CHACHA_CTR_LEN, + req->iv, CCP_AEAD_IV_SIZE); + } + + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, CCP_CTRNONCE_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->iv_dma)) { + ret = -ENOMEM; + goto out_free_iv; + } + + /* Resolve key reference */ + idx = 0; + + rctx->key_dma = tctx->key.raw.dma; + rctx->keylen = tctx->key.raw.len; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_CCP); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + /* AEAD_INIT */ + vcq_add_ccp_aead_init(&cmds[idx++], core_id, key_ref, + (u64)rctx->iv_dma, keylen, ccp_op); + + /* AAD_FINAL if we have associated data */ + if (rctx->assoclen > 0) + vcq_add_ccp_aad_final(&cmds[idx++], core_id, + (u64)rctx->aad_dma, rctx->assoclen); + + /* FINAL with tag */ + vcq_add_ccp_aead_final(&cmds[idx++], core_id, + cryptlen > 0 ? (u64)rctx->in_dma : 0, + cryptlen > 0 ? (u64)rctx->out_dma : 0, + (u64)rctx->tag_dma, cryptlen, authsize); + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_CCP_AEAD_MAX_PACKED, + target_mbx, + cmh_ccp_aead_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + cmh_dma_unmap_single(rctx->iv_dma, CCP_CTRNONCE_SIZE, DMA_TO_DEVICE); +out_free_iv: + kfree(rctx->iv_buf); +out_unmap_tag: + cmh_dma_unmap_single(rctx->tag_dma, authsize, + rctx->encrypting ? DMA_FROM_DEVICE : + DMA_TO_DEVICE); +out_free_tag: + kfree(rctx->tag_buf); +out_unmap_out: + if (cryptlen > 0) + cmh_dma_unmap_single(rctx->out_dma, cryptlen, DMA_FROM_DEVICE); +out_free_out: + kfree_sensitive(rctx->out_buf); +out_unmap_in: + if (cryptlen > 0) + cmh_dma_unmap_single(rctx->in_dma, cryptlen, DMA_TO_DEVICE); +out_free_in: + kfree_sensitive(rctx->in_buf); +out_unmap_aad: + if (rctx->assoclen > 0) + cmh_dma_unmap_single(rctx->aad_dma, rctx->assoclen, + DMA_TO_DEVICE); +out_free_aad: + kfree(rctx->aad_buf); + return ret; +} + +static int cmh_ccp_aead_encrypt(struct aead_request *req) +{ + return cmh_ccp_aead_crypt(req, CCP_OP_ENCRYPT); +} + +static int cmh_ccp_aead_decrypt(struct aead_request *req) +{ + return cmh_ccp_aead_crypt(req, CCP_OP_DECRYPT); +} + +/* -- rfc7539esp: ESP variant with 4-byte salt + 8-byte IV --------------- */ + +/* + * ESP setkey: 36 bytes = 32-byte ChaCha20 key + 4-byte salt. + * The salt is prepended to the 8-byte per-packet IV from the ESP header + * to form the 12-byte RFC 7539 nonce. + */ +static int cmh_ccp_esp_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_ccp_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + + if (keylen != CHACHA_KEY_SIZE + CCP_ESP_SALT_SIZE) + return -EINVAL; + + memcpy(tctx->salt, key + CHACHA_KEY_SIZE, CCP_ESP_SALT_SIZE); + return cmh_key_setkey_raw(&tctx->key, key, CHACHA_KEY_SIZE, CORE_ID_CCP); +} + +/* Registration */ + +static struct aead_alg cmh_rfc7539_alg = { + .setkey = cmh_ccp_aead_setkey, + .setauthsize = cmh_ccp_aead_setauthsize, + .encrypt = cmh_ccp_aead_encrypt, + .decrypt = cmh_ccp_aead_decrypt, + .init = cmh_ccp_aead_init_tfm, + .exit = cmh_ccp_aead_exit_tfm, + .ivsize = CCP_AEAD_IV_SIZE, + .maxauthsize = CCP_AEAD_TAG_SIZE, + .base = { + .cra_name = "rfc7539(chacha20,poly1305)", + .cra_driver_name = "cri-cmh-rfc7539-chacha20-poly1305", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct cmh_ccp_aead_tfm_ctx), + .cra_module = THIS_MODULE, + }, +}; + +static struct aead_alg cmh_rfc7539esp_alg = { + .setkey = cmh_ccp_esp_setkey, + .setauthsize = cmh_ccp_aead_setauthsize, + .encrypt = cmh_ccp_aead_encrypt, + .decrypt = cmh_ccp_aead_decrypt, + .init = cmh_ccp_aead_init_tfm, + .exit = cmh_ccp_aead_exit_tfm, + .ivsize = CCP_ESP_IV_SIZE, + .maxauthsize = CCP_AEAD_TAG_SIZE, + .base = { + .cra_name = "rfc7539esp(chacha20,poly1305)", + .cra_driver_name = "cri-cmh-rfc7539esp-chacha20-poly1305", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct cmh_ccp_aead_tfm_ctx), + .cra_module = THIS_MODULE, + }, +}; + +/** + * cmh_ccp_aead_register() - Register ChaCha20-Poly1305 AEAD algorithm with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_ccp_aead_register(void) +{ + int ret; + + ret = crypto_register_aead(&cmh_rfc7539_alg); + if (ret) { + dev_err(cmh_dev(), "cmh_ccp_aead: failed to register rfc7539 (rc=%d)\n", + ret); + return ret; + } + dev_dbg(cmh_dev(), "cmh_ccp_aead: registered rfc7539(chacha20,poly1305)\n"); + + ret = crypto_register_aead(&cmh_rfc7539esp_alg); + if (ret) { + dev_err(cmh_dev(), "cmh_ccp_aead: failed to register rfc7539esp (rc=%d)\n", + ret); + crypto_unregister_aead(&cmh_rfc7539_alg); + return ret; + } + dev_dbg(cmh_dev(), "cmh_ccp_aead: registered rfc7539esp(chacha20,poly1305)\n"); + + return 0; +} + +/** + * cmh_ccp_aead_unregister() - Unregister ChaCha20-Poly1305 AEAD algorithms + */ +void cmh_ccp_aead_unregister(void) +{ + crypto_unregister_aead(&cmh_rfc7539esp_alg); + crypto_unregister_aead(&cmh_rfc7539_alg); + dev_dbg(cmh_dev(), "cmh_ccp_aead: unregistered rfc7539/rfc7539esp\n"); +} diff --git a/drivers/crypto/cmh/cmh_ccp_poly.c b/drivers/crypto/cmh/cmh_ccp_poly.c new file mode 100644 index 000000000000..020a98fbe607 --- /dev/null +++ b/drivers/crypto/cmh/cmh_ccp_poly.c @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API Poly1305 (ahash) Driver + * + * Registers "poly1305" as an ahash algorithm with the Linux crypto + * subsystem, backed by the CMH CCP core. + * + * Poly1305 is a one-time authenticator that produces a 16-byte MAC. + * It requires two 16-byte keys: r (clamped multiplier) and s (nonce). + * + * Key format: 32 bytes = r_key[0..15] || s_key[16..31] + * This matches the Poly1305 key layout in RFC 7539 S2.5. + * + * VCQ sequence: + * SYS_CMD_WRITE(s_key) + SYS_CMD_WRITE(r_key) + * + CCP_CMD_POLY1305_INIT + CCP_CMD_FINAL + CCP_CMD_FLUSH + * + * Both keys are written to SYS_REF_TEMP; the CMH eSW stacks them + * so that POLY1305_INIT finds r_key (most recent) as rkey and + * s_key (previous) as skey. + * + * The ahash interface accumulates data via .update() and submits the + * full VCQ asynchronously in .final(). + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/crypto.h> +#include <crypto/internal/hash.h> +#include <crypto/scatterwalk.h> +#include <linux/slab.h> +#include <linux/string.h> + +#include "cmh_ccp.h" +#include "cmh_vcq.h" +#include "cmh_ccp_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +#define POLY1305_DIGEST_SIZE 16U +#define POLY1305_BLOCK_SIZE 16U +#define POLY1305_KEY_SIZE 32U /* r(16) + s(16) */ + +/* + * Maximum accumulated data for Poly1305 -- driver-imposed, not HW. + * + * The CCP core does not expose external save/restore VCQ commands, + * so the driver must accumulate all data in kernel memory via + * .update() and submit it atomically in .final(). This cap limits + * the per-request kernel allocation. + */ +#define POLY_MAX_DATA (64 * 1024) + +/* + * Per-transform context -- stores the raw 32-byte key (r || s). + * + * Only the raw-key path is supported for standalone Poly1305. + */ +struct cmh_poly_tfm_ctx { + u8 key[POLY1305_KEY_SIZE]; + dma_addr_t rkey_dma; + dma_addr_t skey_dma; + u32 keylen; + bool has_key; + spinlock_t chunk_lock; /* protects all_chunks */ + struct list_head all_chunks; /* orphan-safe chunk tracking */ +}; + +/* Chunk node for O(1) update() appends */ +struct cmh_poly_chunk { + struct list_head list; + struct list_head tfm_node; /* per-tfm orphan tracking */ + u32 len; + u8 data[]; +}; + +/* Per-request context (lives in ahash_request::__ctx) */ + +/* + * Maximum payload commands: + * SYS_CMD_WRITE(s) + SYS_CMD_WRITE(r) + POLY1305_INIT + * + CCP_CMD_FINAL + FLUSH = 5 + */ +#define CMH_POLY_MAX_PAYLOAD 5 +#define CMH_POLY_MAX_PACKED (CMH_POLY_MAX_PAYLOAD * 2) + +struct cmh_poly_reqctx { + struct list_head chunks; + u32 total_len; + u8 *buf; /* linearised in final() */ + /* DMA state for async final */ + dma_addr_t in_dma; + dma_addr_t tag_dma; + u8 *tag_buf; + struct vcq_cmd packed[CMH_POLY_MAX_PACKED]; +}; + +/* + * Export/import: not supported. + * + * The CCP core lacks external save/restore VCQ commands, so there is + * no way to checkpoint intermediate Poly1305 state to host memory. + * Pending eSW ABI extension to add save/restore for the CCP core. + */ + +static void vcq_add_ccp_poly_init(struct vcq_cmd *slot, u32 core_id, + u64 rkey_ref, u32 rkeylen, + u64 skey_ref, u32 skeylen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_POLY1305_INIT); + slot->hwc.ccp.cmd_poly.rkey = rkey_ref; + slot->hwc.ccp.cmd_poly.rkeylen = rkeylen; + slot->hwc.ccp.cmd_poly.skey = skey_ref; + slot->hwc.ccp.cmd_poly.skeylen = skeylen; +} + +static void vcq_add_ccp_poly_final(struct vcq_cmd *slot, u32 core_id, + u64 input_dma, u64 tag_dma, + u32 iolen, u32 taglen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, CCP_CMD_FINAL); + slot->hwc.ccp.cmd_final.input = input_dma; + slot->hwc.ccp.cmd_final.output = 0; + slot->hwc.ccp.cmd_final.tag = tag_dma; + slot->hwc.ccp.cmd_final.iolen = iolen; + slot->hwc.ccp.cmd_final.taglen = taglen; +} + +static int cmh_poly_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + + /* Poly1305: exactly 32 bytes (r[16] + s[16]) */ + if (keylen != POLY1305_KEY_SIZE) + return -EINVAL; + + /* Unmap old key DMA if re-keying */ + if (tctx->has_key) { + cmh_dma_unmap_single(tctx->rkey_dma, CCP_POLY_KEY_SIZE, + DMA_TO_DEVICE); + cmh_dma_unmap_single(tctx->skey_dma, CCP_POLY_KEY_SIZE, + DMA_TO_DEVICE); + } + + memcpy(tctx->key, key, POLY1305_KEY_SIZE); + tctx->keylen = POLY1305_KEY_SIZE; + + /* + * Pre-map both key halves for DMA. The key buffer lives in + * the tfm context and is stable until exit_tfm() or re-setkey. + */ + tctx->skey_dma = cmh_dma_map_single(tctx->key + CCP_POLY_KEY_SIZE, + CCP_POLY_KEY_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(tctx->skey_dma)) { + tctx->has_key = false; + return -ENOMEM; + } + + tctx->rkey_dma = cmh_dma_map_single(tctx->key, CCP_POLY_KEY_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(tctx->rkey_dma)) { + cmh_dma_unmap_single(tctx->skey_dma, CCP_POLY_KEY_SIZE, + DMA_TO_DEVICE); + tctx->has_key = false; + return -ENOMEM; + } + + tctx->has_key = true; + return 0; +} + +static void cmh_poly_free_chunks(struct cmh_poly_reqctx *rctx, + struct cmh_poly_tfm_ctx *tctx) +{ + struct cmh_poly_chunk *c, *tmp; + + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(c, tmp, &rctx->chunks, list) { + list_del(&c->list); + list_del(&c->tfm_node); + kfree_sensitive(c); + } + spin_unlock_bh(&tctx->chunk_lock); +} + +static int cmh_poly_init(struct ahash_request *req) +{ + struct cmh_poly_reqctx *rctx = ahash_request_ctx(req); + + memset(rctx, 0, sizeof(*rctx)); + INIT_LIST_HEAD(&rctx->chunks); + return 0; +} + +static int cmh_poly_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_poly_reqctx *rctx = ahash_request_ctx(req); + struct cmh_poly_chunk *chunk; + gfp_t gfp; + int ret; + + if (!req->nbytes) + return 0; + + if (req->nbytes > POLY_MAX_DATA - rctx->total_len) { + ret = -EINVAL; + goto err_free_chunks; + } + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + chunk = kmalloc(sizeof(*chunk) + req->nbytes, gfp); + if (!chunk) { + ret = -ENOMEM; + goto err_free_chunks; + } + + chunk->len = req->nbytes; + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(chunk->data, req->svirt, req->nbytes); + else + scatterwalk_map_and_copy(chunk->data, req->src, + 0, req->nbytes, 0); + list_add_tail(&chunk->list, &rctx->chunks); + spin_lock_bh(&tctx->chunk_lock); + list_add_tail(&chunk->tfm_node, &tctx->all_chunks); + spin_unlock_bh(&tctx->chunk_lock); + rctx->total_len += req->nbytes; + return 0; + +err_free_chunks: + /* + * Terminal error -- free all previously accumulated chunks. + * Callers may not call .final() on error, so they would leak. + */ + cmh_poly_free_chunks(rctx, tctx); + return ret; +} + +static void cmh_poly_complete(void *data, int error) +{ + struct ahash_request *req = data; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_poly_reqctx *rctx = ahash_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + if (rctx->total_len > 0) + cmh_dma_unmap_single(rctx->in_dma, rctx->total_len, + DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->tag_dma, POLY1305_DIGEST_SIZE, + DMA_FROM_DEVICE); + + if (!error) + memcpy(req->result, rctx->tag_buf, POLY1305_DIGEST_SIZE); + + kfree(rctx->tag_buf); + rctx->tag_buf = NULL; + cmh_poly_free_chunks(rctx, tctx); + kfree_sensitive(rctx->buf); + rctx->buf = NULL; + rctx->total_len = 0; + cmh_complete(&req->base, error); +} + +static int cmh_poly_final(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_poly_reqctx *rctx = ahash_request_ctx(req); + struct vcq_cmd cmds[CMH_POLY_MAX_PAYLOAD]; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + if (!tctx->has_key) { + ret = -ENOKEY; + goto out_free_chunks; + } + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + /* Linearise chunks into a single contiguous buffer for DMA */ + if (rctx->total_len > 0) { + struct cmh_poly_chunk *c; + u32 off = 0; + + rctx->buf = kmalloc(rctx->total_len, gfp); + if (!rctx->buf) { + ret = -ENOMEM; + goto out_free_chunks; + } + list_for_each_entry(c, &rctx->chunks, list) { + memcpy(rctx->buf + off, c->data, c->len); + off += c->len; + } + } + + /* Tag output buffer */ + rctx->tag_buf = kzalloc(POLY1305_DIGEST_SIZE, gfp); + if (!rctx->tag_buf) { + ret = -ENOMEM; + goto out_free_buf; + } + + rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, + POLY1305_DIGEST_SIZE, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->tag_dma)) { + ret = -ENOMEM; + goto out_free_tag; + } + + /* Map input data */ + if (rctx->total_len > 0) { + rctx->in_dma = cmh_dma_map_single(rctx->buf, rctx->total_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->in_dma)) { + ret = -ENOMEM; + goto out_unmap_tag; + } + } + + /* + * Key DMA handles are pre-mapped in setkey() and live in + * the tfm context. Use them directly for the VCQ writes. + */ + + d = cmh_core_select_instance(CMH_CORE_CCP); + target_mbx = d.mbx_idx; + core_id = d.core_id; + idx = 0; + + /* Write s_key to SYS_REF_TEMP first (bottom of stack) */ + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)tctx->skey_dma, SYS_REF_NONE, + CCP_POLY_KEY_SIZE, + SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_CCP)); + + /* Write r_key to SYS_REF_TEMP second (top of stack) */ + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)tctx->rkey_dma, SYS_REF_NONE, + CCP_POLY_KEY_SIZE, + SYS_TYPE_SET(SYS_TYPE_FLAG_PT, CORE_ID_CCP)); + + /* POLY1305_INIT: rkey=TEMP (top), skey=TEMP (next) */ + vcq_add_ccp_poly_init(&cmds[idx++], core_id, SYS_REF_TEMP, + CCP_POLY_KEY_SIZE, SYS_REF_TEMP, + CCP_POLY_KEY_SIZE); + + /* FINAL: data -> tag */ + vcq_add_ccp_poly_final(&cmds[idx++], core_id, + rctx->total_len > 0 ? (u64)rctx->in_dma : 0, + (u64)rctx->tag_dma, rctx->total_len, + POLY1305_DIGEST_SIZE); + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_POLY_MAX_PACKED, target_mbx, + cmh_poly_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_unmap_in; + + return -EINPROGRESS; + +out_unmap_in: + if (rctx->total_len > 0 && rctx->in_dma) + cmh_dma_unmap_single(rctx->in_dma, rctx->total_len, + DMA_TO_DEVICE); +out_unmap_tag: + cmh_dma_unmap_single(rctx->tag_dma, POLY1305_DIGEST_SIZE, + DMA_FROM_DEVICE); +out_free_tag: + kfree(rctx->tag_buf); +out_free_buf: + kfree_sensitive(rctx->buf); + rctx->buf = NULL; +out_free_chunks: + cmh_poly_free_chunks(rctx, tctx); + rctx->total_len = 0; + return ret; +} + +static int cmh_poly_export(struct ahash_request *req, void *out) +{ + return -EOPNOTSUPP; +} + +static int cmh_poly_import(struct ahash_request *req, const void *in) +{ + return -EOPNOTSUPP; +} + +static int cmh_poly_finup(struct ahash_request *req) +{ + int err; + + err = cmh_poly_update(req); + if (err) + return err; + return cmh_poly_final(req); +} + +static int cmh_poly_digest(struct ahash_request *req) +{ + int err; + + err = cmh_poly_init(req); + if (err) + return err; + return cmh_poly_finup(req); +} + +static int cmh_poly_init_tfm(struct crypto_ahash *tfm) +{ + struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + + memset(tctx, 0, sizeof(*tctx)); + spin_lock_init(&tctx->chunk_lock); + INIT_LIST_HEAD(&tctx->all_chunks); + crypto_ahash_set_reqsize(tfm, sizeof(struct cmh_poly_reqctx)); + return 0; +} + +static void cmh_poly_exit_tfm(struct crypto_ahash *tfm) +{ + struct cmh_poly_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_poly_chunk *c, *tmp; + + /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */ + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(c, tmp, &tctx->all_chunks, tfm_node) { + list_del(&c->tfm_node); + kfree_sensitive(c); + } + spin_unlock_bh(&tctx->chunk_lock); + + if (tctx->has_key) { + cmh_dma_unmap_single(tctx->rkey_dma, CCP_POLY_KEY_SIZE, + DMA_TO_DEVICE); + cmh_dma_unmap_single(tctx->skey_dma, CCP_POLY_KEY_SIZE, + DMA_TO_DEVICE); + } + memzero_explicit(tctx->key, POLY1305_KEY_SIZE); +} + +static struct ahash_alg cmh_poly1305_alg = { + .init = cmh_poly_init, + .update = cmh_poly_update, + .final = cmh_poly_final, + .finup = cmh_poly_finup, + .digest = cmh_poly_digest, + .export = cmh_poly_export, + .import = cmh_poly_import, + .setkey = cmh_poly_setkey, + .init_tfm = cmh_poly_init_tfm, + .exit_tfm = cmh_poly_exit_tfm, + .halg = { + .digestsize = POLY1305_DIGEST_SIZE, + .statesize = sizeof(struct cmh_poly_reqctx), + .base = { + .cra_name = "poly1305", + .cra_driver_name = "cri-cmh-poly1305", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NO_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_REQ_VIRT, + .cra_blocksize = POLY1305_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct cmh_poly_tfm_ctx), + .cra_module = THIS_MODULE, + }, + }, +}; + +/** + * cmh_ccp_poly_register() - Register Poly1305 hash algorithm with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_ccp_poly_register(void) +{ + int ret; + + ret = crypto_register_ahash(&cmh_poly1305_alg); + if (ret) + dev_err(cmh_dev(), "cmh_ccp_poly: failed to register poly1305 (rc=%d)\n", + ret); + else + dev_dbg(cmh_dev(), "cmh_ccp_poly: registered poly1305\n"); + + return ret; +} + +/** + * cmh_ccp_poly_unregister() - Unregister Poly1305 hash algorithm from the crypto framework + */ +void cmh_ccp_poly_unregister(void) +{ + crypto_unregister_ahash(&cmh_poly1305_alg); + dev_dbg(cmh_dev(), "cmh_ccp_poly: unregistered poly1305\n"); +} diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index 0a73964b9785..e5b2a86c3149 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -36,6 +36,7 @@ #include "cmh_sm3.h" #include "cmh_aes.h" #include "cmh_sm4.h" +#include "cmh_ccp.h" #include "cmh_mgmt.h" #include "cmh_registers.h" #include "cmh_debugfs.h" @@ -259,6 +260,21 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_sm4_cmac_register; + /* Register CCP ChaCha20 skcipher algorithm */ + ret = cmh_ccp_register(); + if (ret) + goto err_ccp_register; + + /* Register CCP ChaCha20-Poly1305 AEAD (RFC 7539) */ + ret = cmh_ccp_aead_register(); + if (ret) + goto err_ccp_aead_register; + + /* Register CCP Poly1305 shash algorithm */ + ret = cmh_ccp_poly_register(); + if (ret) + goto err_ccp_poly_register; + /* Register key management device (/dev/cmh_mgmt) */ ret = cmh_mgmt_register(); if (ret) @@ -271,6 +287,12 @@ static int cmh_probe(struct platform_device *pdev) return 0; err_mgmt_register: + cmh_ccp_poly_unregister(); +err_ccp_poly_register: + cmh_ccp_aead_unregister(); +err_ccp_aead_register: + cmh_ccp_unregister(); +err_ccp_register: cmh_sm4_cmac_unregister(); err_sm4_cmac_register: cmh_sm4_aead_unregister(); @@ -319,6 +341,9 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; cmh_mgmt_unregister(); + cmh_ccp_poly_unregister(); + cmh_ccp_aead_unregister(); + cmh_ccp_unregister(); cmh_sm4_cmac_unregister(); cmh_sm4_aead_unregister(); cmh_sm4_unregister(); diff --git a/drivers/crypto/cmh/include/cmh_ccp.h b/drivers/crypto/cmh/include/cmh_ccp.h new file mode 100644 index 000000000000..363d208cbceb --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_ccp.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- CCP Crypto API Drivers + * + * Registers CCP algorithms with the Linux crypto subsystem: + * skcipher: chacha20 + * shash: poly1305 + * aead: rfc7539(chacha20poly1305) + */ + +#ifndef CMH_CCP_H +#define CMH_CCP_H + +int cmh_ccp_register(void); +void cmh_ccp_unregister(void); + +int cmh_ccp_aead_register(void); +void cmh_ccp_aead_unregister(void); + +int cmh_ccp_poly_register(void); +void cmh_ccp_poly_unregister(void); + +#endif /* CMH_CCP_H */ -- 2.43.7
