On 6/16/2015 8:54 AM, Herbert Xu wrote:
> This patch converts the caam GCM implementations to the new AEAD
> interface. This is compile-tested only.
>
> Note that all IV generation for GCM algorithms have been removed.
> The reason is that the current generation uses purely random IVs
> which is not appropriate for counter-based algorithms where we
> first and foremost require uniqueness.
>
> Of course there is no reason why you couldn't implement seqiv or
> seqniv within caam since all they do is xor the sequence number
> with a salt, but since I can't test this on actual hardware I'll
> leave it alone for now.
>
> Signed-off-by: Herbert Xu <[email protected]>
> ---
>
> drivers/crypto/caam/caamalg.c | 1430
> +++++++++++++++++++++---------------------
> 1 file changed, 741 insertions(+), 689 deletions(-)
>
> diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
> index 3c37fe6..f206521 100644
> --- a/drivers/crypto/caam/caamalg.c
> +++ b/drivers/crypto/caam/caamalg.c
> @@ -65,6 +65,10 @@
> /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
> #define CAAM_MAX_IV_LENGTH 16
>
> +#define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)
> +#define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \
> + CAAM_CMD_SZ * 4)
> +
> /* length of descriptors text */
> #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ)
> #define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ)
> @@ -79,18 +83,16 @@
> #define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 17 *
> CAAM_CMD_SZ)
>
> #define DESC_GCM_BASE (3 * CAAM_CMD_SZ)
> -#define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 23 * CAAM_CMD_SZ)
> -#define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 19 * CAAM_CMD_SZ)
> +#define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ)
> +#define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ)
>
> #define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ)
> -#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 15 * CAAM_CMD_SZ)
> -#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 14 * CAAM_CMD_SZ)
> -#define DESC_RFC4106_GIVENC_LEN (DESC_RFC4106_BASE + 21 *
> CAAM_CMD_SZ)
> +#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 10 * CAAM_CMD_SZ)
> +#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 10 * CAAM_CMD_SZ)
>
> #define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ)
> -#define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 25 * CAAM_CMD_SZ)
> -#define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 27 * CAAM_CMD_SZ)
> -#define DESC_RFC4543_GIVENC_LEN (DESC_RFC4543_BASE + 30 *
> CAAM_CMD_SZ)
> +#define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ)
> +#define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ)
>
> #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ)
> #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \
> @@ -98,9 +100,7 @@
> #define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \
> 15 * CAAM_CMD_SZ)
>
> -#define DESC_MAX_USED_BYTES (DESC_RFC4543_GIVENC_LEN + \
> - CAAM_MAX_KEY_SIZE)
> -#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
> +#define DESC_MAX_USED_LEN (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN)
This is going to increase the size of caam_ctx struct, but I agree
previous approach was error-prone.
>
> #ifdef DEBUG
> /* for print_hex_dumps with line references */
> @@ -273,7 +273,7 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead)
> ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
> keys_fit_inline = true;
>
> - /* aead_encrypt shared descriptor */
> + /* old_aead_encrypt shared descriptor */
> desc = ctx->sh_desc_enc;
>
> init_sh_desc(desc, HDR_SHARE_SERIAL);
> @@ -362,7 +362,7 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead)
>
> desc = ctx->sh_desc_dec;
>
> - /* aead_decrypt shared descriptor */
> + /* old_aead_decrypt shared descriptor */
> init_sh_desc(desc, HDR_SHARE_SERIAL);
>
> /* Skip if already shared */
> @@ -496,7 +496,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
> CAAM_DESC_BYTES_MAX)
> keys_fit_inline = true;
>
> - /* aead_encrypt shared descriptor */
> + /* old_aead_encrypt shared descriptor */
> desc = ctx->sh_desc_enc;
>
> /* Note: Context registers are saved. */
> @@ -565,7 +565,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
> CAAM_DESC_BYTES_MAX)
> keys_fit_inline = true;
>
> - /* aead_decrypt shared descriptor */
> + /* old_aead_decrypt shared descriptor */
> desc = ctx->sh_desc_dec;
>
> /* Note: Context registers are saved. */
> @@ -738,7 +738,6 @@ static int aead_setauthsize(struct crypto_aead *authenc,
>
> static int gcm_set_sh_desc(struct crypto_aead *aead)
> {
> - unsigned int ivsize = crypto_aead_ivsize(aead);
> struct caam_ctx *ctx = crypto_aead_ctx(aead);
> struct device *jrdev = ctx->jrdev;
> bool keys_fit_inline = false;
> @@ -754,7 +753,7 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
> * Job Descriptor and Shared Descriptor
> * must fit into the 64-word Descriptor h/w Buffer
> */
> - if (DESC_GCM_ENC_LEN + DESC_JOB_IO_LEN +
> + if (DESC_GCM_ENC_LEN + GCM_DESC_JOB_IO_LEN +
> ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
> keys_fit_inline = true;
>
> @@ -777,34 +776,34 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
> append_operation(desc, ctx->class1_alg_type |
> OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
>
> - /* cryptlen = seqoutlen - authsize */
> - append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
> + /* if assoclen + cryptlen is ZERO, skip to ICV write */
> + append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
> + zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
> + JUMP_COND_MATH_Z);
>
> - /* assoclen + cryptlen = seqinlen - ivsize */
> - append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, ivsize);
> + /* if assoclen is ZERO, skip reading the assoc data */
> + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
> + zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
> + JUMP_COND_MATH_Z);
>
> - /* assoclen = (assoclen + cryptlen) - cryptlen */
> - append_math_sub(desc, REG1, REG2, REG3, CAAM_CMD_SZ);
> + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
> +
> + /* skip assoc data */
> + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
This wasn't previously needed. I assume it's related to your comment:
"This series converts various GCM implementations to the new AEAD
interface. The main changes [...] both src/dst now contain space at the
head equal to assoclen, but only src has the actual AD."
> +
> + /* cryptlen = seqinlen - assoclen */
> + append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
>
> /* if cryptlen is ZERO jump to zero-payload commands */
> - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
> zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
> JUMP_COND_MATH_Z);
> - /* read IV */
> - append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
> - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
> -
> - /* if assoclen is ZERO, skip reading the assoc data */
> - append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
> - zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
> - JUMP_COND_MATH_Z);
>
> /* read assoc data */
> append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
> FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
> set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
>
> - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
> + append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
>
> /* write encrypted data */
> append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
> @@ -814,31 +813,17 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
> FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
>
> /* jump the zero-payload commands */
> - append_jump(desc, JUMP_TEST_ALL | 7);
> + append_jump(desc, JUMP_TEST_ALL | 2);
>
> /* zero-payload commands */
> set_jump_tgt_here(desc, zero_payload_jump_cmd);
>
> - /* if assoclen is ZERO, jump to IV reading - is the only input data */
> - append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
> - zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
> - JUMP_COND_MATH_Z);
> - /* read IV */
> - append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
> - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
> -
> /* read assoc data */
> append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
> FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
>
> - /* jump to ICV writing */
> - append_jump(desc, JUMP_TEST_ALL | 2);
> -
> - /* read IV - is the only input data */
> + /* There is no input data */
> set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
> - append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
> - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
> - FIFOLD_TYPE_LAST1);
>
> /* write ICV */
> append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
> @@ -862,7 +847,7 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
> * must all fit into the 64-word Descriptor h/w Buffer
> */
> keys_fit_inline = false;
> - if (DESC_GCM_DEC_LEN + DESC_JOB_IO_LEN +
> + if (DESC_GCM_DEC_LEN + GCM_DESC_JOB_IO_LEN +
> ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
> keys_fit_inline = true;
>
> @@ -886,33 +871,30 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
> append_operation(desc, ctx->class1_alg_type |
> OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
>
> - /* assoclen + cryptlen = seqinlen - ivsize - icvsize */
> - append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
> - ctx->authsize + ivsize);
> -
> - /* assoclen = (assoclen + cryptlen) - cryptlen */
> - append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
> - append_math_sub(desc, REG1, REG3, REG2, CAAM_CMD_SZ);
> + /* if assoclen is ZERO, skip reading the assoc data */
> + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
> + zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
> + JUMP_COND_MATH_Z);
>
> - /* read IV */
> - append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
> - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
> + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
>
> - /* jump to zero-payload command if cryptlen is zero */
> - append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
> - zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
> - JUMP_COND_MATH_Z);
> + /* skip assoc data */
> + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
>
> - append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
> - /* if asoclen is ZERO, skip reading assoc data */
> - zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
> - JUMP_COND_MATH_Z);
> /* read assoc data */
> append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
> FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
> +
> set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
>
> - append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
> + /* cryptlen = seqoutlen - assoclen */
> + append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
> +
> + /* jump to zero-payload command if cryptlen is zero */
> + zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
> + JUMP_COND_MATH_Z);
> +
> + append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
>
> /* store encrypted data */
> append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
> @@ -921,21 +903,9 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
> append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
> FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
>
> - /* jump the zero-payload commands */
> - append_jump(desc, JUMP_TEST_ALL | 4);
> -
> /* zero-payload command */
> set_jump_tgt_here(desc, zero_payload_jump_cmd);
>
> - /* if assoclen is ZERO, jump to ICV reading */
> - append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
> - zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
> - JUMP_COND_MATH_Z);
> - /* read assoc data */
> - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
> - FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
> - set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
> -
> /* read ICV */
> append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
> FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
[...]
> +static void init_gcm_job(struct aead_request *req,
> + struct aead_edesc *edesc,
> + bool all_contig, bool encrypt)
> +{
> + struct crypto_aead *aead = crypto_aead_reqtfm(req);
> + struct caam_ctx *ctx = crypto_aead_ctx(aead);
> + unsigned int ivsize = crypto_aead_ivsize(aead);
> + u32 *desc = edesc->hw_desc;
> + bool generic_gcm = (ivsize == 12);
> + unsigned int last;
> +
> + init_aead_job(req, edesc, all_contig, encrypt);
> +
> + /* BUG This should not be specific to generic GCM. */
AFAICT, for non-generic GCM uses (RFC4106, RFC4543), cryptlen and/or
assoclen are always > 0. That's why the descriptors do not address these
cases.
> + last = 0;
> + if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen))
> + last = FIFOLD_TYPE_LAST1;
> +
> + /* Read GCM IV */
> + append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
> + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last);
> + /* Append Salt */
> + if (!generic_gcm)
> + append_data(desc, ctx->key + ctx->enckeylen, 4);
> + /* Append IV */
> + append_data(desc, req->iv, ivsize);
> + /* End of blank commands */
> +}
> +
> +/*
> * Fill in aead givencrypt job descriptor
> */
> static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr,
> @@ -2608,9 +2391,10 @@ static void init_ablkcipher_giv_job(u32 *sh_desc,
> dma_addr_t ptr,
> /*
> * allocate and map the aead extended descriptor
> */
> -static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
> - int desc_bytes, bool *all_contig_ptr,
> - bool encrypt)
> +static struct aead_edesc *old_aead_edesc_alloc(struct aead_request *req,
> + int desc_bytes,
> + bool *all_contig_ptr,
> + bool encrypt)
> {
> struct crypto_aead *aead = crypto_aead_reqtfm(req);
> struct caam_ctx *ctx = crypto_aead_ctx(aead);
> @@ -2661,29 +2445,132 @@ static struct aead_edesc *aead_edesc_alloc(struct
> aead_request *req,
> return ERR_PTR(-ENOMEM);
> }
>
> - if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) ==
> - OP_ALG_ALGSEL_AES) &&
> - ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM))
> - is_gcm = true;
> + if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) ==
> + OP_ALG_ALGSEL_AES) &&
> + ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM))
> + is_gcm = true;
Now that GCM is handled separately, is_gcm logic should be removed from
all old_aead_* functions.
> +
> + /*
> + * Check if data are contiguous.
> + * GCM expected input sequence: IV, AAD, text
> + * All other - expected input sequence: AAD, IV, text
> + */
> + if (is_gcm)
> + all_contig = (!assoc_nents &&
> + iv_dma + ivsize == sg_dma_address(req->assoc) &&
> + !src_nents && sg_dma_address(req->assoc) +
> + req->assoclen == sg_dma_address(req->src));
> + else
> + all_contig = (!assoc_nents && sg_dma_address(req->assoc) +
> + req->assoclen == iv_dma && !src_nents &&
> + iv_dma + ivsize == sg_dma_address(req->src));
> + if (!all_contig) {
> + assoc_nents = assoc_nents ? : 1;
> + src_nents = src_nents ? : 1;
> + sec4_sg_len = assoc_nents + 1 + src_nents;
> + }
> +
> + sec4_sg_len += dst_nents;
> +
> + sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
> +
> + /* allocate space for base edesc and hw desc commands, link tables */
> + edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
> + sec4_sg_bytes, GFP_DMA | flags);
> + if (!edesc) {
> + dev_err(jrdev, "could not allocate extended descriptor\n");
> + return ERR_PTR(-ENOMEM);
> + }
> +
> + edesc->assoc_nents = assoc_nents;
> + edesc->assoc_chained = assoc_chained;
> + edesc->src_nents = src_nents;
> + edesc->src_chained = src_chained;
> + edesc->dst_nents = dst_nents;
> + edesc->dst_chained = dst_chained;
> + edesc->iv_dma = iv_dma;
> + edesc->sec4_sg_bytes = sec4_sg_bytes;
> + edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
> + desc_bytes;
> + *all_contig_ptr = all_contig;
> +
> + sec4_sg_index = 0;
> + if (!all_contig) {
> + if (!is_gcm) {
> + sg_to_sec4_sg_len(req->assoc, req->assoclen,
> + edesc->sec4_sg + sec4_sg_index);
> + sec4_sg_index += assoc_nents;
> + }
> +
> + dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
> + iv_dma, ivsize, 0);
> + sec4_sg_index += 1;
> +
> + if (is_gcm) {
> + sg_to_sec4_sg_len(req->assoc, req->assoclen,
> + edesc->sec4_sg + sec4_sg_index);
> + sec4_sg_index += assoc_nents;
> + }
> +
> + sg_to_sec4_sg_last(req->src,
> + src_nents,
> + edesc->sec4_sg +
> + sec4_sg_index, 0);
> + sec4_sg_index += src_nents;
> + }
> + if (dst_nents) {
> + sg_to_sec4_sg_last(req->dst, dst_nents,
> + edesc->sec4_sg + sec4_sg_index, 0);
> + }
> + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
> + sec4_sg_bytes, DMA_TO_DEVICE);
> + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
> + dev_err(jrdev, "unable to map S/G table\n");
> + return ERR_PTR(-ENOMEM);
> + }
> +
> + return edesc;
> +}
> +
> +/*
> + * allocate and map the aead extended descriptor
> + */
> +static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
> + int desc_bytes, bool *all_contig_ptr,
> + bool encrypt)
> +{
> + struct crypto_aead *aead = crypto_aead_reqtfm(req);
> + struct caam_ctx *ctx = crypto_aead_ctx(aead);
> + struct device *jrdev = ctx->jrdev;
> + gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
> + CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
> + int src_nents, dst_nents = 0;
> + struct aead_edesc *edesc;
> + int sgc;
> + bool all_contig = true;
> + bool src_chained = false, dst_chained = false;
> + int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
> + unsigned int authsize = ctx->authsize;
> +
> + if (unlikely(req->dst != req->src)) {
> + src_nents = sg_count(req->src, req->assoclen + req->cryptlen,
> + &src_chained);
> + dst_nents = sg_count(req->dst,
> + req->assoclen + req->cryptlen +
> + (encrypt ? authsize : (-authsize)),
> + &dst_chained);
> + } else {
> + src_nents = sg_count(req->src,
> + req->assoclen + req->cryptlen +
> + (encrypt ? authsize : 0),
> + &src_chained);
> + }
>
> - /*
> - * Check if data are contiguous.
> - * GCM expected input sequence: IV, AAD, text
> - * All other - expected input sequence: AAD, IV, text
> - */
> - if (is_gcm)
> - all_contig = (!assoc_nents &&
> - iv_dma + ivsize == sg_dma_address(req->assoc) &&
> - !src_nents && sg_dma_address(req->assoc) +
> - req->assoclen == sg_dma_address(req->src));
> - else
> - all_contig = (!assoc_nents && sg_dma_address(req->assoc) +
> - req->assoclen == iv_dma && !src_nents &&
> - iv_dma + ivsize == sg_dma_address(req->src));
> + /* Check if data are contiguous. */
> + all_contig = !src_nents;
> if (!all_contig) {
> - assoc_nents = assoc_nents ? : 1;
> src_nents = src_nents ? : 1;
> - sec4_sg_len = assoc_nents + 1 + src_nents;
> + sec4_sg_len = src_nents;
> }
>
> sec4_sg_len += dst_nents;
> @@ -2691,64 +2578,78 @@ static struct aead_edesc *aead_edesc_alloc(struct
> aead_request *req,
> sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
>
> /* allocate space for base edesc and hw desc commands, link tables */
> - edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
> + edesc = kzalloc(sizeof(struct aead_edesc) + desc_bytes +
> sec4_sg_bytes, GFP_DMA | flags);
> if (!edesc) {
> dev_err(jrdev, "could not allocate extended descriptor\n");
> return ERR_PTR(-ENOMEM);
> }
>
> - edesc->assoc_nents = assoc_nents;
> - edesc->assoc_chained = assoc_chained;
> + if (likely(req->src == req->dst)) {
> + sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
> + DMA_BIDIRECTIONAL, src_chained);
> + if (unlikely(!sgc)) {
> + dev_err(jrdev, "unable to map source\n");
> + kfree(edesc);
> + return ERR_PTR(-ENOMEM);
> + }
> + } else {
> + sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
> + DMA_TO_DEVICE, src_chained);
> + if (unlikely(!sgc)) {
> + dev_err(jrdev, "unable to map source\n");
> + kfree(edesc);
> + return ERR_PTR(-ENOMEM);
> + }
> +
> + sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
> + DMA_FROM_DEVICE, dst_chained);
> + if (unlikely(!sgc)) {
> + dev_err(jrdev, "unable to map destination\n");
> + dma_unmap_sg_chained(jrdev, req->src, src_nents ? : 1,
> + DMA_TO_DEVICE, src_chained);
> + kfree(edesc);
> + return ERR_PTR(-ENOMEM);
> + }
> + }
> +
> edesc->src_nents = src_nents;
> edesc->src_chained = src_chained;
> edesc->dst_nents = dst_nents;
> edesc->dst_chained = dst_chained;
> - edesc->iv_dma = iv_dma;
> - edesc->sec4_sg_bytes = sec4_sg_bytes;
> edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
> desc_bytes;
> *all_contig_ptr = all_contig;
>
> sec4_sg_index = 0;
> if (!all_contig) {
> - if (!is_gcm) {
> - sg_to_sec4_sg_len(req->assoc, req->assoclen,
> - edesc->sec4_sg + sec4_sg_index);
> - sec4_sg_index += assoc_nents;
> - }
> -
> - dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
> - iv_dma, ivsize, 0);
> - sec4_sg_index += 1;
> -
> - if (is_gcm) {
> - sg_to_sec4_sg_len(req->assoc, req->assoclen,
> - edesc->sec4_sg + sec4_sg_index);
> - sec4_sg_index += assoc_nents;
> - }
> -
> - sg_to_sec4_sg_last(req->src,
> - src_nents,
> - edesc->sec4_sg +
> - sec4_sg_index, 0);
> + sg_to_sec4_sg(req->src, src_nents,
> + edesc->sec4_sg + sec4_sg_index, 0);
Need to mark end of input S/G, use sg_to_sec4_sg_last() instead.
Thanks,
Horia
--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
