On Wed, Aug 21, 2019 at 05:32:51PM +0300, Ard Biesheuvel wrote:
> Instead of relying on the CTS template to wrap the accelerated CBC
> skcipher, implement the ciphertext stealing part directly.
>
> Signed-off-by: Ard Biesheuvel <[email protected]>
For arm:allmodconfig built with gcc 9.2.0, this patch results in
arch/arm/crypto/aes-ce-core.S: Assembler messages:
arch/arm/crypto/aes-ce-core.S:299: Error: selected processor does not support
`movw ip,:lower16:.Lcts_permute_table' in ARM mode
arch/arm/crypto/aes-ce-core.S:300: Error: selected processor does not support
`movt ip,:upper16:.Lcts_permute_table' in ARM mode
arch/arm/crypto/aes-ce-core.S:337: Error: selected processor does not support
`movw ip,:lower16:.Lcts_permute_table' in ARM mode
arch/arm/crypto/aes-ce-core.S:338: Error: selected processor does not support
`movt ip,:upper16:.Lcts_permute_table' in ARM mode
arch/arm/crypto/aes-ce-core.S:552: Error: selected processor does not support
`movw ip,:lower16:.Lcts_permute_table' in ARM mode
arch/arm/crypto/aes-ce-core.S:553: Error: selected processor does not support
`movt ip,:upper16:.Lcts_permute_table' in ARM mode
arch/arm/crypto/aes-ce-core.S:638: Error: selected processor does not support
`movw ip,:lower16:.Lcts_permute_table' in ARM mode
arch/arm/crypto/aes-ce-core.S:639: Error: selected processor does not support
`movt ip,:upper16:.Lcts_permute_table' in ARM mode
Any idea how to avoid that ?
Guenter
> ---
> arch/arm/crypto/aes-ce-core.S | 85 +++++++++
> arch/arm/crypto/aes-ce-glue.c | 188 ++++++++++++++++++--
> 2 files changed, 256 insertions(+), 17 deletions(-)
>
> diff --git a/arch/arm/crypto/aes-ce-core.S b/arch/arm/crypto/aes-ce-core.S
> index 763e51604ab6..b978cdf133af 100644
> --- a/arch/arm/crypto/aes-ce-core.S
> +++ b/arch/arm/crypto/aes-ce-core.S
> @@ -284,6 +284,91 @@ ENTRY(ce_aes_cbc_decrypt)
> pop {r4-r6, pc}
> ENDPROC(ce_aes_cbc_decrypt)
>
> +
> + /*
> + * ce_aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
> + * int rounds, int bytes, u8 const iv[])
> + * ce_aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
> + * int rounds, int bytes, u8 const iv[])
> + */
> +
> +ENTRY(ce_aes_cbc_cts_encrypt)
> + push {r4-r6, lr}
> + ldrd r4, r5, [sp, #16]
> +
> + movw ip, :lower16:.Lcts_permute_table
> + movt ip, :upper16:.Lcts_permute_table
> + sub r4, r4, #16
> + add lr, ip, #32
> + add ip, ip, r4
> + sub lr, lr, r4
> + vld1.8 {q5}, [ip]
> + vld1.8 {q6}, [lr]
> +
> + add ip, r1, r4
> + vld1.8 {q0}, [r1] @ overlapping loads
> + vld1.8 {q3}, [ip]
> +
> + vld1.8 {q1}, [r5] @ get iv
> + prepare_key r2, r3
> +
> + veor q0, q0, q1 @ xor with iv
> + bl aes_encrypt
> +
> + vtbl.8 d4, {d0-d1}, d10
> + vtbl.8 d5, {d0-d1}, d11
> + vtbl.8 d2, {d6-d7}, d12
> + vtbl.8 d3, {d6-d7}, d13
> +
> + veor q0, q0, q1
> + bl aes_encrypt
> +
> + add r4, r0, r4
> + vst1.8 {q2}, [r4] @ overlapping stores
> + vst1.8 {q0}, [r0]
> +
> + pop {r4-r6, pc}
> +ENDPROC(ce_aes_cbc_cts_encrypt)
> +
> +ENTRY(ce_aes_cbc_cts_decrypt)
> + push {r4-r6, lr}
> + ldrd r4, r5, [sp, #16]
> +
> + movw ip, :lower16:.Lcts_permute_table
> + movt ip, :upper16:.Lcts_permute_table
> + sub r4, r4, #16
> + add lr, ip, #32
> + add ip, ip, r4
> + sub lr, lr, r4
> + vld1.8 {q5}, [ip]
> + vld1.8 {q6}, [lr]
> +
> + add ip, r1, r4
> + vld1.8 {q0}, [r1] @ overlapping loads
> + vld1.8 {q1}, [ip]
> +
> + vld1.8 {q3}, [r5] @ get iv
> + prepare_key r2, r3
> +
> + bl aes_decrypt
> +
> + vtbl.8 d4, {d0-d1}, d10
> + vtbl.8 d5, {d0-d1}, d11
> + vtbx.8 d0, {d2-d3}, d12
> + vtbx.8 d1, {d2-d3}, d13
> +
> + veor q1, q1, q2
> + bl aes_decrypt
> + veor q0, q0, q3 @ xor with iv
> +
> + add r4, r0, r4
> + vst1.8 {q1}, [r4] @ overlapping stores
> + vst1.8 {q0}, [r0]
> +
> + pop {r4-r6, pc}
> +ENDPROC(ce_aes_cbc_cts_decrypt)
> +
> +
> /*
> * aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], int rounds,
> * int blocks, u8 ctr[])
> diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c
> index c215792a2494..cdb1a07e7ad0 100644
> --- a/arch/arm/crypto/aes-ce-glue.c
> +++ b/arch/arm/crypto/aes-ce-glue.c
> @@ -35,6 +35,10 @@ asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const
> in[], u32 const rk[],
> int rounds, int blocks, u8 iv[]);
> asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
> int rounds, int blocks, u8 iv[]);
> +asmlinkage void ce_aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const
> rk[],
> + int rounds, int bytes, u8 const iv[]);
> +asmlinkage void ce_aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const
> rk[],
> + int rounds, int bytes, u8 const iv[]);
>
> asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
> int rounds, int blocks, u8 ctr[]);
> @@ -210,48 +214,182 @@ static int ecb_decrypt(struct skcipher_request *req)
> return err;
> }
>
> -static int cbc_encrypt(struct skcipher_request *req)
> +static int cbc_encrypt_walk(struct skcipher_request *req,
> + struct skcipher_walk *walk)
> {
> struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
> struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
> - struct skcipher_walk walk;
> unsigned int blocks;
> - int err;
> + int err = 0;
>
> - err = skcipher_walk_virt(&walk, req, false);
> -
> - while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
> + while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
> kernel_neon_begin();
> - ce_aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
> + ce_aes_cbc_encrypt(walk->dst.virt.addr, walk->src.virt.addr,
> ctx->key_enc, num_rounds(ctx), blocks,
> - walk.iv);
> + walk->iv);
> kernel_neon_end();
> - err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
> + err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
> }
> return err;
> }
>
> -static int cbc_decrypt(struct skcipher_request *req)
> +static int cbc_encrypt(struct skcipher_request *req)
> {
> - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
> - struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
> struct skcipher_walk walk;
> - unsigned int blocks;
> int err;
>
> err = skcipher_walk_virt(&walk, req, false);
> + if (err)
> + return err;
> + return cbc_encrypt_walk(req, &walk);
> +}
>
> - while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
> +static int cbc_decrypt_walk(struct skcipher_request *req,
> + struct skcipher_walk *walk)
> +{
> + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
> + struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
> + unsigned int blocks;
> + int err = 0;
> +
> + while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
> kernel_neon_begin();
> - ce_aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
> + ce_aes_cbc_decrypt(walk->dst.virt.addr, walk->src.virt.addr,
> ctx->key_dec, num_rounds(ctx), blocks,
> - walk.iv);
> + walk->iv);
> kernel_neon_end();
> - err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
> + err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
> }
> return err;
> }
>
> +static int cbc_decrypt(struct skcipher_request *req)
> +{
> + struct skcipher_walk walk;
> + int err;
> +
> + err = skcipher_walk_virt(&walk, req, false);
> + if (err)
> + return err;
> + return cbc_decrypt_walk(req, &walk);
> +}
> +
> +static int cts_cbc_encrypt(struct skcipher_request *req)
> +{
> + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
> + struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
> + int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
> + struct scatterlist *src = req->src, *dst = req->dst;
> + struct scatterlist sg_src[2], sg_dst[2];
> + struct skcipher_request subreq;
> + struct skcipher_walk walk;
> + int err;
> +
> + skcipher_request_set_tfm(&subreq, tfm);
> + skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
> + NULL, NULL);
> +
> + if (req->cryptlen <= AES_BLOCK_SIZE) {
> + if (req->cryptlen < AES_BLOCK_SIZE)
> + return -EINVAL;
> + cbc_blocks = 1;
> + }
> +
> + if (cbc_blocks > 0) {
> + skcipher_request_set_crypt(&subreq, req->src, req->dst,
> + cbc_blocks * AES_BLOCK_SIZE,
> + req->iv);
> +
> + err = skcipher_walk_virt(&walk, &subreq, false) ?:
> + cbc_encrypt_walk(&subreq, &walk);
> + if (err)
> + return err;
> +
> + if (req->cryptlen == AES_BLOCK_SIZE)
> + return 0;
> +
> + dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
> + if (req->dst != req->src)
> + dst = scatterwalk_ffwd(sg_dst, req->dst,
> + subreq.cryptlen);
> + }
> +
> + /* handle ciphertext stealing */
> + skcipher_request_set_crypt(&subreq, src, dst,
> + req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
> + req->iv);
> +
> + err = skcipher_walk_virt(&walk, &subreq, false);
> + if (err)
> + return err;
> +
> + kernel_neon_begin();
> + ce_aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
> + ctx->key_enc, num_rounds(ctx), walk.nbytes,
> + walk.iv);
> + kernel_neon_end();
> +
> + return skcipher_walk_done(&walk, 0);
> +}
> +
> +static int cts_cbc_decrypt(struct skcipher_request *req)
> +{
> + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
> + struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
> + int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
> + struct scatterlist *src = req->src, *dst = req->dst;
> + struct scatterlist sg_src[2], sg_dst[2];
> + struct skcipher_request subreq;
> + struct skcipher_walk walk;
> + int err;
> +
> + skcipher_request_set_tfm(&subreq, tfm);
> + skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
> + NULL, NULL);
> +
> + if (req->cryptlen <= AES_BLOCK_SIZE) {
> + if (req->cryptlen < AES_BLOCK_SIZE)
> + return -EINVAL;
> + cbc_blocks = 1;
> + }
> +
> + if (cbc_blocks > 0) {
> + skcipher_request_set_crypt(&subreq, req->src, req->dst,
> + cbc_blocks * AES_BLOCK_SIZE,
> + req->iv);
> +
> + err = skcipher_walk_virt(&walk, &subreq, false) ?:
> + cbc_decrypt_walk(&subreq, &walk);
> + if (err)
> + return err;
> +
> + if (req->cryptlen == AES_BLOCK_SIZE)
> + return 0;
> +
> + dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
> + if (req->dst != req->src)
> + dst = scatterwalk_ffwd(sg_dst, req->dst,
> + subreq.cryptlen);
> + }
> +
> + /* handle ciphertext stealing */
> + skcipher_request_set_crypt(&subreq, src, dst,
> + req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
> + req->iv);
> +
> + err = skcipher_walk_virt(&walk, &subreq, false);
> + if (err)
> + return err;
> +
> + kernel_neon_begin();
> + ce_aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
> + ctx->key_dec, num_rounds(ctx), walk.nbytes,
> + walk.iv);
> + kernel_neon_end();
> +
> + return skcipher_walk_done(&walk, 0);
> +}
> +
> static int ctr_encrypt(struct skcipher_request *req)
> {
> struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
> @@ -486,6 +624,22 @@ static struct skcipher_alg aes_algs[] = { {
> .setkey = ce_aes_setkey,
> .encrypt = cbc_encrypt,
> .decrypt = cbc_decrypt,
> +}, {
> + .base.cra_name = "__cts(cbc(aes))",
> + .base.cra_driver_name = "__cts-cbc-aes-ce",
> + .base.cra_priority = 300,
> + .base.cra_flags = CRYPTO_ALG_INTERNAL,
> + .base.cra_blocksize = AES_BLOCK_SIZE,
> + .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
> + .base.cra_module = THIS_MODULE,
> +
> + .min_keysize = AES_MIN_KEY_SIZE,
> + .max_keysize = AES_MAX_KEY_SIZE,
> + .ivsize = AES_BLOCK_SIZE,
> + .walksize = 2 * AES_BLOCK_SIZE,
> + .setkey = ce_aes_setkey,
> + .encrypt = cts_cbc_encrypt,
> + .decrypt = cts_cbc_decrypt,
> }, {
> .base.cra_name = "__ctr(aes)",
> .base.cra_driver_name = "__ctr-aes-ce",
