Merge branch 'for-4.14' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq

[sfrench/cifs-2.6.git] / arch / arm64 / crypto / aes-glue.c

diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index bcf596b0197ef31cfffa60e04c69cd652f5eac09..998ba519a02666d51ef518f3533b649de1c371ac 100644 (file)
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -10,6 +10,7 @@
 
 #include <asm/neon.h>
 #include <asm/hwcap.h>
+#include <asm/simd.h>
 #include <crypto/aes.h>
 #include <crypto/internal/hash.h>
 #include <crypto/internal/simd.h>
@@ -19,6 +20,7 @@
 #include <crypto/xts.h>
 
 #include "aes-ce-setkey.h"
+#include "aes-ctr-fallback.h"
 
 #ifdef USE_V8_CRYPTO_EXTENSIONS
 #define MODE                   "ce"
@@ -241,9 +243,7 @@ static int ctr_encrypt(struct skcipher_request *req)
 
                aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc, rounds,
                                blocks, walk.iv, first);
-               if (tdst != tsrc)
-                       memcpy(tdst, tsrc, nbytes);
-               crypto_xor(tdst, tail, nbytes);
+               crypto_xor_cpy(tdst, tsrc, tail, nbytes);
                err = skcipher_walk_done(&walk, 0);
        }
        kernel_neon_end();
@@ -251,6 +251,17 @@ static int ctr_encrypt(struct skcipher_request *req)
        return err;
 }
 
+static int ctr_encrypt_sync(struct skcipher_request *req)
+{
+       struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+       struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+       if (!may_use_simd())
+               return aes_ctr_encrypt_fallback(ctx, req);
+
+       return ctr_encrypt(req);
+}
+
 static int xts_encrypt(struct skcipher_request *req)
 {
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -357,8 +368,8 @@ static struct skcipher_alg aes_algs[] = { {
        .ivsize         = AES_BLOCK_SIZE,
        .chunksize      = AES_BLOCK_SIZE,
        .setkey         = skcipher_aes_setkey,
-       .encrypt        = ctr_encrypt,
-       .decrypt        = ctr_encrypt,
+       .encrypt        = ctr_encrypt_sync,
+       .decrypt        = ctr_encrypt_sync,
 }, {
        .base = {
                .cra_name               = "__xts(aes)",
@@ -460,11 +471,35 @@ static int mac_init(struct shash_desc *desc)
        return 0;
 }
 
+static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks,
+                         u8 dg[], int enc_before, int enc_after)
+{
+       int rounds = 6 + ctx->key_length / 4;
+
+       if (may_use_simd()) {
+               kernel_neon_begin();
+               aes_mac_update(in, ctx->key_enc, rounds, blocks, dg, enc_before,
+                              enc_after);
+               kernel_neon_end();
+       } else {
+               if (enc_before)
+                       __aes_arm64_encrypt(ctx->key_enc, dg, dg, rounds);
+
+               while (blocks--) {
+                       crypto_xor(dg, in, AES_BLOCK_SIZE);
+                       in += AES_BLOCK_SIZE;
+
+                       if (blocks || enc_after)
+                               __aes_arm64_encrypt(ctx->key_enc, dg, dg,
+                                                   rounds);
+               }
+       }
+}
+
 static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len)
 {
        struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
        struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
-       int rounds = 6 + tctx->key.key_length / 4;
 
        while (len > 0) {
                unsigned int l;
@@ -476,10 +511,8 @@ static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len)
 
                        len %= AES_BLOCK_SIZE;
 
-                       kernel_neon_begin();
-                       aes_mac_update(p, tctx->key.key_enc, rounds, blocks,
-                                      ctx->dg, (ctx->len != 0), (len != 0));
-                       kernel_neon_end();
+                       mac_do_update(&tctx->key, p, blocks, ctx->dg,
+                                     (ctx->len != 0), (len != 0));
 
                        p += blocks * AES_BLOCK_SIZE;
 
@@ -507,11 +540,8 @@ static int cbcmac_final(struct shash_desc *desc, u8 *out)
 {
        struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
        struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
-       int rounds = 6 + tctx->key.key_length / 4;
 
-       kernel_neon_begin();
-       aes_mac_update(NULL, tctx->key.key_enc, rounds, 0, ctx->dg, 1, 0);
-       kernel_neon_end();
+       mac_do_update(&tctx->key, NULL, 0, ctx->dg, 1, 0);
 
        memcpy(out, ctx->dg, AES_BLOCK_SIZE);
 
@@ -522,7 +552,6 @@ static int cmac_final(struct shash_desc *desc, u8 *out)
 {
        struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
        struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
-       int rounds = 6 + tctx->key.key_length / 4;
        u8 *consts = tctx->consts;
 
        if (ctx->len != AES_BLOCK_SIZE) {
@@ -530,9 +559,7 @@ static int cmac_final(struct shash_desc *desc, u8 *out)
                consts += AES_BLOCK_SIZE;
        }
 
-       kernel_neon_begin();
-       aes_mac_update(consts, tctx->key.key_enc, rounds, 1, ctx->dg, 0, 1);
-       kernel_neon_end();
+       mac_do_update(&tctx->key, consts, 1, ctx->dg, 0, 1);
 
        memcpy(out, ctx->dg, AES_BLOCK_SIZE);