lib/crypto: optimize aes_gcm_128

- We avoid variables in order to do a lazy cleanup
  in aes_ccm_128_digest() via ZERO_STRUCTP(ctx)
- We use the optimized aes_block_{xor,rshift}() functions
- Align AES_BLOCK_SIZE arrays to 8 bytes

BUG: https://bugzilla.samba.org/show_bug.cgi?id=11451

Signed-off-by: Stefan Metzmacher <metze@samba.org>
Reviewed-by: Jeremy Allison <jra@samba.org>

diff --git a/lib/crypto/aes_gcm_128.c b/lib/crypto/aes_gcm_128.c
index f59d65949edcb39621f593e60eac991e34996e0e..bfbf176e53dc26d9a6d8e6f0811141a74e0de971 100644 (file)
--- a/lib/crypto/aes_gcm_128.c
+++ b/lib/crypto/aes_gcm_128.c
@@ -30,35 +30,12 @@ static inline void aes_gcm_128_inc32(uint8_t inout[AES_BLOCK_SIZE])
        RSIVAL(inout, AES_BLOCK_SIZE - 4, v);
 }
 
-static inline void aes_gcm_128_xor(const uint8_t in1[AES_BLOCK_SIZE],
-                                  const uint8_t in2[AES_BLOCK_SIZE],
-                                  uint8_t out[AES_BLOCK_SIZE])
-{
-       uint8_t i;
-
-       for (i = 0; i < AES_BLOCK_SIZE; i++) {
-               out[i] = in1[i] ^ in2[i];
-       }
-}
-
-static inline void aes_gcm_128_rightshift(uint8_t x[AES_BLOCK_SIZE])
-{
-       int8_t i;
-
-       for (i = AES_BLOCK_SIZE - 1; i >=0; i--) {
-               x[i] >>= 1;
-               if (i > 0) {
-                       x[i] |= (x[i-1] & 1) << 7;
-               }
-       }
-}
-
 static inline void aes_gcm_128_mul(const uint8_t x[AES_BLOCK_SIZE],
                                   const uint8_t y[AES_BLOCK_SIZE],
+                                  uint8_t v[AES_BLOCK_SIZE],
                                   uint8_t z[AES_BLOCK_SIZE])
 {
        uint8_t i;
-       uint8_t v[AES_BLOCK_SIZE];
        /* 11100001 || 0^120 */
        static const uint8_t r[AES_BLOCK_SIZE] = {
                0xE1, 0x00, 0x00, 0x00,
@@ -75,12 +52,12 @@ static inline void aes_gcm_128_mul(const uint8_t x[AES_BLOCK_SIZE],
                for (mask = 0x80; mask != 0 ; mask >>= 1) {
                        uint8_t v_lsb = v[AES_BLOCK_SIZE-1] & 1;
                        if (x[i] & mask) {
-                               aes_gcm_128_xor(z, v, z);
+                               aes_block_xor(z, v, z);
                        }
 
-                       aes_gcm_128_rightshift(v);
+                       aes_block_rshift(v, v);
                        if (v_lsb != 0) {
-                               aes_gcm_128_xor(v, r, v);
+                               aes_block_xor(v, r, v);
                        }
                }
        }
@@ -89,8 +66,8 @@ static inline void aes_gcm_128_mul(const uint8_t x[AES_BLOCK_SIZE],
 static inline void aes_gcm_128_ghash_block(struct aes_gcm_128_context *ctx,
                                           const uint8_t in[AES_BLOCK_SIZE])
 {
-       aes_gcm_128_xor(ctx->Y, in, ctx->y.block);
-       aes_gcm_128_mul(ctx->y.block, ctx->H, ctx->Y);
+       aes_block_xor(ctx->Y, in, ctx->y.block);
+       aes_gcm_128_mul(ctx->y.block, ctx->H, ctx->v.block, ctx->Y);
 }
 
 void aes_gcm_128_init(struct aes_gcm_128_context *ctx,
@@ -184,6 +161,15 @@ static inline void aes_gcm_128_crypt_tmp(struct aes_gcm_128_context *ctx,
                        tmp->ofs = 0;
                }
 
+               if (likely(tmp->ofs == 0 && m_len >= AES_BLOCK_SIZE)) {
+                       aes_block_xor(m, tmp->block, m);
+                       m += AES_BLOCK_SIZE;
+                       m_len -= AES_BLOCK_SIZE;
+                       aes_gcm_128_inc32(ctx->CB);
+                       AES_encrypt(ctx->CB, tmp->block, &ctx->aes_key);
+                       continue;
+               }
+
                m[0] ^= tmp->block[tmp->ofs];
                m += 1;
                m_len -= 1;
@@ -215,7 +201,7 @@ void aes_gcm_128_digest(struct aes_gcm_128_context *ctx,
        aes_gcm_128_ghash_block(ctx, ctx->AC);
 
        AES_encrypt(ctx->J0, ctx->c.block, &ctx->aes_key);
-       aes_gcm_128_xor(ctx->c.block, ctx->Y, T);
+       aes_block_xor(ctx->c.block, ctx->Y, T);
 
        ZERO_STRUCTP(ctx);
 }

diff --git a/lib/crypto/aes_gcm_128.h b/lib/crypto/aes_gcm_128.h
index 278b6db03082adfbbc8aabb0ae38b7c0df61fe59..8df11c2f6bd7c224f545010e5609e355b9ba684c 100644 (file)
--- a/lib/crypto/aes_gcm_128.h
+++ b/lib/crypto/aes_gcm_128.h
@@ -24,17 +24,20 @@
 
 struct aes_gcm_128_context {
        AES_KEY aes_key;
+
+       uint64_t __align;
+
+       struct aes_gcm_128_tmp {
+               size_t ofs;
+               size_t total;
+               uint8_t block[AES_BLOCK_SIZE];
+       } A, C, c, v, y;
+
        uint8_t H[AES_BLOCK_SIZE];
        uint8_t J0[AES_BLOCK_SIZE];
        uint8_t CB[AES_BLOCK_SIZE];
        uint8_t Y[AES_BLOCK_SIZE];
        uint8_t AC[AES_BLOCK_SIZE];
-
-       struct aes_gcm_128_tmp {
-               uint8_t block[AES_BLOCK_SIZE];
-               size_t ofs;
-               size_t total;
-       } A, C, c, y;
 };
 
 void aes_gcm_128_init(struct aes_gcm_128_context *ctx,