arch/x86/crypto/aesni-intel_glue.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Support for Intel AES-NI instructions. This file contains glue
   4  * code, the real AES implementation is in intel-aes_asm.S.
   5  *
   6  * Copyright (C) 2008, Intel Corp.
   7  *    Author: Huang Ying <ying.huang@intel.com>
   8  *
   9  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
  10  * interface for 64-bit kernels.
  11  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  12  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  13  *             Tadeusz Struk (tadeusz.struk@intel.com)
  14  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  15  *    Copyright (c) 2010, Intel Corporation.
  16  */
  17
  18 #include <linux/hardirq.h>
  19 #include <linux/types.h>
  20 #include <linux/module.h>
  21 #include <linux/err.h>
  22 #include <crypto/algapi.h>
  23 #include <crypto/aes.h>
  24 #include <crypto/ctr.h>
  25 #include <crypto/b128ops.h>
  26 #include <crypto/gcm.h>
  27 #include <crypto/xts.h>
  28 #include <asm/cpu_device_id.h>
  29 #include <asm/simd.h>
  30 #include <crypto/scatterwalk.h>
  31 #include <crypto/internal/aead.h>
  32 #include <crypto/internal/simd.h>
  33 #include <crypto/internal/skcipher.h>
  34 #include <linux/jump_label.h>
  35 #include <linux/workqueue.h>
  36 #include <linux/spinlock.h>
  37 #include <linux/static_call.h>
  38
  39
  40 #define AESNI_ALIGN     16
  41 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
  42 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE - 1))
  43 #define RFC4106_HASH_SUBKEY_SIZE 16
  44 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
  45 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
  46 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
  47
  48 /* This data is stored at the end of the crypto_tfm struct.
  49  * It's a type of per "session" data storage location.
  50  * This needs to be 16 byte aligned.
  51  */
  52 struct aesni_rfc4106_gcm_ctx {
  53         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  54         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  55         u8 nonce[4];
  56 };
  57
  58 struct generic_gcmaes_ctx {
  59         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  60         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  61 };
  62
  63 struct aesni_xts_ctx {
  64         struct crypto_aes_ctx tweak_ctx AESNI_ALIGN_ATTR;
  65         struct crypto_aes_ctx crypt_ctx AESNI_ALIGN_ATTR;
  66 };
  67
  68 #define GCM_BLOCK_LEN 16
  69
  70 struct gcm_context_data {
  71         /* init, update and finalize context data */
  72         u8 aad_hash[GCM_BLOCK_LEN];
  73         u64 aad_length;
  74         u64 in_length;
  75         u8 partial_block_enc_key[GCM_BLOCK_LEN];
  76         u8 orig_IV[GCM_BLOCK_LEN];
  77         u8 current_counter[GCM_BLOCK_LEN];
  78         u64 partial_block_len;
  79         u64 unused;
  80         u8 hash_keys[GCM_BLOCK_LEN * 16];
  81 };
  82
  83 static inline void *aes_align_addr(void *addr)
  84 {
  85         if (crypto_tfm_ctx_alignment() >= AESNI_ALIGN)
  86                 return addr;
  87         return PTR_ALIGN(addr, AESNI_ALIGN);
  88 }
  89
  90 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
  91                              unsigned int key_len);
  92 asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
  93 asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in);
  94 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
  95                               const u8 *in, unsigned int len);
  96 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
  97                               const u8 *in, unsigned int len);
  98 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  99                               const u8 *in, unsigned int len, u8 *iv);
 100 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
 101                               const u8 *in, unsigned int len, u8 *iv);
 102 asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
 103                                   const u8 *in, unsigned int len, u8 *iv);
 104 asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
 105                                   const u8 *in, unsigned int len, u8 *iv);
 106
 107 #define AVX_GEN2_OPTSIZE 640
 108 #define AVX_GEN4_OPTSIZE 4096
 109
 110 asmlinkage void aesni_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *out,
 111                                   const u8 *in, unsigned int len, u8 *iv);
 112
 113 asmlinkage void aesni_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *out,
 114                                   const u8 *in, unsigned int len, u8 *iv);
 115
 116 #ifdef CONFIG_X86_64
 117
 118 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
 119                               const u8 *in, unsigned int len, u8 *iv);
 120 DEFINE_STATIC_CALL(aesni_ctr_enc_tfm, aesni_ctr_enc);
 121
 122 /* Scatter / Gather routines, with args similar to above */
 123 asmlinkage void aesni_gcm_init(void *ctx,
 124                                struct gcm_context_data *gdata,
 125                                u8 *iv,
 126                                u8 *hash_subkey, const u8 *aad,
 127                                unsigned long aad_len);
 128 asmlinkage void aesni_gcm_enc_update(void *ctx,
 129                                      struct gcm_context_data *gdata, u8 *out,
 130                                      const u8 *in, unsigned long plaintext_len);
 131 asmlinkage void aesni_gcm_dec_update(void *ctx,
 132                                      struct gcm_context_data *gdata, u8 *out,
 133                                      const u8 *in,
 134                                      unsigned long ciphertext_len);
 135 asmlinkage void aesni_gcm_finalize(void *ctx,
 136                                    struct gcm_context_data *gdata,
 137                                    u8 *auth_tag, unsigned long auth_tag_len);
 138
 139 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
 140                 void *keys, u8 *out, unsigned int num_bytes);
 141 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
 142                 void *keys, u8 *out, unsigned int num_bytes);
 143 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
 144                 void *keys, u8 *out, unsigned int num_bytes);
 145
 146
 147 asmlinkage void aes_xctr_enc_128_avx_by8(const u8 *in, const u8 *iv,
 148         const void *keys, u8 *out, unsigned int num_bytes,
 149         unsigned int byte_ctr);
 150
 151 asmlinkage void aes_xctr_enc_192_avx_by8(const u8 *in, const u8 *iv,
 152         const void *keys, u8 *out, unsigned int num_bytes,
 153         unsigned int byte_ctr);
 154
 155 asmlinkage void aes_xctr_enc_256_avx_by8(const u8 *in, const u8 *iv,
 156         const void *keys, u8 *out, unsigned int num_bytes,
 157         unsigned int byte_ctr);
 158
 159 /*
 160  * asmlinkage void aesni_gcm_init_avx_gen2()
 161  * gcm_data *my_ctx_data, context data
 162  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 163  */
 164 asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data,
 165                                         struct gcm_context_data *gdata,
 166                                         u8 *iv,
 167                                         u8 *hash_subkey,
 168                                         const u8 *aad,
 169                                         unsigned long aad_len);
 170
 171 asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx,
 172                                      struct gcm_context_data *gdata, u8 *out,
 173                                      const u8 *in, unsigned long plaintext_len);
 174 asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx,
 175                                      struct gcm_context_data *gdata, u8 *out,
 176                                      const u8 *in,
 177                                      unsigned long ciphertext_len);
 178 asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx,
 179                                    struct gcm_context_data *gdata,
 180                                    u8 *auth_tag, unsigned long auth_tag_len);
 181
 182 /*
 183  * asmlinkage void aesni_gcm_init_avx_gen4()
 184  * gcm_data *my_ctx_data, context data
 185  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 186  */
 187 asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data,
 188                                         struct gcm_context_data *gdata,
 189                                         u8 *iv,
 190                                         u8 *hash_subkey,
 191                                         const u8 *aad,
 192                                         unsigned long aad_len);
 193
 194 asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx,
 195                                      struct gcm_context_data *gdata, u8 *out,
 196                                      const u8 *in, unsigned long plaintext_len);
 197 asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx,
 198                                      struct gcm_context_data *gdata, u8 *out,
 199                                      const u8 *in,
 200                                      unsigned long ciphertext_len);
 201 asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx,
 202                                    struct gcm_context_data *gdata,
 203                                    u8 *auth_tag, unsigned long auth_tag_len);
 204
 205 static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx);
 206 static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx2);
 207
 208 static inline struct
 209 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
 210 {
 211         return aes_align_addr(crypto_aead_ctx(tfm));
 212 }
 213
 214 static inline struct
 215 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
 216 {
 217         return aes_align_addr(crypto_aead_ctx(tfm));
 218 }
 219 #endif
 220
 221 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 222 {
 223         return aes_align_addr(raw_ctx);
 224 }
 225
 226 static inline struct aesni_xts_ctx *aes_xts_ctx(struct crypto_skcipher *tfm)
 227 {
 228         return aes_align_addr(crypto_skcipher_ctx(tfm));
 229 }
 230
 231 static int aes_set_key_common(struct crypto_aes_ctx *ctx,
 232                               const u8 *in_key, unsigned int key_len)
 233 {
 234         int err;
 235
 236         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
 237             key_len != AES_KEYSIZE_256)
 238                 return -EINVAL;
 239
 240         if (!crypto_simd_usable())
 241                 err = aes_expandkey(ctx, in_key, key_len);
 242         else {
 243                 kernel_fpu_begin();
 244                 err = aesni_set_key(ctx, in_key, key_len);
 245                 kernel_fpu_end();
 246         }
 247
 248         return err;
 249 }
 250
 251 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 252                        unsigned int key_len)
 253 {
 254         return aes_set_key_common(aes_ctx(crypto_tfm_ctx(tfm)), in_key,
 255                                   key_len);
 256 }
 257
 258 static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 259 {
 260         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 261
 262         if (!crypto_simd_usable()) {
 263                 aes_encrypt(ctx, dst, src);
 264         } else {
 265                 kernel_fpu_begin();
 266                 aesni_enc(ctx, dst, src);
 267                 kernel_fpu_end();
 268         }
 269 }
 270
 271 static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 272 {
 273         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 274
 275         if (!crypto_simd_usable()) {
 276                 aes_decrypt(ctx, dst, src);
 277         } else {
 278                 kernel_fpu_begin();
 279                 aesni_dec(ctx, dst, src);
 280                 kernel_fpu_end();
 281         }
 282 }
 283
 284 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 285                                  unsigned int len)
 286 {
 287         return aes_set_key_common(aes_ctx(crypto_skcipher_ctx(tfm)), key, len);
 288 }
 289
 290 static int ecb_encrypt(struct skcipher_request *req)
 291 {
 292         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 293         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 294         struct skcipher_walk walk;
 295         unsigned int nbytes;
 296         int err;
 297
 298         err = skcipher_walk_virt(&walk, req, false);
 299
 300         while ((nbytes = walk.nbytes)) {
 301                 kernel_fpu_begin();
 302                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 303                               nbytes & AES_BLOCK_MASK);
 304                 kernel_fpu_end();
 305                 nbytes &= AES_BLOCK_SIZE - 1;
 306                 err = skcipher_walk_done(&walk, nbytes);
 307         }
 308
 309         return err;
 310 }
 311
 312 static int ecb_decrypt(struct skcipher_request *req)
 313 {
 314         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 315         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 316         struct skcipher_walk walk;
 317         unsigned int nbytes;
 318         int err;
 319
 320         err = skcipher_walk_virt(&walk, req, false);
 321
 322         while ((nbytes = walk.nbytes)) {
 323                 kernel_fpu_begin();
 324                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 325                               nbytes & AES_BLOCK_MASK);
 326                 kernel_fpu_end();
 327                 nbytes &= AES_BLOCK_SIZE - 1;
 328                 err = skcipher_walk_done(&walk, nbytes);
 329         }
 330
 331         return err;
 332 }
 333
 334 static int cbc_encrypt(struct skcipher_request *req)
 335 {
 336         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 337         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 338         struct skcipher_walk walk;
 339         unsigned int nbytes;
 340         int err;
 341
 342         err = skcipher_walk_virt(&walk, req, false);
 343
 344         while ((nbytes = walk.nbytes)) {
 345                 kernel_fpu_begin();
 346                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 347                               nbytes & AES_BLOCK_MASK, walk.iv);
 348                 kernel_fpu_end();
 349                 nbytes &= AES_BLOCK_SIZE - 1;
 350                 err = skcipher_walk_done(&walk, nbytes);
 351         }
 352
 353         return err;
 354 }
 355
 356 static int cbc_decrypt(struct skcipher_request *req)
 357 {
 358         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 359         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 360         struct skcipher_walk walk;
 361         unsigned int nbytes;
 362         int err;
 363
 364         err = skcipher_walk_virt(&walk, req, false);
 365
 366         while ((nbytes = walk.nbytes)) {
 367                 kernel_fpu_begin();
 368                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 369                               nbytes & AES_BLOCK_MASK, walk.iv);
 370                 kernel_fpu_end();
 371                 nbytes &= AES_BLOCK_SIZE - 1;
 372                 err = skcipher_walk_done(&walk, nbytes);
 373         }
 374
 375         return err;
 376 }
 377
 378 static int cts_cbc_encrypt(struct skcipher_request *req)
 379 {
 380         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 381         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 382         int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
 383         struct scatterlist *src = req->src, *dst = req->dst;
 384         struct scatterlist sg_src[2], sg_dst[2];
 385         struct skcipher_request subreq;
 386         struct skcipher_walk walk;
 387         int err;
 388
 389         skcipher_request_set_tfm(&subreq, tfm);
 390         skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
 391                                       NULL, NULL);
 392
 393         if (req->cryptlen <= AES_BLOCK_SIZE) {
 394                 if (req->cryptlen < AES_BLOCK_SIZE)
 395                         return -EINVAL;
 396                 cbc_blocks = 1;
 397         }
 398
 399         if (cbc_blocks > 0) {
 400                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 401                                            cbc_blocks * AES_BLOCK_SIZE,
 402                                            req->iv);
 403
 404                 err = cbc_encrypt(&subreq);
 405                 if (err)
 406                         return err;
 407
 408                 if (req->cryptlen == AES_BLOCK_SIZE)
 409                         return 0;
 410
 411                 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
 412                 if (req->dst != req->src)
 413                         dst = scatterwalk_ffwd(sg_dst, req->dst,
 414                                                subreq.cryptlen);
 415         }
 416
 417         /* handle ciphertext stealing */
 418         skcipher_request_set_crypt(&subreq, src, dst,
 419                                    req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
 420                                    req->iv);
 421
 422         err = skcipher_walk_virt(&walk, &subreq, false);
 423         if (err)
 424                 return err;
 425
 426         kernel_fpu_begin();
 427         aesni_cts_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 428                           walk.nbytes, walk.iv);
 429         kernel_fpu_end();
 430
 431         return skcipher_walk_done(&walk, 0);
 432 }
 433
 434 static int cts_cbc_decrypt(struct skcipher_request *req)
 435 {
 436         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 437         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 438         int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
 439         struct scatterlist *src = req->src, *dst = req->dst;
 440         struct scatterlist sg_src[2], sg_dst[2];
 441         struct skcipher_request subreq;
 442         struct skcipher_walk walk;
 443         int err;
 444
 445         skcipher_request_set_tfm(&subreq, tfm);
 446         skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
 447                                       NULL, NULL);
 448
 449         if (req->cryptlen <= AES_BLOCK_SIZE) {
 450                 if (req->cryptlen < AES_BLOCK_SIZE)
 451                         return -EINVAL;
 452                 cbc_blocks = 1;
 453         }
 454
 455         if (cbc_blocks > 0) {
 456                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 457                                            cbc_blocks * AES_BLOCK_SIZE,
 458                                            req->iv);
 459
 460                 err = cbc_decrypt(&subreq);
 461                 if (err)
 462                         return err;
 463
 464                 if (req->cryptlen == AES_BLOCK_SIZE)
 465                         return 0;
 466
 467                 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
 468                 if (req->dst != req->src)
 469                         dst = scatterwalk_ffwd(sg_dst, req->dst,
 470                                                subreq.cryptlen);
 471         }
 472
 473         /* handle ciphertext stealing */
 474         skcipher_request_set_crypt(&subreq, src, dst,
 475                                    req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
 476                                    req->iv);
 477
 478         err = skcipher_walk_virt(&walk, &subreq, false);
 479         if (err)
 480                 return err;
 481
 482         kernel_fpu_begin();
 483         aesni_cts_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 484                           walk.nbytes, walk.iv);
 485         kernel_fpu_end();
 486
 487         return skcipher_walk_done(&walk, 0);
 488 }
 489
 490 #ifdef CONFIG_X86_64
 491 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
 492                               const u8 *in, unsigned int len, u8 *iv)
 493 {
 494         /*
 495          * based on key length, override with the by8 version
 496          * of ctr mode encryption/decryption for improved performance
 497          * aes_set_key_common() ensures that key length is one of
 498          * {128,192,256}
 499          */
 500         if (ctx->key_length == AES_KEYSIZE_128)
 501                 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
 502         else if (ctx->key_length == AES_KEYSIZE_192)
 503                 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
 504         else
 505                 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
 506 }
 507
 508 static int ctr_crypt(struct skcipher_request *req)
 509 {
 510         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 511         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 512         u8 keystream[AES_BLOCK_SIZE];
 513         struct skcipher_walk walk;
 514         unsigned int nbytes;
 515         int err;
 516
 517         err = skcipher_walk_virt(&walk, req, false);
 518
 519         while ((nbytes = walk.nbytes) > 0) {
 520                 kernel_fpu_begin();
 521                 if (nbytes & AES_BLOCK_MASK)
 522                         static_call(aesni_ctr_enc_tfm)(ctx, walk.dst.virt.addr,
 523                                                        walk.src.virt.addr,
 524                                                        nbytes & AES_BLOCK_MASK,
 525                                                        walk.iv);
 526                 nbytes &= ~AES_BLOCK_MASK;
 527
 528                 if (walk.nbytes == walk.total && nbytes > 0) {
 529                         aesni_enc(ctx, keystream, walk.iv);
 530                         crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes - nbytes,
 531                                        walk.src.virt.addr + walk.nbytes - nbytes,
 532                                        keystream, nbytes);
 533                         crypto_inc(walk.iv, AES_BLOCK_SIZE);
 534                         nbytes = 0;
 535                 }
 536                 kernel_fpu_end();
 537                 err = skcipher_walk_done(&walk, nbytes);
 538         }
 539         return err;
 540 }
 541
 542 static void aesni_xctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
 543                                    const u8 *in, unsigned int len, u8 *iv,
 544                                    unsigned int byte_ctr)
 545 {
 546         if (ctx->key_length == AES_KEYSIZE_128)
 547                 aes_xctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len,
 548                                          byte_ctr);
 549         else if (ctx->key_length == AES_KEYSIZE_192)
 550                 aes_xctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len,
 551                                          byte_ctr);
 552         else
 553                 aes_xctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len,
 554                                          byte_ctr);
 555 }
 556
 557 static int xctr_crypt(struct skcipher_request *req)
 558 {
 559         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 560         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 561         u8 keystream[AES_BLOCK_SIZE];
 562         struct skcipher_walk walk;
 563         unsigned int nbytes;
 564         unsigned int byte_ctr = 0;
 565         int err;
 566         __le32 block[AES_BLOCK_SIZE / sizeof(__le32)];
 567
 568         err = skcipher_walk_virt(&walk, req, false);
 569
 570         while ((nbytes = walk.nbytes) > 0) {
 571                 kernel_fpu_begin();
 572                 if (nbytes & AES_BLOCK_MASK)
 573                         aesni_xctr_enc_avx_tfm(ctx, walk.dst.virt.addr,
 574                                 walk.src.virt.addr, nbytes & AES_BLOCK_MASK,
 575                                 walk.iv, byte_ctr);
 576                 nbytes &= ~AES_BLOCK_MASK;
 577                 byte_ctr += walk.nbytes - nbytes;
 578
 579                 if (walk.nbytes == walk.total && nbytes > 0) {
 580                         memcpy(block, walk.iv, AES_BLOCK_SIZE);
 581                         block[0] ^= cpu_to_le32(1 + byte_ctr / AES_BLOCK_SIZE);
 582                         aesni_enc(ctx, keystream, (u8 *)block);
 583                         crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes -
 584                                        nbytes, walk.src.virt.addr + walk.nbytes
 585                                        - nbytes, keystream, nbytes);
 586                         byte_ctr += nbytes;
 587                         nbytes = 0;
 588                 }
 589                 kernel_fpu_end();
 590                 err = skcipher_walk_done(&walk, nbytes);
 591         }
 592         return err;
 593 }
 594
 595 static int
 596 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
 597 {
 598         struct crypto_aes_ctx ctx;
 599         int ret;
 600
 601         ret = aes_expandkey(&ctx, key, key_len);
 602         if (ret)
 603                 return ret;
 604
 605         /* Clear the data in the hash sub key container to zero.*/
 606         /* We want to cipher all zeros to create the hash sub key. */
 607         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
 608
 609         aes_encrypt(&ctx, hash_subkey, hash_subkey);
 610
 611         memzero_explicit(&ctx, sizeof(ctx));
 612         return 0;
 613 }
 614
 615 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
 616                                   unsigned int key_len)
 617 {
 618         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
 619
 620         if (key_len < 4)
 621                 return -EINVAL;
 622
 623         /*Account for 4 byte nonce at the end.*/
 624         key_len -= 4;
 625
 626         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
 627
 628         return aes_set_key_common(&ctx->aes_key_expanded, key, key_len) ?:
 629                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 630 }
 631
 632 /* This is the Integrity Check Value (aka the authentication tag) length and can
 633  * be 8, 12 or 16 bytes long. */
 634 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
 635                                        unsigned int authsize)
 636 {
 637         switch (authsize) {
 638         case 8:
 639         case 12:
 640         case 16:
 641                 break;
 642         default:
 643                 return -EINVAL;
 644         }
 645
 646         return 0;
 647 }
 648
 649 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
 650                                        unsigned int authsize)
 651 {
 652         switch (authsize) {
 653         case 4:
 654         case 8:
 655         case 12:
 656         case 13:
 657         case 14:
 658         case 15:
 659         case 16:
 660                 break;
 661         default:
 662                 return -EINVAL;
 663         }
 664
 665         return 0;
 666 }
 667
 668 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
 669                               unsigned int assoclen, u8 *hash_subkey,
 670                               u8 *iv, void *aes_ctx, u8 *auth_tag,
 671                               unsigned long auth_tag_len)
 672 {
 673         u8 databuf[sizeof(struct gcm_context_data) + (AESNI_ALIGN - 8)] __aligned(8);
 674         struct gcm_context_data *data = PTR_ALIGN((void *)databuf, AESNI_ALIGN);
 675         unsigned long left = req->cryptlen;
 676         struct scatter_walk assoc_sg_walk;
 677         struct skcipher_walk walk;
 678         bool do_avx, do_avx2;
 679         u8 *assocmem = NULL;
 680         u8 *assoc;
 681         int err;
 682
 683         if (!enc)
 684                 left -= auth_tag_len;
 685
 686         do_avx = (left >= AVX_GEN2_OPTSIZE);
 687         do_avx2 = (left >= AVX_GEN4_OPTSIZE);
 688
 689         /* Linearize assoc, if not already linear */
 690         if (req->src->length >= assoclen && req->src->length) {
 691                 scatterwalk_start(&assoc_sg_walk, req->src);
 692                 assoc = scatterwalk_map(&assoc_sg_walk);
 693         } else {
 694                 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 695                               GFP_KERNEL : GFP_ATOMIC;
 696
 697                 /* assoc can be any length, so must be on heap */
 698                 assocmem = kmalloc(assoclen, flags);
 699                 if (unlikely(!assocmem))
 700                         return -ENOMEM;
 701                 assoc = assocmem;
 702
 703                 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
 704         }
 705
 706         kernel_fpu_begin();
 707         if (static_branch_likely(&gcm_use_avx2) && do_avx2)
 708                 aesni_gcm_init_avx_gen4(aes_ctx, data, iv, hash_subkey, assoc,
 709                                         assoclen);
 710         else if (static_branch_likely(&gcm_use_avx) && do_avx)
 711                 aesni_gcm_init_avx_gen2(aes_ctx, data, iv, hash_subkey, assoc,
 712                                         assoclen);
 713         else
 714                 aesni_gcm_init(aes_ctx, data, iv, hash_subkey, assoc, assoclen);
 715         kernel_fpu_end();
 716
 717         if (!assocmem)
 718                 scatterwalk_unmap(assoc);
 719         else
 720                 kfree(assocmem);
 721
 722         err = enc ? skcipher_walk_aead_encrypt(&walk, req, false)
 723                   : skcipher_walk_aead_decrypt(&walk, req, false);
 724
 725         while (walk.nbytes > 0) {
 726                 kernel_fpu_begin();
 727                 if (static_branch_likely(&gcm_use_avx2) && do_avx2) {
 728                         if (enc)
 729                                 aesni_gcm_enc_update_avx_gen4(aes_ctx, data,
 730                                                               walk.dst.virt.addr,
 731                                                               walk.src.virt.addr,
 732                                                               walk.nbytes);
 733                         else
 734                                 aesni_gcm_dec_update_avx_gen4(aes_ctx, data,
 735                                                               walk.dst.virt.addr,
 736                                                               walk.src.virt.addr,
 737                                                               walk.nbytes);
 738                 } else if (static_branch_likely(&gcm_use_avx) && do_avx) {
 739                         if (enc)
 740                                 aesni_gcm_enc_update_avx_gen2(aes_ctx, data,
 741                                                               walk.dst.virt.addr,
 742                                                               walk.src.virt.addr,
 743                                                               walk.nbytes);
 744                         else
 745                                 aesni_gcm_dec_update_avx_gen2(aes_ctx, data,
 746                                                               walk.dst.virt.addr,
 747                                                               walk.src.virt.addr,
 748                                                               walk.nbytes);
 749                 } else if (enc) {
 750                         aesni_gcm_enc_update(aes_ctx, data, walk.dst.virt.addr,
 751                                              walk.src.virt.addr, walk.nbytes);
 752                 } else {
 753                         aesni_gcm_dec_update(aes_ctx, data, walk.dst.virt.addr,
 754                                              walk.src.virt.addr, walk.nbytes);
 755                 }
 756                 kernel_fpu_end();
 757
 758                 err = skcipher_walk_done(&walk, 0);
 759         }
 760
 761         if (err)
 762                 return err;
 763
 764         kernel_fpu_begin();
 765         if (static_branch_likely(&gcm_use_avx2) && do_avx2)
 766                 aesni_gcm_finalize_avx_gen4(aes_ctx, data, auth_tag,
 767                                             auth_tag_len);
 768         else if (static_branch_likely(&gcm_use_avx) && do_avx)
 769                 aesni_gcm_finalize_avx_gen2(aes_ctx, data, auth_tag,
 770                                             auth_tag_len);
 771         else
 772                 aesni_gcm_finalize(aes_ctx, data, auth_tag, auth_tag_len);
 773         kernel_fpu_end();
 774
 775         return 0;
 776 }
 777
 778 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
 779                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 780 {
 781         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 782         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 783         u8 auth_tag[16];
 784         int err;
 785
 786         err = gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv, aes_ctx,
 787                                  auth_tag, auth_tag_len);
 788         if (err)
 789                 return err;
 790
 791         scatterwalk_map_and_copy(auth_tag, req->dst,
 792                                  req->assoclen + req->cryptlen,
 793                                  auth_tag_len, 1);
 794         return 0;
 795 }
 796
 797 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
 798                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 799 {
 800         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 801         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 802         u8 auth_tag_msg[16];
 803         u8 auth_tag[16];
 804         int err;
 805
 806         err = gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv, aes_ctx,
 807                                  auth_tag, auth_tag_len);
 808         if (err)
 809                 return err;
 810
 811         /* Copy out original auth_tag */
 812         scatterwalk_map_and_copy(auth_tag_msg, req->src,
 813                                  req->assoclen + req->cryptlen - auth_tag_len,
 814                                  auth_tag_len, 0);
 815
 816         /* Compare generated tag with passed in tag. */
 817         if (crypto_memneq(auth_tag_msg, auth_tag, auth_tag_len)) {
 818                 memzero_explicit(auth_tag, sizeof(auth_tag));
 819                 return -EBADMSG;
 820         }
 821         return 0;
 822 }
 823
 824 static int helper_rfc4106_encrypt(struct aead_request *req)
 825 {
 826         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 827         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 828         void *aes_ctx = &(ctx->aes_key_expanded);
 829         u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
 830         u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
 831         unsigned int i;
 832         __be32 counter = cpu_to_be32(1);
 833
 834         /* Assuming we are supporting rfc4106 64-bit extended */
 835         /* sequence numbers We need to have the AAD length equal */
 836         /* to 16 or 20 bytes */
 837         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 838                 return -EINVAL;
 839
 840         /* IV below built */
 841         for (i = 0; i < 4; i++)
 842                 *(iv+i) = ctx->nonce[i];
 843         for (i = 0; i < 8; i++)
 844                 *(iv+4+i) = req->iv[i];
 845         *((__be32 *)(iv+12)) = counter;
 846
 847         return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 848                               aes_ctx);
 849 }
 850
 851 static int helper_rfc4106_decrypt(struct aead_request *req)
 852 {
 853         __be32 counter = cpu_to_be32(1);
 854         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 855         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 856         void *aes_ctx = &(ctx->aes_key_expanded);
 857         u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
 858         u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
 859         unsigned int i;
 860
 861         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 862                 return -EINVAL;
 863
 864         /* Assuming we are supporting rfc4106 64-bit extended */
 865         /* sequence numbers We need to have the AAD length */
 866         /* equal to 16 or 20 bytes */
 867
 868         /* IV below built */
 869         for (i = 0; i < 4; i++)
 870                 *(iv+i) = ctx->nonce[i];
 871         for (i = 0; i < 8; i++)
 872                 *(iv+4+i) = req->iv[i];
 873         *((__be32 *)(iv+12)) = counter;
 874
 875         return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 876                               aes_ctx);
 877 }
 878 #endif
 879
 880 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
 881                             unsigned int keylen)
 882 {
 883         struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
 884         int err;
 885
 886         err = xts_verify_key(tfm, key, keylen);
 887         if (err)
 888                 return err;
 889
 890         keylen /= 2;
 891
 892         /* first half of xts-key is for crypt */
 893         err = aes_set_key_common(&ctx->crypt_ctx, key, keylen);
 894         if (err)
 895                 return err;
 896
 897         /* second half of xts-key is for tweak */
 898         return aes_set_key_common(&ctx->tweak_ctx, key + keylen, keylen);
 899 }
 900
 901 static int xts_crypt(struct skcipher_request *req, bool encrypt)
 902 {
 903         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 904         struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
 905         int tail = req->cryptlen % AES_BLOCK_SIZE;
 906         struct skcipher_request subreq;
 907         struct skcipher_walk walk;
 908         int err;
 909
 910         if (req->cryptlen < AES_BLOCK_SIZE)
 911                 return -EINVAL;
 912
 913         err = skcipher_walk_virt(&walk, req, false);
 914         if (!walk.nbytes)
 915                 return err;
 916
 917         if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
 918                 int blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
 919
 920                 skcipher_walk_abort(&walk);
 921
 922                 skcipher_request_set_tfm(&subreq, tfm);
 923                 skcipher_request_set_callback(&subreq,
 924                                               skcipher_request_flags(req),
 925                                               NULL, NULL);
 926                 skcipher_request_set_crypt(&subreq, req->src, req->dst,
 927                                            blocks * AES_BLOCK_SIZE, req->iv);
 928                 req = &subreq;
 929
 930                 err = skcipher_walk_virt(&walk, req, false);
 931                 if (!walk.nbytes)
 932                         return err;
 933         } else {
 934                 tail = 0;
 935         }
 936
 937         kernel_fpu_begin();
 938
 939         /* calculate first value of T */
 940         aesni_enc(&ctx->tweak_ctx, walk.iv, walk.iv);
 941
 942         while (walk.nbytes > 0) {
 943                 int nbytes = walk.nbytes;
 944
 945                 if (nbytes < walk.total)
 946                         nbytes &= ~(AES_BLOCK_SIZE - 1);
 947
 948                 if (encrypt)
 949                         aesni_xts_encrypt(&ctx->crypt_ctx,
 950                                           walk.dst.virt.addr, walk.src.virt.addr,
 951                                           nbytes, walk.iv);
 952                 else
 953                         aesni_xts_decrypt(&ctx->crypt_ctx,
 954                                           walk.dst.virt.addr, walk.src.virt.addr,
 955                                           nbytes, walk.iv);
 956                 kernel_fpu_end();
 957
 958                 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 959
 960                 if (walk.nbytes > 0)
 961                         kernel_fpu_begin();
 962         }
 963
 964         if (unlikely(tail > 0 && !err)) {
 965                 struct scatterlist sg_src[2], sg_dst[2];
 966                 struct scatterlist *src, *dst;
 967
 968                 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
 969                 if (req->dst != req->src)
 970                         dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
 971
 972                 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
 973                                            req->iv);
 974
 975                 err = skcipher_walk_virt(&walk, &subreq, false);
 976                 if (err)
 977                         return err;
 978
 979                 kernel_fpu_begin();
 980                 if (encrypt)
 981                         aesni_xts_encrypt(&ctx->crypt_ctx,
 982                                           walk.dst.virt.addr, walk.src.virt.addr,
 983                                           walk.nbytes, walk.iv);
 984                 else
 985                         aesni_xts_decrypt(&ctx->crypt_ctx,
 986                                           walk.dst.virt.addr, walk.src.virt.addr,
 987                                           walk.nbytes, walk.iv);
 988                 kernel_fpu_end();
 989
 990                 err = skcipher_walk_done(&walk, 0);
 991         }
 992         return err;
 993 }
 994
 995 static int xts_encrypt(struct skcipher_request *req)
 996 {
 997         return xts_crypt(req, true);
 998 }
 999
1000 static int xts_decrypt(struct skcipher_request *req)
1001 {
1002         return xts_crypt(req, false);
1003 }
1004
1005 static struct crypto_alg aesni_cipher_alg = {
1006         .cra_name               = "aes",
1007         .cra_driver_name        = "aes-aesni",
1008         .cra_priority           = 300,
1009         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
1010         .cra_blocksize          = AES_BLOCK_SIZE,
1011         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1012         .cra_module             = THIS_MODULE,
1013         .cra_u  = {
1014                 .cipher = {
1015                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
1016                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
1017                         .cia_setkey             = aes_set_key,
1018                         .cia_encrypt            = aesni_encrypt,
1019                         .cia_decrypt            = aesni_decrypt
1020                 }
1021         }
1022 };
1023
1024 static struct skcipher_alg aesni_skciphers[] = {
1025         {
1026                 .base = {
1027                         .cra_name               = "__ecb(aes)",
1028                         .cra_driver_name        = "__ecb-aes-aesni",
1029                         .cra_priority           = 400,
1030                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1031                         .cra_blocksize          = AES_BLOCK_SIZE,
1032                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1033                         .cra_module             = THIS_MODULE,
1034                 },
1035                 .min_keysize    = AES_MIN_KEY_SIZE,
1036                 .max_keysize    = AES_MAX_KEY_SIZE,
1037                 .setkey         = aesni_skcipher_setkey,
1038                 .encrypt        = ecb_encrypt,
1039                 .decrypt        = ecb_decrypt,
1040         }, {
1041                 .base = {
1042                         .cra_name               = "__cbc(aes)",
1043                         .cra_driver_name        = "__cbc-aes-aesni",
1044                         .cra_priority           = 400,
1045                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1046                         .cra_blocksize          = AES_BLOCK_SIZE,
1047                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1048                         .cra_module             = THIS_MODULE,
1049                 },
1050                 .min_keysize    = AES_MIN_KEY_SIZE,
1051                 .max_keysize    = AES_MAX_KEY_SIZE,
1052                 .ivsize         = AES_BLOCK_SIZE,
1053                 .setkey         = aesni_skcipher_setkey,
1054                 .encrypt        = cbc_encrypt,
1055                 .decrypt        = cbc_decrypt,
1056         }, {
1057                 .base = {
1058                         .cra_name               = "__cts(cbc(aes))",
1059                         .cra_driver_name        = "__cts-cbc-aes-aesni",
1060                         .cra_priority           = 400,
1061                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1062                         .cra_blocksize          = AES_BLOCK_SIZE,
1063                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1064                         .cra_module             = THIS_MODULE,
1065                 },
1066                 .min_keysize    = AES_MIN_KEY_SIZE,
1067                 .max_keysize    = AES_MAX_KEY_SIZE,
1068                 .ivsize         = AES_BLOCK_SIZE,
1069                 .walksize       = 2 * AES_BLOCK_SIZE,
1070                 .setkey         = aesni_skcipher_setkey,
1071                 .encrypt        = cts_cbc_encrypt,
1072                 .decrypt        = cts_cbc_decrypt,
1073 #ifdef CONFIG_X86_64
1074         }, {
1075                 .base = {
1076                         .cra_name               = "__ctr(aes)",
1077                         .cra_driver_name        = "__ctr-aes-aesni",
1078                         .cra_priority           = 400,
1079                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1080                         .cra_blocksize          = 1,
1081                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1082                         .cra_module             = THIS_MODULE,
1083                 },
1084                 .min_keysize    = AES_MIN_KEY_SIZE,
1085                 .max_keysize    = AES_MAX_KEY_SIZE,
1086                 .ivsize         = AES_BLOCK_SIZE,
1087                 .chunksize      = AES_BLOCK_SIZE,
1088                 .setkey         = aesni_skcipher_setkey,
1089                 .encrypt        = ctr_crypt,
1090                 .decrypt        = ctr_crypt,
1091 #endif
1092         }, {
1093                 .base = {
1094                         .cra_name               = "__xts(aes)",
1095                         .cra_driver_name        = "__xts-aes-aesni",
1096                         .cra_priority           = 401,
1097                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1098                         .cra_blocksize          = AES_BLOCK_SIZE,
1099                         .cra_ctxsize            = XTS_AES_CTX_SIZE,
1100                         .cra_module             = THIS_MODULE,
1101                 },
1102                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
1103                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
1104                 .ivsize         = AES_BLOCK_SIZE,
1105                 .walksize       = 2 * AES_BLOCK_SIZE,
1106                 .setkey         = xts_aesni_setkey,
1107                 .encrypt        = xts_encrypt,
1108                 .decrypt        = xts_decrypt,
1109         }
1110 };
1111
1112 static
1113 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
1114
1115 #ifdef CONFIG_X86_64
1116 /*
1117  * XCTR does not have a non-AVX implementation, so it must be enabled
1118  * conditionally.
1119  */
1120 static struct skcipher_alg aesni_xctr = {
1121         .base = {
1122                 .cra_name               = "__xctr(aes)",
1123                 .cra_driver_name        = "__xctr-aes-aesni",
1124                 .cra_priority           = 400,
1125                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1126                 .cra_blocksize          = 1,
1127                 .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1128                 .cra_module             = THIS_MODULE,
1129         },
1130         .min_keysize    = AES_MIN_KEY_SIZE,
1131         .max_keysize    = AES_MAX_KEY_SIZE,
1132         .ivsize         = AES_BLOCK_SIZE,
1133         .chunksize      = AES_BLOCK_SIZE,
1134         .setkey         = aesni_skcipher_setkey,
1135         .encrypt        = xctr_crypt,
1136         .decrypt        = xctr_crypt,
1137 };
1138
1139 static struct simd_skcipher_alg *aesni_simd_xctr;
1140 #endif /* CONFIG_X86_64 */
1141
1142 #ifdef CONFIG_X86_64
1143 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
1144                                   unsigned int key_len)
1145 {
1146         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
1147
1148         return aes_set_key_common(&ctx->aes_key_expanded, key, key_len) ?:
1149                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
1150 }
1151
1152 static int generic_gcmaes_encrypt(struct aead_request *req)
1153 {
1154         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1155         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1156         void *aes_ctx = &(ctx->aes_key_expanded);
1157         u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
1158         u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
1159         __be32 counter = cpu_to_be32(1);
1160
1161         memcpy(iv, req->iv, 12);
1162         *((__be32 *)(iv+12)) = counter;
1163
1164         return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
1165                               aes_ctx);
1166 }
1167
1168 static int generic_gcmaes_decrypt(struct aead_request *req)
1169 {
1170         __be32 counter = cpu_to_be32(1);
1171         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1172         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1173         void *aes_ctx = &(ctx->aes_key_expanded);
1174         u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8);
1175         u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN);
1176
1177         memcpy(iv, req->iv, 12);
1178         *((__be32 *)(iv+12)) = counter;
1179
1180         return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1181                               aes_ctx);
1182 }
1183
1184 static struct aead_alg aesni_aeads[] = { {
1185         .setkey                 = common_rfc4106_set_key,
1186         .setauthsize            = common_rfc4106_set_authsize,
1187         .encrypt                = helper_rfc4106_encrypt,
1188         .decrypt                = helper_rfc4106_decrypt,
1189         .ivsize                 = GCM_RFC4106_IV_SIZE,
1190         .maxauthsize            = 16,
1191         .base = {
1192                 .cra_name               = "__rfc4106(gcm(aes))",
1193                 .cra_driver_name        = "__rfc4106-gcm-aesni",
1194                 .cra_priority           = 400,
1195                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1196                 .cra_blocksize          = 1,
1197                 .cra_ctxsize            = sizeof(struct aesni_rfc4106_gcm_ctx),
1198                 .cra_alignmask          = 0,
1199                 .cra_module             = THIS_MODULE,
1200         },
1201 }, {
1202         .setkey                 = generic_gcmaes_set_key,
1203         .setauthsize            = generic_gcmaes_set_authsize,
1204         .encrypt                = generic_gcmaes_encrypt,
1205         .decrypt                = generic_gcmaes_decrypt,
1206         .ivsize                 = GCM_AES_IV_SIZE,
1207         .maxauthsize            = 16,
1208         .base = {
1209                 .cra_name               = "__gcm(aes)",
1210                 .cra_driver_name        = "__generic-gcm-aesni",
1211                 .cra_priority           = 400,
1212                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1213                 .cra_blocksize          = 1,
1214                 .cra_ctxsize            = sizeof(struct generic_gcmaes_ctx),
1215                 .cra_alignmask          = 0,
1216                 .cra_module             = THIS_MODULE,
1217         },
1218 } };
1219 #else
1220 static struct aead_alg aesni_aeads[0];
1221 #endif
1222
1223 static struct simd_aead_alg *aesni_simd_aeads[ARRAY_SIZE(aesni_aeads)];
1224
1225 static const struct x86_cpu_id aesni_cpu_id[] = {
1226         X86_MATCH_FEATURE(X86_FEATURE_AES, NULL),
1227         {}
1228 };
1229 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1230
1231 static int __init aesni_init(void)
1232 {
1233         int err;
1234
1235         if (!x86_match_cpu(aesni_cpu_id))
1236                 return -ENODEV;
1237 #ifdef CONFIG_X86_64
1238         if (boot_cpu_has(X86_FEATURE_AVX2)) {
1239                 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1240                 static_branch_enable(&gcm_use_avx);
1241                 static_branch_enable(&gcm_use_avx2);
1242         } else
1243         if (boot_cpu_has(X86_FEATURE_AVX)) {
1244                 pr_info("AVX version of gcm_enc/dec engaged.\n");
1245                 static_branch_enable(&gcm_use_avx);
1246         } else {
1247                 pr_info("SSE version of gcm_enc/dec engaged.\n");
1248         }
1249         if (boot_cpu_has(X86_FEATURE_AVX)) {
1250                 /* optimize performance of ctr mode encryption transform */
1251                 static_call_update(aesni_ctr_enc_tfm, aesni_ctr_enc_avx_tfm);
1252                 pr_info("AES CTR mode by8 optimization enabled\n");
1253         }
1254 #endif /* CONFIG_X86_64 */
1255
1256         err = crypto_register_alg(&aesni_cipher_alg);
1257         if (err)
1258                 return err;
1259
1260         err = simd_register_skciphers_compat(aesni_skciphers,
1261                                              ARRAY_SIZE(aesni_skciphers),
1262                                              aesni_simd_skciphers);
1263         if (err)
1264                 goto unregister_cipher;
1265
1266         err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1267                                          aesni_simd_aeads);
1268         if (err)
1269                 goto unregister_skciphers;
1270
1271 #ifdef CONFIG_X86_64
1272         if (boot_cpu_has(X86_FEATURE_AVX))
1273                 err = simd_register_skciphers_compat(&aesni_xctr, 1,
1274                                                      &aesni_simd_xctr);
1275         if (err)
1276                 goto unregister_aeads;
1277 #endif /* CONFIG_X86_64 */
1278
1279         return 0;
1280
1281 #ifdef CONFIG_X86_64
1282 unregister_aeads:
1283         simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1284                                 aesni_simd_aeads);
1285 #endif /* CONFIG_X86_64 */
1286
1287 unregister_skciphers:
1288         simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1289                                   aesni_simd_skciphers);
1290 unregister_cipher:
1291         crypto_unregister_alg(&aesni_cipher_alg);
1292         return err;
1293 }
1294
1295 static void __exit aesni_exit(void)
1296 {
1297         simd_unregister_aeads(aesni_aeads, ARRAY_SIZE(aesni_aeads),
1298                               aesni_simd_aeads);
1299         simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
1300                                   aesni_simd_skciphers);
1301         crypto_unregister_alg(&aesni_cipher_alg);
1302 #ifdef CONFIG_X86_64
1303         if (boot_cpu_has(X86_FEATURE_AVX))
1304                 simd_unregister_skciphers(&aesni_xctr, 1, &aesni_simd_xctr);
1305 #endif /* CONFIG_X86_64 */
1306 }
1307
1308 late_initcall(aesni_init);
1309 module_exit(aesni_exit);
1310
1311 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1312 MODULE_LICENSE("GPL");
1313 MODULE_ALIAS_CRYPTO("aes");