arch/x86/crypto/aesni-intel_glue.c

   1 /*
   2  * Support for Intel AES-NI instructions. This file contains glue
   3  * code, the real AES implementation is in intel-aes_asm.S.
   4  *
   5  * Copyright (C) 2008, Intel Corp.
   6  *    Author: Huang Ying <ying.huang@intel.com>
   7  *
   8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
   9  * interface for 64-bit kernels.
  10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  12  *             Tadeusz Struk (tadeusz.struk@intel.com)
  13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  14  *    Copyright (c) 2010, Intel Corporation.
  15  *
  16  * This program is free software; you can redistribute it and/or modify
  17  * it under the terms of the GNU General Public License as published by
  18  * the Free Software Foundation; either version 2 of the License, or
  19  * (at your option) any later version.
  20  */
  21
  22 #include <linux/hardirq.h>
  23 #include <linux/types.h>
  24 #include <linux/module.h>
  25 #include <linux/err.h>
  26 #include <crypto/algapi.h>
  27 #include <crypto/aes.h>
  28 #include <crypto/cryptd.h>
  29 #include <crypto/ctr.h>
  30 #include <crypto/b128ops.h>
  31 #include <crypto/gcm.h>
  32 #include <crypto/xts.h>
  33 #include <asm/cpu_device_id.h>
  34 #include <asm/fpu/api.h>
  35 #include <asm/crypto/aes.h>
  36 #include <crypto/scatterwalk.h>
  37 #include <crypto/internal/aead.h>
  38 #include <crypto/internal/simd.h>
  39 #include <crypto/internal/skcipher.h>
  40 #include <linux/workqueue.h>
  41 #include <linux/spinlock.h>
  42 #ifdef CONFIG_X86_64
  43 #include <asm/crypto/glue_helper.h>
  44 #endif
  45
  46
  47 #define AESNI_ALIGN     16
  48 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
  49 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE - 1))
  50 #define RFC4106_HASH_SUBKEY_SIZE 16
  51 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
  52 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
  53 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
  54
  55 /* This data is stored at the end of the crypto_tfm struct.
  56  * It's a type of per "session" data storage location.
  57  * This needs to be 16 byte aligned.
  58  */
  59 struct aesni_rfc4106_gcm_ctx {
  60         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  61         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  62         u8 nonce[4];
  63 };
  64
  65 struct generic_gcmaes_ctx {
  66         u8 hash_subkey[16] AESNI_ALIGN_ATTR;
  67         struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
  68 };
  69
  70 struct aesni_xts_ctx {
  71         u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
  72         u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
  73 };
  74
  75 #define GCM_BLOCK_LEN 16
  76
  77 struct gcm_context_data {
  78         /* init, update and finalize context data */
  79         u8 aad_hash[GCM_BLOCK_LEN];
  80         u64 aad_length;
  81         u64 in_length;
  82         u8 partial_block_enc_key[GCM_BLOCK_LEN];
  83         u8 orig_IV[GCM_BLOCK_LEN];
  84         u8 current_counter[GCM_BLOCK_LEN];
  85         u64 partial_block_len;
  86         u64 unused;
  87         u8 hash_keys[GCM_BLOCK_LEN * 16];
  88 };
  89
  90 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
  91                              unsigned int key_len);
  92 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
  93                           const u8 *in);
  94 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
  95                           const u8 *in);
  96 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
  97                               const u8 *in, unsigned int len);
  98 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
  99                               const u8 *in, unsigned int len);
 100 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
 101                               const u8 *in, unsigned int len, u8 *iv);
 102 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
 103                               const u8 *in, unsigned int len, u8 *iv);
 104
 105 #define AVX_GEN2_OPTSIZE 640
 106 #define AVX_GEN4_OPTSIZE 4096
 107
 108 #ifdef CONFIG_X86_64
 109
 110 static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
 111                               const u8 *in, unsigned int len, u8 *iv);
 112 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
 113                               const u8 *in, unsigned int len, u8 *iv);
 114
 115 asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
 116                                  const u8 *in, bool enc, u8 *iv);
 117
 118 /* asmlinkage void aesni_gcm_enc()
 119  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
 120  * struct gcm_context_data.  May be uninitialized.
 121  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
 122  * const u8 *in, Plaintext input
 123  * unsigned long plaintext_len, Length of data in bytes for encryption.
 124  * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
 125  *         16-byte aligned pointer.
 126  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 127  * const u8 *aad, Additional Authentication Data (AAD)
 128  * unsigned long aad_len, Length of AAD in bytes.
 129  * u8 *auth_tag, Authenticated Tag output.
 130  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
 131  *          Valid values are 16 (most likely), 12 or 8.
 132  */
 133 asmlinkage void aesni_gcm_enc(void *ctx,
 134                         struct gcm_context_data *gdata, u8 *out,
 135                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 136                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 137                         u8 *auth_tag, unsigned long auth_tag_len);
 138
 139 /* asmlinkage void aesni_gcm_dec()
 140  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
 141  * struct gcm_context_data.  May be uninitialized.
 142  * u8 *out, Plaintext output. Decrypt in-place is allowed.
 143  * const u8 *in, Ciphertext input
 144  * unsigned long ciphertext_len, Length of data in bytes for decryption.
 145  * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
 146  *         16-byte aligned pointer.
 147  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 148  * const u8 *aad, Additional Authentication Data (AAD)
 149  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
 150  * to be 8 or 12 bytes
 151  * u8 *auth_tag, Authenticated Tag output.
 152  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
 153  * Valid values are 16 (most likely), 12 or 8.
 154  */
 155 asmlinkage void aesni_gcm_dec(void *ctx,
 156                         struct gcm_context_data *gdata, u8 *out,
 157                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 158                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 159                         u8 *auth_tag, unsigned long auth_tag_len);
 160
 161 /* Scatter / Gather routines, with args similar to above */
 162 asmlinkage void aesni_gcm_init(void *ctx,
 163                                struct gcm_context_data *gdata,
 164                                u8 *iv,
 165                                u8 *hash_subkey, const u8 *aad,
 166                                unsigned long aad_len);
 167 asmlinkage void aesni_gcm_enc_update(void *ctx,
 168                                      struct gcm_context_data *gdata, u8 *out,
 169                                      const u8 *in, unsigned long plaintext_len);
 170 asmlinkage void aesni_gcm_dec_update(void *ctx,
 171                                      struct gcm_context_data *gdata, u8 *out,
 172                                      const u8 *in,
 173                                      unsigned long ciphertext_len);
 174 asmlinkage void aesni_gcm_finalize(void *ctx,
 175                                    struct gcm_context_data *gdata,
 176                                    u8 *auth_tag, unsigned long auth_tag_len);
 177
 178 static struct aesni_gcm_tfm_s {
 179 void (*init)(void *ctx,
 180                                 struct gcm_context_data *gdata,
 181                                 u8 *iv,
 182                                 u8 *hash_subkey, const u8 *aad,
 183                                 unsigned long aad_len);
 184 void (*enc_update)(void *ctx,
 185                                         struct gcm_context_data *gdata, u8 *out,
 186                                         const u8 *in,
 187                                         unsigned long plaintext_len);
 188 void (*dec_update)(void *ctx,
 189                                         struct gcm_context_data *gdata, u8 *out,
 190                                         const u8 *in,
 191                                         unsigned long ciphertext_len);
 192 void (*finalize)(void *ctx,
 193                                 struct gcm_context_data *gdata,
 194                                 u8 *auth_tag, unsigned long auth_tag_len);
 195 } *aesni_gcm_tfm;
 196
 197 struct aesni_gcm_tfm_s aesni_gcm_tfm_sse = {
 198         .init = &aesni_gcm_init,
 199         .enc_update = &aesni_gcm_enc_update,
 200         .dec_update = &aesni_gcm_dec_update,
 201         .finalize = &aesni_gcm_finalize,
 202 };
 203
 204 #ifdef CONFIG_AS_AVX
 205 asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
 206                 void *keys, u8 *out, unsigned int num_bytes);
 207 asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
 208                 void *keys, u8 *out, unsigned int num_bytes);
 209 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
 210                 void *keys, u8 *out, unsigned int num_bytes);
 211 /*
 212  * asmlinkage void aesni_gcm_init_avx_gen2()
 213  * gcm_data *my_ctx_data, context data
 214  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 215  */
 216 asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data,
 217                                         struct gcm_context_data *gdata,
 218                                         u8 *iv,
 219                                         u8 *hash_subkey,
 220                                         const u8 *aad,
 221                                         unsigned long aad_len);
 222
 223 asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx,
 224                                      struct gcm_context_data *gdata, u8 *out,
 225                                      const u8 *in, unsigned long plaintext_len);
 226 asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx,
 227                                      struct gcm_context_data *gdata, u8 *out,
 228                                      const u8 *in,
 229                                      unsigned long ciphertext_len);
 230 asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx,
 231                                    struct gcm_context_data *gdata,
 232                                    u8 *auth_tag, unsigned long auth_tag_len);
 233
 234 asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx,
 235                                 struct gcm_context_data *gdata, u8 *out,
 236                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 237                         const u8 *aad, unsigned long aad_len,
 238                         u8 *auth_tag, unsigned long auth_tag_len);
 239
 240 asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx,
 241                                 struct gcm_context_data *gdata, u8 *out,
 242                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 243                         const u8 *aad, unsigned long aad_len,
 244                         u8 *auth_tag, unsigned long auth_tag_len);
 245
 246 struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen2 = {
 247         .init = &aesni_gcm_init_avx_gen2,
 248         .enc_update = &aesni_gcm_enc_update_avx_gen2,
 249         .dec_update = &aesni_gcm_dec_update_avx_gen2,
 250         .finalize = &aesni_gcm_finalize_avx_gen2,
 251 };
 252
 253 #endif
 254
 255 #ifdef CONFIG_AS_AVX2
 256 /*
 257  * asmlinkage void aesni_gcm_init_avx_gen4()
 258  * gcm_data *my_ctx_data, context data
 259  * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 260  */
 261 asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data,
 262                                         struct gcm_context_data *gdata,
 263                                         u8 *iv,
 264                                         u8 *hash_subkey,
 265                                         const u8 *aad,
 266                                         unsigned long aad_len);
 267
 268 asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx,
 269                                      struct gcm_context_data *gdata, u8 *out,
 270                                      const u8 *in, unsigned long plaintext_len);
 271 asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx,
 272                                      struct gcm_context_data *gdata, u8 *out,
 273                                      const u8 *in,
 274                                      unsigned long ciphertext_len);
 275 asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx,
 276                                    struct gcm_context_data *gdata,
 277                                    u8 *auth_tag, unsigned long auth_tag_len);
 278
 279 asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx,
 280                                 struct gcm_context_data *gdata, u8 *out,
 281                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 282                         const u8 *aad, unsigned long aad_len,
 283                         u8 *auth_tag, unsigned long auth_tag_len);
 284
 285 asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx,
 286                                 struct gcm_context_data *gdata, u8 *out,
 287                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 288                         const u8 *aad, unsigned long aad_len,
 289                         u8 *auth_tag, unsigned long auth_tag_len);
 290
 291 struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen4 = {
 292         .init = &aesni_gcm_init_avx_gen4,
 293         .enc_update = &aesni_gcm_enc_update_avx_gen4,
 294         .dec_update = &aesni_gcm_dec_update_avx_gen4,
 295         .finalize = &aesni_gcm_finalize_avx_gen4,
 296 };
 297
 298 #endif
 299
 300 static inline struct
 301 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
 302 {
 303         unsigned long align = AESNI_ALIGN;
 304
 305         if (align <= crypto_tfm_ctx_alignment())
 306                 align = 1;
 307         return PTR_ALIGN(crypto_aead_ctx(tfm), align);
 308 }
 309
 310 static inline struct
 311 generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
 312 {
 313         unsigned long align = AESNI_ALIGN;
 314
 315         if (align <= crypto_tfm_ctx_alignment())
 316                 align = 1;
 317         return PTR_ALIGN(crypto_aead_ctx(tfm), align);
 318 }
 319 #endif
 320
 321 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 322 {
 323         unsigned long addr = (unsigned long)raw_ctx;
 324         unsigned long align = AESNI_ALIGN;
 325
 326         if (align <= crypto_tfm_ctx_alignment())
 327                 align = 1;
 328         return (struct crypto_aes_ctx *)ALIGN(addr, align);
 329 }
 330
 331 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
 332                               const u8 *in_key, unsigned int key_len)
 333 {
 334         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
 335         u32 *flags = &tfm->crt_flags;
 336         int err;
 337
 338         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
 339             key_len != AES_KEYSIZE_256) {
 340                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 341                 return -EINVAL;
 342         }
 343
 344         if (!irq_fpu_usable())
 345                 err = crypto_aes_expand_key(ctx, in_key, key_len);
 346         else {
 347                 kernel_fpu_begin();
 348                 err = aesni_set_key(ctx, in_key, key_len);
 349                 kernel_fpu_end();
 350         }
 351
 352         return err;
 353 }
 354
 355 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 356                        unsigned int key_len)
 357 {
 358         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
 359 }
 360
 361 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 362 {
 363         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 364
 365         if (!irq_fpu_usable())
 366                 crypto_aes_encrypt_x86(ctx, dst, src);
 367         else {
 368                 kernel_fpu_begin();
 369                 aesni_enc(ctx, dst, src);
 370                 kernel_fpu_end();
 371         }
 372 }
 373
 374 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 375 {
 376         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 377
 378         if (!irq_fpu_usable())
 379                 crypto_aes_decrypt_x86(ctx, dst, src);
 380         else {
 381                 kernel_fpu_begin();
 382                 aesni_dec(ctx, dst, src);
 383                 kernel_fpu_end();
 384         }
 385 }
 386
 387 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 388 {
 389         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 390
 391         aesni_enc(ctx, dst, src);
 392 }
 393
 394 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 395 {
 396         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 397
 398         aesni_dec(ctx, dst, src);
 399 }
 400
 401 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 402                                  unsigned int len)
 403 {
 404         return aes_set_key_common(crypto_skcipher_tfm(tfm),
 405                                   crypto_skcipher_ctx(tfm), key, len);
 406 }
 407
 408 static int ecb_encrypt(struct skcipher_request *req)
 409 {
 410         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 411         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 412         struct skcipher_walk walk;
 413         unsigned int nbytes;
 414         int err;
 415
 416         err = skcipher_walk_virt(&walk, req, true);
 417
 418         kernel_fpu_begin();
 419         while ((nbytes = walk.nbytes)) {
 420                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 421                               nbytes & AES_BLOCK_MASK);
 422                 nbytes &= AES_BLOCK_SIZE - 1;
 423                 err = skcipher_walk_done(&walk, nbytes);
 424         }
 425         kernel_fpu_end();
 426
 427         return err;
 428 }
 429
 430 static int ecb_decrypt(struct skcipher_request *req)
 431 {
 432         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 433         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 434         struct skcipher_walk walk;
 435         unsigned int nbytes;
 436         int err;
 437
 438         err = skcipher_walk_virt(&walk, req, true);
 439
 440         kernel_fpu_begin();
 441         while ((nbytes = walk.nbytes)) {
 442                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 443                               nbytes & AES_BLOCK_MASK);
 444                 nbytes &= AES_BLOCK_SIZE - 1;
 445                 err = skcipher_walk_done(&walk, nbytes);
 446         }
 447         kernel_fpu_end();
 448
 449         return err;
 450 }
 451
 452 static int cbc_encrypt(struct skcipher_request *req)
 453 {
 454         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 455         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 456         struct skcipher_walk walk;
 457         unsigned int nbytes;
 458         int err;
 459
 460         err = skcipher_walk_virt(&walk, req, true);
 461
 462         kernel_fpu_begin();
 463         while ((nbytes = walk.nbytes)) {
 464                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 465                               nbytes & AES_BLOCK_MASK, walk.iv);
 466                 nbytes &= AES_BLOCK_SIZE - 1;
 467                 err = skcipher_walk_done(&walk, nbytes);
 468         }
 469         kernel_fpu_end();
 470
 471         return err;
 472 }
 473
 474 static int cbc_decrypt(struct skcipher_request *req)
 475 {
 476         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 477         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 478         struct skcipher_walk walk;
 479         unsigned int nbytes;
 480         int err;
 481
 482         err = skcipher_walk_virt(&walk, req, true);
 483
 484         kernel_fpu_begin();
 485         while ((nbytes = walk.nbytes)) {
 486                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 487                               nbytes & AES_BLOCK_MASK, walk.iv);
 488                 nbytes &= AES_BLOCK_SIZE - 1;
 489                 err = skcipher_walk_done(&walk, nbytes);
 490         }
 491         kernel_fpu_end();
 492
 493         return err;
 494 }
 495
 496 #ifdef CONFIG_X86_64
 497 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
 498                             struct skcipher_walk *walk)
 499 {
 500         u8 *ctrblk = walk->iv;
 501         u8 keystream[AES_BLOCK_SIZE];
 502         u8 *src = walk->src.virt.addr;
 503         u8 *dst = walk->dst.virt.addr;
 504         unsigned int nbytes = walk->nbytes;
 505
 506         aesni_enc(ctx, keystream, ctrblk);
 507         crypto_xor_cpy(dst, keystream, src, nbytes);
 508
 509         crypto_inc(ctrblk, AES_BLOCK_SIZE);
 510 }
 511
 512 #ifdef CONFIG_AS_AVX
 513 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
 514                               const u8 *in, unsigned int len, u8 *iv)
 515 {
 516         /*
 517          * based on key length, override with the by8 version
 518          * of ctr mode encryption/decryption for improved performance
 519          * aes_set_key_common() ensures that key length is one of
 520          * {128,192,256}
 521          */
 522         if (ctx->key_length == AES_KEYSIZE_128)
 523                 aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len);
 524         else if (ctx->key_length == AES_KEYSIZE_192)
 525                 aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len);
 526         else
 527                 aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
 528 }
 529 #endif
 530
 531 static int ctr_crypt(struct skcipher_request *req)
 532 {
 533         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 534         struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
 535         struct skcipher_walk walk;
 536         unsigned int nbytes;
 537         int err;
 538
 539         err = skcipher_walk_virt(&walk, req, true);
 540
 541         kernel_fpu_begin();
 542         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
 543                 aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 544                                       nbytes & AES_BLOCK_MASK, walk.iv);
 545                 nbytes &= AES_BLOCK_SIZE - 1;
 546                 err = skcipher_walk_done(&walk, nbytes);
 547         }
 548         if (walk.nbytes) {
 549                 ctr_crypt_final(ctx, &walk);
 550                 err = skcipher_walk_done(&walk, 0);
 551         }
 552         kernel_fpu_end();
 553
 554         return err;
 555 }
 556
 557 static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
 558                             unsigned int keylen)
 559 {
 560         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 561         int err;
 562
 563         err = xts_verify_key(tfm, key, keylen);
 564         if (err)
 565                 return err;
 566
 567         keylen /= 2;
 568
 569         /* first half of xts-key is for crypt */
 570         err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx,
 571                                  key, keylen);
 572         if (err)
 573                 return err;
 574
 575         /* second half of xts-key is for tweak */
 576         return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx,
 577                                   key + keylen, keylen);
 578 }
 579
 580
 581 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
 582 {
 583         aesni_enc(ctx, out, in);
 584 }
 585
 586 static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 587 {
 588         glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));
 589 }
 590
 591 static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 592 {
 593         glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec));
 594 }
 595
 596 static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 597 {
 598         aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);
 599 }
 600
 601 static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 602 {
 603         aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv);
 604 }
 605
 606 static const struct common_glue_ctx aesni_enc_xts = {
 607         .num_funcs = 2,
 608         .fpu_blocks_limit = 1,
 609
 610         .funcs = { {
 611                 .num_blocks = 8,
 612                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) }
 613         }, {
 614                 .num_blocks = 1,
 615                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) }
 616         } }
 617 };
 618
 619 static const struct common_glue_ctx aesni_dec_xts = {
 620         .num_funcs = 2,
 621         .fpu_blocks_limit = 1,
 622
 623         .funcs = { {
 624                 .num_blocks = 8,
 625                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) }
 626         }, {
 627                 .num_blocks = 1,
 628                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) }
 629         } }
 630 };
 631
 632 static int xts_encrypt(struct skcipher_request *req)
 633 {
 634         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 635         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 636
 637         return glue_xts_req_128bit(&aesni_enc_xts, req,
 638                                    XTS_TWEAK_CAST(aesni_xts_tweak),
 639                                    aes_ctx(ctx->raw_tweak_ctx),
 640                                    aes_ctx(ctx->raw_crypt_ctx));
 641 }
 642
 643 static int xts_decrypt(struct skcipher_request *req)
 644 {
 645         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 646         struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 647
 648         return glue_xts_req_128bit(&aesni_dec_xts, req,
 649                                    XTS_TWEAK_CAST(aesni_xts_tweak),
 650                                    aes_ctx(ctx->raw_tweak_ctx),
 651                                    aes_ctx(ctx->raw_crypt_ctx));
 652 }
 653
 654 static int rfc4106_init(struct crypto_aead *aead)
 655 {
 656         struct cryptd_aead *cryptd_tfm;
 657         struct cryptd_aead **ctx = crypto_aead_ctx(aead);
 658
 659         cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni",
 660                                        CRYPTO_ALG_INTERNAL,
 661                                        CRYPTO_ALG_INTERNAL);
 662         if (IS_ERR(cryptd_tfm))
 663                 return PTR_ERR(cryptd_tfm);
 664
 665         *ctx = cryptd_tfm;
 666         crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
 667         return 0;
 668 }
 669
 670 static void rfc4106_exit(struct crypto_aead *aead)
 671 {
 672         struct cryptd_aead **ctx = crypto_aead_ctx(aead);
 673
 674         cryptd_free_aead(*ctx);
 675 }
 676
 677 static int
 678 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
 679 {
 680         struct crypto_cipher *tfm;
 681         int ret;
 682
 683         tfm = crypto_alloc_cipher("aes", 0, 0);
 684         if (IS_ERR(tfm))
 685                 return PTR_ERR(tfm);
 686
 687         ret = crypto_cipher_setkey(tfm, key, key_len);
 688         if (ret)
 689                 goto out_free_cipher;
 690
 691         /* Clear the data in the hash sub key container to zero.*/
 692         /* We want to cipher all zeros to create the hash sub key. */
 693         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
 694
 695         crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey);
 696
 697 out_free_cipher:
 698         crypto_free_cipher(tfm);
 699         return ret;
 700 }
 701
 702 static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
 703                                   unsigned int key_len)
 704 {
 705         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead);
 706
 707         if (key_len < 4) {
 708                 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 709                 return -EINVAL;
 710         }
 711         /*Account for 4 byte nonce at the end.*/
 712         key_len -= 4;
 713
 714         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
 715
 716         return aes_set_key_common(crypto_aead_tfm(aead),
 717                                   &ctx->aes_key_expanded, key, key_len) ?:
 718                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 719 }
 720
 721 static int gcmaes_wrapper_set_key(struct crypto_aead *parent, const u8 *key,
 722                                   unsigned int key_len)
 723 {
 724         struct cryptd_aead **ctx = crypto_aead_ctx(parent);
 725         struct cryptd_aead *cryptd_tfm = *ctx;
 726
 727         return crypto_aead_setkey(&cryptd_tfm->base, key, key_len);
 728 }
 729
 730 static int common_rfc4106_set_authsize(struct crypto_aead *aead,
 731                                        unsigned int authsize)
 732 {
 733         switch (authsize) {
 734         case 8:
 735         case 12:
 736         case 16:
 737                 break;
 738         default:
 739                 return -EINVAL;
 740         }
 741
 742         return 0;
 743 }
 744
 745 /* This is the Integrity Check Value (aka the authentication tag length and can
 746  * be 8, 12 or 16 bytes long. */
 747 static int gcmaes_wrapper_set_authsize(struct crypto_aead *parent,
 748                                        unsigned int authsize)
 749 {
 750         struct cryptd_aead **ctx = crypto_aead_ctx(parent);
 751         struct cryptd_aead *cryptd_tfm = *ctx;
 752
 753         return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
 754 }
 755
 756 static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
 757                                        unsigned int authsize)
 758 {
 759         switch (authsize) {
 760         case 4:
 761         case 8:
 762         case 12:
 763         case 13:
 764         case 14:
 765         case 15:
 766         case 16:
 767                 break;
 768         default:
 769                 return -EINVAL;
 770         }
 771
 772         return 0;
 773 }
 774
 775 static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
 776                               unsigned int assoclen, u8 *hash_subkey,
 777                               u8 *iv, void *aes_ctx)
 778 {
 779         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 780         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 781         struct aesni_gcm_tfm_s *gcm_tfm = aesni_gcm_tfm;
 782         struct gcm_context_data data AESNI_ALIGN_ATTR;
 783         struct scatter_walk dst_sg_walk = {};
 784         unsigned long left = req->cryptlen;
 785         unsigned long len, srclen, dstlen;
 786         struct scatter_walk assoc_sg_walk;
 787         struct scatter_walk src_sg_walk;
 788         struct scatterlist src_start[2];
 789         struct scatterlist dst_start[2];
 790         struct scatterlist *src_sg;
 791         struct scatterlist *dst_sg;
 792         u8 *src, *dst, *assoc;
 793         u8 *assocmem = NULL;
 794         u8 authTag[16];
 795
 796         if (!enc)
 797                 left -= auth_tag_len;
 798
 799 #ifdef CONFIG_AS_AVX2
 800         if (left < AVX_GEN4_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen4)
 801                 gcm_tfm = &aesni_gcm_tfm_avx_gen2;
 802 #endif
 803 #ifdef CONFIG_AS_AVX
 804         if (left < AVX_GEN2_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen2)
 805                 gcm_tfm = &aesni_gcm_tfm_sse;
 806 #endif
 807
 808         /* Linearize assoc, if not already linear */
 809         if (req->src->length >= assoclen && req->src->length &&
 810                 (!PageHighMem(sg_page(req->src)) ||
 811                         req->src->offset + req->src->length <= PAGE_SIZE)) {
 812                 scatterwalk_start(&assoc_sg_walk, req->src);
 813                 assoc = scatterwalk_map(&assoc_sg_walk);
 814         } else {
 815                 /* assoc can be any length, so must be on heap */
 816                 assocmem = kmalloc(assoclen, GFP_ATOMIC);
 817                 if (unlikely(!assocmem))
 818                         return -ENOMEM;
 819                 assoc = assocmem;
 820
 821                 scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
 822         }
 823
 824         src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen);
 825         scatterwalk_start(&src_sg_walk, src_sg);
 826         if (req->src != req->dst) {
 827                 dst_sg = scatterwalk_ffwd(dst_start, req->dst, req->assoclen);
 828                 scatterwalk_start(&dst_sg_walk, dst_sg);
 829         }
 830
 831         kernel_fpu_begin();
 832         gcm_tfm->init(aes_ctx, &data, iv,
 833                 hash_subkey, assoc, assoclen);
 834         if (req->src != req->dst) {
 835                 while (left) {
 836                         src = scatterwalk_map(&src_sg_walk);
 837                         dst = scatterwalk_map(&dst_sg_walk);
 838                         srclen = scatterwalk_clamp(&src_sg_walk, left);
 839                         dstlen = scatterwalk_clamp(&dst_sg_walk, left);
 840                         len = min(srclen, dstlen);
 841                         if (len) {
 842                                 if (enc)
 843                                         gcm_tfm->enc_update(aes_ctx, &data,
 844                                                              dst, src, len);
 845                                 else
 846                                         gcm_tfm->dec_update(aes_ctx, &data,
 847                                                              dst, src, len);
 848                         }
 849                         left -= len;
 850
 851                         scatterwalk_unmap(src);
 852                         scatterwalk_unmap(dst);
 853                         scatterwalk_advance(&src_sg_walk, len);
 854                         scatterwalk_advance(&dst_sg_walk, len);
 855                         scatterwalk_done(&src_sg_walk, 0, left);
 856                         scatterwalk_done(&dst_sg_walk, 1, left);
 857                 }
 858         } else {
 859                 while (left) {
 860                         dst = src = scatterwalk_map(&src_sg_walk);
 861                         len = scatterwalk_clamp(&src_sg_walk, left);
 862                         if (len) {
 863                                 if (enc)
 864                                         gcm_tfm->enc_update(aes_ctx, &data,
 865                                                              src, src, len);
 866                                 else
 867                                         gcm_tfm->dec_update(aes_ctx, &data,
 868                                                              src, src, len);
 869                         }
 870                         left -= len;
 871                         scatterwalk_unmap(src);
 872                         scatterwalk_advance(&src_sg_walk, len);
 873                         scatterwalk_done(&src_sg_walk, 1, left);
 874                 }
 875         }
 876         gcm_tfm->finalize(aes_ctx, &data, authTag, auth_tag_len);
 877         kernel_fpu_end();
 878
 879         if (!assocmem)
 880                 scatterwalk_unmap(assoc);
 881         else
 882                 kfree(assocmem);
 883
 884         if (!enc) {
 885                 u8 authTagMsg[16];
 886
 887                 /* Copy out original authTag */
 888                 scatterwalk_map_and_copy(authTagMsg, req->src,
 889                                          req->assoclen + req->cryptlen -
 890                                          auth_tag_len,
 891                                          auth_tag_len, 0);
 892
 893                 /* Compare generated tag with passed in tag. */
 894                 return crypto_memneq(authTagMsg, authTag, auth_tag_len) ?
 895                         -EBADMSG : 0;
 896         }
 897
 898         /* Copy in the authTag */
 899         scatterwalk_map_and_copy(authTag, req->dst,
 900                                  req->assoclen + req->cryptlen,
 901                                  auth_tag_len, 1);
 902
 903         return 0;
 904 }
 905
 906 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
 907                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 908 {
 909         return gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv,
 910                                 aes_ctx);
 911 }
 912
 913 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
 914                           u8 *hash_subkey, u8 *iv, void *aes_ctx)
 915 {
 916         return gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv,
 917                                 aes_ctx);
 918 }
 919
 920 static int helper_rfc4106_encrypt(struct aead_request *req)
 921 {
 922         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 923         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 924         void *aes_ctx = &(ctx->aes_key_expanded);
 925         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
 926         unsigned int i;
 927         __be32 counter = cpu_to_be32(1);
 928
 929         /* Assuming we are supporting rfc4106 64-bit extended */
 930         /* sequence numbers We need to have the AAD length equal */
 931         /* to 16 or 20 bytes */
 932         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 933                 return -EINVAL;
 934
 935         /* IV below built */
 936         for (i = 0; i < 4; i++)
 937                 *(iv+i) = ctx->nonce[i];
 938         for (i = 0; i < 8; i++)
 939                 *(iv+4+i) = req->iv[i];
 940         *((__be32 *)(iv+12)) = counter;
 941
 942         return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 943                               aes_ctx);
 944 }
 945
 946 static int helper_rfc4106_decrypt(struct aead_request *req)
 947 {
 948         __be32 counter = cpu_to_be32(1);
 949         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 950         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
 951         void *aes_ctx = &(ctx->aes_key_expanded);
 952         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
 953         unsigned int i;
 954
 955         if (unlikely(req->assoclen != 16 && req->assoclen != 20))
 956                 return -EINVAL;
 957
 958         /* Assuming we are supporting rfc4106 64-bit extended */
 959         /* sequence numbers We need to have the AAD length */
 960         /* equal to 16 or 20 bytes */
 961
 962         /* IV below built */
 963         for (i = 0; i < 4; i++)
 964                 *(iv+i) = ctx->nonce[i];
 965         for (i = 0; i < 8; i++)
 966                 *(iv+4+i) = req->iv[i];
 967         *((__be32 *)(iv+12)) = counter;
 968
 969         return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
 970                               aes_ctx);
 971 }
 972
 973 static int gcmaes_wrapper_encrypt(struct aead_request *req)
 974 {
 975         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 976         struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
 977         struct cryptd_aead *cryptd_tfm = *ctx;
 978
 979         tfm = &cryptd_tfm->base;
 980         if (irq_fpu_usable() && (!in_atomic() ||
 981                                  !cryptd_aead_queued(cryptd_tfm)))
 982                 tfm = cryptd_aead_child(cryptd_tfm);
 983
 984         aead_request_set_tfm(req, tfm);
 985
 986         return crypto_aead_encrypt(req);
 987 }
 988
 989 static int gcmaes_wrapper_decrypt(struct aead_request *req)
 990 {
 991         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 992         struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
 993         struct cryptd_aead *cryptd_tfm = *ctx;
 994
 995         tfm = &cryptd_tfm->base;
 996         if (irq_fpu_usable() && (!in_atomic() ||
 997                                  !cryptd_aead_queued(cryptd_tfm)))
 998                 tfm = cryptd_aead_child(cryptd_tfm);
 999
1000         aead_request_set_tfm(req, tfm);
1001
1002         return crypto_aead_decrypt(req);
1003 }
1004 #endif
1005
1006 static struct crypto_alg aesni_algs[] = { {
1007         .cra_name               = "aes",
1008         .cra_driver_name        = "aes-aesni",
1009         .cra_priority           = 300,
1010         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
1011         .cra_blocksize          = AES_BLOCK_SIZE,
1012         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1013         .cra_module             = THIS_MODULE,
1014         .cra_u  = {
1015                 .cipher = {
1016                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
1017                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
1018                         .cia_setkey             = aes_set_key,
1019                         .cia_encrypt            = aes_encrypt,
1020                         .cia_decrypt            = aes_decrypt
1021                 }
1022         }
1023 }, {
1024         .cra_name               = "__aes",
1025         .cra_driver_name        = "__aes-aesni",
1026         .cra_priority           = 300,
1027         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL,
1028         .cra_blocksize          = AES_BLOCK_SIZE,
1029         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1030         .cra_module             = THIS_MODULE,
1031         .cra_u  = {
1032                 .cipher = {
1033                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
1034                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
1035                         .cia_setkey             = aes_set_key,
1036                         .cia_encrypt            = __aes_encrypt,
1037                         .cia_decrypt            = __aes_decrypt
1038                 }
1039         }
1040 } };
1041
1042 static struct skcipher_alg aesni_skciphers[] = {
1043         {
1044                 .base = {
1045                         .cra_name               = "__ecb(aes)",
1046                         .cra_driver_name        = "__ecb-aes-aesni",
1047                         .cra_priority           = 400,
1048                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1049                         .cra_blocksize          = AES_BLOCK_SIZE,
1050                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1051                         .cra_module             = THIS_MODULE,
1052                 },
1053                 .min_keysize    = AES_MIN_KEY_SIZE,
1054                 .max_keysize    = AES_MAX_KEY_SIZE,
1055                 .setkey         = aesni_skcipher_setkey,
1056                 .encrypt        = ecb_encrypt,
1057                 .decrypt        = ecb_decrypt,
1058         }, {
1059                 .base = {
1060                         .cra_name               = "__cbc(aes)",
1061                         .cra_driver_name        = "__cbc-aes-aesni",
1062                         .cra_priority           = 400,
1063                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1064                         .cra_blocksize          = AES_BLOCK_SIZE,
1065                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1066                         .cra_module             = THIS_MODULE,
1067                 },
1068                 .min_keysize    = AES_MIN_KEY_SIZE,
1069                 .max_keysize    = AES_MAX_KEY_SIZE,
1070                 .ivsize         = AES_BLOCK_SIZE,
1071                 .setkey         = aesni_skcipher_setkey,
1072                 .encrypt        = cbc_encrypt,
1073                 .decrypt        = cbc_decrypt,
1074 #ifdef CONFIG_X86_64
1075         }, {
1076                 .base = {
1077                         .cra_name               = "__ctr(aes)",
1078                         .cra_driver_name        = "__ctr-aes-aesni",
1079                         .cra_priority           = 400,
1080                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1081                         .cra_blocksize          = 1,
1082                         .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
1083                         .cra_module             = THIS_MODULE,
1084                 },
1085                 .min_keysize    = AES_MIN_KEY_SIZE,
1086                 .max_keysize    = AES_MAX_KEY_SIZE,
1087                 .ivsize         = AES_BLOCK_SIZE,
1088                 .chunksize      = AES_BLOCK_SIZE,
1089                 .setkey         = aesni_skcipher_setkey,
1090                 .encrypt        = ctr_crypt,
1091                 .decrypt        = ctr_crypt,
1092         }, {
1093                 .base = {
1094                         .cra_name               = "__xts(aes)",
1095                         .cra_driver_name        = "__xts-aes-aesni",
1096                         .cra_priority           = 401,
1097                         .cra_flags              = CRYPTO_ALG_INTERNAL,
1098                         .cra_blocksize          = AES_BLOCK_SIZE,
1099                         .cra_ctxsize            = XTS_AES_CTX_SIZE,
1100                         .cra_module             = THIS_MODULE,
1101                 },
1102                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
1103                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
1104                 .ivsize         = AES_BLOCK_SIZE,
1105                 .setkey         = xts_aesni_setkey,
1106                 .encrypt        = xts_encrypt,
1107                 .decrypt        = xts_decrypt,
1108 #endif
1109         }
1110 };
1111
1112 static
1113 struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];
1114
1115 #ifdef CONFIG_X86_64
1116 static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
1117                                   unsigned int key_len)
1118 {
1119         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
1120
1121         return aes_set_key_common(crypto_aead_tfm(aead),
1122                                   &ctx->aes_key_expanded, key, key_len) ?:
1123                rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
1124 }
1125
1126 static int generic_gcmaes_encrypt(struct aead_request *req)
1127 {
1128         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1129         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1130         void *aes_ctx = &(ctx->aes_key_expanded);
1131         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1132         __be32 counter = cpu_to_be32(1);
1133
1134         memcpy(iv, req->iv, 12);
1135         *((__be32 *)(iv+12)) = counter;
1136
1137         return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
1138                               aes_ctx);
1139 }
1140
1141 static int generic_gcmaes_decrypt(struct aead_request *req)
1142 {
1143         __be32 counter = cpu_to_be32(1);
1144         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1145         struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
1146         void *aes_ctx = &(ctx->aes_key_expanded);
1147         u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
1148
1149         memcpy(iv, req->iv, 12);
1150         *((__be32 *)(iv+12)) = counter;
1151
1152         return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
1153                               aes_ctx);
1154 }
1155
1156 static int generic_gcmaes_init(struct crypto_aead *aead)
1157 {
1158         struct cryptd_aead *cryptd_tfm;
1159         struct cryptd_aead **ctx = crypto_aead_ctx(aead);
1160
1161         cryptd_tfm = cryptd_alloc_aead("__driver-generic-gcm-aes-aesni",
1162                                        CRYPTO_ALG_INTERNAL,
1163                                        CRYPTO_ALG_INTERNAL);
1164         if (IS_ERR(cryptd_tfm))
1165                 return PTR_ERR(cryptd_tfm);
1166
1167         *ctx = cryptd_tfm;
1168         crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
1169
1170         return 0;
1171 }
1172
1173 static void generic_gcmaes_exit(struct crypto_aead *aead)
1174 {
1175         struct cryptd_aead **ctx = crypto_aead_ctx(aead);
1176
1177         cryptd_free_aead(*ctx);
1178 }
1179
1180 static struct aead_alg aesni_aead_algs[] = { {
1181         .setkey                 = common_rfc4106_set_key,
1182         .setauthsize            = common_rfc4106_set_authsize,
1183         .encrypt                = helper_rfc4106_encrypt,
1184         .decrypt                = helper_rfc4106_decrypt,
1185         .ivsize                 = GCM_RFC4106_IV_SIZE,
1186         .maxauthsize            = 16,
1187         .base = {
1188                 .cra_name               = "__gcm-aes-aesni",
1189                 .cra_driver_name        = "__driver-gcm-aes-aesni",
1190                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1191                 .cra_blocksize          = 1,
1192                 .cra_ctxsize            = sizeof(struct aesni_rfc4106_gcm_ctx),
1193                 .cra_alignmask          = AESNI_ALIGN - 1,
1194                 .cra_module             = THIS_MODULE,
1195         },
1196 }, {
1197         .init                   = rfc4106_init,
1198         .exit                   = rfc4106_exit,
1199         .setkey                 = gcmaes_wrapper_set_key,
1200         .setauthsize            = gcmaes_wrapper_set_authsize,
1201         .encrypt                = gcmaes_wrapper_encrypt,
1202         .decrypt                = gcmaes_wrapper_decrypt,
1203         .ivsize                 = GCM_RFC4106_IV_SIZE,
1204         .maxauthsize            = 16,
1205         .base = {
1206                 .cra_name               = "rfc4106(gcm(aes))",
1207                 .cra_driver_name        = "rfc4106-gcm-aesni",
1208                 .cra_priority           = 400,
1209                 .cra_flags              = CRYPTO_ALG_ASYNC,
1210                 .cra_blocksize          = 1,
1211                 .cra_ctxsize            = sizeof(struct cryptd_aead *),
1212                 .cra_module             = THIS_MODULE,
1213         },
1214 }, {
1215         .setkey                 = generic_gcmaes_set_key,
1216         .setauthsize            = generic_gcmaes_set_authsize,
1217         .encrypt                = generic_gcmaes_encrypt,
1218         .decrypt                = generic_gcmaes_decrypt,
1219         .ivsize                 = GCM_AES_IV_SIZE,
1220         .maxauthsize            = 16,
1221         .base = {
1222                 .cra_name               = "__generic-gcm-aes-aesni",
1223                 .cra_driver_name        = "__driver-generic-gcm-aes-aesni",
1224                 .cra_priority           = 0,
1225                 .cra_flags              = CRYPTO_ALG_INTERNAL,
1226                 .cra_blocksize          = 1,
1227                 .cra_ctxsize            = sizeof(struct generic_gcmaes_ctx),
1228                 .cra_alignmask          = AESNI_ALIGN - 1,
1229                 .cra_module             = THIS_MODULE,
1230         },
1231 }, {
1232         .init                   = generic_gcmaes_init,
1233         .exit                   = generic_gcmaes_exit,
1234         .setkey                 = gcmaes_wrapper_set_key,
1235         .setauthsize            = gcmaes_wrapper_set_authsize,
1236         .encrypt                = gcmaes_wrapper_encrypt,
1237         .decrypt                = gcmaes_wrapper_decrypt,
1238         .ivsize                 = GCM_AES_IV_SIZE,
1239         .maxauthsize            = 16,
1240         .base = {
1241                 .cra_name               = "gcm(aes)",
1242                 .cra_driver_name        = "generic-gcm-aesni",
1243                 .cra_priority           = 400,
1244                 .cra_flags              = CRYPTO_ALG_ASYNC,
1245                 .cra_blocksize          = 1,
1246                 .cra_ctxsize            = sizeof(struct cryptd_aead *),
1247                 .cra_module             = THIS_MODULE,
1248         },
1249 } };
1250 #else
1251 static struct aead_alg aesni_aead_algs[0];
1252 #endif
1253
1254
1255 static const struct x86_cpu_id aesni_cpu_id[] = {
1256         X86_FEATURE_MATCH(X86_FEATURE_AES),
1257         {}
1258 };
1259 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1260
1261 static void aesni_free_simds(void)
1262 {
1263         int i;
1264
1265         for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers) &&
1266                     aesni_simd_skciphers[i]; i++)
1267                 simd_skcipher_free(aesni_simd_skciphers[i]);
1268 }
1269
1270 static int __init aesni_init(void)
1271 {
1272         struct simd_skcipher_alg *simd;
1273         const char *basename;
1274         const char *algname;
1275         const char *drvname;
1276         int err;
1277         int i;
1278
1279         if (!x86_match_cpu(aesni_cpu_id))
1280                 return -ENODEV;
1281 #ifdef CONFIG_X86_64
1282 #ifdef CONFIG_AS_AVX2
1283         if (boot_cpu_has(X86_FEATURE_AVX2)) {
1284                 pr_info("AVX2 version of gcm_enc/dec engaged.\n");
1285                 aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen4;
1286         } else
1287 #endif
1288 #ifdef CONFIG_AS_AVX
1289         if (boot_cpu_has(X86_FEATURE_AVX)) {
1290                 pr_info("AVX version of gcm_enc/dec engaged.\n");
1291                 aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen2;
1292         } else
1293 #endif
1294         {
1295                 pr_info("SSE version of gcm_enc/dec engaged.\n");
1296                 aesni_gcm_tfm = &aesni_gcm_tfm_sse;
1297         }
1298         aesni_ctr_enc_tfm = aesni_ctr_enc;
1299 #ifdef CONFIG_AS_AVX
1300         if (boot_cpu_has(X86_FEATURE_AVX)) {
1301                 /* optimize performance of ctr mode encryption transform */
1302                 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
1303                 pr_info("AES CTR mode by8 optimization enabled\n");
1304         }
1305 #endif
1306 #endif
1307
1308         err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1309         if (err)
1310                 return err;
1311
1312         err = crypto_register_skciphers(aesni_skciphers,
1313                                         ARRAY_SIZE(aesni_skciphers));
1314         if (err)
1315                 goto unregister_algs;
1316
1317         err = crypto_register_aeads(aesni_aead_algs,
1318                                     ARRAY_SIZE(aesni_aead_algs));
1319         if (err)
1320                 goto unregister_skciphers;
1321
1322         for (i = 0; i < ARRAY_SIZE(aesni_skciphers); i++) {
1323                 algname = aesni_skciphers[i].base.cra_name + 2;
1324                 drvname = aesni_skciphers[i].base.cra_driver_name + 2;
1325                 basename = aesni_skciphers[i].base.cra_driver_name;
1326                 simd = simd_skcipher_create_compat(algname, drvname, basename);
1327                 err = PTR_ERR(simd);
1328                 if (IS_ERR(simd))
1329                         goto unregister_simds;
1330
1331                 aesni_simd_skciphers[i] = simd;
1332         }
1333
1334         return 0;
1335
1336 unregister_simds:
1337         aesni_free_simds();
1338         crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1339 unregister_skciphers:
1340         crypto_unregister_skciphers(aesni_skciphers,
1341                                     ARRAY_SIZE(aesni_skciphers));
1342 unregister_algs:
1343         crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1344         return err;
1345 }
1346
1347 static void __exit aesni_exit(void)
1348 {
1349         aesni_free_simds();
1350         crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
1351         crypto_unregister_skciphers(aesni_skciphers,
1352                                     ARRAY_SIZE(aesni_skciphers));
1353         crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1354 }
1355
1356 late_initcall(aesni_init);
1357 module_exit(aesni_exit);
1358
1359 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1360 MODULE_LICENSE("GPL");
1361 MODULE_ALIAS_CRYPTO("aes");