drivers/crypto/vmx/aes_xts.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /**
   3  * AES XTS routines supporting VMX In-core instructions on Power 8
   4  *
   5  * Copyright (C) 2015 International Business Machines Inc.
   6  *
   7  * Author: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com>
   8  */
   9
  10 #include <linux/types.h>
  11 #include <linux/err.h>
  12 #include <linux/crypto.h>
  13 #include <linux/delay.h>
  14 #include <asm/simd.h>
  15 #include <asm/switch_to.h>
  16 #include <crypto/aes.h>
  17 #include <crypto/internal/simd.h>
  18 #include <crypto/scatterwalk.h>
  19 #include <crypto/xts.h>
  20 #include <crypto/skcipher.h>
  21
  22 #include "aesp8-ppc.h"
  23
  24 struct p8_aes_xts_ctx {
  25         struct crypto_sync_skcipher *fallback;
  26         struct aes_key enc_key;
  27         struct aes_key dec_key;
  28         struct aes_key tweak_key;
  29 };
  30
  31 static int p8_aes_xts_init(struct crypto_tfm *tfm)
  32 {
  33         const char *alg = crypto_tfm_alg_name(tfm);
  34         struct crypto_sync_skcipher *fallback;
  35         struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
  36
  37         fallback = crypto_alloc_sync_skcipher(alg, 0,
  38                                               CRYPTO_ALG_NEED_FALLBACK);
  39         if (IS_ERR(fallback)) {
  40                 printk(KERN_ERR
  41                         "Failed to allocate transformation for '%s': %ld\n",
  42                         alg, PTR_ERR(fallback));
  43                 return PTR_ERR(fallback);
  44         }
  45
  46         crypto_sync_skcipher_set_flags(
  47                 fallback,
  48                 crypto_skcipher_get_flags((struct crypto_skcipher *)tfm));
  49         ctx->fallback = fallback;
  50
  51         return 0;
  52 }
  53
  54 static void p8_aes_xts_exit(struct crypto_tfm *tfm)
  55 {
  56         struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
  57
  58         if (ctx->fallback) {
  59                 crypto_free_sync_skcipher(ctx->fallback);
  60                 ctx->fallback = NULL;
  61         }
  62 }
  63
  64 static int p8_aes_xts_setkey(struct crypto_tfm *tfm, const u8 *key,
  65                              unsigned int keylen)
  66 {
  67         int ret;
  68         struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
  69
  70         ret = xts_check_key(tfm, key, keylen);
  71         if (ret)
  72                 return ret;
  73
  74         preempt_disable();
  75         pagefault_disable();
  76         enable_kernel_vsx();
  77         ret = aes_p8_set_encrypt_key(key + keylen/2, (keylen/2) * 8, &ctx->tweak_key);
  78         ret |= aes_p8_set_encrypt_key(key, (keylen/2) * 8, &ctx->enc_key);
  79         ret |= aes_p8_set_decrypt_key(key, (keylen/2) * 8, &ctx->dec_key);
  80         disable_kernel_vsx();
  81         pagefault_enable();
  82         preempt_enable();
  83
  84         ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen);
  85
  86         return ret ? -EINVAL : 0;
  87 }
  88
  89 static int p8_aes_xts_crypt(struct blkcipher_desc *desc,
  90                             struct scatterlist *dst,
  91                             struct scatterlist *src,
  92                             unsigned int nbytes, int enc)
  93 {
  94         int ret;
  95         u8 tweak[AES_BLOCK_SIZE];
  96         u8 *iv;
  97         struct blkcipher_walk walk;
  98         struct p8_aes_xts_ctx *ctx =
  99                 crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
 100
 101         if (!crypto_simd_usable()) {
 102                 SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
 103                 skcipher_request_set_sync_tfm(req, ctx->fallback);
 104                 skcipher_request_set_callback(req, desc->flags, NULL, NULL);
 105                 skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
 106                 ret = enc? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req);
 107                 skcipher_request_zero(req);
 108         } else {
 109                 blkcipher_walk_init(&walk, dst, src, nbytes);
 110
 111                 ret = blkcipher_walk_virt(desc, &walk);
 112
 113                 preempt_disable();
 114                 pagefault_disable();
 115                 enable_kernel_vsx();
 116
 117                 iv = walk.iv;
 118                 memset(tweak, 0, AES_BLOCK_SIZE);
 119                 aes_p8_encrypt(iv, tweak, &ctx->tweak_key);
 120
 121                 disable_kernel_vsx();
 122                 pagefault_enable();
 123                 preempt_enable();
 124
 125                 while ((nbytes = walk.nbytes)) {
 126                         preempt_disable();
 127                         pagefault_disable();
 128                         enable_kernel_vsx();
 129                         if (enc)
 130                                 aes_p8_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
 131                                                 nbytes & AES_BLOCK_MASK, &ctx->enc_key, NULL, tweak);
 132                         else
 133                                 aes_p8_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr,
 134                                                 nbytes & AES_BLOCK_MASK, &ctx->dec_key, NULL, tweak);
 135                         disable_kernel_vsx();
 136                         pagefault_enable();
 137                         preempt_enable();
 138
 139                         nbytes &= AES_BLOCK_SIZE - 1;
 140                         ret = blkcipher_walk_done(desc, &walk, nbytes);
 141                 }
 142         }
 143         return ret;
 144 }
 145
 146 static int p8_aes_xts_encrypt(struct blkcipher_desc *desc,
 147                               struct scatterlist *dst,
 148                               struct scatterlist *src, unsigned int nbytes)
 149 {
 150         return p8_aes_xts_crypt(desc, dst, src, nbytes, 1);
 151 }
 152
 153 static int p8_aes_xts_decrypt(struct blkcipher_desc *desc,
 154                               struct scatterlist *dst,
 155                               struct scatterlist *src, unsigned int nbytes)
 156 {
 157         return p8_aes_xts_crypt(desc, dst, src, nbytes, 0);
 158 }
 159
 160 struct crypto_alg p8_aes_xts_alg = {
 161         .cra_name = "xts(aes)",
 162         .cra_driver_name = "p8_aes_xts",
 163         .cra_module = THIS_MODULE,
 164         .cra_priority = 2000,
 165         .cra_type = &crypto_blkcipher_type,
 166         .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK,
 167         .cra_alignmask = 0,
 168         .cra_blocksize = AES_BLOCK_SIZE,
 169         .cra_ctxsize = sizeof(struct p8_aes_xts_ctx),
 170         .cra_init = p8_aes_xts_init,
 171         .cra_exit = p8_aes_xts_exit,
 172         .cra_blkcipher = {
 173                         .ivsize = AES_BLOCK_SIZE,
 174                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
 175                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
 176                         .setkey  = p8_aes_xts_setkey,
 177                         .encrypt = p8_aes_xts_encrypt,
 178                         .decrypt = p8_aes_xts_decrypt,
 179         }
 180 };