arch/x86/crypto/serpent_sse2_glue.c

   1 /*
   2  * Glue Code for SSE2 assembler versions of Serpent Cipher
   3  *
   4  * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   5  *
   6  * Glue code based on aesni-intel_glue.c by:
   7  *  Copyright (C) 2008, Intel Corp.
   8  *    Author: Huang Ying <ying.huang@intel.com>
   9  *
  10  * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
  11  *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  12  * CTR part based on code (crypto/ctr.c) by:
  13  *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
  14  *
  15  * This program is free software; you can redistribute it and/or modify
  16  * it under the terms of the GNU General Public License as published by
  17  * the Free Software Foundation; either version 2 of the License, or
  18  * (at your option) any later version.
  19  *
  20  * This program is distributed in the hope that it will be useful,
  21  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  22  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  23  * GNU General Public License for more details.
  24  *
  25  * You should have received a copy of the GNU General Public License
  26  * along with this program; if not, write to the Free Software
  27  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  28  * USA
  29  *
  30  */
  31
  32 #include <linux/module.h>
  33 #include <linux/hardirq.h>
  34 #include <linux/types.h>
  35 #include <linux/crypto.h>
  36 #include <linux/err.h>
  37 #include <crypto/algapi.h>
  38 #include <crypto/serpent.h>
  39 #include <crypto/cryptd.h>
  40 #include <crypto/b128ops.h>
  41 #include <crypto/ctr.h>
  42 #include <crypto/lrw.h>
  43 #include <crypto/xts.h>
  44 #include <asm/i387.h>
  45 #include <asm/serpent.h>
  46 #include <crypto/scatterwalk.h>
  47 #include <linux/workqueue.h>
  48 #include <linux/spinlock.h>
  49
  50 struct async_serpent_ctx {
  51         struct cryptd_ablkcipher *cryptd_tfm;
  52 };
  53
  54 static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
  55 {
  56         if (fpu_enabled)
  57                 return true;
  58
  59         /* SSE2 is only used when chunk to be processed is large enough, so
  60          * do not enable FPU until it is necessary.
  61          */
  62         if (nbytes < SERPENT_BLOCK_SIZE * SERPENT_PARALLEL_BLOCKS)
  63                 return false;
  64
  65         kernel_fpu_begin();
  66         return true;
  67 }
  68
  69 static inline void serpent_fpu_end(bool fpu_enabled)
  70 {
  71         if (fpu_enabled)
  72                 kernel_fpu_end();
  73 }
  74
  75 static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
  76                      bool enc)
  77 {
  78         bool fpu_enabled = false;
  79         struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  80         const unsigned int bsize = SERPENT_BLOCK_SIZE;
  81         unsigned int nbytes;
  82         int err;
  83
  84         err = blkcipher_walk_virt(desc, walk);
  85         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  86
  87         while ((nbytes = walk->nbytes)) {
  88                 u8 *wsrc = walk->src.virt.addr;
  89                 u8 *wdst = walk->dst.virt.addr;
  90
  91                 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
  92
  93                 /* Process multi-block batch */
  94                 if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
  95                         do {
  96                                 if (enc)
  97                                         serpent_enc_blk_xway(ctx, wdst, wsrc);
  98                                 else
  99                                         serpent_dec_blk_xway(ctx, wdst, wsrc);
 100
 101                                 wsrc += bsize * SERPENT_PARALLEL_BLOCKS;
 102                                 wdst += bsize * SERPENT_PARALLEL_BLOCKS;
 103                                 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
 104                         } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
 105
 106                         if (nbytes < bsize)
 107                                 goto done;
 108                 }
 109
 110                 /* Handle leftovers */
 111                 do {
 112                         if (enc)
 113                                 __serpent_encrypt(ctx, wdst, wsrc);
 114                         else
 115                                 __serpent_decrypt(ctx, wdst, wsrc);
 116
 117                         wsrc += bsize;
 118                         wdst += bsize;
 119                         nbytes -= bsize;
 120                 } while (nbytes >= bsize);
 121
 122 done:
 123                 err = blkcipher_walk_done(desc, walk, nbytes);
 124         }
 125
 126         serpent_fpu_end(fpu_enabled);
 127         return err;
 128 }
 129
 130 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 131                        struct scatterlist *src, unsigned int nbytes)
 132 {
 133         struct blkcipher_walk walk;
 134
 135         blkcipher_walk_init(&walk, dst, src, nbytes);
 136         return ecb_crypt(desc, &walk, true);
 137 }
 138
 139 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 140                        struct scatterlist *src, unsigned int nbytes)
 141 {
 142         struct blkcipher_walk walk;
 143
 144         blkcipher_walk_init(&walk, dst, src, nbytes);
 145         return ecb_crypt(desc, &walk, false);
 146 }
 147
 148 static struct crypto_alg blk_ecb_alg = {
 149         .cra_name               = "__ecb-serpent-sse2",
 150         .cra_driver_name        = "__driver-ecb-serpent-sse2",
 151         .cra_priority           = 0,
 152         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 153         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 154         .cra_ctxsize            = sizeof(struct serpent_ctx),
 155         .cra_alignmask          = 0,
 156         .cra_type               = &crypto_blkcipher_type,
 157         .cra_module             = THIS_MODULE,
 158         .cra_list               = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
 159         .cra_u = {
 160                 .blkcipher = {
 161                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 162                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 163                         .setkey         = serpent_setkey,
 164                         .encrypt        = ecb_encrypt,
 165                         .decrypt        = ecb_decrypt,
 166                 },
 167         },
 168 };
 169
 170 static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
 171                                   struct blkcipher_walk *walk)
 172 {
 173         struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 174         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 175         unsigned int nbytes = walk->nbytes;
 176         u128 *src = (u128 *)walk->src.virt.addr;
 177         u128 *dst = (u128 *)walk->dst.virt.addr;
 178         u128 *iv = (u128 *)walk->iv;
 179
 180         do {
 181                 u128_xor(dst, src, iv);
 182                 __serpent_encrypt(ctx, (u8 *)dst, (u8 *)dst);
 183                 iv = dst;
 184
 185                 src += 1;
 186                 dst += 1;
 187                 nbytes -= bsize;
 188         } while (nbytes >= bsize);
 189
 190         u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
 191         return nbytes;
 192 }
 193
 194 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 195                        struct scatterlist *src, unsigned int nbytes)
 196 {
 197         struct blkcipher_walk walk;
 198         int err;
 199
 200         blkcipher_walk_init(&walk, dst, src, nbytes);
 201         err = blkcipher_walk_virt(desc, &walk);
 202
 203         while ((nbytes = walk.nbytes)) {
 204                 nbytes = __cbc_encrypt(desc, &walk);
 205                 err = blkcipher_walk_done(desc, &walk, nbytes);
 206         }
 207
 208         return err;
 209 }
 210
 211 static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
 212                                   struct blkcipher_walk *walk)
 213 {
 214         struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 215         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 216         unsigned int nbytes = walk->nbytes;
 217         u128 *src = (u128 *)walk->src.virt.addr;
 218         u128 *dst = (u128 *)walk->dst.virt.addr;
 219         u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
 220         u128 last_iv;
 221         int i;
 222
 223         /* Start of the last block. */
 224         src += nbytes / bsize - 1;
 225         dst += nbytes / bsize - 1;
 226
 227         last_iv = *src;
 228
 229         /* Process multi-block batch */
 230         if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
 231                 do {
 232                         nbytes -= bsize * (SERPENT_PARALLEL_BLOCKS - 1);
 233                         src -= SERPENT_PARALLEL_BLOCKS - 1;
 234                         dst -= SERPENT_PARALLEL_BLOCKS - 1;
 235
 236                         for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++)
 237                                 ivs[i] = src[i];
 238
 239                         serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
 240
 241                         for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++)
 242                                 u128_xor(dst + (i + 1), dst + (i + 1), ivs + i);
 243
 244                         nbytes -= bsize;
 245                         if (nbytes < bsize)
 246                                 goto done;
 247
 248                         u128_xor(dst, dst, src - 1);
 249                         src -= 1;
 250                         dst -= 1;
 251                 } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
 252
 253                 if (nbytes < bsize)
 254                         goto done;
 255         }
 256
 257         /* Handle leftovers */
 258         for (;;) {
 259                 __serpent_decrypt(ctx, (u8 *)dst, (u8 *)src);
 260
 261                 nbytes -= bsize;
 262                 if (nbytes < bsize)
 263                         break;
 264
 265                 u128_xor(dst, dst, src - 1);
 266                 src -= 1;
 267                 dst -= 1;
 268         }
 269
 270 done:
 271         u128_xor(dst, dst, (u128 *)walk->iv);
 272         *(u128 *)walk->iv = last_iv;
 273
 274         return nbytes;
 275 }
 276
 277 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 278                        struct scatterlist *src, unsigned int nbytes)
 279 {
 280         bool fpu_enabled = false;
 281         struct blkcipher_walk walk;
 282         int err;
 283
 284         blkcipher_walk_init(&walk, dst, src, nbytes);
 285         err = blkcipher_walk_virt(desc, &walk);
 286         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 287
 288         while ((nbytes = walk.nbytes)) {
 289                 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
 290                 nbytes = __cbc_decrypt(desc, &walk);
 291                 err = blkcipher_walk_done(desc, &walk, nbytes);
 292         }
 293
 294         serpent_fpu_end(fpu_enabled);
 295         return err;
 296 }
 297
 298 static struct crypto_alg blk_cbc_alg = {
 299         .cra_name               = "__cbc-serpent-sse2",
 300         .cra_driver_name        = "__driver-cbc-serpent-sse2",
 301         .cra_priority           = 0,
 302         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 303         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 304         .cra_ctxsize            = sizeof(struct serpent_ctx),
 305         .cra_alignmask          = 0,
 306         .cra_type               = &crypto_blkcipher_type,
 307         .cra_module             = THIS_MODULE,
 308         .cra_list               = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
 309         .cra_u = {
 310                 .blkcipher = {
 311                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 312                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 313                         .setkey         = serpent_setkey,
 314                         .encrypt        = cbc_encrypt,
 315                         .decrypt        = cbc_decrypt,
 316                 },
 317         },
 318 };
 319
 320 static inline void u128_to_be128(be128 *dst, const u128 *src)
 321 {
 322         dst->a = cpu_to_be64(src->a);
 323         dst->b = cpu_to_be64(src->b);
 324 }
 325
 326 static inline void be128_to_u128(u128 *dst, const be128 *src)
 327 {
 328         dst->a = be64_to_cpu(src->a);
 329         dst->b = be64_to_cpu(src->b);
 330 }
 331
 332 static inline void u128_inc(u128 *i)
 333 {
 334         i->b++;
 335         if (!i->b)
 336                 i->a++;
 337 }
 338
 339 static void ctr_crypt_final(struct blkcipher_desc *desc,
 340                             struct blkcipher_walk *walk)
 341 {
 342         struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 343         u8 *ctrblk = walk->iv;
 344         u8 keystream[SERPENT_BLOCK_SIZE];
 345         u8 *src = walk->src.virt.addr;
 346         u8 *dst = walk->dst.virt.addr;
 347         unsigned int nbytes = walk->nbytes;
 348
 349         __serpent_encrypt(ctx, keystream, ctrblk);
 350         crypto_xor(keystream, src, nbytes);
 351         memcpy(dst, keystream, nbytes);
 352
 353         crypto_inc(ctrblk, SERPENT_BLOCK_SIZE);
 354 }
 355
 356 static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
 357                                 struct blkcipher_walk *walk)
 358 {
 359         struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 360         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 361         unsigned int nbytes = walk->nbytes;
 362         u128 *src = (u128 *)walk->src.virt.addr;
 363         u128 *dst = (u128 *)walk->dst.virt.addr;
 364         u128 ctrblk;
 365         be128 ctrblocks[SERPENT_PARALLEL_BLOCKS];
 366         int i;
 367
 368         be128_to_u128(&ctrblk, (be128 *)walk->iv);
 369
 370         /* Process multi-block batch */
 371         if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
 372                 do {
 373                         /* create ctrblks for parallel encrypt */
 374                         for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
 375                                 if (dst != src)
 376                                         dst[i] = src[i];
 377
 378                                 u128_to_be128(&ctrblocks[i], &ctrblk);
 379                                 u128_inc(&ctrblk);
 380                         }
 381
 382                         serpent_enc_blk_xway_xor(ctx, (u8 *)dst,
 383                                                  (u8 *)ctrblocks);
 384
 385                         src += SERPENT_PARALLEL_BLOCKS;
 386                         dst += SERPENT_PARALLEL_BLOCKS;
 387                         nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
 388                 } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
 389
 390                 if (nbytes < bsize)
 391                         goto done;
 392         }
 393
 394         /* Handle leftovers */
 395         do {
 396                 if (dst != src)
 397                         *dst = *src;
 398
 399                 u128_to_be128(&ctrblocks[0], &ctrblk);
 400                 u128_inc(&ctrblk);
 401
 402                 __serpent_encrypt(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
 403                 u128_xor(dst, dst, (u128 *)ctrblocks);
 404
 405                 src += 1;
 406                 dst += 1;
 407                 nbytes -= bsize;
 408         } while (nbytes >= bsize);
 409
 410 done:
 411         u128_to_be128((be128 *)walk->iv, &ctrblk);
 412         return nbytes;
 413 }
 414
 415 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 416                      struct scatterlist *src, unsigned int nbytes)
 417 {
 418         bool fpu_enabled = false;
 419         struct blkcipher_walk walk;
 420         int err;
 421
 422         blkcipher_walk_init(&walk, dst, src, nbytes);
 423         err = blkcipher_walk_virt_block(desc, &walk, SERPENT_BLOCK_SIZE);
 424         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 425
 426         while ((nbytes = walk.nbytes) >= SERPENT_BLOCK_SIZE) {
 427                 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
 428                 nbytes = __ctr_crypt(desc, &walk);
 429                 err = blkcipher_walk_done(desc, &walk, nbytes);
 430         }
 431
 432         serpent_fpu_end(fpu_enabled);
 433
 434         if (walk.nbytes) {
 435                 ctr_crypt_final(desc, &walk);
 436                 err = blkcipher_walk_done(desc, &walk, 0);
 437         }
 438
 439         return err;
 440 }
 441
 442 static struct crypto_alg blk_ctr_alg = {
 443         .cra_name               = "__ctr-serpent-sse2",
 444         .cra_driver_name        = "__driver-ctr-serpent-sse2",
 445         .cra_priority           = 0,
 446         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 447         .cra_blocksize          = 1,
 448         .cra_ctxsize            = sizeof(struct serpent_ctx),
 449         .cra_alignmask          = 0,
 450         .cra_type               = &crypto_blkcipher_type,
 451         .cra_module             = THIS_MODULE,
 452         .cra_list               = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
 453         .cra_u = {
 454                 .blkcipher = {
 455                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 456                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 457                         .ivsize         = SERPENT_BLOCK_SIZE,
 458                         .setkey         = serpent_setkey,
 459                         .encrypt        = ctr_crypt,
 460                         .decrypt        = ctr_crypt,
 461                 },
 462         },
 463 };
 464
 465 struct crypt_priv {
 466         struct serpent_ctx *ctx;
 467         bool fpu_enabled;
 468 };
 469
 470 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 471 {
 472         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 473         struct crypt_priv *ctx = priv;
 474         int i;
 475
 476         ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
 477
 478         if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
 479                 serpent_enc_blk_xway(ctx->ctx, srcdst, srcdst);
 480                 return;
 481         }
 482
 483         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 484                 __serpent_encrypt(ctx->ctx, srcdst, srcdst);
 485 }
 486
 487 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 488 {
 489         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 490         struct crypt_priv *ctx = priv;
 491         int i;
 492
 493         ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
 494
 495         if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
 496                 serpent_dec_blk_xway(ctx->ctx, srcdst, srcdst);
 497                 return;
 498         }
 499
 500         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 501                 __serpent_decrypt(ctx->ctx, srcdst, srcdst);
 502 }
 503
 504 struct serpent_lrw_ctx {
 505         struct lrw_table_ctx lrw_table;
 506         struct serpent_ctx serpent_ctx;
 507 };
 508
 509 static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
 510                               unsigned int keylen)
 511 {
 512         struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
 513         int err;
 514
 515         err = __serpent_setkey(&ctx->serpent_ctx, key, keylen -
 516                                                         SERPENT_BLOCK_SIZE);
 517         if (err)
 518                 return err;
 519
 520         return lrw_init_table(&ctx->lrw_table, key + keylen -
 521                                                 SERPENT_BLOCK_SIZE);
 522 }
 523
 524 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 525                        struct scatterlist *src, unsigned int nbytes)
 526 {
 527         struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 528         be128 buf[SERPENT_PARALLEL_BLOCKS];
 529         struct crypt_priv crypt_ctx = {
 530                 .ctx = &ctx->serpent_ctx,
 531                 .fpu_enabled = false,
 532         };
 533         struct lrw_crypt_req req = {
 534                 .tbuf = buf,
 535                 .tbuflen = sizeof(buf),
 536
 537                 .table_ctx = &ctx->lrw_table,
 538                 .crypt_ctx = &crypt_ctx,
 539                 .crypt_fn = encrypt_callback,
 540         };
 541         int ret;
 542
 543         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 544         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 545         serpent_fpu_end(crypt_ctx.fpu_enabled);
 546
 547         return ret;
 548 }
 549
 550 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 551                        struct scatterlist *src, unsigned int nbytes)
 552 {
 553         struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 554         be128 buf[SERPENT_PARALLEL_BLOCKS];
 555         struct crypt_priv crypt_ctx = {
 556                 .ctx = &ctx->serpent_ctx,
 557                 .fpu_enabled = false,
 558         };
 559         struct lrw_crypt_req req = {
 560                 .tbuf = buf,
 561                 .tbuflen = sizeof(buf),
 562
 563                 .table_ctx = &ctx->lrw_table,
 564                 .crypt_ctx = &crypt_ctx,
 565                 .crypt_fn = decrypt_callback,
 566         };
 567         int ret;
 568
 569         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 570         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 571         serpent_fpu_end(crypt_ctx.fpu_enabled);
 572
 573         return ret;
 574 }
 575
 576 static void lrw_exit_tfm(struct crypto_tfm *tfm)
 577 {
 578         struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
 579
 580         lrw_free_table(&ctx->lrw_table);
 581 }
 582
 583 static struct crypto_alg blk_lrw_alg = {
 584         .cra_name               = "__lrw-serpent-sse2",
 585         .cra_driver_name        = "__driver-lrw-serpent-sse2",
 586         .cra_priority           = 0,
 587         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 588         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 589         .cra_ctxsize            = sizeof(struct serpent_lrw_ctx),
 590         .cra_alignmask          = 0,
 591         .cra_type               = &crypto_blkcipher_type,
 592         .cra_module             = THIS_MODULE,
 593         .cra_list               = LIST_HEAD_INIT(blk_lrw_alg.cra_list),
 594         .cra_exit               = lrw_exit_tfm,
 595         .cra_u = {
 596                 .blkcipher = {
 597                         .min_keysize    = SERPENT_MIN_KEY_SIZE +
 598                                           SERPENT_BLOCK_SIZE,
 599                         .max_keysize    = SERPENT_MAX_KEY_SIZE +
 600                                           SERPENT_BLOCK_SIZE,
 601                         .ivsize         = SERPENT_BLOCK_SIZE,
 602                         .setkey         = lrw_serpent_setkey,
 603                         .encrypt        = lrw_encrypt,
 604                         .decrypt        = lrw_decrypt,
 605                 },
 606         },
 607 };
 608
 609 struct serpent_xts_ctx {
 610         struct serpent_ctx tweak_ctx;
 611         struct serpent_ctx crypt_ctx;
 612 };
 613
 614 static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
 615                               unsigned int keylen)
 616 {
 617         struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm);
 618         u32 *flags = &tfm->crt_flags;
 619         int err;
 620
 621         /* key consists of keys of equal size concatenated, therefore
 622          * the length must be even
 623          */
 624         if (keylen % 2) {
 625                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 626                 return -EINVAL;
 627         }
 628
 629         /* first half of xts-key is for crypt */
 630         err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2);
 631         if (err)
 632                 return err;
 633
 634         /* second half of xts-key is for tweak */
 635         return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
 636 }
 637
 638 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 639                        struct scatterlist *src, unsigned int nbytes)
 640 {
 641         struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 642         be128 buf[SERPENT_PARALLEL_BLOCKS];
 643         struct crypt_priv crypt_ctx = {
 644                 .ctx = &ctx->crypt_ctx,
 645                 .fpu_enabled = false,
 646         };
 647         struct xts_crypt_req req = {
 648                 .tbuf = buf,
 649                 .tbuflen = sizeof(buf),
 650
 651                 .tweak_ctx = &ctx->tweak_ctx,
 652                 .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
 653                 .crypt_ctx = &crypt_ctx,
 654                 .crypt_fn = encrypt_callback,
 655         };
 656         int ret;
 657
 658         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 659         ret = xts_crypt(desc, dst, src, nbytes, &req);
 660         serpent_fpu_end(crypt_ctx.fpu_enabled);
 661
 662         return ret;
 663 }
 664
 665 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 666                        struct scatterlist *src, unsigned int nbytes)
 667 {
 668         struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 669         be128 buf[SERPENT_PARALLEL_BLOCKS];
 670         struct crypt_priv crypt_ctx = {
 671                 .ctx = &ctx->crypt_ctx,
 672                 .fpu_enabled = false,
 673         };
 674         struct xts_crypt_req req = {
 675                 .tbuf = buf,
 676                 .tbuflen = sizeof(buf),
 677
 678                 .tweak_ctx = &ctx->tweak_ctx,
 679                 .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
 680                 .crypt_ctx = &crypt_ctx,
 681                 .crypt_fn = decrypt_callback,
 682         };
 683         int ret;
 684
 685         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 686         ret = xts_crypt(desc, dst, src, nbytes, &req);
 687         serpent_fpu_end(crypt_ctx.fpu_enabled);
 688
 689         return ret;
 690 }
 691
 692 static struct crypto_alg blk_xts_alg = {
 693         .cra_name               = "__xts-serpent-sse2",
 694         .cra_driver_name        = "__driver-xts-serpent-sse2",
 695         .cra_priority           = 0,
 696         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 697         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 698         .cra_ctxsize            = sizeof(struct serpent_xts_ctx),
 699         .cra_alignmask          = 0,
 700         .cra_type               = &crypto_blkcipher_type,
 701         .cra_module             = THIS_MODULE,
 702         .cra_list               = LIST_HEAD_INIT(blk_xts_alg.cra_list),
 703         .cra_u = {
 704                 .blkcipher = {
 705                         .min_keysize    = SERPENT_MIN_KEY_SIZE * 2,
 706                         .max_keysize    = SERPENT_MAX_KEY_SIZE * 2,
 707                         .ivsize         = SERPENT_BLOCK_SIZE,
 708                         .setkey         = xts_serpent_setkey,
 709                         .encrypt        = xts_encrypt,
 710                         .decrypt        = xts_decrypt,
 711                 },
 712         },
 713 };
 714
 715 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
 716                         unsigned int key_len)
 717 {
 718         struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 719         struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
 720         int err;
 721
 722         crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 723         crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
 724                                     & CRYPTO_TFM_REQ_MASK);
 725         err = crypto_ablkcipher_setkey(child, key, key_len);
 726         crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
 727                                     & CRYPTO_TFM_RES_MASK);
 728         return err;
 729 }
 730
 731 static int __ablk_encrypt(struct ablkcipher_request *req)
 732 {
 733         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 734         struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 735         struct blkcipher_desc desc;
 736
 737         desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
 738         desc.info = req->info;
 739         desc.flags = 0;
 740
 741         return crypto_blkcipher_crt(desc.tfm)->encrypt(
 742                 &desc, req->dst, req->src, req->nbytes);
 743 }
 744
 745 static int ablk_encrypt(struct ablkcipher_request *req)
 746 {
 747         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 748         struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 749
 750         if (!irq_fpu_usable()) {
 751                 struct ablkcipher_request *cryptd_req =
 752                         ablkcipher_request_ctx(req);
 753
 754                 memcpy(cryptd_req, req, sizeof(*req));
 755                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 756
 757                 return crypto_ablkcipher_encrypt(cryptd_req);
 758         } else {
 759                 return __ablk_encrypt(req);
 760         }
 761 }
 762
 763 static int ablk_decrypt(struct ablkcipher_request *req)
 764 {
 765         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 766         struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 767
 768         if (!irq_fpu_usable()) {
 769                 struct ablkcipher_request *cryptd_req =
 770                         ablkcipher_request_ctx(req);
 771
 772                 memcpy(cryptd_req, req, sizeof(*req));
 773                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 774
 775                 return crypto_ablkcipher_decrypt(cryptd_req);
 776         } else {
 777                 struct blkcipher_desc desc;
 778
 779                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
 780                 desc.info = req->info;
 781                 desc.flags = 0;
 782
 783                 return crypto_blkcipher_crt(desc.tfm)->decrypt(
 784                         &desc, req->dst, req->src, req->nbytes);
 785         }
 786 }
 787
 788 static void ablk_exit(struct crypto_tfm *tfm)
 789 {
 790         struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm);
 791
 792         cryptd_free_ablkcipher(ctx->cryptd_tfm);
 793 }
 794
 795 static void ablk_init_common(struct crypto_tfm *tfm,
 796                              struct cryptd_ablkcipher *cryptd_tfm)
 797 {
 798         struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm);
 799
 800         ctx->cryptd_tfm = cryptd_tfm;
 801         tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
 802                 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
 803 }
 804
 805 static int ablk_ecb_init(struct crypto_tfm *tfm)
 806 {
 807         struct cryptd_ablkcipher *cryptd_tfm;
 808
 809         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-serpent-sse2", 0, 0);
 810         if (IS_ERR(cryptd_tfm))
 811                 return PTR_ERR(cryptd_tfm);
 812         ablk_init_common(tfm, cryptd_tfm);
 813         return 0;
 814 }
 815
 816 static struct crypto_alg ablk_ecb_alg = {
 817         .cra_name               = "ecb(serpent)",
 818         .cra_driver_name        = "ecb-serpent-sse2",
 819         .cra_priority           = 400,
 820         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 821         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 822         .cra_ctxsize            = sizeof(struct async_serpent_ctx),
 823         .cra_alignmask          = 0,
 824         .cra_type               = &crypto_ablkcipher_type,
 825         .cra_module             = THIS_MODULE,
 826         .cra_list               = LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
 827         .cra_init               = ablk_ecb_init,
 828         .cra_exit               = ablk_exit,
 829         .cra_u = {
 830                 .ablkcipher = {
 831                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 832                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 833                         .setkey         = ablk_set_key,
 834                         .encrypt        = ablk_encrypt,
 835                         .decrypt        = ablk_decrypt,
 836                 },
 837         },
 838 };
 839
 840 static int ablk_cbc_init(struct crypto_tfm *tfm)
 841 {
 842         struct cryptd_ablkcipher *cryptd_tfm;
 843
 844         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-serpent-sse2", 0, 0);
 845         if (IS_ERR(cryptd_tfm))
 846                 return PTR_ERR(cryptd_tfm);
 847         ablk_init_common(tfm, cryptd_tfm);
 848         return 0;
 849 }
 850
 851 static struct crypto_alg ablk_cbc_alg = {
 852         .cra_name               = "cbc(serpent)",
 853         .cra_driver_name        = "cbc-serpent-sse2",
 854         .cra_priority           = 400,
 855         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 856         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 857         .cra_ctxsize            = sizeof(struct async_serpent_ctx),
 858         .cra_alignmask          = 0,
 859         .cra_type               = &crypto_ablkcipher_type,
 860         .cra_module             = THIS_MODULE,
 861         .cra_list               = LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
 862         .cra_init               = ablk_cbc_init,
 863         .cra_exit               = ablk_exit,
 864         .cra_u = {
 865                 .ablkcipher = {
 866                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 867                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 868                         .ivsize         = SERPENT_BLOCK_SIZE,
 869                         .setkey         = ablk_set_key,
 870                         .encrypt        = __ablk_encrypt,
 871                         .decrypt        = ablk_decrypt,
 872                 },
 873         },
 874 };
 875
 876 static int ablk_ctr_init(struct crypto_tfm *tfm)
 877 {
 878         struct cryptd_ablkcipher *cryptd_tfm;
 879
 880         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-serpent-sse2", 0, 0);
 881         if (IS_ERR(cryptd_tfm))
 882                 return PTR_ERR(cryptd_tfm);
 883         ablk_init_common(tfm, cryptd_tfm);
 884         return 0;
 885 }
 886
 887 static struct crypto_alg ablk_ctr_alg = {
 888         .cra_name               = "ctr(serpent)",
 889         .cra_driver_name        = "ctr-serpent-sse2",
 890         .cra_priority           = 400,
 891         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 892         .cra_blocksize          = 1,
 893         .cra_ctxsize            = sizeof(struct async_serpent_ctx),
 894         .cra_alignmask          = 0,
 895         .cra_type               = &crypto_ablkcipher_type,
 896         .cra_module             = THIS_MODULE,
 897         .cra_list               = LIST_HEAD_INIT(ablk_ctr_alg.cra_list),
 898         .cra_init               = ablk_ctr_init,
 899         .cra_exit               = ablk_exit,
 900         .cra_u = {
 901                 .ablkcipher = {
 902                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 903                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 904                         .ivsize         = SERPENT_BLOCK_SIZE,
 905                         .setkey         = ablk_set_key,
 906                         .encrypt        = ablk_encrypt,
 907                         .decrypt        = ablk_encrypt,
 908                         .geniv          = "chainiv",
 909                 },
 910         },
 911 };
 912
 913 static int ablk_lrw_init(struct crypto_tfm *tfm)
 914 {
 915         struct cryptd_ablkcipher *cryptd_tfm;
 916
 917         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-lrw-serpent-sse2", 0, 0);
 918         if (IS_ERR(cryptd_tfm))
 919                 return PTR_ERR(cryptd_tfm);
 920         ablk_init_common(tfm, cryptd_tfm);
 921         return 0;
 922 }
 923
 924 static struct crypto_alg ablk_lrw_alg = {
 925         .cra_name               = "lrw(serpent)",
 926         .cra_driver_name        = "lrw-serpent-sse2",
 927         .cra_priority           = 400,
 928         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 929         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 930         .cra_ctxsize            = sizeof(struct async_serpent_ctx),
 931         .cra_alignmask          = 0,
 932         .cra_type               = &crypto_ablkcipher_type,
 933         .cra_module             = THIS_MODULE,
 934         .cra_list               = LIST_HEAD_INIT(ablk_lrw_alg.cra_list),
 935         .cra_init               = ablk_lrw_init,
 936         .cra_exit               = ablk_exit,
 937         .cra_u = {
 938                 .ablkcipher = {
 939                         .min_keysize    = SERPENT_MIN_KEY_SIZE +
 940                                           SERPENT_BLOCK_SIZE,
 941                         .max_keysize    = SERPENT_MAX_KEY_SIZE +
 942                                           SERPENT_BLOCK_SIZE,
 943                         .ivsize         = SERPENT_BLOCK_SIZE,
 944                         .setkey         = ablk_set_key,
 945                         .encrypt        = ablk_encrypt,
 946                         .decrypt        = ablk_decrypt,
 947                 },
 948         },
 949 };
 950
 951 static int ablk_xts_init(struct crypto_tfm *tfm)
 952 {
 953         struct cryptd_ablkcipher *cryptd_tfm;
 954
 955         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-xts-serpent-sse2", 0, 0);
 956         if (IS_ERR(cryptd_tfm))
 957                 return PTR_ERR(cryptd_tfm);
 958         ablk_init_common(tfm, cryptd_tfm);
 959         return 0;
 960 }
 961
 962 static struct crypto_alg ablk_xts_alg = {
 963         .cra_name               = "xts(serpent)",
 964         .cra_driver_name        = "xts-serpent-sse2",
 965         .cra_priority           = 400,
 966         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 967         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 968         .cra_ctxsize            = sizeof(struct async_serpent_ctx),
 969         .cra_alignmask          = 0,
 970         .cra_type               = &crypto_ablkcipher_type,
 971         .cra_module             = THIS_MODULE,
 972         .cra_list               = LIST_HEAD_INIT(ablk_xts_alg.cra_list),
 973         .cra_init               = ablk_xts_init,
 974         .cra_exit               = ablk_exit,
 975         .cra_u = {
 976                 .ablkcipher = {
 977                         .min_keysize    = SERPENT_MIN_KEY_SIZE * 2,
 978                         .max_keysize    = SERPENT_MAX_KEY_SIZE * 2,
 979                         .ivsize         = SERPENT_BLOCK_SIZE,
 980                         .setkey         = ablk_set_key,
 981                         .encrypt        = ablk_encrypt,
 982                         .decrypt        = ablk_decrypt,
 983                 },
 984         },
 985 };
 986
 987 static int __init serpent_sse2_init(void)
 988 {
 989         int err;
 990
 991         if (!cpu_has_xmm2) {
 992                 printk(KERN_INFO "SSE2 instructions are not detected.\n");
 993                 return -ENODEV;
 994         }
 995
 996         err = crypto_register_alg(&blk_ecb_alg);
 997         if (err)
 998                 goto blk_ecb_err;
 999         err = crypto_register_alg(&blk_cbc_alg);
1000         if (err)
1001                 goto blk_cbc_err;
1002         err = crypto_register_alg(&blk_ctr_alg);
1003         if (err)
1004                 goto blk_ctr_err;
1005         err = crypto_register_alg(&ablk_ecb_alg);
1006         if (err)
1007                 goto ablk_ecb_err;
1008         err = crypto_register_alg(&ablk_cbc_alg);
1009         if (err)
1010                 goto ablk_cbc_err;
1011         err = crypto_register_alg(&ablk_ctr_alg);
1012         if (err)
1013                 goto ablk_ctr_err;
1014         err = crypto_register_alg(&blk_lrw_alg);
1015         if (err)
1016                 goto blk_lrw_err;
1017         err = crypto_register_alg(&ablk_lrw_alg);
1018         if (err)
1019                 goto ablk_lrw_err;
1020         err = crypto_register_alg(&blk_xts_alg);
1021         if (err)
1022                 goto blk_xts_err;
1023         err = crypto_register_alg(&ablk_xts_alg);
1024         if (err)
1025                 goto ablk_xts_err;
1026         return err;
1027
1028         crypto_unregister_alg(&ablk_xts_alg);
1029 ablk_xts_err:
1030         crypto_unregister_alg(&blk_xts_alg);
1031 blk_xts_err:
1032         crypto_unregister_alg(&ablk_lrw_alg);
1033 ablk_lrw_err:
1034         crypto_unregister_alg(&blk_lrw_alg);
1035 blk_lrw_err:
1036         crypto_unregister_alg(&ablk_ctr_alg);
1037 ablk_ctr_err:
1038         crypto_unregister_alg(&ablk_cbc_alg);
1039 ablk_cbc_err:
1040         crypto_unregister_alg(&ablk_ecb_alg);
1041 ablk_ecb_err:
1042         crypto_unregister_alg(&blk_ctr_alg);
1043 blk_ctr_err:
1044         crypto_unregister_alg(&blk_cbc_alg);
1045 blk_cbc_err:
1046         crypto_unregister_alg(&blk_ecb_alg);
1047 blk_ecb_err:
1048         return err;
1049 }
1050
1051 static void __exit serpent_sse2_exit(void)
1052 {
1053         crypto_unregister_alg(&ablk_xts_alg);
1054         crypto_unregister_alg(&blk_xts_alg);
1055         crypto_unregister_alg(&ablk_lrw_alg);
1056         crypto_unregister_alg(&blk_lrw_alg);
1057         crypto_unregister_alg(&ablk_ctr_alg);
1058         crypto_unregister_alg(&ablk_cbc_alg);
1059         crypto_unregister_alg(&ablk_ecb_alg);
1060         crypto_unregister_alg(&blk_ctr_alg);
1061         crypto_unregister_alg(&blk_cbc_alg);
1062         crypto_unregister_alg(&blk_ecb_alg);
1063 }
1064
1065 module_init(serpent_sse2_init);
1066 module_exit(serpent_sse2_exit);
1067
1068 MODULE_DESCRIPTION("Serpent Cipher Algorithm, SSE2 optimized");
1069 MODULE_LICENSE("GPL");
1070 MODULE_ALIAS("serpent");