arch/x86/crypto/sha1_ssse3_glue.c

   1 /*
   2  * Cryptographic API.
   3  *
   4  * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using
   5  * Supplemental SSE3 instructions.
   6  *
   7  * This file is based on sha1_generic.c
   8  *
   9  * Copyright (c) Alan Smithee.
  10  * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
  11  * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
  12  * Copyright (c) Mathias Krause <minipli@googlemail.com>
  13  * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com>
  14  *
  15  * This program is free software; you can redistribute it and/or modify it
  16  * under the terms of the GNU General Public License as published by the Free
  17  * Software Foundation; either version 2 of the License, or (at your option)
  18  * any later version.
  19  *
  20  */
  21
  22 #define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
  23
  24 #include <crypto/internal/hash.h>
  25 #include <crypto/internal/simd.h>
  26 #include <linux/init.h>
  27 #include <linux/module.h>
  28 #include <linux/mm.h>
  29 #include <linux/cryptohash.h>
  30 #include <linux/types.h>
  31 #include <crypto/sha.h>
  32 #include <crypto/sha1_base.h>
  33 #include <asm/simd.h>
  34
  35 typedef void (sha1_transform_fn)(u32 *digest, const char *data,
  36                                 unsigned int rounds);
  37
  38 static int sha1_update(struct shash_desc *desc, const u8 *data,
  39                              unsigned int len, sha1_transform_fn *sha1_xform)
  40 {
  41         struct sha1_state *sctx = shash_desc_ctx(desc);
  42
  43         if (!crypto_simd_usable() ||
  44             (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE)
  45                 return crypto_sha1_update(desc, data, len);
  46
  47         /* make sure casting to sha1_block_fn() is safe */
  48         BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0);
  49
  50         kernel_fpu_begin();
  51         sha1_base_do_update(desc, data, len,
  52                             (sha1_block_fn *)sha1_xform);
  53         kernel_fpu_end();
  54
  55         return 0;
  56 }
  57
  58 static int sha1_finup(struct shash_desc *desc, const u8 *data,
  59                       unsigned int len, u8 *out, sha1_transform_fn *sha1_xform)
  60 {
  61         if (!crypto_simd_usable())
  62                 return crypto_sha1_finup(desc, data, len, out);
  63
  64         kernel_fpu_begin();
  65         if (len)
  66                 sha1_base_do_update(desc, data, len,
  67                                     (sha1_block_fn *)sha1_xform);
  68         sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_xform);
  69         kernel_fpu_end();
  70
  71         return sha1_base_finish(desc, out);
  72 }
  73
  74 asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data,
  75                                      unsigned int rounds);
  76
  77 static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data,
  78                              unsigned int len)
  79 {
  80         return sha1_update(desc, data, len,
  81                         (sha1_transform_fn *) sha1_transform_ssse3);
  82 }
  83
  84 static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data,
  85                               unsigned int len, u8 *out)
  86 {
  87         return sha1_finup(desc, data, len, out,
  88                         (sha1_transform_fn *) sha1_transform_ssse3);
  89 }
  90
  91 /* Add padding and return the message digest. */
  92 static int sha1_ssse3_final(struct shash_desc *desc, u8 *out)
  93 {
  94         return sha1_ssse3_finup(desc, NULL, 0, out);
  95 }
  96
  97 static struct shash_alg sha1_ssse3_alg = {
  98         .digestsize     =       SHA1_DIGEST_SIZE,
  99         .init           =       sha1_base_init,
 100         .update         =       sha1_ssse3_update,
 101         .final          =       sha1_ssse3_final,
 102         .finup          =       sha1_ssse3_finup,
 103         .descsize       =       sizeof(struct sha1_state),
 104         .base           =       {
 105                 .cra_name       =       "sha1",
 106                 .cra_driver_name =      "sha1-ssse3",
 107                 .cra_priority   =       150,
 108                 .cra_blocksize  =       SHA1_BLOCK_SIZE,
 109                 .cra_module     =       THIS_MODULE,
 110         }
 111 };
 112
 113 static int register_sha1_ssse3(void)
 114 {
 115         if (boot_cpu_has(X86_FEATURE_SSSE3))
 116                 return crypto_register_shash(&sha1_ssse3_alg);
 117         return 0;
 118 }
 119
 120 static void unregister_sha1_ssse3(void)
 121 {
 122         if (boot_cpu_has(X86_FEATURE_SSSE3))
 123                 crypto_unregister_shash(&sha1_ssse3_alg);
 124 }
 125
 126 #ifdef CONFIG_AS_AVX
 127 asmlinkage void sha1_transform_avx(u32 *digest, const char *data,
 128                                    unsigned int rounds);
 129
 130 static int sha1_avx_update(struct shash_desc *desc, const u8 *data,
 131                              unsigned int len)
 132 {
 133         return sha1_update(desc, data, len,
 134                         (sha1_transform_fn *) sha1_transform_avx);
 135 }
 136
 137 static int sha1_avx_finup(struct shash_desc *desc, const u8 *data,
 138                               unsigned int len, u8 *out)
 139 {
 140         return sha1_finup(desc, data, len, out,
 141                         (sha1_transform_fn *) sha1_transform_avx);
 142 }
 143
 144 static int sha1_avx_final(struct shash_desc *desc, u8 *out)
 145 {
 146         return sha1_avx_finup(desc, NULL, 0, out);
 147 }
 148
 149 static struct shash_alg sha1_avx_alg = {
 150         .digestsize     =       SHA1_DIGEST_SIZE,
 151         .init           =       sha1_base_init,
 152         .update         =       sha1_avx_update,
 153         .final          =       sha1_avx_final,
 154         .finup          =       sha1_avx_finup,
 155         .descsize       =       sizeof(struct sha1_state),
 156         .base           =       {
 157                 .cra_name       =       "sha1",
 158                 .cra_driver_name =      "sha1-avx",
 159                 .cra_priority   =       160,
 160                 .cra_blocksize  =       SHA1_BLOCK_SIZE,
 161                 .cra_module     =       THIS_MODULE,
 162         }
 163 };
 164
 165 static bool avx_usable(void)
 166 {
 167         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
 168                 if (boot_cpu_has(X86_FEATURE_AVX))
 169                         pr_info("AVX detected but unusable.\n");
 170                 return false;
 171         }
 172
 173         return true;
 174 }
 175
 176 static int register_sha1_avx(void)
 177 {
 178         if (avx_usable())
 179                 return crypto_register_shash(&sha1_avx_alg);
 180         return 0;
 181 }
 182
 183 static void unregister_sha1_avx(void)
 184 {
 185         if (avx_usable())
 186                 crypto_unregister_shash(&sha1_avx_alg);
 187 }
 188
 189 #else  /* CONFIG_AS_AVX */
 190 static inline int register_sha1_avx(void) { return 0; }
 191 static inline void unregister_sha1_avx(void) { }
 192 #endif /* CONFIG_AS_AVX */
 193
 194
 195 #if defined(CONFIG_AS_AVX2) && (CONFIG_AS_AVX)
 196 #define SHA1_AVX2_BLOCK_OPTSIZE 4       /* optimal 4*64 bytes of SHA1 blocks */
 197
 198 asmlinkage void sha1_transform_avx2(u32 *digest, const char *data,
 199                                     unsigned int rounds);
 200
 201 static bool avx2_usable(void)
 202 {
 203         if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2)
 204                 && boot_cpu_has(X86_FEATURE_BMI1)
 205                 && boot_cpu_has(X86_FEATURE_BMI2))
 206                 return true;
 207
 208         return false;
 209 }
 210
 211 static void sha1_apply_transform_avx2(u32 *digest, const char *data,
 212                                 unsigned int rounds)
 213 {
 214         /* Select the optimal transform based on data block size */
 215         if (rounds >= SHA1_AVX2_BLOCK_OPTSIZE)
 216                 sha1_transform_avx2(digest, data, rounds);
 217         else
 218                 sha1_transform_avx(digest, data, rounds);
 219 }
 220
 221 static int sha1_avx2_update(struct shash_desc *desc, const u8 *data,
 222                              unsigned int len)
 223 {
 224         return sha1_update(desc, data, len,
 225                 (sha1_transform_fn *) sha1_apply_transform_avx2);
 226 }
 227
 228 static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data,
 229                               unsigned int len, u8 *out)
 230 {
 231         return sha1_finup(desc, data, len, out,
 232                 (sha1_transform_fn *) sha1_apply_transform_avx2);
 233 }
 234
 235 static int sha1_avx2_final(struct shash_desc *desc, u8 *out)
 236 {
 237         return sha1_avx2_finup(desc, NULL, 0, out);
 238 }
 239
 240 static struct shash_alg sha1_avx2_alg = {
 241         .digestsize     =       SHA1_DIGEST_SIZE,
 242         .init           =       sha1_base_init,
 243         .update         =       sha1_avx2_update,
 244         .final          =       sha1_avx2_final,
 245         .finup          =       sha1_avx2_finup,
 246         .descsize       =       sizeof(struct sha1_state),
 247         .base           =       {
 248                 .cra_name       =       "sha1",
 249                 .cra_driver_name =      "sha1-avx2",
 250                 .cra_priority   =       170,
 251                 .cra_blocksize  =       SHA1_BLOCK_SIZE,
 252                 .cra_module     =       THIS_MODULE,
 253         }
 254 };
 255
 256 static int register_sha1_avx2(void)
 257 {
 258         if (avx2_usable())
 259                 return crypto_register_shash(&sha1_avx2_alg);
 260         return 0;
 261 }
 262
 263 static void unregister_sha1_avx2(void)
 264 {
 265         if (avx2_usable())
 266                 crypto_unregister_shash(&sha1_avx2_alg);
 267 }
 268
 269 #else
 270 static inline int register_sha1_avx2(void) { return 0; }
 271 static inline void unregister_sha1_avx2(void) { }
 272 #endif
 273
 274 #ifdef CONFIG_AS_SHA1_NI
 275 asmlinkage void sha1_ni_transform(u32 *digest, const char *data,
 276                                    unsigned int rounds);
 277
 278 static int sha1_ni_update(struct shash_desc *desc, const u8 *data,
 279                              unsigned int len)
 280 {
 281         return sha1_update(desc, data, len,
 282                 (sha1_transform_fn *) sha1_ni_transform);
 283 }
 284
 285 static int sha1_ni_finup(struct shash_desc *desc, const u8 *data,
 286                               unsigned int len, u8 *out)
 287 {
 288         return sha1_finup(desc, data, len, out,
 289                 (sha1_transform_fn *) sha1_ni_transform);
 290 }
 291
 292 static int sha1_ni_final(struct shash_desc *desc, u8 *out)
 293 {
 294         return sha1_ni_finup(desc, NULL, 0, out);
 295 }
 296
 297 static struct shash_alg sha1_ni_alg = {
 298         .digestsize     =       SHA1_DIGEST_SIZE,
 299         .init           =       sha1_base_init,
 300         .update         =       sha1_ni_update,
 301         .final          =       sha1_ni_final,
 302         .finup          =       sha1_ni_finup,
 303         .descsize       =       sizeof(struct sha1_state),
 304         .base           =       {
 305                 .cra_name       =       "sha1",
 306                 .cra_driver_name =      "sha1-ni",
 307                 .cra_priority   =       250,
 308                 .cra_blocksize  =       SHA1_BLOCK_SIZE,
 309                 .cra_module     =       THIS_MODULE,
 310         }
 311 };
 312
 313 static int register_sha1_ni(void)
 314 {
 315         if (boot_cpu_has(X86_FEATURE_SHA_NI))
 316                 return crypto_register_shash(&sha1_ni_alg);
 317         return 0;
 318 }
 319
 320 static void unregister_sha1_ni(void)
 321 {
 322         if (boot_cpu_has(X86_FEATURE_SHA_NI))
 323                 crypto_unregister_shash(&sha1_ni_alg);
 324 }
 325
 326 #else
 327 static inline int register_sha1_ni(void) { return 0; }
 328 static inline void unregister_sha1_ni(void) { }
 329 #endif
 330
 331 static int __init sha1_ssse3_mod_init(void)
 332 {
 333         if (register_sha1_ssse3())
 334                 goto fail;
 335
 336         if (register_sha1_avx()) {
 337                 unregister_sha1_ssse3();
 338                 goto fail;
 339         }
 340
 341         if (register_sha1_avx2()) {
 342                 unregister_sha1_avx();
 343                 unregister_sha1_ssse3();
 344                 goto fail;
 345         }
 346
 347         if (register_sha1_ni()) {
 348                 unregister_sha1_avx2();
 349                 unregister_sha1_avx();
 350                 unregister_sha1_ssse3();
 351                 goto fail;
 352         }
 353
 354         return 0;
 355 fail:
 356         return -ENODEV;
 357 }
 358
 359 static void __exit sha1_ssse3_mod_fini(void)
 360 {
 361         unregister_sha1_ni();
 362         unregister_sha1_avx2();
 363         unregister_sha1_avx();
 364         unregister_sha1_ssse3();
 365 }
 366
 367 module_init(sha1_ssse3_mod_init);
 368 module_exit(sha1_ssse3_mod_fini);
 369
 370 MODULE_LICENSE("GPL");
 371 MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");
 372
 373 MODULE_ALIAS_CRYPTO("sha1");
 374 MODULE_ALIAS_CRYPTO("sha1-ssse3");
 375 MODULE_ALIAS_CRYPTO("sha1-avx");
 376 MODULE_ALIAS_CRYPTO("sha1-avx2");
 377 #ifdef CONFIG_AS_SHA1_NI
 378 MODULE_ALIAS_CRYPTO("sha1-ni");
 379 #endif