arch/arm64/crypto/crct10dif-ce-glue.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
   4  *
   5  * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
   6  */
   7
   8 #include <linux/cpufeature.h>
   9 #include <linux/crc-t10dif.h>
  10 #include <linux/init.h>
  11 #include <linux/kernel.h>
  12 #include <linux/module.h>
  13 #include <linux/string.h>
  14
  15 #include <crypto/internal/hash.h>
  16 #include <crypto/internal/simd.h>
  17
  18 #include <asm/neon.h>
  19 #include <asm/simd.h>
  20
  21 #define CRC_T10DIF_PMULL_CHUNK_SIZE     16U
  22
  23 asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len);
  24 asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len);
  25
  26 static int crct10dif_init(struct shash_desc *desc)
  27 {
  28         u16 *crc = shash_desc_ctx(desc);
  29
  30         *crc = 0;
  31         return 0;
  32 }
  33
  34 static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data,
  35                             unsigned int length)
  36 {
  37         u16 *crc = shash_desc_ctx(desc);
  38
  39         if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
  40                 do {
  41                         unsigned int chunk = length;
  42
  43                         if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE)
  44                                 chunk = SZ_4K;
  45
  46                         kernel_neon_begin();
  47                         *crc = crc_t10dif_pmull_p8(*crc, data, chunk);
  48                         kernel_neon_end();
  49                         data += chunk;
  50                         length -= chunk;
  51                 } while (length);
  52         } else {
  53                 *crc = crc_t10dif_generic(*crc, data, length);
  54         }
  55
  56         return 0;
  57 }
  58
  59 static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data,
  60                             unsigned int length)
  61 {
  62         u16 *crc = shash_desc_ctx(desc);
  63
  64         if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
  65                 do {
  66                         unsigned int chunk = length;
  67
  68                         if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE)
  69                                 chunk = SZ_4K;
  70
  71                         kernel_neon_begin();
  72                         *crc = crc_t10dif_pmull_p64(*crc, data, chunk);
  73                         kernel_neon_end();
  74                         data += chunk;
  75                         length -= chunk;
  76                 } while (length);
  77         } else {
  78                 *crc = crc_t10dif_generic(*crc, data, length);
  79         }
  80
  81         return 0;
  82 }
  83
  84 static int crct10dif_final(struct shash_desc *desc, u8 *out)
  85 {
  86         u16 *crc = shash_desc_ctx(desc);
  87
  88         *(u16 *)out = *crc;
  89         return 0;
  90 }
  91
  92 static struct shash_alg crc_t10dif_alg[] = {{
  93         .digestsize             = CRC_T10DIF_DIGEST_SIZE,
  94         .init                   = crct10dif_init,
  95         .update                 = crct10dif_update_pmull_p8,
  96         .final                  = crct10dif_final,
  97         .descsize               = CRC_T10DIF_DIGEST_SIZE,
  98
  99         .base.cra_name          = "crct10dif",
 100         .base.cra_driver_name   = "crct10dif-arm64-neon",
 101         .base.cra_priority      = 100,
 102         .base.cra_blocksize     = CRC_T10DIF_BLOCK_SIZE,
 103         .base.cra_module        = THIS_MODULE,
 104 }, {
 105         .digestsize             = CRC_T10DIF_DIGEST_SIZE,
 106         .init                   = crct10dif_init,
 107         .update                 = crct10dif_update_pmull_p64,
 108         .final                  = crct10dif_final,
 109         .descsize               = CRC_T10DIF_DIGEST_SIZE,
 110
 111         .base.cra_name          = "crct10dif",
 112         .base.cra_driver_name   = "crct10dif-arm64-ce",
 113         .base.cra_priority      = 200,
 114         .base.cra_blocksize     = CRC_T10DIF_BLOCK_SIZE,
 115         .base.cra_module        = THIS_MODULE,
 116 }};
 117
 118 static int __init crc_t10dif_mod_init(void)
 119 {
 120         if (cpu_have_named_feature(PMULL))
 121                 return crypto_register_shashes(crc_t10dif_alg,
 122                                                ARRAY_SIZE(crc_t10dif_alg));
 123         else
 124                 /* only register the first array element */
 125                 return crypto_register_shash(crc_t10dif_alg);
 126 }
 127
 128 static void __exit crc_t10dif_mod_exit(void)
 129 {
 130         if (cpu_have_named_feature(PMULL))
 131                 crypto_unregister_shashes(crc_t10dif_alg,
 132                                           ARRAY_SIZE(crc_t10dif_alg));
 133         else
 134                 crypto_unregister_shash(crc_t10dif_alg);
 135 }
 136
 137 module_cpu_feature_match(ASIMD, crc_t10dif_mod_init);
 138 module_exit(crc_t10dif_mod_exit);
 139
 140 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 141 MODULE_LICENSE("GPL v2");
 142 MODULE_ALIAS_CRYPTO("crct10dif");
 143 MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce");