arch/arm/crypto/chacha20-neon-glue.c

   1 /*
   2  * ChaCha20 256-bit cipher algorithm, RFC7539, ARM NEON functions
   3  *
   4  * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  *
  10  * Based on:
  11  * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
  12  *
  13  * Copyright (C) 2015 Martin Willi
  14  *
  15  * This program is free software; you can redistribute it and/or modify
  16  * it under the terms of the GNU General Public License as published by
  17  * the Free Software Foundation; either version 2 of the License, or
  18  * (at your option) any later version.
  19  */
  20
  21 #include <crypto/algapi.h>
  22 #include <crypto/chacha20.h>
  23 #include <linux/crypto.h>
  24 #include <linux/kernel.h>
  25 #include <linux/module.h>
  26
  27 #include <asm/hwcap.h>
  28 #include <asm/neon.h>
  29 #include <asm/simd.h>
  30
  31 asmlinkage void chacha20_block_xor_neon(u32 *state, u8 *dst, const u8 *src);
  32 asmlinkage void chacha20_4block_xor_neon(u32 *state, u8 *dst, const u8 *src);
  33
  34 static void chacha20_dosimd(u32 *state, u8 *dst, const u8 *src,
  35                             unsigned int bytes)
  36 {
  37         u8 buf[CHACHA20_BLOCK_SIZE];
  38
  39         while (bytes >= CHACHA20_BLOCK_SIZE * 4) {
  40                 chacha20_4block_xor_neon(state, dst, src);
  41                 bytes -= CHACHA20_BLOCK_SIZE * 4;
  42                 src += CHACHA20_BLOCK_SIZE * 4;
  43                 dst += CHACHA20_BLOCK_SIZE * 4;
  44                 state[12] += 4;
  45         }
  46         while (bytes >= CHACHA20_BLOCK_SIZE) {
  47                 chacha20_block_xor_neon(state, dst, src);
  48                 bytes -= CHACHA20_BLOCK_SIZE;
  49                 src += CHACHA20_BLOCK_SIZE;
  50                 dst += CHACHA20_BLOCK_SIZE;
  51                 state[12]++;
  52         }
  53         if (bytes) {
  54                 memcpy(buf, src, bytes);
  55                 chacha20_block_xor_neon(state, buf, buf);
  56                 memcpy(dst, buf, bytes);
  57         }
  58 }
  59
  60 static int chacha20_simd(struct blkcipher_desc *desc, struct scatterlist *dst,
  61                          struct scatterlist *src, unsigned int nbytes)
  62 {
  63         struct blkcipher_walk walk;
  64         u32 state[16];
  65         int err;
  66
  67         if (nbytes <= CHACHA20_BLOCK_SIZE || !may_use_simd())
  68                 return crypto_chacha20_crypt(desc, dst, src, nbytes);
  69
  70         blkcipher_walk_init(&walk, dst, src, nbytes);
  71         err = blkcipher_walk_virt_block(desc, &walk, CHACHA20_BLOCK_SIZE);
  72
  73         crypto_chacha20_init(state, crypto_blkcipher_ctx(desc->tfm), walk.iv);
  74
  75         kernel_neon_begin();
  76
  77         while (walk.nbytes >= CHACHA20_BLOCK_SIZE) {
  78                 chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
  79                                 rounddown(walk.nbytes, CHACHA20_BLOCK_SIZE));
  80                 err = blkcipher_walk_done(desc, &walk,
  81                                           walk.nbytes % CHACHA20_BLOCK_SIZE);
  82         }
  83
  84         if (walk.nbytes) {
  85                 chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
  86                                 walk.nbytes);
  87                 err = blkcipher_walk_done(desc, &walk, 0);
  88         }
  89
  90         kernel_neon_end();
  91
  92         return err;
  93 }
  94
  95 static struct crypto_alg alg = {
  96         .cra_name               = "chacha20",
  97         .cra_driver_name        = "chacha20-neon",
  98         .cra_priority           = 300,
  99         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 100         .cra_blocksize          = 1,
 101         .cra_type               = &crypto_blkcipher_type,
 102         .cra_ctxsize            = sizeof(struct chacha20_ctx),
 103         .cra_alignmask          = sizeof(u32) - 1,
 104         .cra_module             = THIS_MODULE,
 105         .cra_u                  = {
 106                 .blkcipher = {
 107                         .min_keysize    = CHACHA20_KEY_SIZE,
 108                         .max_keysize    = CHACHA20_KEY_SIZE,
 109                         .ivsize         = CHACHA20_IV_SIZE,
 110                         .geniv          = "seqiv",
 111                         .setkey         = crypto_chacha20_setkey,
 112                         .encrypt        = chacha20_simd,
 113                         .decrypt        = chacha20_simd,
 114                 },
 115         },
 116 };
 117
 118 static int __init chacha20_simd_mod_init(void)
 119 {
 120         if (!(elf_hwcap & HWCAP_NEON))
 121                 return -ENODEV;
 122
 123         return crypto_register_alg(&alg);
 124 }
 125
 126 static void __exit chacha20_simd_mod_fini(void)
 127 {
 128         crypto_unregister_alg(&alg);
 129 }
 130
 131 module_init(chacha20_simd_mod_init);
 132 module_exit(chacha20_simd_mod_fini);
 133
 134 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 135 MODULE_LICENSE("GPL v2");
 136 MODULE_ALIAS_CRYPTO("chacha20");