drivers/crypto/caam/caamalg.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * caam - Freescale FSL CAAM support for crypto API
   4  *
   5  * Copyright 2008-2011 Freescale Semiconductor, Inc.
   6  * Copyright 2016-2019 NXP
   7  *
   8  * Based on talitos crypto API driver.
   9  *
  10  * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
  11  *
  12  * ---------------                     ---------------
  13  * | JobDesc #1  |-------------------->|  ShareDesc  |
  14  * | *(packet 1) |                     |   (PDB)     |
  15  * ---------------      |------------->|  (hashKey)  |
  16  *       .              |              | (cipherKey) |
  17  *       .              |    |-------->| (operation) |
  18  * ---------------      |    |         ---------------
  19  * | JobDesc #2  |------|    |
  20  * | *(packet 2) |           |
  21  * ---------------           |
  22  *       .                   |
  23  *       .                   |
  24  * ---------------           |
  25  * | JobDesc #3  |------------
  26  * | *(packet 3) |
  27  * ---------------
  28  *
  29  * The SharedDesc never changes for a connection unless rekeyed, but
  30  * each packet will likely be in a different place. So all we need
  31  * to know to process the packet is where the input is, where the
  32  * output goes, and what context we want to process with. Context is
  33  * in the SharedDesc, packet references in the JobDesc.
  34  *
  35  * So, a job desc looks like:
  36  *
  37  * ---------------------
  38  * | Header            |
  39  * | ShareDesc Pointer |
  40  * | SEQ_OUT_PTR       |
  41  * | (output buffer)   |
  42  * | (output length)   |
  43  * | SEQ_IN_PTR        |
  44  * | (input buffer)    |
  45  * | (input length)    |
  46  * ---------------------
  47  */
  48
  49 #include "compat.h"
  50
  51 #include "regs.h"
  52 #include "intern.h"
  53 #include "desc_constr.h"
  54 #include "jr.h"
  55 #include "error.h"
  56 #include "sg_sw_sec4.h"
  57 #include "key_gen.h"
  58 #include "caamalg_desc.h"
  59
  60 /*
  61  * crypto alg
  62  */
  63 #define CAAM_CRA_PRIORITY               3000
  64 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  65 #define CAAM_MAX_KEY_SIZE               (AES_MAX_KEY_SIZE + \
  66                                          CTR_RFC3686_NONCE_SIZE + \
  67                                          SHA512_DIGEST_SIZE * 2)
  68
  69 #define AEAD_DESC_JOB_IO_LEN            (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)
  70 #define GCM_DESC_JOB_IO_LEN             (AEAD_DESC_JOB_IO_LEN + \
  71                                          CAAM_CMD_SZ * 4)
  72 #define AUTHENC_DESC_JOB_IO_LEN         (AEAD_DESC_JOB_IO_LEN + \
  73                                          CAAM_CMD_SZ * 5)
  74
  75 #define CHACHAPOLY_DESC_JOB_IO_LEN      (AEAD_DESC_JOB_IO_LEN + CAAM_CMD_SZ * 6)
  76
  77 #define DESC_MAX_USED_BYTES             (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN)
  78 #define DESC_MAX_USED_LEN               (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
  79
  80 #ifdef DEBUG
  81 /* for print_hex_dumps with line references */
  82 #define debug(format, arg...) printk(format, arg)
  83 #else
  84 #define debug(format, arg...)
  85 #endif
  86
  87 struct caam_alg_entry {
  88         int class1_alg_type;
  89         int class2_alg_type;
  90         bool rfc3686;
  91         bool geniv;
  92 };
  93
  94 struct caam_aead_alg {
  95         struct aead_alg aead;
  96         struct caam_alg_entry caam;
  97         bool registered;
  98 };
  99
 100 struct caam_skcipher_alg {
 101         struct skcipher_alg skcipher;
 102         struct caam_alg_entry caam;
 103         bool registered;
 104 };
 105
 106 /*
 107  * per-session context
 108  */
 109 struct caam_ctx {
 110         u32 sh_desc_enc[DESC_MAX_USED_LEN];
 111         u32 sh_desc_dec[DESC_MAX_USED_LEN];
 112         u8 key[CAAM_MAX_KEY_SIZE];
 113         dma_addr_t sh_desc_enc_dma;
 114         dma_addr_t sh_desc_dec_dma;
 115         dma_addr_t key_dma;
 116         enum dma_data_direction dir;
 117         struct device *jrdev;
 118         struct alginfo adata;
 119         struct alginfo cdata;
 120         unsigned int authsize;
 121 };
 122
 123 static int aead_null_set_sh_desc(struct crypto_aead *aead)
 124 {
 125         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 126         struct device *jrdev = ctx->jrdev;
 127         struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 128         u32 *desc;
 129         int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN -
 130                         ctx->adata.keylen_pad;
 131
 132         /*
 133          * Job Descriptor and Shared Descriptors
 134          * must all fit into the 64-word Descriptor h/w Buffer
 135          */
 136         if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) {
 137                 ctx->adata.key_inline = true;
 138                 ctx->adata.key_virt = ctx->key;
 139         } else {
 140                 ctx->adata.key_inline = false;
 141                 ctx->adata.key_dma = ctx->key_dma;
 142         }
 143
 144         /* aead_encrypt shared descriptor */
 145         desc = ctx->sh_desc_enc;
 146         cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize,
 147                                     ctrlpriv->era);
 148         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 149                                    desc_bytes(desc), ctx->dir);
 150
 151         /*
 152          * Job Descriptor and Shared Descriptors
 153          * must all fit into the 64-word Descriptor h/w Buffer
 154          */
 155         if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) {
 156                 ctx->adata.key_inline = true;
 157                 ctx->adata.key_virt = ctx->key;
 158         } else {
 159                 ctx->adata.key_inline = false;
 160                 ctx->adata.key_dma = ctx->key_dma;
 161         }
 162
 163         /* aead_decrypt shared descriptor */
 164         desc = ctx->sh_desc_dec;
 165         cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize,
 166                                     ctrlpriv->era);
 167         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 168                                    desc_bytes(desc), ctx->dir);
 169
 170         return 0;
 171 }
 172
 173 static int aead_set_sh_desc(struct crypto_aead *aead)
 174 {
 175         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
 176                                                  struct caam_aead_alg, aead);
 177         unsigned int ivsize = crypto_aead_ivsize(aead);
 178         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 179         struct device *jrdev = ctx->jrdev;
 180         struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 181         u32 ctx1_iv_off = 0;
 182         u32 *desc, *nonce = NULL;
 183         u32 inl_mask;
 184         unsigned int data_len[2];
 185         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 186                                OP_ALG_AAI_CTR_MOD128);
 187         const bool is_rfc3686 = alg->caam.rfc3686;
 188
 189         if (!ctx->authsize)
 190                 return 0;
 191
 192         /* NULL encryption / decryption */
 193         if (!ctx->cdata.keylen)
 194                 return aead_null_set_sh_desc(aead);
 195
 196         /*
 197          * AES-CTR needs to load IV in CONTEXT1 reg
 198          * at an offset of 128bits (16bytes)
 199          * CONTEXT1[255:128] = IV
 200          */
 201         if (ctr_mode)
 202                 ctx1_iv_off = 16;
 203
 204         /*
 205          * RFC3686 specific:
 206          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 207          */
 208         if (is_rfc3686) {
 209                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 210                 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
 211                                 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
 212         }
 213
 214         data_len[0] = ctx->adata.keylen_pad;
 215         data_len[1] = ctx->cdata.keylen;
 216
 217         if (alg->caam.geniv)
 218                 goto skip_enc;
 219
 220         /*
 221          * Job Descriptor and Shared Descriptors
 222          * must all fit into the 64-word Descriptor h/w Buffer
 223          */
 224         if (desc_inline_query(DESC_AEAD_ENC_LEN +
 225                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 226                               AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
 227                               ARRAY_SIZE(data_len)) < 0)
 228                 return -EINVAL;
 229
 230         if (inl_mask & 1)
 231                 ctx->adata.key_virt = ctx->key;
 232         else
 233                 ctx->adata.key_dma = ctx->key_dma;
 234
 235         if (inl_mask & 2)
 236                 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 237         else
 238                 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 239
 240         ctx->adata.key_inline = !!(inl_mask & 1);
 241         ctx->cdata.key_inline = !!(inl_mask & 2);
 242
 243         /* aead_encrypt shared descriptor */
 244         desc = ctx->sh_desc_enc;
 245         cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize,
 246                                ctx->authsize, is_rfc3686, nonce, ctx1_iv_off,
 247                                false, ctrlpriv->era);
 248         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 249                                    desc_bytes(desc), ctx->dir);
 250
 251 skip_enc:
 252         /*
 253          * Job Descriptor and Shared Descriptors
 254          * must all fit into the 64-word Descriptor h/w Buffer
 255          */
 256         if (desc_inline_query(DESC_AEAD_DEC_LEN +
 257                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 258                               AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
 259                               ARRAY_SIZE(data_len)) < 0)
 260                 return -EINVAL;
 261
 262         if (inl_mask & 1)
 263                 ctx->adata.key_virt = ctx->key;
 264         else
 265                 ctx->adata.key_dma = ctx->key_dma;
 266
 267         if (inl_mask & 2)
 268                 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 269         else
 270                 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 271
 272         ctx->adata.key_inline = !!(inl_mask & 1);
 273         ctx->cdata.key_inline = !!(inl_mask & 2);
 274
 275         /* aead_decrypt shared descriptor */
 276         desc = ctx->sh_desc_dec;
 277         cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize,
 278                                ctx->authsize, alg->caam.geniv, is_rfc3686,
 279                                nonce, ctx1_iv_off, false, ctrlpriv->era);
 280         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 281                                    desc_bytes(desc), ctx->dir);
 282
 283         if (!alg->caam.geniv)
 284                 goto skip_givenc;
 285
 286         /*
 287          * Job Descriptor and Shared Descriptors
 288          * must all fit into the 64-word Descriptor h/w Buffer
 289          */
 290         if (desc_inline_query(DESC_AEAD_GIVENC_LEN +
 291                               (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
 292                               AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
 293                               ARRAY_SIZE(data_len)) < 0)
 294                 return -EINVAL;
 295
 296         if (inl_mask & 1)
 297                 ctx->adata.key_virt = ctx->key;
 298         else
 299                 ctx->adata.key_dma = ctx->key_dma;
 300
 301         if (inl_mask & 2)
 302                 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
 303         else
 304                 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
 305
 306         ctx->adata.key_inline = !!(inl_mask & 1);
 307         ctx->cdata.key_inline = !!(inl_mask & 2);
 308
 309         /* aead_givencrypt shared descriptor */
 310         desc = ctx->sh_desc_enc;
 311         cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize,
 312                                   ctx->authsize, is_rfc3686, nonce,
 313                                   ctx1_iv_off, false, ctrlpriv->era);
 314         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 315                                    desc_bytes(desc), ctx->dir);
 316
 317 skip_givenc:
 318         return 0;
 319 }
 320
 321 static int aead_setauthsize(struct crypto_aead *authenc,
 322                                     unsigned int authsize)
 323 {
 324         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 325
 326         ctx->authsize = authsize;
 327         aead_set_sh_desc(authenc);
 328
 329         return 0;
 330 }
 331
 332 static int gcm_set_sh_desc(struct crypto_aead *aead)
 333 {
 334         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 335         struct device *jrdev = ctx->jrdev;
 336         unsigned int ivsize = crypto_aead_ivsize(aead);
 337         u32 *desc;
 338         int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
 339                         ctx->cdata.keylen;
 340
 341         if (!ctx->cdata.keylen || !ctx->authsize)
 342                 return 0;
 343
 344         /*
 345          * AES GCM encrypt shared descriptor
 346          * Job Descriptor and Shared Descriptor
 347          * must fit into the 64-word Descriptor h/w Buffer
 348          */
 349         if (rem_bytes >= DESC_GCM_ENC_LEN) {
 350                 ctx->cdata.key_inline = true;
 351                 ctx->cdata.key_virt = ctx->key;
 352         } else {
 353                 ctx->cdata.key_inline = false;
 354                 ctx->cdata.key_dma = ctx->key_dma;
 355         }
 356
 357         desc = ctx->sh_desc_enc;
 358         cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false);
 359         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 360                                    desc_bytes(desc), ctx->dir);
 361
 362         /*
 363          * Job Descriptor and Shared Descriptors
 364          * must all fit into the 64-word Descriptor h/w Buffer
 365          */
 366         if (rem_bytes >= DESC_GCM_DEC_LEN) {
 367                 ctx->cdata.key_inline = true;
 368                 ctx->cdata.key_virt = ctx->key;
 369         } else {
 370                 ctx->cdata.key_inline = false;
 371                 ctx->cdata.key_dma = ctx->key_dma;
 372         }
 373
 374         desc = ctx->sh_desc_dec;
 375         cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false);
 376         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 377                                    desc_bytes(desc), ctx->dir);
 378
 379         return 0;
 380 }
 381
 382 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
 383 {
 384         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 385
 386         ctx->authsize = authsize;
 387         gcm_set_sh_desc(authenc);
 388
 389         return 0;
 390 }
 391
 392 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 393 {
 394         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 395         struct device *jrdev = ctx->jrdev;
 396         unsigned int ivsize = crypto_aead_ivsize(aead);
 397         u32 *desc;
 398         int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
 399                         ctx->cdata.keylen;
 400
 401         if (!ctx->cdata.keylen || !ctx->authsize)
 402                 return 0;
 403
 404         /*
 405          * RFC4106 encrypt shared descriptor
 406          * Job Descriptor and Shared Descriptor
 407          * must fit into the 64-word Descriptor h/w Buffer
 408          */
 409         if (rem_bytes >= DESC_RFC4106_ENC_LEN) {
 410                 ctx->cdata.key_inline = true;
 411                 ctx->cdata.key_virt = ctx->key;
 412         } else {
 413                 ctx->cdata.key_inline = false;
 414                 ctx->cdata.key_dma = ctx->key_dma;
 415         }
 416
 417         desc = ctx->sh_desc_enc;
 418         cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 419                                   false);
 420         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 421                                    desc_bytes(desc), ctx->dir);
 422
 423         /*
 424          * Job Descriptor and Shared Descriptors
 425          * must all fit into the 64-word Descriptor h/w Buffer
 426          */
 427         if (rem_bytes >= DESC_RFC4106_DEC_LEN) {
 428                 ctx->cdata.key_inline = true;
 429                 ctx->cdata.key_virt = ctx->key;
 430         } else {
 431                 ctx->cdata.key_inline = false;
 432                 ctx->cdata.key_dma = ctx->key_dma;
 433         }
 434
 435         desc = ctx->sh_desc_dec;
 436         cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 437                                   false);
 438         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 439                                    desc_bytes(desc), ctx->dir);
 440
 441         return 0;
 442 }
 443
 444 static int rfc4106_setauthsize(struct crypto_aead *authenc,
 445                                unsigned int authsize)
 446 {
 447         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 448
 449         ctx->authsize = authsize;
 450         rfc4106_set_sh_desc(authenc);
 451
 452         return 0;
 453 }
 454
 455 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 456 {
 457         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 458         struct device *jrdev = ctx->jrdev;
 459         unsigned int ivsize = crypto_aead_ivsize(aead);
 460         u32 *desc;
 461         int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
 462                         ctx->cdata.keylen;
 463
 464         if (!ctx->cdata.keylen || !ctx->authsize)
 465                 return 0;
 466
 467         /*
 468          * RFC4543 encrypt shared descriptor
 469          * Job Descriptor and Shared Descriptor
 470          * must fit into the 64-word Descriptor h/w Buffer
 471          */
 472         if (rem_bytes >= DESC_RFC4543_ENC_LEN) {
 473                 ctx->cdata.key_inline = true;
 474                 ctx->cdata.key_virt = ctx->key;
 475         } else {
 476                 ctx->cdata.key_inline = false;
 477                 ctx->cdata.key_dma = ctx->key_dma;
 478         }
 479
 480         desc = ctx->sh_desc_enc;
 481         cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize,
 482                                   false);
 483         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 484                                    desc_bytes(desc), ctx->dir);
 485
 486         /*
 487          * Job Descriptor and Shared Descriptors
 488          * must all fit into the 64-word Descriptor h/w Buffer
 489          */
 490         if (rem_bytes >= DESC_RFC4543_DEC_LEN) {
 491                 ctx->cdata.key_inline = true;
 492                 ctx->cdata.key_virt = ctx->key;
 493         } else {
 494                 ctx->cdata.key_inline = false;
 495                 ctx->cdata.key_dma = ctx->key_dma;
 496         }
 497
 498         desc = ctx->sh_desc_dec;
 499         cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize,
 500                                   false);
 501         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 502                                    desc_bytes(desc), ctx->dir);
 503
 504         return 0;
 505 }
 506
 507 static int rfc4543_setauthsize(struct crypto_aead *authenc,
 508                                unsigned int authsize)
 509 {
 510         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 511
 512         ctx->authsize = authsize;
 513         rfc4543_set_sh_desc(authenc);
 514
 515         return 0;
 516 }
 517
 518 static int chachapoly_set_sh_desc(struct crypto_aead *aead)
 519 {
 520         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 521         struct device *jrdev = ctx->jrdev;
 522         unsigned int ivsize = crypto_aead_ivsize(aead);
 523         u32 *desc;
 524
 525         if (!ctx->cdata.keylen || !ctx->authsize)
 526                 return 0;
 527
 528         desc = ctx->sh_desc_enc;
 529         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 530                                ctx->authsize, true, false);
 531         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 532                                    desc_bytes(desc), ctx->dir);
 533
 534         desc = ctx->sh_desc_dec;
 535         cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize,
 536                                ctx->authsize, false, false);
 537         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 538                                    desc_bytes(desc), ctx->dir);
 539
 540         return 0;
 541 }
 542
 543 static int chachapoly_setauthsize(struct crypto_aead *aead,
 544                                   unsigned int authsize)
 545 {
 546         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 547
 548         if (authsize != POLY1305_DIGEST_SIZE)
 549                 return -EINVAL;
 550
 551         ctx->authsize = authsize;
 552         return chachapoly_set_sh_desc(aead);
 553 }
 554
 555 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
 556                              unsigned int keylen)
 557 {
 558         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 559         unsigned int ivsize = crypto_aead_ivsize(aead);
 560         unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize;
 561
 562         if (keylen != CHACHA_KEY_SIZE + saltlen) {
 563                 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 564                 return -EINVAL;
 565         }
 566
 567         ctx->cdata.key_virt = key;
 568         ctx->cdata.keylen = keylen - saltlen;
 569
 570         return chachapoly_set_sh_desc(aead);
 571 }
 572
 573 static int aead_setkey(struct crypto_aead *aead,
 574                                const u8 *key, unsigned int keylen)
 575 {
 576         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 577         struct device *jrdev = ctx->jrdev;
 578         struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 579         struct crypto_authenc_keys keys;
 580         int ret = 0;
 581
 582         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 583                 goto badkey;
 584
 585 #ifdef DEBUG
 586         printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
 587                keys.authkeylen + keys.enckeylen, keys.enckeylen,
 588                keys.authkeylen);
 589         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 590                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 591 #endif
 592
 593         /*
 594          * If DKP is supported, use it in the shared descriptor to generate
 595          * the split key.
 596          */
 597         if (ctrlpriv->era >= 6) {
 598                 ctx->adata.keylen = keys.authkeylen;
 599                 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
 600                                                       OP_ALG_ALGSEL_MASK);
 601
 602                 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
 603                         goto badkey;
 604
 605                 memcpy(ctx->key, keys.authkey, keys.authkeylen);
 606                 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
 607                        keys.enckeylen);
 608                 dma_sync_single_for_device(jrdev, ctx->key_dma,
 609                                            ctx->adata.keylen_pad +
 610                                            keys.enckeylen, ctx->dir);
 611                 goto skip_split_key;
 612         }
 613
 614         ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey,
 615                             keys.authkeylen, CAAM_MAX_KEY_SIZE -
 616                             keys.enckeylen);
 617         if (ret) {
 618                 goto badkey;
 619         }
 620
 621         /* postpend encryption key to auth split key */
 622         memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 623         dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
 624                                    keys.enckeylen, ctx->dir);
 625 #ifdef DEBUG
 626         print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
 627                        DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 628                        ctx->adata.keylen_pad + keys.enckeylen, 1);
 629 #endif
 630
 631 skip_split_key:
 632         ctx->cdata.keylen = keys.enckeylen;
 633         memzero_explicit(&keys, sizeof(keys));
 634         return aead_set_sh_desc(aead);
 635 badkey:
 636         crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 637         memzero_explicit(&keys, sizeof(keys));
 638         return -EINVAL;
 639 }
 640
 641 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
 642                             unsigned int keylen)
 643 {
 644         struct crypto_authenc_keys keys;
 645         u32 flags;
 646         int err;
 647
 648         err = crypto_authenc_extractkeys(&keys, key, keylen);
 649         if (unlikely(err))
 650                 goto badkey;
 651
 652         err = -EINVAL;
 653         if (keys.enckeylen != DES3_EDE_KEY_SIZE)
 654                 goto badkey;
 655
 656         flags = crypto_aead_get_flags(aead);
 657         err = __des3_verify_key(&flags, keys.enckey);
 658         if (unlikely(err)) {
 659                 crypto_aead_set_flags(aead, flags);
 660                 goto out;
 661         }
 662
 663         err = aead_setkey(aead, key, keylen);
 664
 665 out:
 666         memzero_explicit(&keys, sizeof(keys));
 667         return err;
 668
 669 badkey:
 670         crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 671         goto out;
 672 }
 673
 674 static int gcm_setkey(struct crypto_aead *aead,
 675                       const u8 *key, unsigned int keylen)
 676 {
 677         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 678         struct device *jrdev = ctx->jrdev;
 679
 680 #ifdef DEBUG
 681         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 682                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 683 #endif
 684
 685         memcpy(ctx->key, key, keylen);
 686         dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
 687         ctx->cdata.keylen = keylen;
 688
 689         return gcm_set_sh_desc(aead);
 690 }
 691
 692 static int rfc4106_setkey(struct crypto_aead *aead,
 693                           const u8 *key, unsigned int keylen)
 694 {
 695         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 696         struct device *jrdev = ctx->jrdev;
 697
 698         if (keylen < 4)
 699                 return -EINVAL;
 700
 701 #ifdef DEBUG
 702         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 703                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 704 #endif
 705
 706         memcpy(ctx->key, key, keylen);
 707
 708         /*
 709          * The last four bytes of the key material are used as the salt value
 710          * in the nonce. Update the AES key length.
 711          */
 712         ctx->cdata.keylen = keylen - 4;
 713         dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
 714                                    ctx->dir);
 715         return rfc4106_set_sh_desc(aead);
 716 }
 717
 718 static int rfc4543_setkey(struct crypto_aead *aead,
 719                           const u8 *key, unsigned int keylen)
 720 {
 721         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 722         struct device *jrdev = ctx->jrdev;
 723
 724         if (keylen < 4)
 725                 return -EINVAL;
 726
 727 #ifdef DEBUG
 728         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 729                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 730 #endif
 731
 732         memcpy(ctx->key, key, keylen);
 733
 734         /*
 735          * The last four bytes of the key material are used as the salt value
 736          * in the nonce. Update the AES key length.
 737          */
 738         ctx->cdata.keylen = keylen - 4;
 739         dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
 740                                    ctx->dir);
 741         return rfc4543_set_sh_desc(aead);
 742 }
 743
 744 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 745                            unsigned int keylen)
 746 {
 747         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 748         struct caam_skcipher_alg *alg =
 749                 container_of(crypto_skcipher_alg(skcipher), typeof(*alg),
 750                              skcipher);
 751         struct device *jrdev = ctx->jrdev;
 752         unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
 753         u32 *desc;
 754         u32 ctx1_iv_off = 0;
 755         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 756                                OP_ALG_AAI_CTR_MOD128);
 757         const bool is_rfc3686 = alg->caam.rfc3686;
 758
 759 #ifdef DEBUG
 760         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 761                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 762 #endif
 763         /*
 764          * AES-CTR needs to load IV in CONTEXT1 reg
 765          * at an offset of 128bits (16bytes)
 766          * CONTEXT1[255:128] = IV
 767          */
 768         if (ctr_mode)
 769                 ctx1_iv_off = 16;
 770
 771         /*
 772          * RFC3686 specific:
 773          *      | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
 774          *      | *key = {KEY, NONCE}
 775          */
 776         if (is_rfc3686) {
 777                 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
 778                 keylen -= CTR_RFC3686_NONCE_SIZE;
 779         }
 780
 781         ctx->cdata.keylen = keylen;
 782         ctx->cdata.key_virt = key;
 783         ctx->cdata.key_inline = true;
 784
 785         /* skcipher_encrypt shared descriptor */
 786         desc = ctx->sh_desc_enc;
 787         cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
 788                                    ctx1_iv_off);
 789         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 790                                    desc_bytes(desc), ctx->dir);
 791
 792         /* skcipher_decrypt shared descriptor */
 793         desc = ctx->sh_desc_dec;
 794         cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
 795                                    ctx1_iv_off);
 796         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 797                                    desc_bytes(desc), ctx->dir);
 798
 799         return 0;
 800 }
 801
 802 static int des_skcipher_setkey(struct crypto_skcipher *skcipher,
 803                                const u8 *key, unsigned int keylen)
 804 {
 805         u32 tmp[DES3_EDE_EXPKEY_WORDS];
 806         struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher);
 807
 808         if (keylen == DES3_EDE_KEY_SIZE &&
 809             __des3_ede_setkey(tmp, &tfm->crt_flags, key, DES3_EDE_KEY_SIZE)) {
 810                 return -EINVAL;
 811         }
 812
 813         if (!des_ekey(tmp, key) && (crypto_skcipher_get_flags(skcipher) &
 814             CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
 815                 crypto_skcipher_set_flags(skcipher,
 816                                           CRYPTO_TFM_RES_WEAK_KEY);
 817                 return -EINVAL;
 818         }
 819
 820         return skcipher_setkey(skcipher, key, keylen);
 821 }
 822
 823 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 824                                unsigned int keylen)
 825 {
 826         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
 827         struct device *jrdev = ctx->jrdev;
 828         u32 *desc;
 829
 830         if (keylen != 2 * AES_MIN_KEY_SIZE  && keylen != 2 * AES_MAX_KEY_SIZE) {
 831                 crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
 832                 dev_err(jrdev, "key size mismatch\n");
 833                 return -EINVAL;
 834         }
 835
 836         ctx->cdata.keylen = keylen;
 837         ctx->cdata.key_virt = key;
 838         ctx->cdata.key_inline = true;
 839
 840         /* xts_skcipher_encrypt shared descriptor */
 841         desc = ctx->sh_desc_enc;
 842         cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata);
 843         dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
 844                                    desc_bytes(desc), ctx->dir);
 845
 846         /* xts_skcipher_decrypt shared descriptor */
 847         desc = ctx->sh_desc_dec;
 848         cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata);
 849         dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
 850                                    desc_bytes(desc), ctx->dir);
 851
 852         return 0;
 853 }
 854
 855 /*
 856  * aead_edesc - s/w-extended aead descriptor
 857  * @src_nents: number of segments in input s/w scatterlist
 858  * @dst_nents: number of segments in output s/w scatterlist
 859  * @mapped_src_nents: number of segments in input h/w link table
 860  * @mapped_dst_nents: number of segments in output h/w link table
 861  * @sec4_sg_bytes: length of dma mapped sec4_sg space
 862  * @sec4_sg_dma: bus physical mapped address of h/w link table
 863  * @sec4_sg: pointer to h/w link table
 864  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 865  */
 866 struct aead_edesc {
 867         int src_nents;
 868         int dst_nents;
 869         int mapped_src_nents;
 870         int mapped_dst_nents;
 871         int sec4_sg_bytes;
 872         dma_addr_t sec4_sg_dma;
 873         struct sec4_sg_entry *sec4_sg;
 874         u32 hw_desc[];
 875 };
 876
 877 /*
 878  * skcipher_edesc - s/w-extended skcipher descriptor
 879  * @src_nents: number of segments in input s/w scatterlist
 880  * @dst_nents: number of segments in output s/w scatterlist
 881  * @mapped_src_nents: number of segments in input h/w link table
 882  * @mapped_dst_nents: number of segments in output h/w link table
 883  * @iv_dma: dma address of iv for checking continuity and link table
 884  * @sec4_sg_bytes: length of dma mapped sec4_sg space
 885  * @sec4_sg_dma: bus physical mapped address of h/w link table
 886  * @sec4_sg: pointer to h/w link table
 887  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 888  *           and IV
 889  */
 890 struct skcipher_edesc {
 891         int src_nents;
 892         int dst_nents;
 893         int mapped_src_nents;
 894         int mapped_dst_nents;
 895         dma_addr_t iv_dma;
 896         int sec4_sg_bytes;
 897         dma_addr_t sec4_sg_dma;
 898         struct sec4_sg_entry *sec4_sg;
 899         u32 hw_desc[0];
 900 };
 901
 902 static void caam_unmap(struct device *dev, struct scatterlist *src,
 903                        struct scatterlist *dst, int src_nents,
 904                        int dst_nents,
 905                        dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
 906                        int sec4_sg_bytes)
 907 {
 908         if (dst != src) {
 909                 if (src_nents)
 910                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 911                 if (dst_nents)
 912                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 913         } else {
 914                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 915         }
 916
 917         if (iv_dma)
 918                 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
 919         if (sec4_sg_bytes)
 920                 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
 921                                  DMA_TO_DEVICE);
 922 }
 923
 924 static void aead_unmap(struct device *dev,
 925                        struct aead_edesc *edesc,
 926                        struct aead_request *req)
 927 {
 928         caam_unmap(dev, req->src, req->dst,
 929                    edesc->src_nents, edesc->dst_nents, 0, 0,
 930                    edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 931 }
 932
 933 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
 934                            struct skcipher_request *req)
 935 {
 936         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
 937         int ivsize = crypto_skcipher_ivsize(skcipher);
 938
 939         caam_unmap(dev, req->src, req->dst,
 940                    edesc->src_nents, edesc->dst_nents,
 941                    edesc->iv_dma, ivsize,
 942                    edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 943 }
 944
 945 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
 946                                    void *context)
 947 {
 948         struct aead_request *req = context;
 949         struct aead_edesc *edesc;
 950
 951 #ifdef DEBUG
 952         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 953 #endif
 954
 955         edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
 956
 957         if (err)
 958                 caam_jr_strstatus(jrdev, err);
 959
 960         aead_unmap(jrdev, edesc, req);
 961
 962         kfree(edesc);
 963
 964         aead_request_complete(req, err);
 965 }
 966
 967 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 968                                    void *context)
 969 {
 970         struct aead_request *req = context;
 971         struct aead_edesc *edesc;
 972
 973 #ifdef DEBUG
 974         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 975 #endif
 976
 977         edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
 978
 979         if (err)
 980                 caam_jr_strstatus(jrdev, err);
 981
 982         aead_unmap(jrdev, edesc, req);
 983
 984         /*
 985          * verify hw auth check passed else return -EBADMSG
 986          */
 987         if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
 988                 err = -EBADMSG;
 989
 990         kfree(edesc);
 991
 992         aead_request_complete(req, err);
 993 }
 994
 995 static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
 996                                   void *context)
 997 {
 998         struct skcipher_request *req = context;
 999         struct skcipher_edesc *edesc;
1000         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1001         int ivsize = crypto_skcipher_ivsize(skcipher);
1002
1003 #ifdef DEBUG
1004         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1005 #endif
1006
1007         edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]);
1008
1009         if (err)
1010                 caam_jr_strstatus(jrdev, err);
1011
1012 #ifdef DEBUG
1013         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
1014                        DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1015                        edesc->src_nents > 1 ? 100 : ivsize, 1);
1016 #endif
1017         caam_dump_sg(KERN_ERR, "dst    @" __stringify(__LINE__)": ",
1018                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1019                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1020
1021         skcipher_unmap(jrdev, edesc, req);
1022
1023         /*
1024          * The crypto API expects us to set the IV (req->iv) to the last
1025          * ciphertext block. This is used e.g. by the CTS mode.
1026          */
1027         if (ivsize)
1028                 scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen -
1029                                          ivsize, ivsize, 0);
1030
1031         kfree(edesc);
1032
1033         skcipher_request_complete(req, err);
1034 }
1035
1036 static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
1037                                   void *context)
1038 {
1039         struct skcipher_request *req = context;
1040         struct skcipher_edesc *edesc;
1041 #ifdef DEBUG
1042         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1043         int ivsize = crypto_skcipher_ivsize(skcipher);
1044
1045         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1046 #endif
1047
1048         edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]);
1049         if (err)
1050                 caam_jr_strstatus(jrdev, err);
1051
1052 #ifdef DEBUG
1053         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
1054                        DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
1055 #endif
1056         caam_dump_sg(KERN_ERR, "dst    @" __stringify(__LINE__)": ",
1057                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1058                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1059
1060         skcipher_unmap(jrdev, edesc, req);
1061         kfree(edesc);
1062
1063         skcipher_request_complete(req, err);
1064 }
1065
1066 /*
1067  * Fill in aead job descriptor
1068  */
1069 static void init_aead_job(struct aead_request *req,
1070                           struct aead_edesc *edesc,
1071                           bool all_contig, bool encrypt)
1072 {
1073         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1074         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1075         int authsize = ctx->authsize;
1076         u32 *desc = edesc->hw_desc;
1077         u32 out_options, in_options;
1078         dma_addr_t dst_dma, src_dma;
1079         int len, sec4_sg_index = 0;
1080         dma_addr_t ptr;
1081         u32 *sh_desc;
1082
1083         sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
1084         ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
1085
1086         len = desc_len(sh_desc);
1087         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1088
1089         if (all_contig) {
1090                 src_dma = edesc->mapped_src_nents ? sg_dma_address(req->src) :
1091                                                     0;
1092                 in_options = 0;
1093         } else {
1094                 src_dma = edesc->sec4_sg_dma;
1095                 sec4_sg_index += edesc->mapped_src_nents;
1096                 in_options = LDST_SGF;
1097         }
1098
1099         append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen,
1100                           in_options);
1101
1102         dst_dma = src_dma;
1103         out_options = in_options;
1104
1105         if (unlikely(req->src != req->dst)) {
1106                 if (!edesc->mapped_dst_nents) {
1107                         dst_dma = 0;
1108                 } else if (edesc->mapped_dst_nents == 1) {
1109                         dst_dma = sg_dma_address(req->dst);
1110                         out_options = 0;
1111                 } else {
1112                         dst_dma = edesc->sec4_sg_dma +
1113                                   sec4_sg_index *
1114                                   sizeof(struct sec4_sg_entry);
1115                         out_options = LDST_SGF;
1116                 }
1117         }
1118
1119         if (encrypt)
1120                 append_seq_out_ptr(desc, dst_dma,
1121                                    req->assoclen + req->cryptlen + authsize,
1122                                    out_options);
1123         else
1124                 append_seq_out_ptr(desc, dst_dma,
1125                                    req->assoclen + req->cryptlen - authsize,
1126                                    out_options);
1127 }
1128
1129 static void init_gcm_job(struct aead_request *req,
1130                          struct aead_edesc *edesc,
1131                          bool all_contig, bool encrypt)
1132 {
1133         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1134         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1135         unsigned int ivsize = crypto_aead_ivsize(aead);
1136         u32 *desc = edesc->hw_desc;
1137         bool generic_gcm = (ivsize == GCM_AES_IV_SIZE);
1138         unsigned int last;
1139
1140         init_aead_job(req, edesc, all_contig, encrypt);
1141         append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1142
1143         /* BUG This should not be specific to generic GCM. */
1144         last = 0;
1145         if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen))
1146                 last = FIFOLD_TYPE_LAST1;
1147
1148         /* Read GCM IV */
1149         append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1150                          FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last);
1151         /* Append Salt */
1152         if (!generic_gcm)
1153                 append_data(desc, ctx->key + ctx->cdata.keylen, 4);
1154         /* Append IV */
1155         append_data(desc, req->iv, ivsize);
1156         /* End of blank commands */
1157 }
1158
1159 static void init_chachapoly_job(struct aead_request *req,
1160                                 struct aead_edesc *edesc, bool all_contig,
1161                                 bool encrypt)
1162 {
1163         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1164         unsigned int ivsize = crypto_aead_ivsize(aead);
1165         unsigned int assoclen = req->assoclen;
1166         u32 *desc = edesc->hw_desc;
1167         u32 ctx_iv_off = 4;
1168
1169         init_aead_job(req, edesc, all_contig, encrypt);
1170
1171         if (ivsize != CHACHAPOLY_IV_SIZE) {
1172                 /* IPsec specific: CONTEXT1[223:128] = {NONCE, IV} */
1173                 ctx_iv_off += 4;
1174
1175                 /*
1176                  * The associated data comes already with the IV but we need
1177                  * to skip it when we authenticate or encrypt...
1178                  */
1179                 assoclen -= ivsize;
1180         }
1181
1182         append_math_add_imm_u32(desc, REG3, ZERO, IMM, assoclen);
1183
1184         /*
1185          * For IPsec load the IV further in the same register.
1186          * For RFC7539 simply load the 12 bytes nonce in a single operation
1187          */
1188         append_load_as_imm(desc, req->iv, ivsize, LDST_CLASS_1_CCB |
1189                            LDST_SRCDST_BYTE_CONTEXT |
1190                            ctx_iv_off << LDST_OFFSET_SHIFT);
1191 }
1192
1193 static void init_authenc_job(struct aead_request *req,
1194                              struct aead_edesc *edesc,
1195                              bool all_contig, bool encrypt)
1196 {
1197         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1198         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
1199                                                  struct caam_aead_alg, aead);
1200         unsigned int ivsize = crypto_aead_ivsize(aead);
1201         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1202         struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
1203         const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
1204                                OP_ALG_AAI_CTR_MOD128);
1205         const bool is_rfc3686 = alg->caam.rfc3686;
1206         u32 *desc = edesc->hw_desc;
1207         u32 ivoffset = 0;
1208
1209         /*
1210          * AES-CTR needs to load IV in CONTEXT1 reg
1211          * at an offset of 128bits (16bytes)
1212          * CONTEXT1[255:128] = IV
1213          */
1214         if (ctr_mode)
1215                 ivoffset = 16;
1216
1217         /*
1218          * RFC3686 specific:
1219          *      CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1220          */
1221         if (is_rfc3686)
1222                 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE;
1223
1224         init_aead_job(req, edesc, all_contig, encrypt);
1225
1226         /*
1227          * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports
1228          * having DPOVRD as destination.
1229          */
1230         if (ctrlpriv->era < 3)
1231                 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1232         else
1233                 append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen);
1234
1235         if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv))
1236                 append_load_as_imm(desc, req->iv, ivsize,
1237                                    LDST_CLASS_1_CCB |
1238                                    LDST_SRCDST_BYTE_CONTEXT |
1239                                    (ivoffset << LDST_OFFSET_SHIFT));
1240 }
1241
1242 /*
1243  * Fill in skcipher job descriptor
1244  */
1245 static void init_skcipher_job(struct skcipher_request *req,
1246                               struct skcipher_edesc *edesc,
1247                               const bool encrypt)
1248 {
1249         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1250         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1251         int ivsize = crypto_skcipher_ivsize(skcipher);
1252         u32 *desc = edesc->hw_desc;
1253         u32 *sh_desc;
1254         u32 in_options = 0, out_options = 0;
1255         dma_addr_t src_dma, dst_dma, ptr;
1256         int len, sec4_sg_index = 0;
1257
1258 #ifdef DEBUG
1259         print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
1260                        DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
1261         pr_err("asked=%d, cryptlen%d\n",
1262                (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen);
1263 #endif
1264         caam_dump_sg(KERN_ERR, "src    @" __stringify(__LINE__)": ",
1265                      DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1266                      edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
1267
1268         sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
1269         ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
1270
1271         len = desc_len(sh_desc);
1272         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1273
1274         if (ivsize || edesc->mapped_src_nents > 1) {
1275                 src_dma = edesc->sec4_sg_dma;
1276                 sec4_sg_index = edesc->mapped_src_nents + !!ivsize;
1277                 in_options = LDST_SGF;
1278         } else {
1279                 src_dma = sg_dma_address(req->src);
1280         }
1281
1282         append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options);
1283
1284         if (likely(req->src == req->dst)) {
1285                 dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry);
1286                 out_options = in_options;
1287         } else if (edesc->mapped_dst_nents == 1) {
1288                 dst_dma = sg_dma_address(req->dst);
1289         } else {
1290                 dst_dma = edesc->sec4_sg_dma + sec4_sg_index *
1291                           sizeof(struct sec4_sg_entry);
1292                 out_options = LDST_SGF;
1293         }
1294
1295         append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options);
1296 }
1297
1298 /*
1299  * allocate and map the aead extended descriptor
1300  */
1301 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
1302                                            int desc_bytes, bool *all_contig_ptr,
1303                                            bool encrypt)
1304 {
1305         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1306         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1307         struct device *jrdev = ctx->jrdev;
1308         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1309                        GFP_KERNEL : GFP_ATOMIC;
1310         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1311         struct aead_edesc *edesc;
1312         int sec4_sg_index, sec4_sg_len, sec4_sg_bytes;
1313         unsigned int authsize = ctx->authsize;
1314
1315         if (unlikely(req->dst != req->src)) {
1316                 src_nents = sg_nents_for_len(req->src, req->assoclen +
1317                                              req->cryptlen);
1318                 if (unlikely(src_nents < 0)) {
1319                         dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1320                                 req->assoclen + req->cryptlen);
1321                         return ERR_PTR(src_nents);
1322                 }
1323
1324                 dst_nents = sg_nents_for_len(req->dst, req->assoclen +
1325                                              req->cryptlen +
1326                                                 (encrypt ? authsize :
1327                                                            (-authsize)));
1328                 if (unlikely(dst_nents < 0)) {
1329                         dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
1330                                 req->assoclen + req->cryptlen +
1331                                 (encrypt ? authsize : (-authsize)));
1332                         return ERR_PTR(dst_nents);
1333                 }
1334         } else {
1335                 src_nents = sg_nents_for_len(req->src, req->assoclen +
1336                                              req->cryptlen +
1337                                              (encrypt ? authsize : 0));
1338                 if (unlikely(src_nents < 0)) {
1339                         dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1340                                 req->assoclen + req->cryptlen +
1341                                 (encrypt ? authsize : 0));
1342                         return ERR_PTR(src_nents);
1343                 }
1344         }
1345
1346         if (likely(req->src == req->dst)) {
1347                 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1348                                               DMA_BIDIRECTIONAL);
1349                 if (unlikely(!mapped_src_nents)) {
1350                         dev_err(jrdev, "unable to map source\n");
1351                         return ERR_PTR(-ENOMEM);
1352                 }
1353         } else {
1354                 /* Cover also the case of null (zero length) input data */
1355                 if (src_nents) {
1356                         mapped_src_nents = dma_map_sg(jrdev, req->src,
1357                                                       src_nents, DMA_TO_DEVICE);
1358                         if (unlikely(!mapped_src_nents)) {
1359                                 dev_err(jrdev, "unable to map source\n");
1360                                 return ERR_PTR(-ENOMEM);
1361                         }
1362                 } else {
1363                         mapped_src_nents = 0;
1364                 }
1365
1366                 /* Cover also the case of null (zero length) output data */
1367                 if (dst_nents) {
1368                         mapped_dst_nents = dma_map_sg(jrdev, req->dst,
1369                                                       dst_nents,
1370                                                       DMA_FROM_DEVICE);
1371                         if (unlikely(!mapped_dst_nents)) {
1372                                 dev_err(jrdev, "unable to map destination\n");
1373                                 dma_unmap_sg(jrdev, req->src, src_nents,
1374                                              DMA_TO_DEVICE);
1375                                 return ERR_PTR(-ENOMEM);
1376                         }
1377                 } else {
1378                         mapped_dst_nents = 0;
1379                 }
1380         }
1381
1382         sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0;
1383         sec4_sg_len += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
1384         sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
1385
1386         /* allocate space for base edesc and hw desc commands, link tables */
1387         edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
1388                         GFP_DMA | flags);
1389         if (!edesc) {
1390                 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
1391                            0, 0, 0);
1392                 return ERR_PTR(-ENOMEM);
1393         }
1394
1395         edesc->src_nents = src_nents;
1396         edesc->dst_nents = dst_nents;
1397         edesc->mapped_src_nents = mapped_src_nents;
1398         edesc->mapped_dst_nents = mapped_dst_nents;
1399         edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
1400                          desc_bytes;
1401         *all_contig_ptr = !(mapped_src_nents > 1);
1402
1403         sec4_sg_index = 0;
1404         if (mapped_src_nents > 1) {
1405                 sg_to_sec4_sg_last(req->src, mapped_src_nents,
1406                                    edesc->sec4_sg + sec4_sg_index, 0);
1407                 sec4_sg_index += mapped_src_nents;
1408         }
1409         if (mapped_dst_nents > 1) {
1410                 sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
1411                                    edesc->sec4_sg + sec4_sg_index, 0);
1412         }
1413
1414         if (!sec4_sg_bytes)
1415                 return edesc;
1416
1417         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1418                                             sec4_sg_bytes, DMA_TO_DEVICE);
1419         if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1420                 dev_err(jrdev, "unable to map S/G table\n");
1421                 aead_unmap(jrdev, edesc, req);
1422                 kfree(edesc);
1423                 return ERR_PTR(-ENOMEM);
1424         }
1425
1426         edesc->sec4_sg_bytes = sec4_sg_bytes;
1427
1428         return edesc;
1429 }
1430
1431 static int gcm_encrypt(struct aead_request *req)
1432 {
1433         struct aead_edesc *edesc;
1434         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1435         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1436         struct device *jrdev = ctx->jrdev;
1437         bool all_contig;
1438         u32 *desc;
1439         int ret = 0;
1440
1441         /* allocate extended descriptor */
1442         edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true);
1443         if (IS_ERR(edesc))
1444                 return PTR_ERR(edesc);
1445
1446         /* Create and submit job descriptor */
1447         init_gcm_job(req, edesc, all_contig, true);
1448 #ifdef DEBUG
1449         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1450                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1451                        desc_bytes(edesc->hw_desc), 1);
1452 #endif
1453
1454         desc = edesc->hw_desc;
1455         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
1456         if (!ret) {
1457                 ret = -EINPROGRESS;
1458         } else {
1459                 aead_unmap(jrdev, edesc, req);
1460                 kfree(edesc);
1461         }
1462
1463         return ret;
1464 }
1465
1466 static int chachapoly_encrypt(struct aead_request *req)
1467 {
1468         struct aead_edesc *edesc;
1469         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1470         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1471         struct device *jrdev = ctx->jrdev;
1472         bool all_contig;
1473         u32 *desc;
1474         int ret;
1475
1476         edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, &all_contig,
1477                                  true);
1478         if (IS_ERR(edesc))
1479                 return PTR_ERR(edesc);
1480
1481         desc = edesc->hw_desc;
1482
1483         init_chachapoly_job(req, edesc, all_contig, true);
1484         print_hex_dump_debug("chachapoly jobdesc@" __stringify(__LINE__)": ",
1485                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1486                              1);
1487
1488         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
1489         if (!ret) {
1490                 ret = -EINPROGRESS;
1491         } else {
1492                 aead_unmap(jrdev, edesc, req);
1493                 kfree(edesc);
1494         }
1495
1496         return ret;
1497 }
1498
1499 static int chachapoly_decrypt(struct aead_request *req)
1500 {
1501         struct aead_edesc *edesc;
1502         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1503         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1504         struct device *jrdev = ctx->jrdev;
1505         bool all_contig;
1506         u32 *desc;
1507         int ret;
1508
1509         edesc = aead_edesc_alloc(req, CHACHAPOLY_DESC_JOB_IO_LEN, &all_contig,
1510                                  false);
1511         if (IS_ERR(edesc))
1512                 return PTR_ERR(edesc);
1513
1514         desc = edesc->hw_desc;
1515
1516         init_chachapoly_job(req, edesc, all_contig, false);
1517         print_hex_dump_debug("chachapoly jobdesc@" __stringify(__LINE__)": ",
1518                              DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1519                              1);
1520
1521         ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
1522         if (!ret) {
1523                 ret = -EINPROGRESS;
1524         } else {
1525                 aead_unmap(jrdev, edesc, req);
1526                 kfree(edesc);
1527         }
1528
1529         return ret;
1530 }
1531
1532 static int ipsec_gcm_encrypt(struct aead_request *req)
1533 {
1534         if (req->assoclen < 8)
1535                 return -EINVAL;
1536
1537         return gcm_encrypt(req);
1538 }
1539
1540 static int aead_encrypt(struct aead_request *req)
1541 {
1542         struct aead_edesc *edesc;
1543         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1544         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1545         struct device *jrdev = ctx->jrdev;
1546         bool all_contig;
1547         u32 *desc;
1548         int ret = 0;
1549
1550         /* allocate extended descriptor */
1551         edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
1552                                  &all_contig, true);
1553         if (IS_ERR(edesc))
1554                 return PTR_ERR(edesc);
1555
1556         /* Create and submit job descriptor */
1557         init_authenc_job(req, edesc, all_contig, true);
1558 #ifdef DEBUG
1559         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1560                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1561                        desc_bytes(edesc->hw_desc), 1);
1562 #endif
1563
1564         desc = edesc->hw_desc;
1565         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
1566         if (!ret) {
1567                 ret = -EINPROGRESS;
1568         } else {
1569                 aead_unmap(jrdev, edesc, req);
1570                 kfree(edesc);
1571         }
1572
1573         return ret;
1574 }
1575
1576 static int gcm_decrypt(struct aead_request *req)
1577 {
1578         struct aead_edesc *edesc;
1579         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1580         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1581         struct device *jrdev = ctx->jrdev;
1582         bool all_contig;
1583         u32 *desc;
1584         int ret = 0;
1585
1586         /* allocate extended descriptor */
1587         edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false);
1588         if (IS_ERR(edesc))
1589                 return PTR_ERR(edesc);
1590
1591         /* Create and submit job descriptor*/
1592         init_gcm_job(req, edesc, all_contig, false);
1593 #ifdef DEBUG
1594         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1595                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1596                        desc_bytes(edesc->hw_desc), 1);
1597 #endif
1598
1599         desc = edesc->hw_desc;
1600         ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
1601         if (!ret) {
1602                 ret = -EINPROGRESS;
1603         } else {
1604                 aead_unmap(jrdev, edesc, req);
1605                 kfree(edesc);
1606         }
1607
1608         return ret;
1609 }
1610
1611 static int ipsec_gcm_decrypt(struct aead_request *req)
1612 {
1613         if (req->assoclen < 8)
1614                 return -EINVAL;
1615
1616         return gcm_decrypt(req);
1617 }
1618
1619 static int aead_decrypt(struct aead_request *req)
1620 {
1621         struct aead_edesc *edesc;
1622         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1623         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1624         struct device *jrdev = ctx->jrdev;
1625         bool all_contig;
1626         u32 *desc;
1627         int ret = 0;
1628
1629         caam_dump_sg(KERN_ERR, "dec src@" __stringify(__LINE__)": ",
1630                      DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1631                      req->assoclen + req->cryptlen, 1);
1632
1633         /* allocate extended descriptor */
1634         edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
1635                                  &all_contig, false);
1636         if (IS_ERR(edesc))
1637                 return PTR_ERR(edesc);
1638
1639         /* Create and submit job descriptor*/
1640         init_authenc_job(req, edesc, all_contig, false);
1641 #ifdef DEBUG
1642         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1643                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1644                        desc_bytes(edesc->hw_desc), 1);
1645 #endif
1646
1647         desc = edesc->hw_desc;
1648         ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
1649         if (!ret) {
1650                 ret = -EINPROGRESS;
1651         } else {
1652                 aead_unmap(jrdev, edesc, req);
1653                 kfree(edesc);
1654         }
1655
1656         return ret;
1657 }
1658
1659 /*
1660  * allocate and map the skcipher extended descriptor for skcipher
1661  */
1662 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
1663                                                    int desc_bytes)
1664 {
1665         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1666         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1667         struct device *jrdev = ctx->jrdev;
1668         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1669                        GFP_KERNEL : GFP_ATOMIC;
1670         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1671         struct skcipher_edesc *edesc;
1672         dma_addr_t iv_dma = 0;
1673         u8 *iv;
1674         int ivsize = crypto_skcipher_ivsize(skcipher);
1675         int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
1676
1677         src_nents = sg_nents_for_len(req->src, req->cryptlen);
1678         if (unlikely(src_nents < 0)) {
1679                 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1680                         req->cryptlen);
1681                 return ERR_PTR(src_nents);
1682         }
1683
1684         if (req->dst != req->src) {
1685                 dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1686                 if (unlikely(dst_nents < 0)) {
1687                         dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
1688                                 req->cryptlen);
1689                         return ERR_PTR(dst_nents);
1690                 }
1691         }
1692
1693         if (likely(req->src == req->dst)) {
1694                 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1695                                               DMA_BIDIRECTIONAL);
1696                 if (unlikely(!mapped_src_nents)) {
1697                         dev_err(jrdev, "unable to map source\n");
1698                         return ERR_PTR(-ENOMEM);
1699                 }
1700         } else {
1701                 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1702                                               DMA_TO_DEVICE);
1703                 if (unlikely(!mapped_src_nents)) {
1704                         dev_err(jrdev, "unable to map source\n");
1705                         return ERR_PTR(-ENOMEM);
1706                 }
1707                 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents,
1708                                               DMA_FROM_DEVICE);
1709                 if (unlikely(!mapped_dst_nents)) {
1710                         dev_err(jrdev, "unable to map destination\n");
1711                         dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1712                         return ERR_PTR(-ENOMEM);
1713                 }
1714         }
1715
1716         if (!ivsize && mapped_src_nents == 1)
1717                 sec4_sg_ents = 0; // no need for an input hw s/g table
1718         else
1719                 sec4_sg_ents = mapped_src_nents + !!ivsize;
1720         dst_sg_idx = sec4_sg_ents;
1721         sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
1722         sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
1723
1724         /*
1725          * allocate space for base edesc and hw desc commands, link tables, IV
1726          */
1727         edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes + ivsize,
1728                         GFP_DMA | flags);
1729         if (!edesc) {
1730                 dev_err(jrdev, "could not allocate extended descriptor\n");
1731                 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
1732                            0, 0, 0);
1733                 return ERR_PTR(-ENOMEM);
1734         }
1735
1736         edesc->src_nents = src_nents;
1737         edesc->dst_nents = dst_nents;
1738         edesc->mapped_src_nents = mapped_src_nents;
1739         edesc->mapped_dst_nents = mapped_dst_nents;
1740         edesc->sec4_sg_bytes = sec4_sg_bytes;
1741         edesc->sec4_sg = (struct sec4_sg_entry *)((u8 *)edesc->hw_desc +
1742                                                   desc_bytes);
1743
1744         /* Make sure IV is located in a DMAable area */
1745         if (ivsize) {
1746                 iv = (u8 *)edesc->hw_desc + desc_bytes + sec4_sg_bytes;
1747                 memcpy(iv, req->iv, ivsize);
1748
1749                 iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_TO_DEVICE);
1750                 if (dma_mapping_error(jrdev, iv_dma)) {
1751                         dev_err(jrdev, "unable to map IV\n");
1752                         caam_unmap(jrdev, req->src, req->dst, src_nents,
1753                                    dst_nents, 0, 0, 0, 0);
1754                         kfree(edesc);
1755                         return ERR_PTR(-ENOMEM);
1756                 }
1757
1758                 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
1759         }
1760         if (dst_sg_idx)
1761                 sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg +
1762                                    !!ivsize, 0);
1763
1764         if (mapped_dst_nents > 1) {
1765                 sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
1766                                    edesc->sec4_sg + dst_sg_idx, 0);
1767         }
1768
1769         if (sec4_sg_bytes) {
1770                 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1771                                                     sec4_sg_bytes,
1772                                                     DMA_TO_DEVICE);
1773                 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1774                         dev_err(jrdev, "unable to map S/G table\n");
1775                         caam_unmap(jrdev, req->src, req->dst, src_nents,
1776                                    dst_nents, iv_dma, ivsize, 0, 0);
1777                         kfree(edesc);
1778                         return ERR_PTR(-ENOMEM);
1779                 }
1780         }
1781
1782         edesc->iv_dma = iv_dma;
1783
1784 #ifdef DEBUG
1785         print_hex_dump(KERN_ERR, "skcipher sec4_sg@" __stringify(__LINE__)": ",
1786                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
1787                        sec4_sg_bytes, 1);
1788 #endif
1789
1790         return edesc;
1791 }
1792
1793 static int skcipher_encrypt(struct skcipher_request *req)
1794 {
1795         struct skcipher_edesc *edesc;
1796         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1797         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1798         struct device *jrdev = ctx->jrdev;
1799         u32 *desc;
1800         int ret = 0;
1801
1802         /* allocate extended descriptor */
1803         edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
1804         if (IS_ERR(edesc))
1805                 return PTR_ERR(edesc);
1806
1807         /* Create and submit job descriptor*/
1808         init_skcipher_job(req, edesc, true);
1809 #ifdef DEBUG
1810         print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ",
1811                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1812                        desc_bytes(edesc->hw_desc), 1);
1813 #endif
1814         desc = edesc->hw_desc;
1815         ret = caam_jr_enqueue(jrdev, desc, skcipher_encrypt_done, req);
1816
1817         if (!ret) {
1818                 ret = -EINPROGRESS;
1819         } else {
1820                 skcipher_unmap(jrdev, edesc, req);
1821                 kfree(edesc);
1822         }
1823
1824         return ret;
1825 }
1826
1827 static int skcipher_decrypt(struct skcipher_request *req)
1828 {
1829         struct skcipher_edesc *edesc;
1830         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1831         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1832         int ivsize = crypto_skcipher_ivsize(skcipher);
1833         struct device *jrdev = ctx->jrdev;
1834         u32 *desc;
1835         int ret = 0;
1836
1837         /* allocate extended descriptor */
1838         edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ);
1839         if (IS_ERR(edesc))
1840                 return PTR_ERR(edesc);
1841
1842         /*
1843          * The crypto API expects us to set the IV (req->iv) to the last
1844          * ciphertext block.
1845          */
1846         if (ivsize)
1847                 scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen -
1848                                          ivsize, ivsize, 0);
1849
1850         /* Create and submit job descriptor*/
1851         init_skcipher_job(req, edesc, false);
1852         desc = edesc->hw_desc;
1853 #ifdef DEBUG
1854         print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ",
1855                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1856                        desc_bytes(edesc->hw_desc), 1);
1857 #endif
1858
1859         ret = caam_jr_enqueue(jrdev, desc, skcipher_decrypt_done, req);
1860         if (!ret) {
1861                 ret = -EINPROGRESS;
1862         } else {
1863                 skcipher_unmap(jrdev, edesc, req);
1864                 kfree(edesc);
1865         }
1866
1867         return ret;
1868 }
1869
1870 static struct caam_skcipher_alg driver_algs[] = {
1871         {
1872                 .skcipher = {
1873                         .base = {
1874                                 .cra_name = "cbc(aes)",
1875                                 .cra_driver_name = "cbc-aes-caam",
1876                                 .cra_blocksize = AES_BLOCK_SIZE,
1877                         },
1878                         .setkey = skcipher_setkey,
1879                         .encrypt = skcipher_encrypt,
1880                         .decrypt = skcipher_decrypt,
1881                         .min_keysize = AES_MIN_KEY_SIZE,
1882                         .max_keysize = AES_MAX_KEY_SIZE,
1883                         .ivsize = AES_BLOCK_SIZE,
1884                 },
1885                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1886         },
1887         {
1888                 .skcipher = {
1889                         .base = {
1890                                 .cra_name = "cbc(des3_ede)",
1891                                 .cra_driver_name = "cbc-3des-caam",
1892                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1893                         },
1894                         .setkey = des_skcipher_setkey,
1895                         .encrypt = skcipher_encrypt,
1896                         .decrypt = skcipher_decrypt,
1897                         .min_keysize = DES3_EDE_KEY_SIZE,
1898                         .max_keysize = DES3_EDE_KEY_SIZE,
1899                         .ivsize = DES3_EDE_BLOCK_SIZE,
1900                 },
1901                 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1902         },
1903         {
1904                 .skcipher = {
1905                         .base = {
1906                                 .cra_name = "cbc(des)",
1907                                 .cra_driver_name = "cbc-des-caam",
1908                                 .cra_blocksize = DES_BLOCK_SIZE,
1909                         },
1910                         .setkey = des_skcipher_setkey,
1911                         .encrypt = skcipher_encrypt,
1912                         .decrypt = skcipher_decrypt,
1913                         .min_keysize = DES_KEY_SIZE,
1914                         .max_keysize = DES_KEY_SIZE,
1915                         .ivsize = DES_BLOCK_SIZE,
1916                 },
1917                 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1918         },
1919         {
1920                 .skcipher = {
1921                         .base = {
1922                                 .cra_name = "ctr(aes)",
1923                                 .cra_driver_name = "ctr-aes-caam",
1924                                 .cra_blocksize = 1,
1925                         },
1926                         .setkey = skcipher_setkey,
1927                         .encrypt = skcipher_encrypt,
1928                         .decrypt = skcipher_decrypt,
1929                         .min_keysize = AES_MIN_KEY_SIZE,
1930                         .max_keysize = AES_MAX_KEY_SIZE,
1931                         .ivsize = AES_BLOCK_SIZE,
1932                         .chunksize = AES_BLOCK_SIZE,
1933                 },
1934                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1935                                         OP_ALG_AAI_CTR_MOD128,
1936         },
1937         {
1938                 .skcipher = {
1939                         .base = {
1940                                 .cra_name = "rfc3686(ctr(aes))",
1941                                 .cra_driver_name = "rfc3686-ctr-aes-caam",
1942                                 .cra_blocksize = 1,
1943                         },
1944                         .setkey = skcipher_setkey,
1945                         .encrypt = skcipher_encrypt,
1946                         .decrypt = skcipher_decrypt,
1947                         .min_keysize = AES_MIN_KEY_SIZE +
1948                                        CTR_RFC3686_NONCE_SIZE,
1949                         .max_keysize = AES_MAX_KEY_SIZE +
1950                                        CTR_RFC3686_NONCE_SIZE,
1951                         .ivsize = CTR_RFC3686_IV_SIZE,
1952                         .chunksize = AES_BLOCK_SIZE,
1953                 },
1954                 .caam = {
1955                         .class1_alg_type = OP_ALG_ALGSEL_AES |
1956                                            OP_ALG_AAI_CTR_MOD128,
1957                         .rfc3686 = true,
1958                 },
1959         },
1960         {
1961                 .skcipher = {
1962                         .base = {
1963                                 .cra_name = "xts(aes)",
1964                                 .cra_driver_name = "xts-aes-caam",
1965                                 .cra_blocksize = AES_BLOCK_SIZE,
1966                         },
1967                         .setkey = xts_skcipher_setkey,
1968                         .encrypt = skcipher_encrypt,
1969                         .decrypt = skcipher_decrypt,
1970                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
1971                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
1972                         .ivsize = AES_BLOCK_SIZE,
1973                 },
1974                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1975         },
1976         {
1977                 .skcipher = {
1978                         .base = {
1979                                 .cra_name = "ecb(des)",
1980                                 .cra_driver_name = "ecb-des-caam",
1981                                 .cra_blocksize = DES_BLOCK_SIZE,
1982                         },
1983                         .setkey = des_skcipher_setkey,
1984                         .encrypt = skcipher_encrypt,
1985                         .decrypt = skcipher_decrypt,
1986                         .min_keysize = DES_KEY_SIZE,
1987                         .max_keysize = DES_KEY_SIZE,
1988                 },
1989                 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_ECB,
1990         },
1991         {
1992                 .skcipher = {
1993                         .base = {
1994                                 .cra_name = "ecb(aes)",
1995                                 .cra_driver_name = "ecb-aes-caam",
1996                                 .cra_blocksize = AES_BLOCK_SIZE,
1997                         },
1998                         .setkey = skcipher_setkey,
1999                         .encrypt = skcipher_encrypt,
2000                         .decrypt = skcipher_decrypt,
2001                         .min_keysize = AES_MIN_KEY_SIZE,
2002                         .max_keysize = AES_MAX_KEY_SIZE,
2003                 },
2004                 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_ECB,
2005         },
2006         {
2007                 .skcipher = {
2008                         .base = {
2009                                 .cra_name = "ecb(des3_ede)",
2010                                 .cra_driver_name = "ecb-des3-caam",
2011                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2012                         },
2013                         .setkey = des_skcipher_setkey,
2014                         .encrypt = skcipher_encrypt,
2015                         .decrypt = skcipher_decrypt,
2016                         .min_keysize = DES3_EDE_KEY_SIZE,
2017                         .max_keysize = DES3_EDE_KEY_SIZE,
2018                 },
2019                 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_ECB,
2020         },
2021         {
2022                 .skcipher = {
2023                         .base = {
2024                                 .cra_name = "ecb(arc4)",
2025                                 .cra_driver_name = "ecb-arc4-caam",
2026                                 .cra_blocksize = ARC4_BLOCK_SIZE,
2027                         },
2028                         .setkey = skcipher_setkey,
2029                         .encrypt = skcipher_encrypt,
2030                         .decrypt = skcipher_decrypt,
2031                         .min_keysize = ARC4_MIN_KEY_SIZE,
2032                         .max_keysize = ARC4_MAX_KEY_SIZE,
2033                 },
2034                 .caam.class1_alg_type = OP_ALG_ALGSEL_ARC4 | OP_ALG_AAI_ECB,
2035         },
2036 };
2037
2038 static struct caam_aead_alg driver_aeads[] = {
2039         {
2040                 .aead = {
2041                         .base = {
2042                                 .cra_name = "rfc4106(gcm(aes))",
2043                                 .cra_driver_name = "rfc4106-gcm-aes-caam",
2044                                 .cra_blocksize = 1,
2045                         },
2046                         .setkey = rfc4106_setkey,
2047                         .setauthsize = rfc4106_setauthsize,
2048                         .encrypt = ipsec_gcm_encrypt,
2049                         .decrypt = ipsec_gcm_decrypt,
2050                         .ivsize = GCM_RFC4106_IV_SIZE,
2051                         .maxauthsize = AES_BLOCK_SIZE,
2052                 },
2053                 .caam = {
2054                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2055                 },
2056         },
2057         {
2058                 .aead = {
2059                         .base = {
2060                                 .cra_name = "rfc4543(gcm(aes))",
2061                                 .cra_driver_name = "rfc4543-gcm-aes-caam",
2062                                 .cra_blocksize = 1,
2063                         },
2064                         .setkey = rfc4543_setkey,
2065                         .setauthsize = rfc4543_setauthsize,
2066                         .encrypt = ipsec_gcm_encrypt,
2067                         .decrypt = ipsec_gcm_decrypt,
2068                         .ivsize = GCM_RFC4543_IV_SIZE,
2069                         .maxauthsize = AES_BLOCK_SIZE,
2070                 },
2071                 .caam = {
2072                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2073                 },
2074         },
2075         /* Galois Counter Mode */
2076         {
2077                 .aead = {
2078                         .base = {
2079                                 .cra_name = "gcm(aes)",
2080                                 .cra_driver_name = "gcm-aes-caam",
2081                                 .cra_blocksize = 1,
2082                         },
2083                         .setkey = gcm_setkey,
2084                         .setauthsize = gcm_setauthsize,
2085                         .encrypt = gcm_encrypt,
2086                         .decrypt = gcm_decrypt,
2087                         .ivsize = GCM_AES_IV_SIZE,
2088                         .maxauthsize = AES_BLOCK_SIZE,
2089                 },
2090                 .caam = {
2091                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2092                 },
2093         },
2094         /* single-pass ipsec_esp descriptor */
2095         {
2096                 .aead = {
2097                         .base = {
2098                                 .cra_name = "authenc(hmac(md5),"
2099                                             "ecb(cipher_null))",
2100                                 .cra_driver_name = "authenc-hmac-md5-"
2101                                                    "ecb-cipher_null-caam",
2102                                 .cra_blocksize = NULL_BLOCK_SIZE,
2103                         },
2104                         .setkey = aead_setkey,
2105                         .setauthsize = aead_setauthsize,
2106                         .encrypt = aead_encrypt,
2107                         .decrypt = aead_decrypt,
2108                         .ivsize = NULL_IV_SIZE,
2109                         .maxauthsize = MD5_DIGEST_SIZE,
2110                 },
2111                 .caam = {
2112                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2113                                            OP_ALG_AAI_HMAC_PRECOMP,
2114                 },
2115         },
2116         {
2117                 .aead = {
2118                         .base = {
2119                                 .cra_name = "authenc(hmac(sha1),"
2120                                             "ecb(cipher_null))",
2121                                 .cra_driver_name = "authenc-hmac-sha1-"
2122                                                    "ecb-cipher_null-caam",
2123                                 .cra_blocksize = NULL_BLOCK_SIZE,
2124                         },
2125                         .setkey = aead_setkey,
2126                         .setauthsize = aead_setauthsize,
2127                         .encrypt = aead_encrypt,
2128                         .decrypt = aead_decrypt,
2129                         .ivsize = NULL_IV_SIZE,
2130                         .maxauthsize = SHA1_DIGEST_SIZE,
2131                 },
2132                 .caam = {
2133                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2134                                            OP_ALG_AAI_HMAC_PRECOMP,
2135                 },
2136         },
2137         {
2138                 .aead = {
2139                         .base = {
2140                                 .cra_name = "authenc(hmac(sha224),"
2141                                             "ecb(cipher_null))",
2142                                 .cra_driver_name = "authenc-hmac-sha224-"
2143                                                    "ecb-cipher_null-caam",
2144                                 .cra_blocksize = NULL_BLOCK_SIZE,
2145                         },
2146                         .setkey = aead_setkey,
2147                         .setauthsize = aead_setauthsize,
2148                         .encrypt = aead_encrypt,
2149                         .decrypt = aead_decrypt,
2150                         .ivsize = NULL_IV_SIZE,
2151                         .maxauthsize = SHA224_DIGEST_SIZE,
2152                 },
2153                 .caam = {
2154                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2155                                            OP_ALG_AAI_HMAC_PRECOMP,
2156                 },
2157         },
2158         {
2159                 .aead = {
2160                         .base = {
2161                                 .cra_name = "authenc(hmac(sha256),"
2162                                             "ecb(cipher_null))",
2163                                 .cra_driver_name = "authenc-hmac-sha256-"
2164                                                    "ecb-cipher_null-caam",
2165                                 .cra_blocksize = NULL_BLOCK_SIZE,
2166                         },
2167                         .setkey = aead_setkey,
2168                         .setauthsize = aead_setauthsize,
2169                         .encrypt = aead_encrypt,
2170                         .decrypt = aead_decrypt,
2171                         .ivsize = NULL_IV_SIZE,
2172                         .maxauthsize = SHA256_DIGEST_SIZE,
2173                 },
2174                 .caam = {
2175                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2176                                            OP_ALG_AAI_HMAC_PRECOMP,
2177                 },
2178         },
2179         {
2180                 .aead = {
2181                         .base = {
2182                                 .cra_name = "authenc(hmac(sha384),"
2183                                             "ecb(cipher_null))",
2184                                 .cra_driver_name = "authenc-hmac-sha384-"
2185                                                    "ecb-cipher_null-caam",
2186                                 .cra_blocksize = NULL_BLOCK_SIZE,
2187                         },
2188                         .setkey = aead_setkey,
2189                         .setauthsize = aead_setauthsize,
2190                         .encrypt = aead_encrypt,
2191                         .decrypt = aead_decrypt,
2192                         .ivsize = NULL_IV_SIZE,
2193                         .maxauthsize = SHA384_DIGEST_SIZE,
2194                 },
2195                 .caam = {
2196                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2197                                            OP_ALG_AAI_HMAC_PRECOMP,
2198                 },
2199         },
2200         {
2201                 .aead = {
2202                         .base = {
2203                                 .cra_name = "authenc(hmac(sha512),"
2204                                             "ecb(cipher_null))",
2205                                 .cra_driver_name = "authenc-hmac-sha512-"
2206                                                    "ecb-cipher_null-caam",
2207                                 .cra_blocksize = NULL_BLOCK_SIZE,
2208                         },
2209                         .setkey = aead_setkey,
2210                         .setauthsize = aead_setauthsize,
2211                         .encrypt = aead_encrypt,
2212                         .decrypt = aead_decrypt,
2213                         .ivsize = NULL_IV_SIZE,
2214                         .maxauthsize = SHA512_DIGEST_SIZE,
2215                 },
2216                 .caam = {
2217                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2218                                            OP_ALG_AAI_HMAC_PRECOMP,
2219                 },
2220         },
2221         {
2222                 .aead = {
2223                         .base = {
2224                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
2225                                 .cra_driver_name = "authenc-hmac-md5-"
2226                                                    "cbc-aes-caam",
2227                                 .cra_blocksize = AES_BLOCK_SIZE,
2228                         },
2229                         .setkey = aead_setkey,
2230                         .setauthsize = aead_setauthsize,
2231                         .encrypt = aead_encrypt,
2232                         .decrypt = aead_decrypt,
2233                         .ivsize = AES_BLOCK_SIZE,
2234                         .maxauthsize = MD5_DIGEST_SIZE,
2235                 },
2236                 .caam = {
2237                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2238                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2239                                            OP_ALG_AAI_HMAC_PRECOMP,
2240                 },
2241         },
2242         {
2243                 .aead = {
2244                         .base = {
2245                                 .cra_name = "echainiv(authenc(hmac(md5),"
2246                                             "cbc(aes)))",
2247                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2248                                                    "cbc-aes-caam",
2249                                 .cra_blocksize = AES_BLOCK_SIZE,
2250                         },
2251                         .setkey = aead_setkey,
2252                         .setauthsize = aead_setauthsize,
2253                         .encrypt = aead_encrypt,
2254                         .decrypt = aead_decrypt,
2255                         .ivsize = AES_BLOCK_SIZE,
2256                         .maxauthsize = MD5_DIGEST_SIZE,
2257                 },
2258                 .caam = {
2259                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2260                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2261                                            OP_ALG_AAI_HMAC_PRECOMP,
2262                         .geniv = true,
2263                 },
2264         },
2265         {
2266                 .aead = {
2267                         .base = {
2268                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2269                                 .cra_driver_name = "authenc-hmac-sha1-"
2270                                                    "cbc-aes-caam",
2271                                 .cra_blocksize = AES_BLOCK_SIZE,
2272                         },
2273                         .setkey = aead_setkey,
2274                         .setauthsize = aead_setauthsize,
2275                         .encrypt = aead_encrypt,
2276                         .decrypt = aead_decrypt,
2277                         .ivsize = AES_BLOCK_SIZE,
2278                         .maxauthsize = SHA1_DIGEST_SIZE,
2279                 },
2280                 .caam = {
2281                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2282                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2283                                            OP_ALG_AAI_HMAC_PRECOMP,
2284                 },
2285         },
2286         {
2287                 .aead = {
2288                         .base = {
2289                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2290                                             "cbc(aes)))",
2291                                 .cra_driver_name = "echainiv-authenc-"
2292                                                    "hmac-sha1-cbc-aes-caam",
2293                                 .cra_blocksize = AES_BLOCK_SIZE,
2294                         },
2295                         .setkey = aead_setkey,
2296                         .setauthsize = aead_setauthsize,
2297                         .encrypt = aead_encrypt,
2298                         .decrypt = aead_decrypt,
2299                         .ivsize = AES_BLOCK_SIZE,
2300                         .maxauthsize = SHA1_DIGEST_SIZE,
2301                 },
2302                 .caam = {
2303                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2304                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2305                                            OP_ALG_AAI_HMAC_PRECOMP,
2306                         .geniv = true,
2307                 },
2308         },
2309         {
2310                 .aead = {
2311                         .base = {
2312                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2313                                 .cra_driver_name = "authenc-hmac-sha224-"
2314                                                    "cbc-aes-caam",
2315                                 .cra_blocksize = AES_BLOCK_SIZE,
2316                         },
2317                         .setkey = aead_setkey,
2318                         .setauthsize = aead_setauthsize,
2319                         .encrypt = aead_encrypt,
2320                         .decrypt = aead_decrypt,
2321                         .ivsize = AES_BLOCK_SIZE,
2322                         .maxauthsize = SHA224_DIGEST_SIZE,
2323                 },
2324                 .caam = {
2325                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2326                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2327                                            OP_ALG_AAI_HMAC_PRECOMP,
2328                 },
2329         },
2330         {
2331                 .aead = {
2332                         .base = {
2333                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2334                                             "cbc(aes)))",
2335                                 .cra_driver_name = "echainiv-authenc-"
2336                                                    "hmac-sha224-cbc-aes-caam",
2337                                 .cra_blocksize = AES_BLOCK_SIZE,
2338                         },
2339                         .setkey = aead_setkey,
2340                         .setauthsize = aead_setauthsize,
2341                         .encrypt = aead_encrypt,
2342                         .decrypt = aead_decrypt,
2343                         .ivsize = AES_BLOCK_SIZE,
2344                         .maxauthsize = SHA224_DIGEST_SIZE,
2345                 },
2346                 .caam = {
2347                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2348                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2349                                            OP_ALG_AAI_HMAC_PRECOMP,
2350                         .geniv = true,
2351                 },
2352         },
2353         {
2354                 .aead = {
2355                         .base = {
2356                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2357                                 .cra_driver_name = "authenc-hmac-sha256-"
2358                                                    "cbc-aes-caam",
2359                                 .cra_blocksize = AES_BLOCK_SIZE,
2360                         },
2361                         .setkey = aead_setkey,
2362                         .setauthsize = aead_setauthsize,
2363                         .encrypt = aead_encrypt,
2364                         .decrypt = aead_decrypt,
2365                         .ivsize = AES_BLOCK_SIZE,
2366                         .maxauthsize = SHA256_DIGEST_SIZE,
2367                 },
2368                 .caam = {
2369                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2370                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2371                                            OP_ALG_AAI_HMAC_PRECOMP,
2372                 },
2373         },
2374         {
2375                 .aead = {
2376                         .base = {
2377                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2378                                             "cbc(aes)))",
2379                                 .cra_driver_name = "echainiv-authenc-"
2380                                                    "hmac-sha256-cbc-aes-caam",
2381                                 .cra_blocksize = AES_BLOCK_SIZE,
2382                         },
2383                         .setkey = aead_setkey,
2384                         .setauthsize = aead_setauthsize,
2385                         .encrypt = aead_encrypt,
2386                         .decrypt = aead_decrypt,
2387                         .ivsize = AES_BLOCK_SIZE,
2388                         .maxauthsize = SHA256_DIGEST_SIZE,
2389                 },
2390                 .caam = {
2391                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2392                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2393                                            OP_ALG_AAI_HMAC_PRECOMP,
2394                         .geniv = true,
2395                 },
2396         },
2397         {
2398                 .aead = {
2399                         .base = {
2400                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2401                                 .cra_driver_name = "authenc-hmac-sha384-"
2402                                                    "cbc-aes-caam",
2403                                 .cra_blocksize = AES_BLOCK_SIZE,
2404                         },
2405                         .setkey = aead_setkey,
2406                         .setauthsize = aead_setauthsize,
2407                         .encrypt = aead_encrypt,
2408                         .decrypt = aead_decrypt,
2409                         .ivsize = AES_BLOCK_SIZE,
2410                         .maxauthsize = SHA384_DIGEST_SIZE,
2411                 },
2412                 .caam = {
2413                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2414                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2415                                            OP_ALG_AAI_HMAC_PRECOMP,
2416                 },
2417         },
2418         {
2419                 .aead = {
2420                         .base = {
2421                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2422                                             "cbc(aes)))",
2423                                 .cra_driver_name = "echainiv-authenc-"
2424                                                    "hmac-sha384-cbc-aes-caam",
2425                                 .cra_blocksize = AES_BLOCK_SIZE,
2426                         },
2427                         .setkey = aead_setkey,
2428                         .setauthsize = aead_setauthsize,
2429                         .encrypt = aead_encrypt,
2430                         .decrypt = aead_decrypt,
2431                         .ivsize = AES_BLOCK_SIZE,
2432                         .maxauthsize = SHA384_DIGEST_SIZE,
2433                 },
2434                 .caam = {
2435                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2436                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2437                                            OP_ALG_AAI_HMAC_PRECOMP,
2438                         .geniv = true,
2439                 },
2440         },
2441         {
2442                 .aead = {
2443                         .base = {
2444                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2445                                 .cra_driver_name = "authenc-hmac-sha512-"
2446                                                    "cbc-aes-caam",
2447                                 .cra_blocksize = AES_BLOCK_SIZE,
2448                         },
2449                         .setkey = aead_setkey,
2450                         .setauthsize = aead_setauthsize,
2451                         .encrypt = aead_encrypt,
2452                         .decrypt = aead_decrypt,
2453                         .ivsize = AES_BLOCK_SIZE,
2454                         .maxauthsize = SHA512_DIGEST_SIZE,
2455                 },
2456                 .caam = {
2457                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2458                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2459                                            OP_ALG_AAI_HMAC_PRECOMP,
2460                 },
2461         },
2462         {
2463                 .aead = {
2464                         .base = {
2465                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2466                                             "cbc(aes)))",
2467                                 .cra_driver_name = "echainiv-authenc-"
2468                                                    "hmac-sha512-cbc-aes-caam",
2469                                 .cra_blocksize = AES_BLOCK_SIZE,
2470                         },
2471                         .setkey = aead_setkey,
2472                         .setauthsize = aead_setauthsize,
2473                         .encrypt = aead_encrypt,
2474                         .decrypt = aead_decrypt,
2475                         .ivsize = AES_BLOCK_SIZE,
2476                         .maxauthsize = SHA512_DIGEST_SIZE,
2477                 },
2478                 .caam = {
2479                         .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2480                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2481                                            OP_ALG_AAI_HMAC_PRECOMP,
2482                         .geniv = true,
2483                 },
2484         },
2485         {
2486                 .aead = {
2487                         .base = {
2488                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2489                                 .cra_driver_name = "authenc-hmac-md5-"
2490                                                    "cbc-des3_ede-caam",
2491                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2492                         },
2493                         .setkey = des3_aead_setkey,
2494                         .setauthsize = aead_setauthsize,
2495                         .encrypt = aead_encrypt,
2496                         .decrypt = aead_decrypt,
2497                         .ivsize = DES3_EDE_BLOCK_SIZE,
2498                         .maxauthsize = MD5_DIGEST_SIZE,
2499                 },
2500                 .caam = {
2501                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2502                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2503                                            OP_ALG_AAI_HMAC_PRECOMP,
2504                 }
2505         },
2506         {
2507                 .aead = {
2508                         .base = {
2509                                 .cra_name = "echainiv(authenc(hmac(md5),"
2510                                             "cbc(des3_ede)))",
2511                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2512                                                    "cbc-des3_ede-caam",
2513                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2514                         },
2515                         .setkey = des3_aead_setkey,
2516                         .setauthsize = aead_setauthsize,
2517                         .encrypt = aead_encrypt,
2518                         .decrypt = aead_decrypt,
2519                         .ivsize = DES3_EDE_BLOCK_SIZE,
2520                         .maxauthsize = MD5_DIGEST_SIZE,
2521                 },
2522                 .caam = {
2523                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2524                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2525                                            OP_ALG_AAI_HMAC_PRECOMP,
2526                         .geniv = true,
2527                 }
2528         },
2529         {
2530                 .aead = {
2531                         .base = {
2532                                 .cra_name = "authenc(hmac(sha1),"
2533                                             "cbc(des3_ede))",
2534                                 .cra_driver_name = "authenc-hmac-sha1-"
2535                                                    "cbc-des3_ede-caam",
2536                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2537                         },
2538                         .setkey = des3_aead_setkey,
2539                         .setauthsize = aead_setauthsize,
2540                         .encrypt = aead_encrypt,
2541                         .decrypt = aead_decrypt,
2542                         .ivsize = DES3_EDE_BLOCK_SIZE,
2543                         .maxauthsize = SHA1_DIGEST_SIZE,
2544                 },
2545                 .caam = {
2546                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2547                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2548                                            OP_ALG_AAI_HMAC_PRECOMP,
2549                 },
2550         },
2551         {
2552                 .aead = {
2553                         .base = {
2554                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2555                                             "cbc(des3_ede)))",
2556                                 .cra_driver_name = "echainiv-authenc-"
2557                                                    "hmac-sha1-"
2558                                                    "cbc-des3_ede-caam",
2559                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2560                         },
2561                         .setkey = des3_aead_setkey,
2562                         .setauthsize = aead_setauthsize,
2563                         .encrypt = aead_encrypt,
2564                         .decrypt = aead_decrypt,
2565                         .ivsize = DES3_EDE_BLOCK_SIZE,
2566                         .maxauthsize = SHA1_DIGEST_SIZE,
2567                 },
2568                 .caam = {
2569                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2570                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2571                                            OP_ALG_AAI_HMAC_PRECOMP,
2572                         .geniv = true,
2573                 },
2574         },
2575         {
2576                 .aead = {
2577                         .base = {
2578                                 .cra_name = "authenc(hmac(sha224),"
2579                                             "cbc(des3_ede))",
2580                                 .cra_driver_name = "authenc-hmac-sha224-"
2581                                                    "cbc-des3_ede-caam",
2582                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2583                         },
2584                         .setkey = des3_aead_setkey,
2585                         .setauthsize = aead_setauthsize,
2586                         .encrypt = aead_encrypt,
2587                         .decrypt = aead_decrypt,
2588                         .ivsize = DES3_EDE_BLOCK_SIZE,
2589                         .maxauthsize = SHA224_DIGEST_SIZE,
2590                 },
2591                 .caam = {
2592                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2593                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2594                                            OP_ALG_AAI_HMAC_PRECOMP,
2595                 },
2596         },
2597         {
2598                 .aead = {
2599                         .base = {
2600                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2601                                             "cbc(des3_ede)))",
2602                                 .cra_driver_name = "echainiv-authenc-"
2603                                                    "hmac-sha224-"
2604                                                    "cbc-des3_ede-caam",
2605                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2606                         },
2607                         .setkey = des3_aead_setkey,
2608                         .setauthsize = aead_setauthsize,
2609                         .encrypt = aead_encrypt,
2610                         .decrypt = aead_decrypt,
2611                         .ivsize = DES3_EDE_BLOCK_SIZE,
2612                         .maxauthsize = SHA224_DIGEST_SIZE,
2613                 },
2614                 .caam = {
2615                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2616                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2617                                            OP_ALG_AAI_HMAC_PRECOMP,
2618                         .geniv = true,
2619                 },
2620         },
2621         {
2622                 .aead = {
2623                         .base = {
2624                                 .cra_name = "authenc(hmac(sha256),"
2625                                             "cbc(des3_ede))",
2626                                 .cra_driver_name = "authenc-hmac-sha256-"
2627                                                    "cbc-des3_ede-caam",
2628                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2629                         },
2630                         .setkey = des3_aead_setkey,
2631                         .setauthsize = aead_setauthsize,
2632                         .encrypt = aead_encrypt,
2633                         .decrypt = aead_decrypt,
2634                         .ivsize = DES3_EDE_BLOCK_SIZE,
2635                         .maxauthsize = SHA256_DIGEST_SIZE,
2636                 },
2637                 .caam = {
2638                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2639                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2640                                            OP_ALG_AAI_HMAC_PRECOMP,
2641                 },
2642         },
2643         {
2644                 .aead = {
2645                         .base = {
2646                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2647                                             "cbc(des3_ede)))",
2648                                 .cra_driver_name = "echainiv-authenc-"
2649                                                    "hmac-sha256-"
2650                                                    "cbc-des3_ede-caam",
2651                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2652                         },
2653                         .setkey = des3_aead_setkey,
2654                         .setauthsize = aead_setauthsize,
2655                         .encrypt = aead_encrypt,
2656                         .decrypt = aead_decrypt,
2657                         .ivsize = DES3_EDE_BLOCK_SIZE,
2658                         .maxauthsize = SHA256_DIGEST_SIZE,
2659                 },
2660                 .caam = {
2661                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2662                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2663                                            OP_ALG_AAI_HMAC_PRECOMP,
2664                         .geniv = true,
2665                 },
2666         },
2667         {
2668                 .aead = {
2669                         .base = {
2670                                 .cra_name = "authenc(hmac(sha384),"
2671                                             "cbc(des3_ede))",
2672                                 .cra_driver_name = "authenc-hmac-sha384-"
2673                                                    "cbc-des3_ede-caam",
2674                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2675                         },
2676                         .setkey = des3_aead_setkey,
2677                         .setauthsize = aead_setauthsize,
2678                         .encrypt = aead_encrypt,
2679                         .decrypt = aead_decrypt,
2680                         .ivsize = DES3_EDE_BLOCK_SIZE,
2681                         .maxauthsize = SHA384_DIGEST_SIZE,
2682                 },
2683                 .caam = {
2684                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2685                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2686                                            OP_ALG_AAI_HMAC_PRECOMP,
2687                 },
2688         },
2689         {
2690                 .aead = {
2691                         .base = {
2692                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2693                                             "cbc(des3_ede)))",
2694                                 .cra_driver_name = "echainiv-authenc-"
2695                                                    "hmac-sha384-"
2696                                                    "cbc-des3_ede-caam",
2697                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2698                         },
2699                         .setkey = des3_aead_setkey,
2700                         .setauthsize = aead_setauthsize,
2701                         .encrypt = aead_encrypt,
2702                         .decrypt = aead_decrypt,
2703                         .ivsize = DES3_EDE_BLOCK_SIZE,
2704                         .maxauthsize = SHA384_DIGEST_SIZE,
2705                 },
2706                 .caam = {
2707                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2708                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2709                                            OP_ALG_AAI_HMAC_PRECOMP,
2710                         .geniv = true,
2711                 },
2712         },
2713         {
2714                 .aead = {
2715                         .base = {
2716                                 .cra_name = "authenc(hmac(sha512),"
2717                                             "cbc(des3_ede))",
2718                                 .cra_driver_name = "authenc-hmac-sha512-"
2719                                                    "cbc-des3_ede-caam",
2720                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2721                         },
2722                         .setkey = des3_aead_setkey,
2723                         .setauthsize = aead_setauthsize,
2724                         .encrypt = aead_encrypt,
2725                         .decrypt = aead_decrypt,
2726                         .ivsize = DES3_EDE_BLOCK_SIZE,
2727                         .maxauthsize = SHA512_DIGEST_SIZE,
2728                 },
2729                 .caam = {
2730                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2731                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2732                                            OP_ALG_AAI_HMAC_PRECOMP,
2733                 },
2734         },
2735         {
2736                 .aead = {
2737                         .base = {
2738                                 .cra_name = "echainiv(authenc(hmac(sha512),"
2739                                             "cbc(des3_ede)))",
2740                                 .cra_driver_name = "echainiv-authenc-"
2741                                                    "hmac-sha512-"
2742                                                    "cbc-des3_ede-caam",
2743                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2744                         },
2745                         .setkey = des3_aead_setkey,
2746                         .setauthsize = aead_setauthsize,
2747                         .encrypt = aead_encrypt,
2748                         .decrypt = aead_decrypt,
2749                         .ivsize = DES3_EDE_BLOCK_SIZE,
2750                         .maxauthsize = SHA512_DIGEST_SIZE,
2751                 },
2752                 .caam = {
2753                         .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2754                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2755                                            OP_ALG_AAI_HMAC_PRECOMP,
2756                         .geniv = true,
2757                 },
2758         },
2759         {
2760                 .aead = {
2761                         .base = {
2762                                 .cra_name = "authenc(hmac(md5),cbc(des))",
2763                                 .cra_driver_name = "authenc-hmac-md5-"
2764                                                    "cbc-des-caam",
2765                                 .cra_blocksize = DES_BLOCK_SIZE,
2766                         },
2767                         .setkey = aead_setkey,
2768                         .setauthsize = aead_setauthsize,
2769                         .encrypt = aead_encrypt,
2770                         .decrypt = aead_decrypt,
2771                         .ivsize = DES_BLOCK_SIZE,
2772                         .maxauthsize = MD5_DIGEST_SIZE,
2773                 },
2774                 .caam = {
2775                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2776                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2777                                            OP_ALG_AAI_HMAC_PRECOMP,
2778                 },
2779         },
2780         {
2781                 .aead = {
2782                         .base = {
2783                                 .cra_name = "echainiv(authenc(hmac(md5),"
2784                                             "cbc(des)))",
2785                                 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2786                                                    "cbc-des-caam",
2787                                 .cra_blocksize = DES_BLOCK_SIZE,
2788                         },
2789                         .setkey = aead_setkey,
2790                         .setauthsize = aead_setauthsize,
2791                         .encrypt = aead_encrypt,
2792                         .decrypt = aead_decrypt,
2793                         .ivsize = DES_BLOCK_SIZE,
2794                         .maxauthsize = MD5_DIGEST_SIZE,
2795                 },
2796                 .caam = {
2797                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2798                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2799                                            OP_ALG_AAI_HMAC_PRECOMP,
2800                         .geniv = true,
2801                 },
2802         },
2803         {
2804                 .aead = {
2805                         .base = {
2806                                 .cra_name = "authenc(hmac(sha1),cbc(des))",
2807                                 .cra_driver_name = "authenc-hmac-sha1-"
2808                                                    "cbc-des-caam",
2809                                 .cra_blocksize = DES_BLOCK_SIZE,
2810                         },
2811                         .setkey = aead_setkey,
2812                         .setauthsize = aead_setauthsize,
2813                         .encrypt = aead_encrypt,
2814                         .decrypt = aead_decrypt,
2815                         .ivsize = DES_BLOCK_SIZE,
2816                         .maxauthsize = SHA1_DIGEST_SIZE,
2817                 },
2818                 .caam = {
2819                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2820                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2821                                            OP_ALG_AAI_HMAC_PRECOMP,
2822                 },
2823         },
2824         {
2825                 .aead = {
2826                         .base = {
2827                                 .cra_name = "echainiv(authenc(hmac(sha1),"
2828                                             "cbc(des)))",
2829                                 .cra_driver_name = "echainiv-authenc-"
2830                                                    "hmac-sha1-cbc-des-caam",
2831                                 .cra_blocksize = DES_BLOCK_SIZE,
2832                         },
2833                         .setkey = aead_setkey,
2834                         .setauthsize = aead_setauthsize,
2835                         .encrypt = aead_encrypt,
2836                         .decrypt = aead_decrypt,
2837                         .ivsize = DES_BLOCK_SIZE,
2838                         .maxauthsize = SHA1_DIGEST_SIZE,
2839                 },
2840                 .caam = {
2841                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2842                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2843                                            OP_ALG_AAI_HMAC_PRECOMP,
2844                         .geniv = true,
2845                 },
2846         },
2847         {
2848                 .aead = {
2849                         .base = {
2850                                 .cra_name = "authenc(hmac(sha224),cbc(des))",
2851                                 .cra_driver_name = "authenc-hmac-sha224-"
2852                                                    "cbc-des-caam",
2853                                 .cra_blocksize = DES_BLOCK_SIZE,
2854                         },
2855                         .setkey = aead_setkey,
2856                         .setauthsize = aead_setauthsize,
2857                         .encrypt = aead_encrypt,
2858                         .decrypt = aead_decrypt,
2859                         .ivsize = DES_BLOCK_SIZE,
2860                         .maxauthsize = SHA224_DIGEST_SIZE,
2861                 },
2862                 .caam = {
2863                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2864                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2865                                            OP_ALG_AAI_HMAC_PRECOMP,
2866                 },
2867         },
2868         {
2869                 .aead = {
2870                         .base = {
2871                                 .cra_name = "echainiv(authenc(hmac(sha224),"
2872                                             "cbc(des)))",
2873                                 .cra_driver_name = "echainiv-authenc-"
2874                                                    "hmac-sha224-cbc-des-caam",
2875                                 .cra_blocksize = DES_BLOCK_SIZE,
2876                         },
2877                         .setkey = aead_setkey,
2878                         .setauthsize = aead_setauthsize,
2879                         .encrypt = aead_encrypt,
2880                         .decrypt = aead_decrypt,
2881                         .ivsize = DES_BLOCK_SIZE,
2882                         .maxauthsize = SHA224_DIGEST_SIZE,
2883                 },
2884                 .caam = {
2885                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2886                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2887                                            OP_ALG_AAI_HMAC_PRECOMP,
2888                         .geniv = true,
2889                 },
2890         },
2891         {
2892                 .aead = {
2893                         .base = {
2894                                 .cra_name = "authenc(hmac(sha256),cbc(des))",
2895                                 .cra_driver_name = "authenc-hmac-sha256-"
2896                                                    "cbc-des-caam",
2897                                 .cra_blocksize = DES_BLOCK_SIZE,
2898                         },
2899                         .setkey = aead_setkey,
2900                         .setauthsize = aead_setauthsize,
2901                         .encrypt = aead_encrypt,
2902                         .decrypt = aead_decrypt,
2903                         .ivsize = DES_BLOCK_SIZE,
2904                         .maxauthsize = SHA256_DIGEST_SIZE,
2905                 },
2906                 .caam = {
2907                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2908                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2909                                            OP_ALG_AAI_HMAC_PRECOMP,
2910                 },
2911         },
2912         {
2913                 .aead = {
2914                         .base = {
2915                                 .cra_name = "echainiv(authenc(hmac(sha256),"
2916                                             "cbc(des)))",
2917                                 .cra_driver_name = "echainiv-authenc-"
2918                                                    "hmac-sha256-cbc-des-caam",
2919                                 .cra_blocksize = DES_BLOCK_SIZE,
2920                         },
2921                         .setkey = aead_setkey,
2922                         .setauthsize = aead_setauthsize,
2923                         .encrypt = aead_encrypt,
2924                         .decrypt = aead_decrypt,
2925                         .ivsize = DES_BLOCK_SIZE,
2926                         .maxauthsize = SHA256_DIGEST_SIZE,
2927                 },
2928                 .caam = {
2929                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2930                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2931                                            OP_ALG_AAI_HMAC_PRECOMP,
2932                         .geniv = true,
2933                 },
2934         },
2935         {
2936                 .aead = {
2937                         .base = {
2938                                 .cra_name = "authenc(hmac(sha384),cbc(des))",
2939                                 .cra_driver_name = "authenc-hmac-sha384-"
2940                                                    "cbc-des-caam",
2941                                 .cra_blocksize = DES_BLOCK_SIZE,
2942                         },
2943                         .setkey = aead_setkey,
2944                         .setauthsize = aead_setauthsize,
2945                         .encrypt = aead_encrypt,
2946                         .decrypt = aead_decrypt,
2947                         .ivsize = DES_BLOCK_SIZE,
2948                         .maxauthsize = SHA384_DIGEST_SIZE,
2949                 },
2950                 .caam = {
2951                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2952                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2953                                            OP_ALG_AAI_HMAC_PRECOMP,
2954                 },
2955         },
2956         {
2957                 .aead = {
2958                         .base = {
2959                                 .cra_name = "echainiv(authenc(hmac(sha384),"
2960                                             "cbc(des)))",
2961                                 .cra_driver_name = "echainiv-authenc-"
2962                                                    "hmac-sha384-cbc-des-caam",
2963                                 .cra_blocksize = DES_BLOCK_SIZE,
2964                         },
2965                         .setkey = aead_setkey,
2966                         .setauthsize = aead_setauthsize,
2967                         .encrypt = aead_encrypt,
2968                         .decrypt = aead_decrypt,
2969                         .ivsize = DES_BLOCK_SIZE,
2970                         .maxauthsize = SHA384_DIGEST_SIZE,
2971                 },
2972                 .caam = {
2973                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2974                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2975                                            OP_ALG_AAI_HMAC_PRECOMP,
2976                         .geniv = true,
2977                 },
2978         },
2979         {
2980                 .aead = {
2981                         .base = {
2982                                 .cra_name = "authenc(hmac(sha512),cbc(des))",
2983                                 .cra_driver_name = "authenc-hmac-sha512-"
2984                                                    "cbc-des-caam",
2985                                 .cra_blocksize = DES_BLOCK_SIZE,
2986                         },
2987                         .setkey = aead_setkey,
2988                         .setauthsize = aead_setauthsize,
2989                         .encrypt = aead_encrypt,
2990                         .decrypt = aead_decrypt,
2991                         .ivsize = DES_BLOCK_SIZE,
2992                         .maxauthsize = SHA512_DIGEST_SIZE,
2993                 },
2994                 .caam = {
2995                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2996                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2997                                            OP_ALG_AAI_HMAC_PRECOMP,
2998                 },
2999         },
3000         {
3001                 .aead = {
3002                         .base = {
3003                                 .cra_name = "echainiv(authenc(hmac(sha512),"
3004                                             "cbc(des)))",
3005                                 .cra_driver_name = "echainiv-authenc-"
3006                                                    "hmac-sha512-cbc-des-caam",
3007                                 .cra_blocksize = DES_BLOCK_SIZE,
3008                         },
3009                         .setkey = aead_setkey,
3010                         .setauthsize = aead_setauthsize,
3011                         .encrypt = aead_encrypt,
3012                         .decrypt = aead_decrypt,
3013                         .ivsize = DES_BLOCK_SIZE,
3014                         .maxauthsize = SHA512_DIGEST_SIZE,
3015                 },
3016                 .caam = {
3017                         .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3018                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3019                                            OP_ALG_AAI_HMAC_PRECOMP,
3020                         .geniv = true,
3021                 },
3022         },
3023         {
3024                 .aead = {
3025                         .base = {
3026                                 .cra_name = "authenc(hmac(md5),"
3027                                             "rfc3686(ctr(aes)))",
3028                                 .cra_driver_name = "authenc-hmac-md5-"
3029                                                    "rfc3686-ctr-aes-caam",
3030                                 .cra_blocksize = 1,
3031                         },
3032                         .setkey = aead_setkey,
3033                         .setauthsize = aead_setauthsize,
3034                         .encrypt = aead_encrypt,
3035                         .decrypt = aead_decrypt,
3036                         .ivsize = CTR_RFC3686_IV_SIZE,
3037                         .maxauthsize = MD5_DIGEST_SIZE,
3038                 },
3039                 .caam = {
3040                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3041                                            OP_ALG_AAI_CTR_MOD128,
3042                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3043                                            OP_ALG_AAI_HMAC_PRECOMP,
3044                         .rfc3686 = true,
3045                 },
3046         },
3047         {
3048                 .aead = {
3049                         .base = {
3050                                 .cra_name = "seqiv(authenc("
3051                                             "hmac(md5),rfc3686(ctr(aes))))",
3052                                 .cra_driver_name = "seqiv-authenc-hmac-md5-"
3053                                                    "rfc3686-ctr-aes-caam",
3054                                 .cra_blocksize = 1,
3055                         },
3056                         .setkey = aead_setkey,
3057                         .setauthsize = aead_setauthsize,
3058                         .encrypt = aead_encrypt,
3059                         .decrypt = aead_decrypt,
3060                         .ivsize = CTR_RFC3686_IV_SIZE,
3061                         .maxauthsize = MD5_DIGEST_SIZE,
3062                 },
3063                 .caam = {
3064                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3065                                            OP_ALG_AAI_CTR_MOD128,
3066                         .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3067                                            OP_ALG_AAI_HMAC_PRECOMP,
3068                         .rfc3686 = true,
3069                         .geniv = true,
3070                 },
3071         },
3072         {
3073                 .aead = {
3074                         .base = {
3075                                 .cra_name = "authenc(hmac(sha1),"
3076                                             "rfc3686(ctr(aes)))",
3077                                 .cra_driver_name = "authenc-hmac-sha1-"
3078                                                    "rfc3686-ctr-aes-caam",
3079                                 .cra_blocksize = 1,
3080                         },
3081                         .setkey = aead_setkey,
3082                         .setauthsize = aead_setauthsize,
3083                         .encrypt = aead_encrypt,
3084                         .decrypt = aead_decrypt,
3085                         .ivsize = CTR_RFC3686_IV_SIZE,
3086                         .maxauthsize = SHA1_DIGEST_SIZE,
3087                 },
3088                 .caam = {
3089                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3090                                            OP_ALG_AAI_CTR_MOD128,
3091                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3092                                            OP_ALG_AAI_HMAC_PRECOMP,
3093                         .rfc3686 = true,
3094                 },
3095         },
3096         {
3097                 .aead = {
3098                         .base = {
3099                                 .cra_name = "seqiv(authenc("
3100                                             "hmac(sha1),rfc3686(ctr(aes))))",
3101                                 .cra_driver_name = "seqiv-authenc-hmac-sha1-"
3102                                                    "rfc3686-ctr-aes-caam",
3103                                 .cra_blocksize = 1,
3104                         },
3105                         .setkey = aead_setkey,
3106                         .setauthsize = aead_setauthsize,
3107                         .encrypt = aead_encrypt,
3108                         .decrypt = aead_decrypt,
3109                         .ivsize = CTR_RFC3686_IV_SIZE,
3110                         .maxauthsize = SHA1_DIGEST_SIZE,
3111                 },
3112                 .caam = {
3113                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3114                                            OP_ALG_AAI_CTR_MOD128,
3115                         .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3116                                            OP_ALG_AAI_HMAC_PRECOMP,
3117                         .rfc3686 = true,
3118                         .geniv = true,
3119                 },
3120         },
3121         {
3122                 .aead = {
3123                         .base = {
3124                                 .cra_name = "authenc(hmac(sha224),"
3125                                             "rfc3686(ctr(aes)))",
3126                                 .cra_driver_name = "authenc-hmac-sha224-"
3127                                                    "rfc3686-ctr-aes-caam",
3128                                 .cra_blocksize = 1,
3129                         },
3130                         .setkey = aead_setkey,
3131                         .setauthsize = aead_setauthsize,
3132                         .encrypt = aead_encrypt,
3133                         .decrypt = aead_decrypt,
3134                         .ivsize = CTR_RFC3686_IV_SIZE,
3135                         .maxauthsize = SHA224_DIGEST_SIZE,
3136                 },
3137                 .caam = {
3138                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3139                                            OP_ALG_AAI_CTR_MOD128,
3140                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3141                                            OP_ALG_AAI_HMAC_PRECOMP,
3142                         .rfc3686 = true,
3143                 },
3144         },
3145         {
3146                 .aead = {
3147                         .base = {
3148                                 .cra_name = "seqiv(authenc("
3149                                             "hmac(sha224),rfc3686(ctr(aes))))",
3150                                 .cra_driver_name = "seqiv-authenc-hmac-sha224-"
3151                                                    "rfc3686-ctr-aes-caam",
3152                                 .cra_blocksize = 1,
3153                         },
3154                         .setkey = aead_setkey,
3155                         .setauthsize = aead_setauthsize,
3156                         .encrypt = aead_encrypt,
3157                         .decrypt = aead_decrypt,
3158                         .ivsize = CTR_RFC3686_IV_SIZE,
3159                         .maxauthsize = SHA224_DIGEST_SIZE,
3160                 },
3161                 .caam = {
3162                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3163                                            OP_ALG_AAI_CTR_MOD128,
3164                         .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3165                                            OP_ALG_AAI_HMAC_PRECOMP,
3166                         .rfc3686 = true,
3167                         .geniv = true,
3168                 },
3169         },
3170         {
3171                 .aead = {
3172                         .base = {
3173                                 .cra_name = "authenc(hmac(sha256),"
3174                                             "rfc3686(ctr(aes)))",
3175                                 .cra_driver_name = "authenc-hmac-sha256-"
3176                                                    "rfc3686-ctr-aes-caam",
3177                                 .cra_blocksize = 1,
3178                         },
3179                         .setkey = aead_setkey,
3180                         .setauthsize = aead_setauthsize,
3181                         .encrypt = aead_encrypt,
3182                         .decrypt = aead_decrypt,
3183                         .ivsize = CTR_RFC3686_IV_SIZE,
3184                         .maxauthsize = SHA256_DIGEST_SIZE,
3185                 },
3186                 .caam = {
3187                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3188                                            OP_ALG_AAI_CTR_MOD128,
3189                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3190                                            OP_ALG_AAI_HMAC_PRECOMP,
3191                         .rfc3686 = true,
3192                 },
3193         },
3194         {
3195                 .aead = {
3196                         .base = {
3197                                 .cra_name = "seqiv(authenc(hmac(sha256),"
3198                                             "rfc3686(ctr(aes))))",
3199                                 .cra_driver_name = "seqiv-authenc-hmac-sha256-"
3200                                                    "rfc3686-ctr-aes-caam",
3201                                 .cra_blocksize = 1,
3202                         },
3203                         .setkey = aead_setkey,
3204                         .setauthsize = aead_setauthsize,
3205                         .encrypt = aead_encrypt,
3206                         .decrypt = aead_decrypt,
3207                         .ivsize = CTR_RFC3686_IV_SIZE,
3208                         .maxauthsize = SHA256_DIGEST_SIZE,
3209                 },
3210                 .caam = {
3211                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3212                                            OP_ALG_AAI_CTR_MOD128,
3213                         .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3214                                            OP_ALG_AAI_HMAC_PRECOMP,
3215                         .rfc3686 = true,
3216                         .geniv = true,
3217                 },
3218         },
3219         {
3220                 .aead = {
3221                         .base = {
3222                                 .cra_name = "authenc(hmac(sha384),"
3223                                             "rfc3686(ctr(aes)))",
3224                                 .cra_driver_name = "authenc-hmac-sha384-"
3225                                                    "rfc3686-ctr-aes-caam",
3226                                 .cra_blocksize = 1,
3227                         },
3228                         .setkey = aead_setkey,
3229                         .setauthsize = aead_setauthsize,
3230                         .encrypt = aead_encrypt,
3231                         .decrypt = aead_decrypt,
3232                         .ivsize = CTR_RFC3686_IV_SIZE,
3233                         .maxauthsize = SHA384_DIGEST_SIZE,
3234                 },
3235                 .caam = {
3236                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3237                                            OP_ALG_AAI_CTR_MOD128,
3238                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3239                                            OP_ALG_AAI_HMAC_PRECOMP,
3240                         .rfc3686 = true,
3241                 },
3242         },
3243         {
3244                 .aead = {
3245                         .base = {
3246                                 .cra_name = "seqiv(authenc(hmac(sha384),"
3247                                             "rfc3686(ctr(aes))))",
3248                                 .cra_driver_name = "seqiv-authenc-hmac-sha384-"
3249                                                    "rfc3686-ctr-aes-caam",
3250                                 .cra_blocksize = 1,
3251                         },
3252                         .setkey = aead_setkey,
3253                         .setauthsize = aead_setauthsize,
3254                         .encrypt = aead_encrypt,
3255                         .decrypt = aead_decrypt,
3256                         .ivsize = CTR_RFC3686_IV_SIZE,
3257                         .maxauthsize = SHA384_DIGEST_SIZE,
3258                 },
3259                 .caam = {
3260                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3261                                            OP_ALG_AAI_CTR_MOD128,
3262                         .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3263                                            OP_ALG_AAI_HMAC_PRECOMP,
3264                         .rfc3686 = true,
3265                         .geniv = true,
3266                 },
3267         },
3268         {
3269                 .aead = {
3270                         .base = {
3271                                 .cra_name = "authenc(hmac(sha512),"
3272                                             "rfc3686(ctr(aes)))",
3273                                 .cra_driver_name = "authenc-hmac-sha512-"
3274                                                    "rfc3686-ctr-aes-caam",
3275                                 .cra_blocksize = 1,
3276                         },
3277                         .setkey = aead_setkey,
3278                         .setauthsize = aead_setauthsize,
3279                         .encrypt = aead_encrypt,
3280                         .decrypt = aead_decrypt,
3281                         .ivsize = CTR_RFC3686_IV_SIZE,
3282                         .maxauthsize = SHA512_DIGEST_SIZE,
3283                 },
3284                 .caam = {
3285                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3286                                            OP_ALG_AAI_CTR_MOD128,
3287                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3288                                            OP_ALG_AAI_HMAC_PRECOMP,
3289                         .rfc3686 = true,
3290                 },
3291         },
3292         {
3293                 .aead = {
3294                         .base = {
3295                                 .cra_name = "seqiv(authenc(hmac(sha512),"
3296                                             "rfc3686(ctr(aes))))",
3297                                 .cra_driver_name = "seqiv-authenc-hmac-sha512-"
3298                                                    "rfc3686-ctr-aes-caam",
3299                                 .cra_blocksize = 1,
3300                         },
3301                         .setkey = aead_setkey,
3302                         .setauthsize = aead_setauthsize,
3303                         .encrypt = aead_encrypt,
3304                         .decrypt = aead_decrypt,
3305                         .ivsize = CTR_RFC3686_IV_SIZE,
3306                         .maxauthsize = SHA512_DIGEST_SIZE,
3307                 },
3308                 .caam = {
3309                         .class1_alg_type = OP_ALG_ALGSEL_AES |
3310                                            OP_ALG_AAI_CTR_MOD128,
3311                         .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3312                                            OP_ALG_AAI_HMAC_PRECOMP,
3313                         .rfc3686 = true,
3314                         .geniv = true,
3315                 },
3316         },
3317         {
3318                 .aead = {
3319                         .base = {
3320                                 .cra_name = "rfc7539(chacha20,poly1305)",
3321                                 .cra_driver_name = "rfc7539-chacha20-poly1305-"
3322                                                    "caam",
3323                                 .cra_blocksize = 1,
3324                         },
3325                         .setkey = chachapoly_setkey,
3326                         .setauthsize = chachapoly_setauthsize,
3327                         .encrypt = chachapoly_encrypt,
3328                         .decrypt = chachapoly_decrypt,
3329                         .ivsize = CHACHAPOLY_IV_SIZE,
3330                         .maxauthsize = POLY1305_DIGEST_SIZE,
3331                 },
3332                 .caam = {
3333                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
3334                                            OP_ALG_AAI_AEAD,
3335                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
3336                                            OP_ALG_AAI_AEAD,
3337                 },
3338         },
3339         {
3340                 .aead = {
3341                         .base = {
3342                                 .cra_name = "rfc7539esp(chacha20,poly1305)",
3343                                 .cra_driver_name = "rfc7539esp-chacha20-"
3344                                                    "poly1305-caam",
3345                                 .cra_blocksize = 1,
3346                         },
3347                         .setkey = chachapoly_setkey,
3348                         .setauthsize = chachapoly_setauthsize,
3349                         .encrypt = chachapoly_encrypt,
3350                         .decrypt = chachapoly_decrypt,
3351                         .ivsize = 8,
3352                         .maxauthsize = POLY1305_DIGEST_SIZE,
3353                 },
3354                 .caam = {
3355                         .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 |
3356                                            OP_ALG_AAI_AEAD,
3357                         .class2_alg_type = OP_ALG_ALGSEL_POLY1305 |
3358                                            OP_ALG_AAI_AEAD,
3359                 },
3360         },
3361 };
3362
3363 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
3364                             bool uses_dkp)
3365 {
3366         dma_addr_t dma_addr;
3367         struct caam_drv_private *priv;
3368
3369         ctx->jrdev = caam_jr_alloc();
3370         if (IS_ERR(ctx->jrdev)) {
3371                 pr_err("Job Ring Device allocation for transform failed\n");
3372                 return PTR_ERR(ctx->jrdev);
3373         }
3374
3375         priv = dev_get_drvdata(ctx->jrdev->parent);
3376         if (priv->era >= 6 && uses_dkp)
3377                 ctx->dir = DMA_BIDIRECTIONAL;
3378         else
3379                 ctx->dir = DMA_TO_DEVICE;
3380
3381         dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc,
3382                                         offsetof(struct caam_ctx,
3383                                                  sh_desc_enc_dma),
3384                                         ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
3385         if (dma_mapping_error(ctx->jrdev, dma_addr)) {
3386                 dev_err(ctx->jrdev, "unable to map key, shared descriptors\n");
3387                 caam_jr_free(ctx->jrdev);
3388                 return -ENOMEM;
3389         }
3390
3391         ctx->sh_desc_enc_dma = dma_addr;
3392         ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx,
3393                                                    sh_desc_dec);
3394         ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key);
3395
3396         /* copy descriptor header template value */
3397         ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
3398         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
3399
3400         return 0;
3401 }
3402
3403 static int caam_cra_init(struct crypto_skcipher *tfm)
3404 {
3405         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
3406         struct caam_skcipher_alg *caam_alg =
3407                 container_of(alg, typeof(*caam_alg), skcipher);
3408
3409         return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam,
3410                                 false);
3411 }
3412
3413 static int caam_aead_init(struct crypto_aead *tfm)
3414 {
3415         struct aead_alg *alg = crypto_aead_alg(tfm);
3416         struct caam_aead_alg *caam_alg =
3417                  container_of(alg, struct caam_aead_alg, aead);
3418         struct caam_ctx *ctx = crypto_aead_ctx(tfm);
3419
3420         return caam_init_common(ctx, &caam_alg->caam,
3421                                 alg->setkey == aead_setkey);
3422 }
3423
3424 static void caam_exit_common(struct caam_ctx *ctx)
3425 {
3426         dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma,
3427                                offsetof(struct caam_ctx, sh_desc_enc_dma),
3428                                ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
3429         caam_jr_free(ctx->jrdev);
3430 }
3431
3432 static void caam_cra_exit(struct crypto_skcipher *tfm)
3433 {
3434         caam_exit_common(crypto_skcipher_ctx(tfm));
3435 }
3436
3437 static void caam_aead_exit(struct crypto_aead *tfm)
3438 {
3439         caam_exit_common(crypto_aead_ctx(tfm));
3440 }
3441
3442 static void __exit caam_algapi_exit(void)
3443 {
3444         int i;
3445
3446         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
3447                 struct caam_aead_alg *t_alg = driver_aeads + i;
3448
3449                 if (t_alg->registered)
3450                         crypto_unregister_aead(&t_alg->aead);
3451         }
3452
3453         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3454                 struct caam_skcipher_alg *t_alg = driver_algs + i;
3455
3456                 if (t_alg->registered)
3457                         crypto_unregister_skcipher(&t_alg->skcipher);
3458         }
3459 }
3460
3461 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
3462 {
3463         struct skcipher_alg *alg = &t_alg->skcipher;
3464
3465         alg->base.cra_module = THIS_MODULE;
3466         alg->base.cra_priority = CAAM_CRA_PRIORITY;
3467         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
3468         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
3469
3470         alg->init = caam_cra_init;
3471         alg->exit = caam_cra_exit;
3472 }
3473
3474 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
3475 {
3476         struct aead_alg *alg = &t_alg->aead;
3477
3478         alg->base.cra_module = THIS_MODULE;
3479         alg->base.cra_priority = CAAM_CRA_PRIORITY;
3480         alg->base.cra_ctxsize = sizeof(struct caam_ctx);
3481         alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
3482
3483         alg->init = caam_aead_init;
3484         alg->exit = caam_aead_exit;
3485 }
3486
3487 static int __init caam_algapi_init(void)
3488 {
3489         struct device_node *dev_node;
3490         struct platform_device *pdev;
3491         struct caam_drv_private *priv;
3492         int i = 0, err = 0;
3493         u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst;
3494         u32 arc4_inst;
3495         unsigned int md_limit = SHA512_DIGEST_SIZE;
3496         bool registered = false, gcm_support;
3497
3498         dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
3499         if (!dev_node) {
3500                 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
3501                 if (!dev_node)
3502                         return -ENODEV;
3503         }
3504
3505         pdev = of_find_device_by_node(dev_node);
3506         if (!pdev) {
3507                 of_node_put(dev_node);
3508                 return -ENODEV;
3509         }
3510
3511         priv = dev_get_drvdata(&pdev->dev);
3512         of_node_put(dev_node);
3513
3514         /*
3515          * If priv is NULL, it's probably because the caam driver wasn't
3516          * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
3517          */
3518         if (!priv) {
3519                 err = -ENODEV;
3520                 goto out_put_dev;
3521         }
3522
3523
3524         /*
3525          * Register crypto algorithms the device supports.
3526          * First, detect presence and attributes of DES, AES, and MD blocks.
3527          */
3528         if (priv->era < 10) {
3529                 u32 cha_vid, cha_inst, aes_rn;
3530
3531                 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
3532                 aes_vid = cha_vid & CHA_ID_LS_AES_MASK;
3533                 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
3534
3535                 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
3536                 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >>
3537                            CHA_ID_LS_DES_SHIFT;
3538                 aes_inst = cha_inst & CHA_ID_LS_AES_MASK;
3539                 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
3540                 arc4_inst = (cha_inst & CHA_ID_LS_ARC4_MASK) >>
3541                             CHA_ID_LS_ARC4_SHIFT;
3542                 ccha_inst = 0;
3543                 ptha_inst = 0;
3544
3545                 aes_rn = rd_reg32(&priv->ctrl->perfmon.cha_rev_ls) &
3546                          CHA_ID_LS_AES_MASK;
3547                 gcm_support = !(aes_vid == CHA_VER_VID_AES_LP && aes_rn < 8);
3548         } else {
3549                 u32 aesa, mdha;
3550
3551                 aesa = rd_reg32(&priv->ctrl->vreg.aesa);
3552                 mdha = rd_reg32(&priv->ctrl->vreg.mdha);
3553
3554                 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
3555                 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
3556
3557                 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK;
3558                 aes_inst = aesa & CHA_VER_NUM_MASK;
3559                 md_inst = mdha & CHA_VER_NUM_MASK;
3560                 ccha_inst = rd_reg32(&priv->ctrl->vreg.ccha) & CHA_VER_NUM_MASK;
3561                 ptha_inst = rd_reg32(&priv->ctrl->vreg.ptha) & CHA_VER_NUM_MASK;
3562                 arc4_inst = rd_reg32(&priv->ctrl->vreg.afha) & CHA_VER_NUM_MASK;
3563
3564                 gcm_support = aesa & CHA_VER_MISC_AES_GCM;
3565         }
3566
3567         /* If MD is present, limit digest size based on LP256 */
3568         if (md_inst && md_vid  == CHA_VER_VID_MD_LP256)
3569                 md_limit = SHA256_DIGEST_SIZE;
3570
3571         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3572                 struct caam_skcipher_alg *t_alg = driver_algs + i;
3573                 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
3574
3575                 /* Skip DES algorithms if not supported by device */
3576                 if (!des_inst &&
3577                     ((alg_sel == OP_ALG_ALGSEL_3DES) ||
3578                      (alg_sel == OP_ALG_ALGSEL_DES)))
3579                                 continue;
3580
3581                 /* Skip AES algorithms if not supported by device */
3582                 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
3583                                 continue;
3584
3585                 /* Skip ARC4 algorithms if not supported by device */
3586                 if (!arc4_inst && alg_sel == OP_ALG_ALGSEL_ARC4)
3587                         continue;
3588
3589                 /*
3590                  * Check support for AES modes not available
3591                  * on LP devices.
3592                  */
3593                 if (aes_vid == CHA_VER_VID_AES_LP &&
3594                     (t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK) ==
3595                     OP_ALG_AAI_XTS)
3596                         continue;
3597
3598                 caam_skcipher_alg_init(t_alg);
3599
3600                 err = crypto_register_skcipher(&t_alg->skcipher);
3601                 if (err) {
3602                         pr_warn("%s alg registration failed\n",
3603                                 t_alg->skcipher.base.cra_driver_name);
3604                         continue;
3605                 }
3606
3607                 t_alg->registered = true;
3608                 registered = true;
3609         }
3610
3611         for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
3612                 struct caam_aead_alg *t_alg = driver_aeads + i;
3613                 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
3614                                  OP_ALG_ALGSEL_MASK;
3615                 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
3616                                  OP_ALG_ALGSEL_MASK;
3617                 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
3618
3619                 /* Skip DES algorithms if not supported by device */
3620                 if (!des_inst &&
3621                     ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
3622                      (c1_alg_sel == OP_ALG_ALGSEL_DES)))
3623                                 continue;
3624
3625                 /* Skip AES algorithms if not supported by device */
3626                 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
3627                                 continue;
3628
3629                 /* Skip CHACHA20 algorithms if not supported by device */
3630                 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && !ccha_inst)
3631                         continue;
3632
3633                 /* Skip POLY1305 algorithms if not supported by device */
3634                 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && !ptha_inst)
3635                         continue;
3636
3637                 /* Skip GCM algorithms if not supported by device */
3638                 if (c1_alg_sel == OP_ALG_ALGSEL_AES &&
3639                     alg_aai == OP_ALG_AAI_GCM && !gcm_support)
3640                         continue;
3641
3642                 /*
3643                  * Skip algorithms requiring message digests
3644                  * if MD or MD size is not supported by device.
3645                  */
3646                 if (is_mdha(c2_alg_sel) &&
3647                     (!md_inst || t_alg->aead.maxauthsize > md_limit))
3648                         continue;
3649
3650                 caam_aead_alg_init(t_alg);
3651
3652                 err = crypto_register_aead(&t_alg->aead);
3653                 if (err) {
3654                         pr_warn("%s alg registration failed\n",
3655                                 t_alg->aead.base.cra_driver_name);
3656                         continue;
3657                 }
3658
3659                 t_alg->registered = true;
3660                 registered = true;
3661         }
3662
3663         if (registered)
3664                 pr_info("caam algorithms registered in /proc/crypto\n");
3665
3666 out_put_dev:
3667         put_device(&pdev->dev);
3668         return err;
3669 }
3670
3671 module_init(caam_algapi_init);
3672 module_exit(caam_algapi_exit);
3673
3674 MODULE_LICENSE("GPL");
3675 MODULE_DESCRIPTION("FSL CAAM support for crypto API");
3676 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");